diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c3be9c4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Oracle snippets must keep LF: the vendored grammars reject a backslash-CRLF +# continuation, and the oracle test skips nothing. +textmask/testdata/** text eol=lf diff --git a/astkit_test.go b/astkit_test.go index abd530d..40221ae 100644 --- a/astkit_test.go +++ b/astkit_test.go @@ -54,6 +54,20 @@ func TestDetectLanguageSniffsAmbiguousHeader(t *testing.T) { if got := astkit.DetectLanguage("point.h", "struct Point { int x; };"); got != astkit.LangC { t.Errorf("plain C header language = %q, want c", got) } + // Comments, `#if 0` branches and the C++-only branch of a __cplusplus + // guard do not make a C header C++; an unguarded extern "C" does. + for src, want := range map[string]astkit.LanguageKey{ + "/*\n class handling for the C API\n */\nint real(void);\n": astkit.LangC, + "#if 0\nclass Old {};\n#endif\nint real(void);\n": astkit.LangC, + "#ifdef __cplusplus\nextern \"C\" {\n#endif\nint real(void);\n#ifdef __cplusplus\n}\n#endif\n": astkit.LangC, + "#ifndef __cplusplus\nint c_only;\n#else\nnamespace n {}\n#endif\n": astkit.LangC, + "extern \"C\" {\nint real(void);\n}\n": astkit.LangCPP, + "#if defined(X)\nclass Shape {};\n#endif\n": astkit.LangCPP, + } { + if got := astkit.DetectLanguage("x.h", src); got != want { + t.Errorf("DetectLanguage(x.h, %q) = %q, want %q", src, got, want) + } + } } func TestIsAST_IsConfigData(t *testing.T) { diff --git a/cmd/decoy/main.go b/cmd/decoy/main.go new file mode 100644 index 0000000..8aa2b8e --- /dev/null +++ b/cmd/decoy/main.go @@ -0,0 +1,184 @@ +// Command decoy injects code-shaped comments (and Python docstrings) into +// every source file under a directory, in place, for metamorphic testing of +// an indexer: a comment cannot change what a program means, so an index of +// the decoyed tree must equal an index of the original (compared by name, +// since inserted lines shift spans). +// +// Usage: decoy +// +// Decoys go only where they cannot change the program: after a line whose +// last character is a code `{` (the newline is then in code, not inside a +// literal or comment), after a one-line Python def/class header, and at the +// end of the file. Insertion points are found on textmask output, so a `{` +// inside a string, comment or JSX text never qualifies. +package main + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/provasign/astkit" + "github.com/provasign/astkit/textmask" +) + +// One line each; no `*/`. The apostrophes and the lone quote are on purpose: +// a lexer that opens a literal on them blanks the code that follows. +const cDecoy = `// decoy: class Fake extends Base implements Iface, Other { private Bar client; ` + + `void ghost() { helper(1); new Fake(); client.go(); } } use Loggable; impl Shape for Wrapper {} ` + + `beta::normalize(x); Widget::draw(x); export default ghost; don't 'quote "unterminated` + +var cBlock = []string{ + `/* decoy block: don't 'quote`, + `#include "dead.h"`, + `class Fake2 extends Base implements Iface {`, + ` Fake helper = new Fake(); helper.run(); self.client = Bar()`, + `func Phantom() {} type Ghost struct{} fn ghost() {} def helper(x):`, + `*/`, +} + +const pyDecoy = `# decoy: class Fake(Base): def helper(x): self.client = Bar(); raise NotImplementedError; ` + + `import dead; don't 'quote "unterminated` + +var pyDoc = []string{ + `"""Decoy docstring. Don't 'quote.`, + ``, + `Attributes`, + `----------`, + `client : Fake`, + ``, + `>>> def helper(x):`, + `... raise NotImplementedError`, + `>>> class Phantom(Base): pass`, + `"""`, +} + +var skipDirs = map[string]bool{".git": true, ".grove": true, "node_modules": true, "vendor": true, + "third_party": true, "target": true, "build": true, "dist": true} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: decoy ") + os.Exit(2) + } + changed := 0 + err := filepath.WalkDir(os.Args[1], func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if skipDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() { + return nil + } + src, err := os.ReadFile(path) + if err != nil { + return err + } + lang := string(astkit.DetectLanguage(path, string(src))) + if !textmask.Supported(lang) || lang == "cobol" || lang == "jcl" { + return nil + } + out := inject(lang, string(src)) + if out == string(src) { + return nil + } + changed++ + return os.WriteFile(path, []byte(out), 0o644) + }) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(changed) +} + +func inject(lang, src string) string { + if strings.Contains(src, "\r\n") { + return src // keep line-ending handling out of it + } + lines := strings.Split(src, "\n") + masked := strings.Split(textmask.MaskFile(lang, src), "\n") + var out []string + for i, line := range lines { + out = append(out, line) + if lang == "python" { + out = append(out, pythonDecoys(lines, masked, i)...) + continue + } + code := strings.TrimRight(masked[i], " \t") + if strings.HasSuffix(code, "{") && strings.HasSuffix(strings.TrimRight(line, " \t"), "{") { + out = append(out, indentAfter(lines, i)+cDecoy) + } + } + tail := cBlock + if lang == "python" { + tail = []string{pyDecoy} + } + if lang == "php" && !strings.Contains(textmask.MaskFile(lang, src+"\n$x;"), "$x;") { + return strings.Join(out, "\n") // file ends in inline HTML: a comment there would be page text + } + if len(out) > 0 && out[len(out)-1] == "" { + out = append(out[:len(out)-1], append(append([]string{}, tail...), "")...) + } else { + out = append(out, tail...) + } + return strings.Join(out, "\n") +} + +// pythonDecoys returns the lines to insert after line i: a comment, plus a +// docstring when line i is a one-line def/class header. +func pythonDecoys(lines, masked []string, i int) []string { + code := strings.TrimSpace(masked[i]) + if !strings.HasSuffix(code, ":") || !strings.HasSuffix(strings.TrimRight(lines[i], " \t"), ":") { + return nil + } + if !(strings.HasPrefix(code, "def ") || strings.HasPrefix(code, "async def ") || strings.HasPrefix(code, "class ")) { + return nil + } + if strings.Count(code, "(") != strings.Count(code, ")") || strings.Count(code, "[") != strings.Count(code, "]") { + return nil + } + j := i + 1 + for j < len(lines) && strings.TrimSpace(lines[j]) == "" { + j++ + } + if j >= len(lines) { + return nil + } + body := leading(lines[j]) + if len(body) <= len(leading(lines[i])) { + return nil + } + out := []string{body + pyDecoy} + for _, l := range pyDoc { + if l == "" { + out = append(out, "") + } else { + out = append(out, body+l) + } + } + return out +} + +func indentAfter(lines []string, i int) string { + for j := i + 1; j < len(lines); j++ { + if strings.TrimSpace(lines[j]) != "" { + if ind := leading(lines[j]); ind != "" { + return ind + } + break + } + } + return leading(lines[i]) + " " +} + +func leading(s string) string { + return s[:len(s)-len(strings.TrimLeft(s, " \t"))] +} diff --git a/engine.go b/engine.go index fdce270..c9ad9ed 100644 --- a/engine.go +++ b/engine.go @@ -63,7 +63,13 @@ func (e *Engine) Parse(ctx context.Context, lang LanguageKey, src []byte) (*sitt p.SetLanguage(tsLang) cctx, cancel := context.WithTimeout(ctx, e.timeout) defer cancel() - tree, err := p.ParseCtx(cctx, nil, src) + var tree *sitter.Tree + var err error + if lang == LangKotlin { + tree, err = parseKotlinWithoutComments(cctx, p, src) + } else { + tree, err = p.ParseCtx(cctx, nil, src) + } if err != nil { return nil, fmt.Errorf("astkit: parse %s: %w", lang, err) } diff --git a/internalast/helpers.go b/internalast/helpers.go index 2adc638..8565212 100644 --- a/internalast/helpers.go +++ b/internalast/helpers.go @@ -4,6 +4,7 @@ package internalast import ( + "sort" "strings" sitter "github.com/smacker/go-tree-sitter" @@ -123,11 +124,21 @@ func FirstLine(s string) string { // SignatureBeforeBody returns the declaration header of n — everything from // the node start up to (not including) its "body" child — with annotation -// nodes excised and whitespace collapsed to single spaces. Unlike FirstLine, -// this survives declarations that wrap `extends`/`implements` clauses or -// parameter lists onto continuation lines, and never returns a leading -// `@Override` line as the signature. Falls back to the whole node text when -// there is no body (e.g. abstract or interface methods). +// nodes, attribute sections and comments excised and whitespace collapsed to +// single spaces. Unlike FirstLine, this survives declarations that wrap +// `extends`/`implements` clauses or parameter lists onto continuation lines, +// and never returns a leading `@Override` line as the signature. Falls back +// to the whole node text when there is no body (e.g. abstract or interface +// methods). +// +// Comments are cut from the tree (they are tree-sitter extras), so a +// trailing line comment is never glued into the collapsed header: jackson's +// +// public final class ManagedReferenceProperty // Changed in 2.9 +// extends SettableBeanProperty.Delegating +// +// stays "… ManagedReferenceProperty extends …", and a Python +// `class A(Base): # was Fake` keeps no Fake. func SignatureBeforeBody(n *sitter.Node, src []byte) string { if n == nil { return "" @@ -146,7 +157,7 @@ func SignatureBeforeBody(n *sitter.Node, src []byte) string { // Annotations live inside the "modifiers" child; excise their byte ranges // so they never masquerade as (or pollute) the signature. They remain // available on the symbol's Annotations field. - var cuts [][2]uint32 + cuts := headerCuts(n, start, end, IsAttributeSection) if mods := FindChildByType(n, "modifiers"); mods != nil { count := int(mods.ChildCount()) for i := 0; i < count; i++ { @@ -159,96 +170,193 @@ func SignatureBeforeBody(n *sitter.Node, src []byte) string { } } } - var b strings.Builder + return strings.Join(strings.Fields(blankRanges(src, start, end, cuts)), " ") +} + +// IsComment reports whether n is a comment node in any astkit grammar. +func IsComment(n *sitter.Node) bool { + switch n.Type() { + case "comment", "line_comment", "block_comment", "multiline_comment": + return true + } + return false +} + +// IsAttributeSection reports whether n is a C# or PHP attribute section +// (`[Obsolete("x")]`, `#[Route("/x")]`). Like Java annotations, these are +// not part of a signature; C# keeps them on the symbol's Annotations. +func IsAttributeSection(n *sitter.Node) bool { + return n.Type() == "attribute_list" +} + +// headerCuts returns the byte ranges, clipped to [start, end), of the +// comment nodes under n and of n's direct children for which drop reports +// true. Only children overlapping the range are descended, so a header +// costs a walk of the header's nodes, not the whole declaration. +func headerCuts(n *sitter.Node, start, end uint32, drop func(*sitter.Node) bool) [][2]uint32 { + var cuts [][2]uint32 + add := func(c *sitter.Node) { + cuts = append(cuts, [2]uint32{max(c.StartByte(), start), min(c.EndByte(), end)}) + } + var walk func(p *sitter.Node, direct bool) + walk = func(p *sitter.Node, direct bool) { + count := int(p.ChildCount()) + for i := 0; i < count; i++ { + c := p.Child(i) + if c == nil || c.EndByte() <= start { + continue + } + if c.StartByte() >= end { + break + } + switch { + case IsComment(c), direct && drop != nil && drop(c): + add(c) + case c.ChildCount() > 0: + walk(c, false) + } + } + } + walk(n, true) + return cuts +} + +// blankRanges returns src[start:end] with the cut ranges replaced by spaces +// (newlines kept), so tokens on either side of a cut stay separated. +func blankRanges(src []byte, start, end uint32, cuts [][2]uint32) string { + if len(cuts) == 0 { + return string(src[start:end]) + } + b := []byte(string(src[start:end])) + for _, c := range cuts { + for i := max(c[0], start); i < min(c[1], end); i++ { + if ch := b[i-start]; ch != '\n' && ch != '\r' { + b[i-start] = ' ' + } + } + } + return string(b) +} + +// HeaderText returns src[start:end] (a declaration header inside n) without +// the comments under n and the direct children of n for which drop (may be +// nil) reports true. Text with nothing to remove is returned unchanged. When +// something was removed, the blanks around each cut on its line collapse to +// one space (none before `,;)]>` or after `([<`), trailing blanks are +// trimmed from each line and lines left empty are dropped, so +// `f(a, // note` reads `f(a,` and `f(app /* T */, b)` reads `f(app, b)`. +func HeaderText(n *sitter.Node, src []byte, start, end uint32, drop func(*sitter.Node) bool) string { + return HeaderTextCut(n, src, start, end, drop, nil) +} + +// HeaderTextCut is HeaderText with extra byte ranges of src removed as +// comments are: for a tree parsed without its comments (astkit parses +// Kotlin that way), the caller finds them in the text. +func HeaderTextCut(n *sitter.Node, src []byte, start, end uint32, drop func(*sitter.Node) bool, extra [][2]uint32) string { + if int(end) > len(src) { + end = uint32(len(src)) + } + if n == nil || start >= end { + return "" + } + cuts := headerCuts(n, start, end, drop) + for _, c := range extra { + if c[1] > start && c[0] < end { + cuts = append(cuts, [2]uint32{max(c[0], start), min(c[1], end)}) + } + } + if len(cuts) == 0 { + return string(src[start:end]) + } + sort.Slice(cuts, func(i, j int) bool { return cuts[i][0] < cuts[j][0] }) + var out []byte + cut := false // a cut sits between out and the next segment + appendSeg := func(seg []byte) { + if cut { + left := len(out) + for left > 0 && (out[left-1] == ' ' || out[left-1] == '\t') { + left-- + } + out = out[:left] + for len(seg) > 0 && (seg[0] == ' ' || seg[0] == '\t') { + seg = seg[1:] + } + if left > 0 && len(seg) > 0 && out[left-1] != '\n' && !strings.ContainsRune("([<", rune(out[left-1])) && + !strings.ContainsRune(",;)]>\r\n", rune(seg[0])) { + out = append(out, ' ') + } + cut = false + } + out = append(out, seg...) + } pos := start - for _, cut := range cuts { - if cut[0] >= end { - break + for _, c := range cuts { + if c[1] <= pos { + continue } - if cut[0] > pos { - b.Write(src[pos:cut[0]]) + if c[0] > pos { + appendSeg(src[pos:c[0]]) } - if cut[1] > pos { - pos = cut[1] + // Keep the newlines a cut spans: a block comment over several + // lines still ends each of them. + for i := max(c[0], pos); i < c[1]; i++ { + if src[i] == '\n' { + appendSeg([]byte{'\n'}) + } } + cut = true + pos = c[1] } if pos < end { - b.Write(src[pos:end]) + appendSeg(src[pos:end]) } - return strings.Join(strings.Fields(stripHeaderComments(b.String())), " ") + return tidyLines(string(out)) } -// stripHeaderComments removes line comments (`// …` to end of physical line) -// and block comments from a declaration header BEFORE whitespace collapsing. -// The collapse joins continuation lines, so a trailing line comment would -// otherwise be glued into the signature: jackson's -// -// public final class ManagedReferenceProperty // Changed in 2.9 -// extends SettableBeanProperty.Delegating -// -// became "… ManagedReferenceProperty // Changed in 2.9 extends …", burying -// the real extends clause behind comment text and corrupting every consumer -// that parses the signature. Quote-aware so `//` inside a string literal -// (annotation arguments) survives. -func stripHeaderComments(s string) string { - var out strings.Builder - out.Grow(len(s)) - inStr, inChar, inBlock := false, false, false - for i := 0; i < len(s); i++ { - c := s[i] - if inBlock { - if c == '*' && i+1 < len(s) && s[i+1] == '/' { - inBlock = false - i++ - // Replace the whole comment with a space so tokens that - // abutted it stay separated: `Foo/*x*/extends Bar` must not - // collapse to `Fooextends Bar` (which hides the extends - // clause from downstream regexes). Later Fields() collapses - // runs of whitespace, so one space is enough. - out.WriteByte(' ') - } - continue +func tidyLines(s string) string { + lines := strings.Split(s, "\n") + kept := lines[:0] + for _, l := range lines { + if l = strings.TrimRight(l, " \t\r"); strings.TrimSpace(l) != "" { + kept = append(kept, l) } - if inStr { - out.WriteByte(c) - if c == '\\' && i+1 < len(s) { - out.WriteByte(s[i+1]) - i++ - } else if c == '"' { - inStr = false - } - continue + } + return strings.TrimSpace(strings.Join(kept, "\n")) +} + +// FirstLineSig is FirstLine(n.Content(src)) without comments and without +// the direct children of n for which drop (may be nil) reports true: the +// first line that still has code once they are removed. With nothing to +// remove it equals FirstLine(n.Content(src)). +func FirstLineSig(n *sitter.Node, src []byte, drop func(*sitter.Node) bool) string { + return FirstLineSigCut(n, src, drop, nil) +} + +// FirstLineSigCut is FirstLineSig with extra byte ranges of src removed as +// comments are (see HeaderTextCut). +func FirstLineSigCut(n *sitter.Node, src []byte, drop func(*sitter.Node) bool, extra [][2]uint32) string { + if n == nil { + return "" + } + start, end := n.StartByte(), n.EndByte() + if int(end) > len(src) { + end = uint32(len(src)) + } + for start < end { + lineEnd := end + if i := strings.IndexByte(string(src[start:end]), '\n'); i >= 0 { + lineEnd = start + uint32(i) } - if inChar { - out.WriteByte(c) - if c == '\\' && i+1 < len(s) { - out.WriteByte(s[i+1]) - i++ - } else if c == '\'' { - inChar = false - } - continue + if lineEnd == n.StartByte() { + return "" // the node starts with a line break, as FirstLine reads it } - switch { - case c == '"': - inStr = true - out.WriteByte(c) - case c == '\'': - inChar = true - out.WriteByte(c) - case c == '/' && i+1 < len(s) && s[i+1] == '/': - for i < len(s) && s[i] != '\n' { - i++ - } - if i < len(s) { - out.WriteByte('\n') + if line := HeaderTextCut(n, src, start, lineEnd, drop, extra); line != "" { + if start != n.StartByte() { + line = strings.TrimSpace(line) // a later line's indentation } - case c == '/' && i+1 < len(s) && s[i+1] == '*': - inBlock = true - i++ - default: - out.WriteByte(c) + return line } + start = lineEnd + 1 } - return out.String() + return "" } diff --git a/kotlinparse.go b/kotlinparse.go new file mode 100644 index 0000000..f2f8717 --- /dev/null +++ b/kotlinparse.go @@ -0,0 +1,121 @@ +package astkit + +import ( + "context" + + sitter "github.com/smacker/go-tree-sitter" + + "github.com/provasign/astkit/textmask" +) + +// parseKotlinWithoutComments parses Kotlin source as if it had no comments. +// +// The vendored Kotlin grammar fails on a good deal of valid modern Kotlin +// (`object : X by y`, `f(x).p = v`, a primary constructor on the line after +// the class name), and how it recovers depends on the comments around the +// failure: comment tokens are extras the recovery can attach anywhere, and +// tree-sitter prices each candidate recovery by the bytes and lines it +// skips, comment lines included. Adding a comment line inside a class could +// turn the class into an ERROR node and drop its members, or surface a class +// from a commented-out block. A comment cannot change what a program means, +// so the parser never sees one: +// +// - every comment is blanked (textmask, offsets kept); +// - every line that held only comment text is cut out, so the parser's +// input, and every recovery cost, is the same with or without them; +// - the cut lines are put back with tree edits, which move node positions +// without re-parsing, so the tree addresses src's bytes, rows and +// columns as a normal parse does. +// +// The tree has no comment nodes: callers reading text between nodes must +// mask it themselves (textmask.MaskComments). +func parseKotlinWithoutComments(ctx context.Context, p *sitter.Parser, src []byte) (*sitter.Tree, error) { + masked := textmask.MaskComments(string(LangKotlin), string(src)) + cuts := kotlinCommentOnlyLines(src, masked) + if len(cuts) == 0 { + return p.ParseCtx(ctx, nil, []byte(masked)) + } + compact := make([]byte, 0, len(masked)) + prev := 0 + for _, c := range cuts { + compact = append(compact, masked[prev:c[0]]...) + prev = c[1] + } + compact = append(compact, masked[prev:]...) + tree, err := p.ParseCtx(ctx, nil, compact) + if err != nil || tree == nil { + return tree, err + } + // Put the cut lines back front to back: everything before a cut is + // already back in place, so its src offset and point are also its + // position in the edited tree. + var at points + for _, c := range cuts { + start := at.advance(src, c[0]) + if !treeEdit(tree, sitter.EditInput{ + StartIndex: uint32(c[0]), OldEndIndex: uint32(c[0]), NewEndIndex: uint32(c[1]), + StartPoint: start, OldEndPoint: start, NewEndPoint: at.advance(src, c[1]), + }) { + // Unreachable while go-tree-sitter keeps its layout (a test + // checks it); positions would be wrong, so parse in place. + tree.Close() + return p.ParseCtx(ctx, nil, []byte(masked)) + } + } + return tree, nil +} + +// kotlinCommentOnlyLines returns the byte ranges of src's lines (each with +// its newline) that hold comment text and nothing else, merged into +// maximal runs. masked is src with comments blanked. +func kotlinCommentOnlyLines(src []byte, masked string) [][2]int { + var cuts [][2]int + for ls := 0; ls < len(src); { + le := ls + for le < len(src) && src[le] != '\n' { + le++ + } + if le < len(src) { + le++ // include the newline + } + if isBlank(masked[ls:le]) && !isBlank(string(src[ls:le])) { + if n := len(cuts); n > 0 && cuts[n-1][1] == ls { + cuts[n-1][1] = le + } else { + cuts = append(cuts, [2]int{ls, le}) + } + } + ls = le + } + return cuts +} + +func isBlank(s string) bool { + for i := 0; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\r', '\n', '\f', '\v': + default: + return false + } + } + return true +} + +// points converts increasing byte offsets of one source to tree-sitter +// row/column points (columns in bytes) in a single forward pass. +type points struct { + off int + pt sitter.Point +} + +func (p *points) advance(src []byte, off int) sitter.Point { + for ; p.off < off; p.off++ { + if src[p.off] == '\n' { + p.pt.Row++ + p.pt.Column = 0 + } else { + p.pt.Column++ + } + } + return p.pt +} diff --git a/kotlinparse_test.go b/kotlinparse_test.go new file mode 100644 index 0000000..08e6aba --- /dev/null +++ b/kotlinparse_test.go @@ -0,0 +1,203 @@ +package astkit_test + +import ( + "context" + "fmt" + "strings" + "testing" + + sitter "github.com/smacker/go-tree-sitter" + "github.com/smacker/go-tree-sitter/kotlin" + + "github.com/provasign/astkit" + "github.com/provasign/astkit/strategies" + "github.com/provasign/astkit/textmask" +) + +// namedNodes lists every named non-comment node with its position. A +// node's end is taken back over trailing blanks in masked (the source with +// comments blanked): a node the grammar ends with its newlines, such as +// import_list, may run on over blank or comment-only lines, which says +// nothing about where its code is. +func namedNodes(n *sitter.Node, masked string, out *[]string) { + switch n.Type() { + case "line_comment", "multiline_comment": + return + } + if n.IsNamed() && n.Type() != "source_file" { + end := int(n.EndByte()) + for end > int(n.StartByte()) && strings.ContainsRune(" \t\r\n", rune(masked[end-1])) { + end-- + } + row := strings.Count(masked[:end], "\n") + col := end - (strings.LastIndexByte(masked[:end], '\n') + 1) + *out = append(*out, fmt.Sprintf("%s %d-%d %v-%d:%d", n.Type(), n.StartByte(), end, n.StartPoint(), row, col)) + } + for i := 0; i < int(n.ChildCount()); i++ { + namedNodes(n.Child(i), masked, out) + } +} + +// A Kotlin tree parsed without its comments addresses the source exactly as +// a parse of the comment-blanked source does: same nodes, same bytes, rows +// and columns. (A parse of the raw source is not the reference: comments +// change how the grammar groups `@Suppress("x")` and an import's extent.) +// It also checks treeEdit still reaches go-tree-sitter's native tree. +func TestKotlinParseWithoutCommentsKeepsPositions(t *testing.T) { + src := []byte(`/* + * License header. + */ +package demo + +// Imports follow. +import kotlin.math.max // trailing + +/** A greeter. */ +@Suppress("x") +// between the annotation and the class +class Greeter( + // the name + private val name: String, /* inline */ val n: Int, +) : Base(), Iface { + // first member + val greeting = "hi /* not a comment */ $name" // trailing + + /* + * A block. + */ + fun greet(times: Int /* count */): String { + // body comment + return greeting.repeat(max(times, 1)) + } +} +// end of file`) + p := sitter.NewParser() + p.SetLanguage(kotlin.GetLanguage()) + masked := textmask.MaskComments("kotlin", string(src)) + plain, err := p.ParseCtx(context.Background(), nil, []byte(masked)) + if err != nil { + t.Fatal(err) + } + defer plain.Close() + if plain.RootNode().HasError() { + t.Fatalf("fixture must parse cleanly: %s", plain.RootNode().String()) + } + tree, err := astkit.NewEngine().Parse(context.Background(), astkit.LangKotlin, src) + if err != nil { + t.Fatal(err) + } + defer tree.Close() + var want, got []string + namedNodes(plain.RootNode(), masked, &want) + namedNodes(tree.RootNode(), masked, &got) + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("nodes differ:\n got %v\nwant %v", got, want) + } +} + +// Comment lines cannot change what is extracted from Kotlin the grammar +// cannot parse. Before comment lines were cut from the parser's input, a +// comment after a `{` moved the error recovery so a class became an ERROR +// node and lost its members, a class body became a lambda, or method +// locals became fields. +func TestKotlinCommentLinesDoNotMoveErrorRecovery(t *testing.T) { + cases := map[string]string{ + // Cut down from okio's ByteString.kt: a primary constructor on the + // line after `expect open class ByteString` (not in the grammar). + "constructor-line": `internal constructor(data: ByteArray) : Comparable { + fun hex(): String +`, + // Cut down from okio's CommonRealBufferedSourceTest.kt, whose + // `object : Source by buffer` the grammar cannot parse. + "unclosed-body": `class CommonRealBufferedSourceTest { + val bufferedSource = ( + assertEquals(6, buffer.size) + assertEquals(-1, bufferedSource.indexOf('e'.code.toByte(), 0, 4)) + assertEquals(2, buffer.size) + } +`, + // Cut down from okio's ZipBuilder.kt. + "unclosed-method": `class ZipBuilder( +) { + fun build(): Path { + val result = process.waitFor() + .apply { environment()["TZ"] = "UTC" } + require(exitCode == 0) + } +`, + // From moshi's LinkedHashTreeMap.kt: assigning through a call + // result is not in the grammar. + "call-receiver-assign": `package moshi + +internal class Tree(comparator: Comparator? = null) : AbstractMutableMap() { + private var size = 0 + + fun removeInternal(node: Node, unlink: Boolean) { + if (unlink) { + knownNotNull(node.prev).next = node.next + node.prev = null + } + var left = node.left + if (left != null) { + size-- + } + } + + fun other(): Int { + return size + } +} +`, + } + for name, clean := range cases { + t.Run(name, func(t *testing.T) { + decoy := injectCommentLines(clean) + if got, want := kotlinFacts(t, decoy), kotlinFacts(t, clean); got != want { + t.Fatalf("comment lines changed the symbols:\n got %s\nwant %s", got, want) + } + }) + } +} + +// injectCommentLines adds a code-shaped comment line after every line +// ending in `{`, indented like the next line, and a block comment at the +// end, the way cmd/decoy does. +func injectCommentLines(src string) string { + lines := strings.Split(src, "\n") + var b strings.Builder + for i, line := range lines { + b.WriteString(line + "\n") + if !strings.HasSuffix(line, "{") { + continue + } + indent := " " + for _, next := range lines[i+1:] { + if strings.TrimSpace(next) != "" { + indent = next[:len(next)-len(strings.TrimLeft(next, " "))] + break + } + } + b.WriteString(indent + "// decoy: class Fake extends Base implements Iface, Other { private Bar client; void ghost() { helper(1); new Fake(); client.go(); } } use Loggable; impl Shape for Wrapper {} beta::normalize(x); Widget::draw(x); export default ghost; don't 'quote \"unterminated\n") + } + b.WriteString("/* decoy block: don't 'quote\n#include \"dead.h\"\nclass Fake2 extends Base implements Iface {\n Fake helper = new Fake(); helper.run(); self.client = Bar()\nfunc Phantom() {} type Ghost struct{} fn ghost() {} def helper(x):\n*/\n") + return b.String() +} + +// kotlinFacts lists the extracted symbols without positions. +func kotlinFacts(t *testing.T, src string) string { + t.Helper() + tree, err := astkit.NewEngine().Parse(context.Background(), astkit.LangKotlin, []byte(src)) + if err != nil { + t.Fatal(err) + } + defer tree.Close() + syms, err := strategies.Default().Extract(astkit.LangKotlin, tree, []byte(src)) + if err != nil { + t.Fatal(err) + } + var out []string + for _, s := range syms { + out = append(out, fmt.Sprintf("%s %s %q", s.Kind, s.QualifiedName, s.Signature)) + } + return strings.Join(out, "\n") +} diff --git a/lang.go b/lang.go index 39c4074..0c2f2ab 100644 --- a/lang.go +++ b/lang.go @@ -4,6 +4,8 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/provasign/astkit/textmask" ) // LanguageKey identifies a programming language or data format known to @@ -45,6 +47,75 @@ const ( var cppHeaderMarker = regexp.MustCompile(`(?m)^\s*(?:template\s*<|namespace\b|class\s+[A-Za-z_]|extern\s+"C")`) +// cppGuard matches a conditional on __cplusplus: group 1 is set for the +// negated forms (`#ifndef __cplusplus`, `#if !defined(__cplusplus)`). +var cppGuard = regexp.MustCompile(`^#\s*(?:ifdef\s+__cplusplus\b|if\s+(?:defined\s*\(?\s*__cplusplus\b|__cplusplus\b)|(ifndef\s+__cplusplus\b|if\s+!\s*defined\s*\(?\s*__cplusplus\b))`) + +// isCPPHeader reports whether a .h header holds a C++-only declaration form +// in code a C compiler would see. Comments and `#if 0` branches are masked, +// and the C++-only branches of __cplusplus conditionals are blanked: the +// C-library idiom `#ifdef __cplusplus / extern "C" { / #endif` keeps a +// header C, while an unguarded `extern "C"` (invalid C) makes it C++. Grove's +// header sniff (internal/parser sniff.go sniffHeaderCode) applies the same +// rule, so both layers agree on a header's language. +func isCPPHeader(content string) bool { + if !cppHeaderMarker.MatchString(content) { + return false + } + code := textmask.MaskComments("cpp", textmask.MaskInactivePreprocessor("cpp", content)) + if strings.Contains(code, "__cplusplus") { + code = blankCPPOnlyBranches(code) + } + return cppHeaderMarker.MatchString(code) +} + +// blankCPPOnlyBranches blanks the lines of code that only a C++ compiler +// sees: the `#ifdef __cplusplus` branch, or the `#else` of an +// `#ifndef __cplusplus`. +func blankCPPOnlyBranches(code string) string { + lines := strings.Split(code, "\n") + type frame struct { + guard int // 1: #ifdef __cplusplus, -1: #ifndef __cplusplus, 0: other + cppOnly bool + } + var stack []frame + for i, line := range lines { + t := strings.TrimSpace(line) + directive := strings.HasPrefix(t, "#") + if directive { + d := strings.TrimSpace(strings.TrimPrefix(t, "#")) + switch { + case strings.HasPrefix(d, "if"): + f := frame{} + if m := cppGuard.FindStringSubmatch(t); m != nil { + f.guard = 1 + if m[1] != "" { + f.guard = -1 + } + f.cppOnly = f.guard == 1 + } + stack = append(stack, f) + case strings.HasPrefix(d, "el"): + if n := len(stack); n > 0 && stack[n-1].guard != 0 { + stack[n-1].cppOnly = stack[n-1].guard == -1 + } + case strings.HasPrefix(d, "endif"): + if n := len(stack); n > 0 { + stack = stack[:n-1] + } + } + continue + } + for _, f := range stack { + if f.cppOnly { + lines[i] = "" + break + } + } + } + return strings.Join(lines, "\n") +} + // DetectLanguage returns the language for a given file path. Ambiguous .h // headers are parsed as C++ when their contents contain a declaration form // that is not valid C; otherwise they retain the conservative C default. @@ -67,7 +138,7 @@ func DetectLanguage(path, content string) LanguageKey { case ".c": return LangC case ".h": - if cppHeaderMarker.MatchString(content) { + if isCPPHeader(content) { return LangCPP } return LangC diff --git a/strategies/cobol.go b/strategies/cobol.go index f43531a..f681943 100644 --- a/strategies/cobol.go +++ b/strategies/cobol.go @@ -7,6 +7,7 @@ import ( sitter "github.com/smacker/go-tree-sitter" "github.com/provasign/astkit" + "github.com/provasign/astkit/textmask" ) // COBOL symbol kinds. Consumers treat unknown kinds as "other", so these are @@ -37,9 +38,15 @@ func (c *cobolStrategy) ExtractsFromText() bool { return true } // srcLine is one normalized line of code: the code-area text with its // original 1-based line number, so every symbol cites a real location. +// COBOL lines also carry two masked views of text, byte-aligned with it: +// code (comments and literals blanked: where verbs and names are read) and +// lits (comments blanked, literals kept: where `CALL 'PROG'` targets are +// read). JCL lines leave them empty. type srcLine struct { text string orig int + code string + lits string } // normalizeCOBOL applies the column model ahead of extraction. Fixed-format @@ -94,9 +101,34 @@ func normalizeCOBOL(src []byte) []srcLine { } out = append(out, srcLine{text: code, orig: i + 1}) } + maskCOBOLLines(out) return out } +// maskCOBOLLines fills each line's code and lits views. The lines are +// already column-normalized (sequence and indicator areas gone), so they +// are masked as free-format source: a `*>` inline comment is blanked in +// either source format, and `DISPLAY 'PLEASE PERFORM BACKUP'` hides its +// PERFORM. A continued literal is joined with a space, so its tail after the +// joint can read as code; that only loses precision on such lines. +func maskCOBOLLines(lines []srcLine) { + if len(lines) == 0 { + return + } + var b strings.Builder + b.WriteString(">>SOURCE FORMAT FREE") + for _, ln := range lines { + b.WriteByte('\n') + b.WriteString(strings.ReplaceAll(ln.text, "\r", " ")) + } + joined := b.String() + code := strings.Split(textmask.Mask("cobol", joined), "\n")[1:] + lits := strings.Split(textmask.MaskComments("cobol", joined), "\n")[1:] + for i := range lines { + lines[i].code, lines[i].lits = code[i], lits[i] + } +} + func detectFixedFormat(lines []string) bool { fixedShaped, sampled := 0, 0 for _, line := range lines { @@ -136,8 +168,9 @@ var ( reFD = regexp.MustCompile(`(?i)^\s*FD\s+([A-Za-z0-9-]+)`) reSelect = regexp.MustCompile(`(?i)\bSELECT\s+(?:OPTIONAL\s+)?([A-Za-z0-9-]+)\s+ASSIGN\s+TO\s+([A-Za-z0-9-]+)`) rePerform = regexp.MustCompile(`(?i)\bPERFORM\s+([A-Za-z0-9][A-Za-z0-9-]*)(?:\s+(?:THRU|THROUGH)\s+([A-Za-z0-9-]+))?`) - reCallLit = regexp.MustCompile(`(?i)\bCALL\s+['"]([^'"]+)['"]`) - reCallVar = regexp.MustCompile(`(?i)\bCALL\s+([A-Za-z][A-Za-z0-9-]*)`) + reCallVerb = regexp.MustCompile(`(?i)\bCALL\s`) + reCallLit = regexp.MustCompile(`(?i)^CALL\s+['"]([^'"]+)['"]`) + reCallVar = regexp.MustCompile(`(?i)^CALL\s+([A-Za-z][A-Za-z0-9-]*)`) // Member names cannot start with a digit (PDS naming): a leading-digit // match is a sequence number or numeric operand, not a member // (field-reported: 68 numeric "members" like 053300). @@ -188,16 +221,18 @@ func (c *cobolStrategy) Extract(tree *sitter.Tree, src []byte) ([]astkit.Symbol, } for _, ln := range lines { - if m := reDivision.FindStringSubmatch(ln.text); m != nil { + // Structure and verbs are matched on ln.code (comments and literals + // blanked); signatures keep the line minus its comments (ln.lits). + if m := reDivision.FindStringSubmatch(ln.code); m != nil { division = strings.ToUpper(m[1]) continue } - if m := reProgramID.FindStringSubmatch(ln.text); m != nil { + if m := reProgramID.FindStringSubmatch(ln.lits); m != nil { flushProc() name := strings.TrimSuffix(m[1], ".") s := astkit.Symbol{ Kind: kindProgram, Name: name, QualifiedName: name, - Signature: strings.TrimSpace(ln.text), + Signature: strings.TrimSpace(ln.lits), Span: astkit.LineRange{Start: ln.orig, End: ln.orig}, Exported: true, } @@ -209,20 +244,20 @@ func (c *cobolStrategy) Extract(tree *sitter.Tree, src []byte) ([]astkit.Symbol, switch division { case "ENVIRONMENT": - if m := reSelect.FindStringSubmatch(ln.text); m != nil { + if m := reSelect.FindStringSubmatch(ln.code); m != nil { syms = append(syms, astkit.Symbol{ Kind: kindLogicalFile, Name: m[1], QualifiedName: m[1], ParentName: programName, - Signature: strings.TrimSpace(ln.text), + Signature: strings.TrimSpace(ln.lits), Span: astkit.LineRange{Start: ln.orig, End: ln.orig}, }) } case "DATA": - if m := reFD.FindStringSubmatch(ln.text); m != nil { + if m := reFD.FindStringSubmatch(ln.code); m != nil { stack = stack[:0] continue } - if m := reDataItem.FindStringSubmatch(ln.text); m != nil { + if m := reDataItem.FindStringSubmatch(ln.code); m != nil { level := parseLevel(m[1]) name := m[2] rest := m[3] @@ -260,7 +295,7 @@ func (c *cobolStrategy) Extract(tree *sitter.Tree, src []byte) ([]astkit.Symbol, qn = lastItem + "." + name } } - sig := strings.TrimSpace(strings.TrimSuffix(ln.text, ".")) + sig := strings.TrimSpace(strings.TrimSuffix(ln.lits, ".")) sym := astkit.Symbol{ Kind: kind, Name: name, QualifiedName: qn, ParentName: parent, Signature: sig, @@ -286,26 +321,26 @@ func (c *cobolStrategy) Extract(tree *sitter.Tree, src []byte) ([]astkit.Symbol, continue } case "PROCEDURE": - if m := reSection.FindStringSubmatch(ln.text); m != nil { + if m := reSection.FindStringSubmatch(ln.code); m != nil { flushProc() name := m[1] s := astkit.Symbol{ Kind: kindSection, Name: name, QualifiedName: name, ParentName: programName, - Signature: strings.TrimSpace(ln.text), + Signature: strings.TrimSpace(ln.lits), Span: astkit.LineRange{Start: ln.orig, End: ln.orig}, Body: strings.TrimSpace(ln.text), } currentProc = &s continue } - if m := reParagraph.FindStringSubmatch(ln.text); m != nil && !reReserved.MatchString(m[1]) { + if m := reParagraph.FindStringSubmatch(ln.code); m != nil && !reReserved.MatchString(m[1]) { flushProc() name := m[1] s := astkit.Symbol{ Kind: kindParagraph, Name: name, QualifiedName: name, ParentName: programName, - Signature: strings.TrimSpace(ln.text), + Signature: strings.TrimSpace(ln.lits), Span: astkit.LineRange{Start: ln.orig, End: ln.orig}, Body: strings.TrimSpace(ln.text), } @@ -317,7 +352,7 @@ func (c *cobolStrategy) Extract(tree *sitter.Tree, src []byte) ([]astkit.Symbol, target = &syms[progIndex] } if target != nil { - for _, pm := range rePerform.FindAllStringSubmatch(ln.text, -1) { + for _, pm := range rePerform.FindAllStringSubmatch(ln.code, -1) { if !reReserved.MatchString(pm[1]) { target.CallSites = append(target.CallSites, astkit.CallSite{Callee: pm[1], Line: ln.orig}) } @@ -326,14 +361,18 @@ func (c *cobolStrategy) Extract(tree *sitter.Tree, src []byte) ([]astkit.Symbol, performRanges = append(performRanges, performRange{target.QualifiedName, pm[1], pm[2], ln.orig}) } } - if cm := reCallLit.FindStringSubmatch(ln.text); cm != nil { - target.CallSites = append(target.CallSites, astkit.CallSite{Callee: cm[1], Line: ln.orig}) - } else if cm := reCallVar.FindStringSubmatch(ln.text); cm != nil && !strings.EqualFold(cm[1], "FUNCTION") { - // Dynamic call through a variable: record the variable name - // so the edge exists as a known-unknown rather than vanishing. - target.CallSites = append(target.CallSites, astkit.CallSite{ - Callee: cm[1], Line: ln.orig, Args: []string{"dynamic"}, - }) + // The CALL verb is found in code; its literal target is read + // at the same offset in lits, where the literal survives. + if loc := reCallVerb.FindStringIndex(ln.code); loc != nil { + if cm := reCallLit.FindStringSubmatch(ln.lits[loc[0]:]); cm != nil { + target.CallSites = append(target.CallSites, astkit.CallSite{Callee: cm[1], Line: ln.orig}) + } else if cm := reCallVar.FindStringSubmatch(ln.code[loc[0]:]); cm != nil && !strings.EqualFold(cm[1], "FUNCTION") { + // Dynamic call through a variable: record the variable name + // so the edge exists as a known-unknown rather than vanishing. + target.CallSites = append(target.CallSites, astkit.CallSite{ + Callee: cm[1], Line: ln.orig, Args: []string{"dynamic"}, + }) + } } } } @@ -383,6 +422,17 @@ func (c *cobolStrategy) Extract(tree *sitter.Tree, src []byte) ([]astkit.Symbol, return syms, nil } +// cobolTrimLike trims masked the way strings.TrimSpace trims text (both are +// byte-aligned), so a joined line's views stay aligned with its text. +func cobolTrimLike(text, masked string) string { + lead := len(text) - len(strings.TrimLeft(text, " \t\r\n")) + trail := len(strings.TrimRight(text, " \t\r\n")) + if trail < lead { + return "" + } + return masked[lead:trail] +} + func joinCOBOLDataClauses(lines []srcLine) []srcLine { out := make([]srcLine, 0, len(lines)) division := "DATA" @@ -403,6 +453,8 @@ func joinCOBOLDataClauses(lines []srcLine) []srcLine { break } line.text += " " + strings.TrimSpace(next.text) + line.code += " " + cobolTrimLike(next.text, next.code) + line.lits += " " + cobolTrimLike(next.text, next.lits) idx++ if strings.HasSuffix(strings.TrimSpace(next.text), ".") { break @@ -418,8 +470,8 @@ var ( // extraction: REPLACING arguments legally contain '&', '#' and even the // word COPY, and a member name harvested from them is confidently wrong // (observed on a real estate: 54 members extracted from REPLACING args). + // Literals and comments are already blanked in srcLine.code. rePseudoText = regexp.MustCompile(`==[^=]*(?:=[^=]+)*==`) - reQuotedLit = regexp.MustCompile(`'[^']*'|"[^"]*"`) reCopyTail = regexp.MustCompile(`(?i)\bCOPY\s*$`) reMemberHead = regexp.MustCompile(`^\s*([A-Za-z@#$][A-Za-z0-9@#$-]*)`) ) @@ -440,8 +492,7 @@ func (c *cobolStrategy) ExtractImports(tree *sitter.Tree, src []byte) ([]astkit. } pendingCopy := false // previous line ended at COPY; member starts this line for _, ln := range normalizeCOBOL(src) { - clean := rePseudoText.ReplaceAllString(ln.text, " ") - clean = reQuotedLit.ReplaceAllString(clean, " ") + clean := rePseudoText.ReplaceAllString(ln.code, " ") if pendingCopy { if m := reMemberHead.FindStringSubmatch(clean); m != nil { emit(m[1], ln.text, ln.orig) diff --git a/strategies/comment_string_test.go b/strategies/comment_string_test.go new file mode 100644 index 0000000..292234c --- /dev/null +++ b/strategies/comment_string_test.go @@ -0,0 +1,205 @@ +package strategies_test + +import ( + "strings" + "testing" + + "github.com/provasign/astkit" + "github.com/provasign/astkit/strategies" +) + +// Comments, string literals and disabled preprocessor branches are not code: +// no symbol, call site, signature, modifier or export flag may come from +// them (parsing audit 2026-10-10, items A1-A8). + +func symByQN(syms []astkit.Symbol, qn string) (astkit.Symbol, bool) { + for _, s := range syms { + if s.QualifiedName == qn { + return s, true + } + } + return astkit.Symbol{}, false +} + +func callees(s astkit.Symbol) string { + var out []string + for _, c := range s.CallSites { + out = append(out, c.Callee) + } + return strings.Join(out, ",") +} + +func TestRustMacroArgsIgnoreCommentsAndLiterals(t *testing.T) { + syms, _ := extract(t, astkit.LangRust, `fn f(c: char) { + assert!(ok(), // note: phantom_one(2) + ); + assert_eq!(c, '"', "see bar(1)", real(2)); +} +`) + f, _ := symByQN(syms, "f") + got := "," + callees(f) + "," + if !strings.Contains(got, ",ok,") || !strings.Contains(got, ",real,") || strings.Contains(got, "phantom_one") || strings.Contains(got, "bar") { + t.Fatalf("f call sites = %s", got) + } +} + +func TestCMacroBodyIgnoresCommentsAndStrings(t *testing.T) { + syms, _ := extract(t, astkit.LangC, `#define LOG(x) printf("value(%d) helper_fn(x)", x, "a,b") +#define LOG2(x) do { \ + printf("%d", (x)); /* other_fn(x) */ \ + } while (0) // other_fn(x) +`) + log, ok := symByQN(syms, "LOG") + if !ok || callees(log) != "printf" || strings.Join(log.CallSites[0].Args, ",") != "#String,x,#String" { + t.Fatalf("LOG = %+v", log) + } + // A block comment mid-body makes the grammar emit an ERROR node; the + // macro is still recovered. + log2, ok := symByQN(syms, "LOG2") + if !ok || callees(log2) != "printf" || log2.Signature != "#define LOG2(x)" || log2.Span.End != 4 { + t.Fatalf("LOG2 = %+v", log2) + } +} + +func TestCIfZeroBranchIsNotIndexed(t *testing.T) { + syms, _ := extract(t, astkit.LangC, "#if 0\nint dead(void);\n#elif 0\nint dead2(void);\n#else\nint live(void);\n#endif\n#if X\nint guarded(void);\n#endif\n") + wantAbsent(t, syms, "dead", "dead2") + for _, qn := range []string{"live", "guarded"} { + if _, ok := symByQN(syms, qn); !ok { + t.Errorf("%s missing: %+v", qn, syms) + } + } +} + +func TestObjCMacroEnumMaskedAndDirectiveMembers(t *testing.T) { + syms, _ := extract(t, astkit.LangObjC, `#import +static NSString *k = @"typedef NS_ENUM(NSInteger, Ghost) { GhostA };"; +#if 0 +typedef NS_ENUM(NSInteger, Dead) { DeadA }; +#endif +typedef NS_ENUM(NSInteger, Mode) { + ModeA, +#if X + ModeB, +#endif + ModeC +}; +`) + wantAbsent(t, syms, "Ghost", "GhostA", "Ghost.GhostA", "Dead", "DeadA", "Dead.DeadA") + for _, qn := range []string{"Mode", "ModeA", "ModeB", "ModeC"} { + if _, ok := symByQN(syms, qn); !ok { + t.Errorf("%s missing: %+v", qn, syms) + } + } +} + +func TestSignaturesDropComments(t *testing.T) { + cases := []struct { + lang astkit.LanguageKey + src, qn string + signature string + }{ + {astkit.LangObjC, "@interface M : Sup /* */ \n@end\n", "M", "@interface M : Sup "}, + {astkit.LangKotlin, "class Foo(\n val a: Int, // : Thing\n) : Base()\n", "Foo#constructor", "Foo( val a: Int, )"}, + {astkit.LangKotlin, "fun g(\n a: Int, // : Thing\n): Int = a\n", "g", "fun g(\n a: Int,\n): Int"}, + {astkit.LangPython, "class A(Base): # was Fake\n pass\n", "A", "class A(Base):"}, + {astkit.LangPython, "def g(a, # first, really (Fake)\n b):\n return a\n", "g", "def g(a, b):"}, + {astkit.LangPython, "def build(\n size,\n) -> Gadget:\n return Gadget()\n", "build", "def build( size, ) -> Gadget:"}, + {astkit.LangPython, "def one(x): return x\n", "one", "def one(x): return x"}, + {astkit.LangCSharp, "class A {\n [Description(\"see Fake\")] public void M() {}\n}\n", "A.M", "public void M()"}, + {astkit.LangCSharp, "class A {\n [Obsolete(\"x\")]\n public int P { get; set; }\n}\n", "A.P", "public int P { get; set; }"}, + {astkit.LangTypeScript, "class S {\n attach(app /*: T */, opts: number) {}\n}\n", "S.attach", "attach(app, opts: number)"}, + {astkit.LangJava, "class A {\n public /* hot */ static void f() {}\n}\n", "A.f", "public static void f()"}, + } + for _, c := range cases { + syms, _ := extract(t, c.lang, c.src) + qn, kind, _ := strings.Cut(c.qn, "#") + var s astkit.Symbol + ok := false + for _, x := range syms { + if x.QualifiedName == qn && (kind == "" || string(x.Kind) == kind) { + s, ok = x, true + break + } + } + if !ok { + t.Errorf("%s %s missing: %+v", c.lang, c.qn, syms) + continue + } + if s.Signature != c.signature { + t.Errorf("%s %s signature = %q, want %q", c.lang, c.qn, s.Signature, c.signature) + } + for _, m := range s.Modifiers { + if strings.Contains(m, "/*") { + t.Errorf("%s %s modifiers = %v", c.lang, c.qn, s.Modifiers) + } + } + } +} + +func TestKotlinFunInterfaceNotFromComment(t *testing.T) { + syms, _ := extract(t, astkit.LangKotlin, "// just for fun\ninterface Plain {\n fun x(): Int\n}\nfun interface Real {\n fun y(): Int\n}\n") + for qn, want := range map[string]bool{"Plain": false, "Real": true} { + s, ok := symByQN(syms, qn) + if !ok || contains(s.Modifiers, "fun") != want { + t.Errorf("%s = %+v, want fun modifier %v", qn, s, want) + } + } +} + +func TestJavaExportedFromModifierKeyword(t *testing.T) { + syms, _ := extract(t, astkit.LangJava, "class A {\n void publication() {}\n String mode = \"public\";\n public int open;\n}\n") + for qn, want := range map[string]bool{"A.publication": false, "A.mode": false, "A.open": true} { + if s, ok := symByQN(syms, qn); !ok || s.Exported != want { + t.Errorf("%s = %+v, want exported %v", qn, s, want) + } + } +} + +func TestLombokAccessorsFromTypeNode(t *testing.T) { + syms, _ := extract(t, astkit.LangJava, `import lombok.Getter; +class A { + @Getter String kind = "boolean x"; + @Getter boolean on; + @Getter Boolean boxed; +} +@lombok.Getter +class B { + String name; +} +`) + for _, qn := range []string{"A.getKind", "A.isOn", "A.getBoxed", "B.getName"} { + if _, ok := symByQN(syms, qn); !ok { + t.Errorf("%s missing", qn) + } + } + wantAbsent(t, syms, "A.isKind", "A.isBoxed") +} + +func TestCOBOLVerbsIgnoreLiteralsAndInlineComments(t *testing.T) { + src := ` IDENTIFICATION DIVISION. + PROGRAM-ID. PROG1. + PROCEDURE DIVISION. + MAIN-PARA. + DISPLAY 'PLEASE PERFORM BACKUP FIRST'. + MOVE 1 TO X *> PERFORM OLD-PARA + DISPLAY 'CALL SUPPORT'. + PERFORM REAL-PARA. + CALL 'SUBPROG' USING X. + CALL WS-PROG. + STOP RUN. + REAL-PARA. + DISPLAY 'R'. +` + syms, err := strategies.NewCOBOL().Extract(nil, []byte(src)) + if err != nil { + t.Fatal(err) + } + main, ok := symByQN(syms, "MAIN-PARA") + if !ok { + t.Fatalf("MAIN-PARA missing: %+v", syms) + } + if got := callees(main); got != "REAL-PARA,SUBPROG,WS-PROG" { + t.Fatalf("MAIN-PARA call sites = %s", got) + } +} diff --git a/strategies/extractors.go b/strategies/extractors.go index 16d5b27..44faae0 100644 --- a/strategies/extractors.go +++ b/strategies/extractors.go @@ -10,6 +10,7 @@ import ( "github.com/provasign/astkit" "github.com/provasign/astkit/internalast" + "github.com/provasign/astkit/textmask" ) // ─── Go ─────────────────────────────────────────────────────────────────────── @@ -160,7 +161,7 @@ func goTypeDecl(n *sitter.Node, filePath, blobSHA string, src []byte, imports [] Kind: kind, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: goStandaloneDecl("type", internalast.FirstLineSig(spec, src, nil)), Span: internalast.NodeSpan(spec), Exported: internalast.IsCapitalized(name), Body: raw, @@ -202,7 +203,7 @@ func goInterfaceMethods(ifaceType *sitter.Node, parent string, typeParams []stri Name: name, QualifiedName: name, ParentName: parent, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(elem, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(elem), Exported: internalast.IsCapitalized(name), Body: raw, @@ -308,7 +309,7 @@ func goVarDecl(n *sitter.Node, filePath, blobSHA string, src []byte, imports []s Kind: astkit.KindVariable, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: goStandaloneDecl("var", internalast.FirstLineSig(spec, src, nil)), Span: internalast.NodeSpan(spec), Exported: internalast.IsCapitalized(name), Body: raw, @@ -561,7 +562,7 @@ func jsAssignTarget(assign, left, value *sitter.Node, filePath, blobSHA, languag Kind: k, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(assign, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(assign), Exported: exported, Body: raw, @@ -592,7 +593,7 @@ func jsAssignTarget(assign, left, value *sitter.Node, filePath, blobSHA, languag } else if exported { *out = append(*out, astkit.Symbol{ Kind: astkit.KindVariable, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(assign), + Signature: internalast.FirstLineSig(assign, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(assign), Exported: true, Body: raw, Modifiers: []string{"module-value"}, }) jsObjectMembers(value, name, false, false, 1, filePath, blobSHA, language, src, imports, out) @@ -601,7 +602,7 @@ func jsAssignTarget(assign, left, value *sitter.Node, filePath, blobSHA, languag if exported && jsIsDeclaredValue(value, src) { *out = append(*out, astkit.Symbol{ Kind: astkit.KindVariable, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(assign), + Signature: internalast.FirstLineSig(assign, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(assign), Exported: true, Body: raw, Modifiers: []string{"module-value"}, }) } @@ -658,7 +659,7 @@ func jsObjectMembers(obj *sitter.Node, parent string, exported, values bool, dep *out = append(*out, astkit.Symbol{ Kind: k, Name: memberName, QualifiedName: qualJoin(parent, memberName), - Signature: internalast.FirstLine(member.Content(src)), + Signature: internalast.FirstLineSig(member, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(member), Body: member.Content(src), Exported: parent == "" && exported, ParentName: qualLast(parent), @@ -670,7 +671,7 @@ func jsObjectMembers(obj *sitter.Node, parent string, exported, values bool, dep } *out = append(*out, astkit.Symbol{ Kind: astkit.KindVariable, Name: memberName, QualifiedName: qualJoin(parent, memberName), - Signature: internalast.FirstLine(member.Content(src)), Span: internalast.NodeSpan(member), + Signature: internalast.FirstLineSig(member, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(member), Body: member.Content(src), ParentName: qualLast(parent), Modifiers: []string{"member-value"}, }) @@ -681,7 +682,7 @@ func jsObjectMembers(obj *sitter.Node, parent string, exported, values bool, dep } *out = append(*out, astkit.Symbol{ Kind: astkit.KindVariable, Name: memberName, QualifiedName: qualJoin(parent, memberName), - Signature: internalast.FirstLine(member.Content(src)), Span: internalast.NodeSpan(member), + Signature: internalast.FirstLineSig(member, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(member), Exported: parent == "" && exported, Body: member.Content(src), ParentName: qualLast(parent), Modifiers: []string{"module-value"}, }) @@ -789,7 +790,7 @@ func jsDefinePropertyCall(n *sitter.Node, src []byte, out *[]astkit.Symbol) { raw := n.Content(src) *out = append(*out, astkit.Symbol{ Kind: astkit.KindMethod, Name: name, QualifiedName: name, ParentName: parent, - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(n), Body: raw, + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Body: raw, CallSites: jsCallSites(accessor.ChildByFieldName("body"), src), }) } @@ -822,7 +823,7 @@ func jsThisFields(body *sitter.Node, owner string, src []byte, out *[]astkit.Sym raw := c.Content(src) *out = append(*out, astkit.Symbol{ Kind: astkit.KindField, Name: name, QualifiedName: qualJoin(owner, name), - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(c), + Signature: internalast.FirstLineSig(c, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(c), Body: raw, ParentName: qualLast(owner), Modifiers: []string{"member-value", "this-assigned"}, }) @@ -902,7 +903,7 @@ func jsParamProperties(ctor *sitter.Node, parentClass string, src []byte, out *[ raw := p.Content(src) *out = append(*out, astkit.Symbol{ Kind: astkit.KindField, Name: name, QualifiedName: qualJoin(parentClass, name), - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(p), Body: raw, + Signature: internalast.FirstLineSig(p, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(p), Body: raw, ParentName: qualLast(parentClass), Modifiers: append(mods, "member-value"), Annotations: jsDecorators(p, src), }) @@ -937,7 +938,7 @@ func jsEnumMembers(n *sitter.Node, enumQual string, exported bool, src []byte, o raw := m.Content(src) *out = append(*out, astkit.Symbol{ Kind: astkit.KindConst, Name: name, QualifiedName: qualJoin(enumQual, name), - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(m), Body: raw, + Signature: internalast.FirstLineSig(m, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(m), Body: raw, Exported: exported, ParentName: qualLast(enumQual), Modifiers: []string{"member-value"}, }) } @@ -972,7 +973,7 @@ func jsInterfaceMembers(n *sitter.Node, filePath, blobSHA, language string, src raw := m.Content(src) *out = append(*out, astkit.Symbol{ Kind: astkit.KindField, Name: name, QualifiedName: qualJoin(ifaceQual, name), - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(m), Body: raw, + Signature: internalast.FirstLineSig(m, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(m), Body: raw, ParentName: qualLast(ifaceQual), Modifiers: append(jsModifiers(m, src), "member-value"), // Like method signatures, a function-typed property // (`onSave: (x) => void`) declares no body to call into. @@ -1133,7 +1134,7 @@ func jsFieldDef(n *sitter.Node, filePath, blobSHA, language string, src []byte, Kind: astkit.KindField, Name: name, QualifiedName: qualJoin(parentClass, name), - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: false, Body: raw, @@ -1231,7 +1232,7 @@ func jsArrowDecl(n *sitter.Node, filePath, blobSHA, language string, src []byte, Kind: k, Name: name, QualifiedName: qualJoin(parentClass, name), - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(decl, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(decl), Exported: exported, Body: raw, @@ -1253,7 +1254,7 @@ func jsArrowDecl(n *sitter.Node, filePath, blobSHA, language string, src []byte, *out = append(*out, astkit.Symbol{ Kind: astkit.KindVariable, Name: objectName, QualifiedName: qualJoin(parentClass, objectName), - Signature: internalast.FirstLine(decl.Content(src)), + Signature: internalast.FirstLineSig(decl, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(decl), Exported: exported, Body: decl.Content(src), ParentName: qualLast(parentClass), }) @@ -1274,7 +1275,7 @@ func jsValueSym(decl, nameNode *sitter.Node, parentClass string, exported bool, raw := decl.Content(src) *out = append(*out, astkit.Symbol{ Kind: astkit.KindVariable, Name: name, QualifiedName: qualJoin(parentClass, name), - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(decl), + Signature: internalast.FirstLineSig(decl, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(decl), Exported: exported, Body: raw, ParentName: qualLast(parentClass), // Lets graph builders tell these apart from the object-literal // variables indexed before 2026-09-26. @@ -1338,7 +1339,7 @@ func pythonVisitDefinition(n *sitter.Node, filePath, blobSHA string, src []byte, Kind: kind, Name: name, QualifiedName: qualified, - Signature: internalast.FirstLine(raw), + Signature: pythonDefSig(n, src), Span: internalast.NodeSpan(n), Exported: !strings.HasPrefix(name, "_"), Body: raw, @@ -1457,7 +1458,7 @@ func pythonVisitDefinition(n *sitter.Node, filePath, blobSHA string, src []byte, if typeNode != nil { sig = name + ": " + typeNode.Content(src) } else if right := a.ChildByFieldName("right"); right != nil { - sig = internalast.FirstLine(raw) + sig = internalast.FirstLineSig(n, src, internalast.IsAttributeSection) } kind := astkit.KindVariable qn := name @@ -1485,6 +1486,23 @@ func pythonVisitDefinition(n *sitter.Node, filePath, blobSHA string, src []byte, // each contained statement so conditionally-defined classes, functions, and // module-level annotated variables are still indexed. Condition and iterable // expressions are skipped — only block-bearing children are descended. +// pythonDefSig is a def's header: its first line, or, when the parameter +// list wraps (`def build(\n size,\n) -> Gadget:`), the whole header up to +// the colon, whitespace collapsed. Comments are cut either way. +func pythonDefSig(n *sitter.Node, src []byte) string { + var colon *sitter.Node + for i := int(n.ChildCount()) - 1; i >= 0; i-- { + if c := n.Child(i); c != nil && c.Type() == ":" { + colon = c + break + } + } + if colon == nil || colon.StartPoint().Row == n.StartPoint().Row { + return internalast.FirstLineSig(n, src, nil) + } + return strings.Join(strings.Fields(internalast.HeaderText(n, src, n.StartByte(), colon.EndByte(), nil)), " ") +} + func pythonVisitBlocks(n *sitter.Node, filePath, blobSHA string, src []byte, imports []string, parentClass, qualifier string, inFunction bool, out *[]astkit.Symbol) { for i := 0; i < int(n.ChildCount()); i++ { c := n.Child(i) @@ -1565,7 +1583,7 @@ func pythonModuleAssignments(root *sitter.Node, src []byte, existing []astkit.Sy Kind: kind, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: !strings.HasPrefix(name, "_"), Body: raw, @@ -1913,13 +1931,10 @@ func javaTypeDecl(n *sitter.Node, kind astkit.SymbolKind, filePath, blobSHA stri // continuation lines, and a leading annotation is not a signature. sig := internalast.SignatureBeforeBody(n, src) modifiers := javaModifiers(n, src) - exports := strings.Contains(sig, "public") - for _, m := range modifiers { - if m == "public" { - exports = true - break - } - } + // Exported comes from the modifiers node's `public` keyword, never + // from text: `void publication()` and `String mode = "public"` are + // package-private. + exports := internalast.HasModifier(modifiers, "public") *out = append(*out, astkit.Symbol{ Kind: kind, Name: className, @@ -1940,8 +1955,34 @@ func javaTypeDecl(n *sitter.Node, kind astkit.SymbolKind, filePath, blobSHA stri if body != nil { before := len(*out) javaVisit(body, filePath, blobSHA, src, imports, qualJoin(parentClass, className), out) - synthesizeLombokAccessors(className, javaAnnotations(n, src), before, out) + synthesizeLombokAccessors(className, javaAnnotations(n, src), javaPrimitiveBooleanFields(body, src), before, out) + } +} + +// javaPrimitiveBooleanFields returns the names of the fields declared in +// class body whose type node is the primitive `boolean`. +func javaPrimitiveBooleanFields(body *sitter.Node, src []byte) map[string]bool { + var out map[string]bool + for i := 0; i < int(body.NamedChildCount()); i++ { + fd := body.NamedChild(i) + if fd == nil || fd.Type() != "field_declaration" { + continue + } + if t := fd.ChildByFieldName("type"); t == nil || t.Type() != "boolean_type" { + continue + } + for j := 0; j < int(fd.NamedChildCount()); j++ { + if d := fd.NamedChild(j); d != nil && d.Type() == "variable_declarator" { + if nm := d.ChildByFieldName("name"); nm != nil { + if out == nil { + out = map[string]bool{} + } + out[nm.Content(src)] = true + } + } + } } + return out } // javaRecordComponents emits the implicit field and public accessor that Java @@ -2006,11 +2047,17 @@ func javaEnumConstantDecl(n *sitter.Node, src []byte, parentClass string, out *[ // queries could not be queried at all). Synthesized symbols carry the // "lombok-generated" modifier and the FIELD's span, so results point at the // declaration a human would edit. -func synthesizeLombokAccessors(className string, classAnn []string, fieldsFrom int, out *[]astkit.Symbol) { +// +// The getter of a primitive `boolean` field is isX; every other type, +// boxed Boolean included, gets getX. boolFields names the primitive boolean +// fields, read from the declarations' type nodes. +func synthesizeLombokAccessors(className string, classAnn []string, boolFields map[string]bool, fieldsFrom int, out *[]astkit.Symbol) { has := func(ann []string, names ...string) bool { // javaAnnotations strips the leading @; match the bare name exactly - // or with an argument list ("Getter", "Getter(...)"). + // or with an argument list ("Getter", "Getter(...)"), plain or + // fully qualified ("lombok.Getter"). for _, a := range ann { + a = strings.TrimPrefix(a, "lombok.") for _, n := range names { if a == n || strings.HasPrefix(a, n+"(") { return true @@ -2045,7 +2092,7 @@ func synthesizeLombokAccessors(className string, classAnn []string, fieldsFrom i name := f.Name cap := strings.ToUpper(name[:1]) + name[1:] getter := "get" + cap - if strings.Contains(f.Signature, "boolean ") { + if boolFields[name] { getter = "is" + cap } mk := func(mname, sig string) astkit.Symbol { @@ -2085,12 +2132,7 @@ func javaMethodDecl(n *sitter.Node, kind astkit.SymbolKind, filePath, blobSHA st sig += javaCompactConstructorParams(n, src) } modifiers := javaModifiers(n, src) - exports := strings.Contains(sig, "public") || n.Type() == "annotation_type_element_declaration" - for _, m := range modifiers { - if m == "public" { - exports = true - } - } + exports := internalast.HasModifier(modifiers, "public") || n.Type() == "annotation_type_element_declaration" body := n.ChildByFieldName("body") *out = append(*out, astkit.Symbol{ Kind: kind, @@ -2212,12 +2254,7 @@ func javaFieldDecl(n *sitter.Node, filePath, blobSHA string, src []byte, imports // field made FirstLine return just "@Deprecated" as the signature. sig := internalast.SignatureBeforeBody(n, src) modifiers := javaModifiers(n, src) - exports := strings.Contains(sig, "public") || n.Type() == "constant_declaration" - for _, m := range modifiers { - if m == "public" { - exports = true - } - } + exports := internalast.HasModifier(modifiers, "public") || n.Type() == "constant_declaration" for i := 0; i < int(n.NamedChildCount()); i++ { decl := n.NamedChild(i) if decl == nil || decl.Type() != "variable_declarator" { @@ -2493,7 +2530,7 @@ func rustEnumVariants(n *sitter.Node, src []byte, out *[]astkit.Symbol) { Kind: astkit.KindField, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(v, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(v), Exported: exported, Body: raw, @@ -2515,7 +2552,7 @@ func rustNamedItem(n *sitter.Node, kind astkit.SymbolKind, filePath, blobSHA str Kind: kind, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: strings.HasPrefix(strings.TrimSpace(raw), "pub"), Body: raw, @@ -2581,7 +2618,7 @@ func rustStructFields(n *sitter.Node, filePath, blobSHA string, src []byte, impo Kind: astkit.KindField, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(fd, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(fd), Exported: strings.HasPrefix(strings.TrimSpace(raw), "pub"), Body: raw, @@ -2690,41 +2727,87 @@ func cMacroSym(n *sitter.Node, filePath, blobSHA, language string, src []byte) * if nameNode == nil { return nil } - name := nameNode.Content(src) + body := "" + if v := n.ChildByFieldName("value"); v != nil { + body = v.Content(src) + } + return cMacroSymbol(nameNode.Content(src), n.ChildByFieldName("parameters"), body, + n.Content(src), internalast.NodeSpan(n), language, src) +} + +// cMacroFromError recovers a `#define` the C grammar could not parse. A +// block comment in the middle of a continued macro body +// (`do { \ f(); /* note */ \ } while (0)`) turns the whole directive into +// an ERROR node that still starts with `#define NAME(params)`; without this +// the macro vanished. The body is the rest of the logical line: physical +// lines joined by a trailing backslash. +func cMacroFromError(n *sitter.Node, language string, src []byte) *astkit.Symbol { + if n.ChildCount() < 2 || n.Child(0).Type() != "#define" || n.Child(1).Type() != "identifier" { + return nil + } + nameNode := n.Child(1) + var params *sitter.Node + bodyStart := nameNode.EndByte() + if c := n.Child(2); c != nil && c.Type() == "preproc_params" { + params = c + bodyStart = c.EndByte() + } + end := int(bodyStart) + for end < len(src) { + nl := strings.IndexByte(string(src[end:]), '\n') + if nl < 0 { + end = len(src) + break + } + line := strings.TrimRight(string(src[end:end+nl]), "\r") + end += nl + if !strings.HasSuffix(line, "\\") { + break + } + end++ + } + start := n.StartPoint() + span := astkit.LineRange{Start: int(start.Row) + 1, End: int(start.Row) + 1 + strings.Count(string(src[n.StartByte():end]), "\n")} + return cMacroSymbol(nameNode.Content(src), params, string(src[bodyStart:end]), + string(src[n.StartByte():end]), span, language, src) +} + +func cMacroSymbol(name string, paramsNode *sitter.Node, body, raw string, span astkit.LineRange, language string, src []byte) *astkit.Symbol { params := "" var paramNames []string - if p := n.ChildByFieldName("parameters"); p != nil { - params = strings.Join(strings.Fields(p.Content(src)), "") + if paramsNode != nil { + params = strings.Join(strings.Fields(paramsNode.Content(src)), "") for _, x := range strings.Split(strings.Trim(params, "()"), ",") { if x = strings.TrimSpace(x); x != "" { paramNames = append(paramNames, x) } } } - body := "" - if v := n.ChildByFieldName("value"); v != nil { - body = v.Content(src) - } isParam := map[string]bool{} for _, p := range paramNames { isParam[p] = true } + // Calls are read from the body with comments and string literals + // blanked: `printf("value(%d)", x)` calls printf, not value, and a + // comment on a continued line is not code. Argument text is still + // classified from the original body (a string argument is "#String"). + code := textmask.Mask(language, body) var sites []astkit.CallSite - for _, m := range cMacroCallRe.FindAllStringSubmatchIndex(body, -1) { - callee := body[m[2]:m[3]] + for _, m := range cMacroCallRe.FindAllStringSubmatchIndex(code, -1) { + callee := code[m[2]:m[3]] if cMacroKeywords[callee] { continue } open := m[1] - 1 // index of '(' - args, argc := cMacroArgs(body, open) + args, argc := cMacroArgs(body, code, open) sites = append(sites, astkit.CallSite{ - Callee: callee, Line: int(n.StartPoint().Row) + 1, Argc: argc, Args: args, + Callee: callee, Line: span.Start, Argc: argc, Args: args, }) } sig := "#define " + name + params return &astkit.Symbol{ Kind: astkit.KindMacro, Name: name, QualifiedName: name, Signature: sig, - Span: internalast.NodeSpan(n), Exported: true, Body: n.Content(src), + Span: span, Exported: true, Body: raw, CallSites: sites, } } @@ -2732,20 +2815,22 @@ func cMacroSym(n *sitter.Node, filePath, blobSHA, language string, src []byte) * // cMacroArgs tokenizes the argument list opening at body[open]: an // identifier stays (a macro parameter name is substituted downstream, a // function name becomes a reference), a string or stringized `#param` -// argument is "#String", a number "#int", anything else "". -func cMacroArgs(body string, open int) ([]string, int) { +// argument is "#String", a number "#int", anything else "". Parentheses and +// commas are read from code (body with comments and literals masked), so a +// comma inside a string does not split an argument. +func cMacroArgs(body, code string, open int) ([]string, int) { depth := 0 start := open + 1 var raw []string - for i := open; i < len(body); i++ { - switch body[i] { + for i := open; i < len(code); i++ { + switch code[i] { case '(': depth++ case ')': depth-- if depth == 0 { raw = append(raw, body[start:i]) - i = len(body) + i = len(code) } case ',': if depth == 1 { @@ -2841,11 +2926,27 @@ func extractCNodes(root *sitter.Node, filePath, blobSHA, language string, src [] // Declarations in conditional branches are ordinary source symbols. // Tree-sitter nests them below the preprocessor node rather than the // translation unit, so explicitly descend into each guarded region. + // An `#if 0` branch is disabled code (often a commented-out + // block): only its `#elif`/`#else` alternative is descended. + if cPreprocDisabled(n, src) { + alt := n.ChildByFieldName("alternative") + for alt != nil && cPreprocDisabled(alt, src) { + alt = alt.ChildByFieldName("alternative") + } + if alt != nil { + out = append(out, extractCNodes(alt, filePath, blobSHA, language, src, imports)...) + } + continue + } out = append(out, extractCNodes(n, filePath, blobSHA, language, src, imports)...) case "preproc_function_def", "preproc_def": if sym := cMacroSym(n, filePath, blobSHA, language, src); sym != nil { out = append(out, *sym) } + case "ERROR": + if sym := cMacroFromError(n, language, src); sym != nil { + out = append(out, *sym) + } case "linkage_specification": // `extern "C" { ... }` — in a C header it sits under `#ifdef // __cplusplus`, and the C grammar hands the whole rest of the @@ -2885,6 +2986,23 @@ func extractCNodes(root *sitter.Node, filePath, blobSHA, language string, src [] return out } +// cPreprocDisabled reports whether n is an `#if 0` / `#elif 0` conditional, +// the same literal-false test textmask.MaskInactivePreprocessor applies. +func cPreprocDisabled(n *sitter.Node, src []byte) bool { + if n.Type() != "preproc_if" && n.Type() != "preproc_elif" { + return false + } + c := n.ChildByFieldName("condition") + if c == nil { + return false + } + switch strings.Join(strings.Fields(c.Content(src)), "") { + case "0", "(0)": + return true + } + return false +} + func cFuncSym(n *sitter.Node, filePath, blobSHA, language string, src []byte, imports []string, parentClass string) *astkit.Symbol { // function_definition: type declarator body declarator := n.ChildByFieldName("declarator") @@ -2905,7 +3023,7 @@ func cFuncSym(n *sitter.Node, filePath, blobSHA, language string, src []byte, im return nil } if language == "cpp" && parentClass == "" { - parentClass = cppDeclaratorOwner(declarator.Content(src)) + parentClass = cppDeclaratorOwner(internalast.HeaderText(declarator, src, declarator.StartByte(), declarator.EndByte(), nil)) } raw := n.Content(src) modifiers := cStorageModifiers(n, src) @@ -3198,7 +3316,7 @@ func cTypedefSyms(n *sitter.Node, filePath, blobSHA, language string, src []byte Kind: kind, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: true, Body: raw, @@ -3216,7 +3334,7 @@ func cTypedefSyms(n *sitter.Node, filePath, blobSHA, language string, src []byte if tag != "" && tag != name { out = append(out, astkit.Symbol{ Kind: kind, Name: tag, QualifiedName: tag, - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(n), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: true, Body: raw, }) } @@ -3513,7 +3631,7 @@ func cNestedRecordSyms(fd *sitter.Node, parent, sep string, src []byte) []astkit raw := spec.Content(src) out := []astkit.Symbol{{ Kind: astkit.KindStruct, Name: tag, QualifiedName: tag, - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(spec), + Signature: internalast.FirstLineSig(spec, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(spec), Exported: true, Body: raw, }} return append(out, cStructFields(spec, tag, src)...) @@ -3550,7 +3668,7 @@ func cTaggedTypeSym(n *sitter.Node, kind astkit.SymbolKind, filePath, blobSHA, l Kind: kind, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: true, Body: raw, @@ -3838,7 +3956,7 @@ func cppNestedTypeSyms(fd *sitter.Node, className, access, filePath, blobSHA, la raw := spec.Content(src) enum := astkit.Symbol{ Kind: astkit.KindEnum, Name: nameNode.Content(src), QualifiedName: nameNode.Content(src), - ParentName: className, Signature: internalast.FirstLine(raw), + ParentName: className, Signature: internalast.FirstLineSig(spec, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(spec), Exported: true, Body: raw, } cppSetMemberAccess(&enum, access) @@ -3899,7 +4017,7 @@ func cppNamespaceSym(n *sitter.Node, filePath, blobSHA, language string, src []b Kind: astkit.KindNamespace, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: true, Body: raw, @@ -3982,7 +4100,7 @@ func csVisit(node *sitter.Node, filePath, blobSHA string, src []byte, imports [] Kind: astkit.KindNamespace, Name: nsName, QualifiedName: nsName, - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: true, Body: raw, @@ -3994,7 +4112,9 @@ func csVisit(node *sitter.Node, filePath, blobSHA string, src []byte, imports [] csTypeDecl(n, astkit.KindClass, filePath, blobSHA, src, imports, parentClass, out) case "record_declaration": kind := astkit.KindClass - if strings.Contains(internalast.SignatureBeforeBody(n, src), "record struct") { + // `record struct` carries an anonymous `struct` keyword token; + // text matching also hit "record struct" in a string argument. + if internalast.FindChildByType(n, "struct") != nil { kind = astkit.KindStruct } csTypeDecl(n, kind, filePath, blobSHA, src, imports, parentClass, out) @@ -4142,7 +4262,9 @@ func csEventDecl(n *sitter.Node, src []byte, parentClass string, out *[]astkit.S name := nameNode.Content(src) raw := n.Content(src) modifiers := append(csModifiers(n, src), "event") - sig := raw + // Attribute sections and comments are cut first: a `{` inside + // `[X("{")]` ended the signature early. + sig := internalast.HeaderText(n, src, n.StartByte(), n.EndByte(), internalast.IsAttributeSection) if i := strings.IndexByte(sig, '{'); i >= 0 { sig = sig[:i] } @@ -4431,7 +4553,7 @@ func csTypeLastName(n *sitter.Node, src []byte) string { // receiver's indexer. func csIndexerDecl(n *sitter.Node, filePath, blobSHA string, src []byte, imports []string, parentClass string, out *[]astkit.Symbol) { raw := n.Content(src) - sig := raw + sig := internalast.HeaderText(n, src, n.StartByte(), n.EndByte(), internalast.IsAttributeSection) if i := strings.IndexByte(sig, '{'); i >= 0 { sig = sig[:i] } @@ -4469,7 +4591,7 @@ func csPropertyDecl(n *sitter.Node, filePath, blobSHA string, src []byte, import Kind: kind, Name: name, QualifiedName: qualJoin(parentClass, name), - Signature: internalast.FirstLine(raw), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: csIsExported(modifiers), Body: raw, @@ -4504,7 +4626,7 @@ func csFieldDecl(n *sitter.Node, filePath, blobSHA string, src []byte, imports [ raw := decl.Content(src) *out = append(*out, astkit.Symbol{ Kind: astkit.KindField, Name: name, QualifiedName: qualJoin(parentClass, name), - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(decl), + Signature: internalast.FirstLineSig(decl, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(decl), Exported: csIsExported(modifiers), Body: raw, ParentName: qualLast(parentClass), Modifiers: modifiers, }) } @@ -5261,7 +5383,7 @@ func swiftProtocolDecl(n *sitter.Node, filePath, blobSHA string, src []byte, imp } *out = append(*out, astkit.Symbol{ Kind: astkit.KindMethod, Name: nameNode.Content(src), QualifiedName: nameNode.Content(src), - Signature: internalast.FirstLine(m.Content(src)), Span: internalast.NodeSpan(m), + Signature: internalast.FirstLineSig(m, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(m), Exported: true, Body: m.Content(src), ParentName: name, }) case "protocol_property_declaration": @@ -5336,7 +5458,7 @@ func swiftPropertyDecl(n *sitter.Node, filePath, blobSHA string, src []byte, imp } *out = append(*out, astkit.Symbol{ Kind: kind, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(raw), Span: internalast.NodeSpan(n), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: swiftIsExported(modifiers), Body: raw, ParentName: qualLast(implType), Modifiers: modifiers, Annotations: swiftAttributes(n, src), }) @@ -5413,7 +5535,7 @@ func swiftSubscriptDecl(n *sitter.Node, filePath, blobSHA string, src []byte, im // "body" field; walking the whole declaration reaches both. *out = append(*out, astkit.Symbol{ Kind: astkit.KindMethod, Name: "subscript", QualifiedName: qualJoin(qualLast(implType), "subscript"), - Signature: internalast.FirstLine(n.Content(src)), Span: internalast.NodeSpan(n), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: swiftIsExported(modifiers), Body: n.Content(src), ParentName: qualLast(implType), Modifiers: modifiers, CallSites: sharedNavCallSites(n, src, true), }) @@ -5428,7 +5550,7 @@ func swiftNamedItem(n *sitter.Node, kind astkit.SymbolKind, filePath, blobSHA st modifiers := swiftModifiers(n, src) *out = append(*out, astkit.Symbol{ Kind: kind, Name: name, QualifiedName: name, - Signature: internalast.FirstLine(n.Content(src)), Span: internalast.NodeSpan(n), + Signature: internalast.FirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: swiftIsExported(modifiers), Body: n.Content(src), Modifiers: modifiers, Annotations: swiftAttributes(n, src), }) @@ -5443,7 +5565,7 @@ func swiftModifiers(n *sitter.Node, src []byte) []string { } for j := 0; j < int(c.ChildCount()); j++ { m := c.Child(j) - if m == nil || !m.IsNamed() || m.Type() == "attribute" { + if m == nil || !m.IsNamed() || m.Type() == "attribute" || internalast.IsComment(m) { continue } out = append(out, strings.TrimSpace(m.Content(src))) @@ -5523,13 +5645,36 @@ func kotlinBlankFunInterface(src []byte) []byte { } // kotlinIsFunInterface reports whether the interface keyword of class -// declaration n is preceded by `fun` in the original source. +// declaration n is preceded by `fun` in the original source. The grammar +// has no `fun interface` (kotlinBlankFunInterface re-parses with the `fun` +// blanked), so the token is read from the gap between the keyword and the +// node before it — the preceding modifiers, or the previous sibling, which +// is a comment node when one precedes the declaration: `// just for fun` +// is not the modifier. func kotlinIsFunInterface(n *sitter.Node, src []byte) bool { for i := 0; i < int(n.ChildCount()); i++ { c := n.Child(i) - if c != nil && !c.IsNamed() && c.Type() == "interface" { - return strings.HasSuffix(strings.TrimRight(string(src[:c.StartByte()]), " \t\r\n"), "fun") + if c == nil || c.IsNamed() || c.Type() != "interface" { + continue + } + var gapStart uint32 // file start unless a node precedes + if i > 0 { + gapStart = n.Child(i - 1).EndByte() + } else { + for a := n; a != nil; a = a.Parent() { + if prev := a.PrevSibling(); prev != nil { + gapStart = prev.EndByte() + break + } + } + } + if gapStart > c.StartByte() { + return false } + // The tree has no comment nodes (see kotlinCommentCuts); the gap + // starts in code, so masking it alone is exact. + gap := strings.Fields(textmask.MaskComments(string(astkit.LangKotlin), string(src[gapStart:c.StartByte()]))) + return len(gap) > 0 && gap[len(gap)-1] == "fun" } return false } @@ -5606,7 +5751,7 @@ func kotlinTypeAlias(n *sitter.Node, src []byte, implType string, out *[]astkit. modifiers := kotlinModifiers(n, src) *out = append(*out, astkit.Symbol{ Kind: astkit.KindType, Name: name, QualifiedName: qualJoin(implType, name), - Signature: strings.Join(strings.Fields(n.Content(src)), " "), Span: internalast.NodeSpan(n), + Signature: strings.Join(strings.Fields(kotlinHeaderText(n, src, n.StartByte(), n.EndByte())), " "), Span: internalast.NodeSpan(n), Exported: kotlinIsExported(modifiers), Body: n.Content(src), ParentName: qualLast(implType), Modifiers: modifiers, TypeParameters: kotlinTypeParameters(n, src), Annotations: kotlinAnnotations(n, src), }) @@ -5653,7 +5798,7 @@ func kotlinPrimaryConstructorSites(pc, body *sitter.Node, src []byte) []astkit.C func kotlinSecondaryConstructor(n *sitter.Node, src []byte, implType string, out *[]astkit.Symbol) { params := "()" if p := internalast.FindChildByType(n, "function_value_parameters"); p != nil { - params = p.Content(src) + params = kotlinHeaderText(p, src, p.StartByte(), p.EndByte()) } kotlinConstructorSymbol(qualLast(implType), implType, params, internalast.NodeSpan(n), n.Content(src), kotlinModifiers(n, src), sharedNavCallSites(n, src, false), out) @@ -5712,7 +5857,7 @@ func kotlinClassDecl(n *sitter.Node, filePath, blobSHA string, src []byte, impor sites := kotlinPrimaryConstructorSites(pc, body, src) switch { case pc != nil: - kotlinConstructorSymbol(name, qualJoin(parentChain, name), pc.Content(src), internalast.NodeSpan(pc), pc.Content(src), modifiers, sites, out) + kotlinConstructorSymbol(name, qualJoin(parentChain, name), kotlinHeaderText(pc, src, pc.StartByte(), pc.EndByte()), internalast.NodeSpan(pc), pc.Content(src), modifiers, sites, out) case body == nil || internalast.FindChildByType(body, "secondary_constructor") == nil: kotlinConstructorSymbol(name, qualJoin(parentChain, name), "()", internalast.NodeSpan(nameNode), name+"()", modifiers, sites, out) } @@ -5812,7 +5957,7 @@ func kotlinPropertyDecl(n *sitter.Node, filePath, blobSHA string, src []byte, im } *out = append(*out, astkit.Symbol{ Kind: kind, Name: nameNode.Content(src), QualifiedName: nameNode.Content(src), - Signature: internalast.FirstLine(n.Content(src)), Span: internalast.NodeSpan(n), + Signature: kotlinFirstLineSig(n, src, internalast.IsAttributeSection), Span: internalast.NodeSpan(n), Exported: kotlinIsExported(modifiers), Body: n.Content(src), ParentName: qualLast(implType), Modifiers: modifiers, Annotations: kotlinAnnotations(n, src), }) @@ -5853,14 +5998,14 @@ func kotlinFunctionDecl(n *sitter.Node, filePath, blobSHA string, src []byte, im func kotlinFuncSig(n *sitter.Node, src []byte) string { body := internalast.FindChildByType(n, "function_body") if body == nil { - return internalast.FirstLine(strings.TrimSpace(n.Content(src))) + return kotlinFirstLineSig(n, src, nil) } start := n.StartByte() bodyStart := body.StartByte() if bodyStart <= start { - return internalast.FirstLine(n.Content(src)) + return kotlinFirstLineSig(n, src, nil) } - sig := strings.TrimSpace(string(src[start:bodyStart])) + sig := strings.TrimSpace(kotlinHeaderText(n, src, start, bodyStart)) sig = strings.TrimRight(sig, " \t\n{=") return strings.TrimSpace(sig) } @@ -5876,15 +6021,14 @@ func kotlinSignatureBeforeBody(n *sitter.Node, src []byte) string { body = internalast.FindChildByType(n, "enum_class_body") } if body == nil { - return internalast.FirstLine(strings.TrimSpace(n.Content(src))) + return kotlinFirstLineSig(n, src, nil) } start := n.StartByte() bodyStart := body.StartByte() if bodyStart <= start { - return internalast.FirstLine(n.Content(src)) + return kotlinFirstLineSig(n, src, nil) } - sig := strings.TrimSpace(string(src[start:bodyStart])) - return strings.Join(strings.Fields(sig), " ") + return strings.Join(strings.Fields(kotlinHeaderText(n, src, start, bodyStart)), " ") } func kotlinModifiers(n *sitter.Node, src []byte) []string { @@ -5895,7 +6039,7 @@ func kotlinModifiers(n *sitter.Node, src []byte) []string { var out []string for i := 0; i < int(mods.ChildCount()); i++ { c := mods.Child(i) - if c == nil || !c.IsNamed() || c.Type() == "annotation" { + if c == nil || !c.IsNamed() || c.Type() == "annotation" || internalast.IsComment(c) { continue } out = append(out, strings.TrimSpace(c.Content(src))) @@ -5944,6 +6088,52 @@ func kotlinTypeParameters(n *sitter.Node, src []byte) []string { return out } +// kotlinCommentCuts returns the byte ranges of the comments in +// src[start:end], which must start in code. astkit parses Kotlin without +// its comments (astkit.Engine.Parse), so the tree has no comment nodes for +// internalast.HeaderText to cut; the text is masked instead. Comments +// separated only by whitespace form one range. +func kotlinCommentCuts(src []byte, start, end uint32) [][2]uint32 { + if int(end) > len(src) { + end = uint32(len(src)) + } + if start >= end { + return nil + } + seg := src[start:end] + masked := textmask.MaskComments(string(astkit.LangKotlin), string(seg)) + var cuts [][2]uint32 + for i := 0; i < len(seg); i++ { + if masked[i] == seg[i] { + continue + } + j := i + 1 + for j < len(seg) && masked[j] != seg[j] { + j++ + } + lo, hi := start+uint32(i), start+uint32(j) + if n := len(cuts); n > 0 && strings.TrimSpace(string(src[cuts[n-1][1]:lo])) == "" { + cuts[n-1][1] = hi + } else { + cuts = append(cuts, [2]uint32{lo, hi}) + } + i = j - 1 + } + return cuts +} + +// kotlinHeaderText is internalast.HeaderText for the comment-free Kotlin +// tree: src[start:end] without its comments. +func kotlinHeaderText(n *sitter.Node, src []byte, start, end uint32) string { + return internalast.HeaderTextCut(n, src, start, end, nil, kotlinCommentCuts(src, start, end)) +} + +// kotlinFirstLineSig is internalast.FirstLineSig for the comment-free +// Kotlin tree. +func kotlinFirstLineSig(n *sitter.Node, src []byte, drop func(*sitter.Node) bool) string { + return internalast.FirstLineSigCut(n, src, drop, kotlinCommentCuts(src, n.StartByte(), n.EndByte())) +} + // ─── Objective-C ──────────────────────────────────────────────────────────── // // Objective-C is a strict superset of C: astkit's existing C extractor @@ -5963,22 +6153,47 @@ func kotlinTypeParameters(n *sitter.Node, src []byte) []string { func extractObjCNodes(root *sitter.Node, filePath, blobSHA string, src []byte, imports []string) []astkit.Symbol { out := extractCNodes(root, filePath, blobSHA, "objc", src, imports) - for i := 0; i < int(root.ChildCount()); i++ { - n := root.Child(i) - if n == nil { + objcContainers(root, src, &out) + out = objcBlockTypedefs(root, src, out) + return objcMacroEnums(src, out) +} + +// objcContainers emits the @interface/@protocol/@implementation blocks +// among n's children, descending into conditional regions and `extern "C"` +// blocks as extractCNodes does: AFNetworking's UIKit headers wrap their +// whole @interface in `#if TARGET_OS_IOS`, and only a parse that broke +// around the #if had let the class surface (as flat recovery junk). +func objcContainers(n *sitter.Node, src []byte, out *[]astkit.Symbol) { + for i := 0; i < int(n.ChildCount()); i++ { + c := n.Child(i) + if c == nil { continue } - switch n.Type() { + switch c.Type() { case "protocol_declaration": - objcProtocolDecl(n, src, &out) + objcProtocolDecl(c, src, out) case "class_interface": - objcInterfaceDecl(n, src, &out) + objcInterfaceDecl(c, src, out) case "class_implementation": - objcImplementationDecl(n, src, &out) + objcImplementationDecl(c, src, out) + case "preproc_if", "preproc_ifdef", "preproc_elif", "preproc_else": + if cPreprocDisabled(c, src) { + alt := c.ChildByFieldName("alternative") + for alt != nil && cPreprocDisabled(alt, src) { + alt = alt.ChildByFieldName("alternative") + } + if alt != nil { + objcContainers(alt, src, out) + } + continue + } + objcContainers(c, src, out) + case "linkage_specification": + if body := c.ChildByFieldName("body"); body != nil && body.Type() == "declaration_list" { + objcContainers(body, src, out) + } } } - out = objcBlockTypedefs(root, src, out) - return objcMacroEnums(src, out) } // objcBlockTypedefs emits `typedef void (^Handler)(int);` as a type named @@ -6023,8 +6238,8 @@ func objcBlockTypedefs(root *sitter.Node, src []byte, out []astkit.Symbol) []ast } // objcMacroEnumRe matches `typedef NS_ENUM(NSInteger, Mode) { ... }` and its -// NS_OPTIONS / NS_CLOSED_ENUM / NS_ERROR_ENUM / CF_* siblings in -// comment-blanked source. Group 2 is the enum name, group 3 the members. +// NS_OPTIONS / NS_CLOSED_ENUM / NS_ERROR_ENUM / CF_* siblings in source +// with comments, string literals and `#if 0` branches masked. Group 2 is the enum name, group 3 the members. var objcMacroEnumRe = regexp.MustCompile(`\btypedef\s+((?:NS|CF)_(?:ENUM|OPTIONS|CLOSED_ENUM|ERROR_ENUM))\s*\(\s*[^,(){};]+?\s*,\s*([A-Za-z_]\w*)\s*\)\s*\{([^{}]*)\}`) var objcEnumMemberRe = regexp.MustCompile(`^\s*([A-Za-z_]\w*)`) @@ -6040,7 +6255,9 @@ func objcMacroEnums(src []byte, out []astkit.Symbol) []astkit.Symbol { if !strings.Contains(string(src), "_ENUM") && !strings.Contains(string(src), "_OPTIONS") { return out } - text := blankCComments(src) + // A typedef inside a string literal, a comment or an `#if 0` branch + // declares nothing. + text := textmask.Mask("objc", textmask.MaskInactivePreprocessor("objc", string(src))) type memberAt struct { name string line int @@ -6059,14 +6276,16 @@ func objcMacroEnums(src []byte, out []astkit.Symbol) []astkit.Symbol { span := astkit.LineRange{Start: startLine, End: endLine} added = append(added, astkit.Symbol{ Kind: astkit.KindEnum, Name: name, QualifiedName: name, - Signature: strings.Join(strings.Fields(string(src[m[0]:m[6]-1])), " "), + Signature: strings.Join(strings.Fields(text[m[0]:m[6]-1]), " "), Span: span, Exported: true, Body: raw, }) var members []memberAt bodyStart := m[6] depth, itemStart := 0, bodyStart flush := func(itemEnd int) { - item := text[itemStart:itemEnd] + // A conditional around a member (`#if X\n ModeB,\n#endif`) + // puts directive lines in the item; they are not members. + item := objcBlankDirectiveLines(text[itemStart:itemEnd]) if mm := objcEnumMemberRe.FindStringSubmatchIndex(item); mm != nil { members = append(members, memberAt{ name: item[mm[2]:mm[3]], @@ -6111,34 +6330,28 @@ func objcMacroEnums(src []byte, out []astkit.Symbol) []astkit.Symbol { return append(kept, added...) } -// blankCComments returns src as a string with // and /* */ comments -// replaced by spaces (newlines kept), so offsets and line numbers match src. -func blankCComments(src []byte) string { - b := []byte(string(src)) - for i := 0; i < len(b); i++ { - switch { - case b[i] == '"': - for i++; i < len(b) && b[i] != '"' && b[i] != '\n'; i++ { - if b[i] == '\\' { - i++ - } - } - case b[i] == '/' && i+1 < len(b) && b[i+1] == '/': - for ; i < len(b) && b[i] != '\n'; i++ { - b[i] = ' ' - } - case b[i] == '/' && i+1 < len(b) && b[i+1] == '*': - j := i - for ; j < len(b) && !(b[j] == '*' && j+1 < len(b) && b[j+1] == '/' && j > i+1); j++ { - if b[j] != '\n' { - b[j] = ' ' +// objcBlankDirectiveLines blanks every line of s whose first non-blank +// character is '#' (a preprocessor directive), keeping offsets. +func objcBlankDirectiveLines(s string) string { + if !strings.Contains(s, "#") { + return s + } + b := []byte(s) + for ls := 0; ls < len(b); { + le := strings.IndexByte(s[ls:], '\n') + if le < 0 { + le = len(b) + } else { + le += ls + } + if strings.HasPrefix(strings.TrimLeft(s[ls:le], " \t"), "#") { + for k := ls; k < le; k++ { + if b[k] != '\r' { + b[k] = ' ' } } - if j+1 < len(b) { - b[j], b[j+1] = ' ', ' ' - } - i = j + 1 } + ls = le + 1 } return string(b) } @@ -6177,9 +6390,11 @@ func objcHeaderText(n *sitter.Node, src []byte) string { end = uint32(len(src)) } if start >= end { - return internalast.FirstLine(n.Content(src)) + return internalast.FirstLineSig(n, src, nil) } - return strings.Join(strings.Fields(string(src[start:end])), " ") + // Comments are cut so `@interface M : Sup /* */

` names + // only P as a protocol. + return strings.Join(strings.Fields(internalast.HeaderText(n, src, start, end, nil)), " ") } // objcCategoryName returns the category name of `@interface Foo (Bar)` — a @@ -6391,31 +6606,31 @@ func objcMethodDefSym(n *sitter.Node, parentName string, src []byte, out *[]astk func objcMethodSigBeforeBody(n *sitter.Node, src []byte) string { body := internalast.FindChildByType(n, "compound_statement") if body == nil { - return internalast.FirstLine(strings.TrimSpace(n.Content(src))) + return internalast.FirstLineSig(n, src, nil) } start := n.StartByte() bodyStart := body.StartByte() if bodyStart <= start { - return internalast.FirstLine(n.Content(src)) + return internalast.FirstLineSig(n, src, nil) } - sig := strings.TrimSpace(string(src[start:bodyStart])) - return strings.Join(strings.Fields(sig), " ") + return strings.Join(strings.Fields(internalast.HeaderText(n, src, start, bodyStart, nil)), " ") } // ─── Shared helpers ─────────────────────────────────────────────────────────── -// funcSig returns the function/method signature without the body. +// funcSig returns the function/method signature without the body, its +// comments, or its C#/PHP attribute sections (those are on Annotations). func funcSig(n *sitter.Node, src []byte) string { body := n.ChildByFieldName("body") if body == nil { - return strings.TrimSpace(n.Content(src)) + return strings.TrimSpace(internalast.HeaderText(n, src, n.StartByte(), n.EndByte(), internalast.IsAttributeSection)) } start := n.StartByte() bodyStart := body.StartByte() if bodyStart <= start { - return internalast.FirstLine(n.Content(src)) + return internalast.FirstLineSig(n, src, internalast.IsAttributeSection) } - sig := strings.TrimSpace(string(src[start:bodyStart])) + sig := strings.TrimSpace(internalast.HeaderText(n, src, start, bodyStart, internalast.IsAttributeSection)) sig = strings.TrimRight(sig, " \t\n{") return strings.TrimSpace(sig) } diff --git a/strategies/metadata.go b/strategies/metadata.go index 8d27091..04f8631 100644 --- a/strategies/metadata.go +++ b/strategies/metadata.go @@ -9,6 +9,7 @@ import ( "github.com/provasign/astkit" "github.com/provasign/astkit/internalast" + "github.com/provasign/astkit/textmask" ) var _ = internalast.NodeSpan // imported by extractors.go callers @@ -815,10 +816,6 @@ func rustCallSites(body *sitter.Node, src []byte) []astkit.CallSite { return out } -// rustMacroStringRe strips string literals from macro token trees so format -// strings can't fabricate call sites. -var rustMacroStringRe = regexp.MustCompile(`"(?:[^"\\]|\\.)*"`) - // rustMacroCallRe finds call-shaped token runs inside a macro's token tree: // "unescape(", "m.start(", "SearcherTester::new(". var rustMacroCallRe = regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*(?:(?:::|\.)[A-Za-z_][A-Za-z0-9_]*)*)\s*\(`) @@ -846,7 +843,9 @@ func rustMacroCallSites(body *sitter.Node, src []byte) []astkit.CallSite { } if n.Type() == "macro_invocation" { if tt := internalast.FindChildByType(n, "token_tree"); tt != nil { - text := rustMacroStringRe.ReplaceAllString(tt.Content(src), `""`) + // Comments and string/char literals in the token tree + // are prose, not calls: `assert!(ok, "see bar(1)")`. + text := textmask.Mask("rust", tt.Content(src)) line := int(n.StartPoint().Row) + 1 for _, m := range rustMacroCallRe.FindAllStringSubmatchIndex(text, -1) { path := text[m[2]:m[3]] @@ -1541,8 +1540,8 @@ func javaModifiers(n *sitter.Node, src []byte) []string { if c == nil { continue } - switch c.Type() { - case "marker_annotation", "annotation": + switch { + case c.Type() == "marker_annotation" || c.Type() == "annotation", internalast.IsComment(c): continue default: text := strings.TrimSpace(c.Content(src)) diff --git a/strategies/objc_round2_test.go b/strategies/objc_round2_test.go index 11be04e..90b7fa9 100644 --- a/strategies/objc_round2_test.go +++ b/strategies/objc_round2_test.go @@ -180,3 +180,49 @@ func TestObjCPlainCFunctionStillExtracted(t *testing.T) { t.Fatalf("plain C function 'add' not extracted; symbols: %+v", syms) } } + +func TestObjCContainersInsideConditionals(t *testing.T) { + // AFNetworking's UIKit headers wrap the @interface in `#if + // TARGET_OS_IOS`; the grammar nests it under preproc_if. + src := `#if TARGET_OS_IOS +@interface Indicator : NSObject +@property (nonatomic, assign) BOOL enabled; +- (void)increment; +@end +#else +@protocol Fallback +- (void)noop; +@end +#endif +#if 0 +@interface Dead : NSObject +@end +#endif +#ifdef __cplusplus +extern "C" { +#endif +@implementation Indicator +- (void)increment { +} +@end +#ifdef __cplusplus +} +#endif +` + syms, _ := extract(t, astkit.LangObjC, src) + names := map[string]bool{} + for _, s := range syms { + names[s.QualifiedName] = true + if s.ParentName != "" { + names[s.ParentName+"."+s.Name] = true + } + } + for _, want := range []string{"Indicator", "Indicator.enabled", "Indicator.increment", "Fallback", "Fallback.noop"} { + if !names[want] { + t.Errorf("%s missing; symbols: %v", want, names) + } + } + if names["Dead"] { + t.Errorf("#if 0 interface extracted: %v", names) + } +} diff --git a/textmask/oracle_test.go b/textmask/oracle_test.go new file mode 100644 index 0000000..bd05e18 --- /dev/null +++ b/textmask/oracle_test.go @@ -0,0 +1,491 @@ +package textmask + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" + + sitter "github.com/smacker/go-tree-sitter" + "github.com/smacker/go-tree-sitter/c" + "github.com/smacker/go-tree-sitter/cpp" + "github.com/smacker/go-tree-sitter/csharp" + "github.com/smacker/go-tree-sitter/golang" + "github.com/smacker/go-tree-sitter/java" + "github.com/smacker/go-tree-sitter/javascript" + "github.com/smacker/go-tree-sitter/kotlin" + "github.com/smacker/go-tree-sitter/php" + "github.com/smacker/go-tree-sitter/python" + "github.com/smacker/go-tree-sitter/rust" + "github.com/smacker/go-tree-sitter/swift" + tstsx "github.com/smacker/go-tree-sitter/typescript/tsx" + tstype "github.com/smacker/go-tree-sitter/typescript/typescript" + + "github.com/provasign/astkit/thirdparty/tsobjc" +) + +// The oracle: tree-sitter's comment and literal nodes say which bytes are +// prose or literal text. Mask must blank every non-space byte inside them +// (minus interpolation holes, which are code) and no non-space byte outside. + +func grammar(lang string) *sitter.Language { + switch lang { + case "go": + return golang.GetLanguage() + case "typescript": + return tstype.GetLanguage() + case "tsx": + return tstsx.GetLanguage() + case "javascript": + return javascript.GetLanguage() + case "python": + return python.GetLanguage() + case "java": + return java.GetLanguage() + case "rust": + return rust.GetLanguage() + case "c": + return c.GetLanguage() + case "cpp": + return cpp.GetLanguage() + case "csharp": + return csharp.GetLanguage() + case "php": + return php.GetLanguage() + case "swift": + return swift.GetLanguage() + case "kotlin": + return kotlin.GetLanguage() + case "objc": + return tsobjc.GetLanguage() + } + return nil +} + +// langOfPath maps a file extension to a language key. +func langOfPath(p string) string { + switch strings.ToLower(filepath.Ext(p)) { + case ".go": + return "go" + case ".ts", ".mts", ".cts": + return "typescript" + case ".tsx": + return "tsx" + case ".js", ".jsx", ".mjs", ".cjs": + return "javascript" + case ".py": + return "python" + case ".java": + return "java" + case ".rs": + return "rust" + case ".c": + return "c" + case ".h": + return "cpp" + case ".cc", ".cpp", ".cxx", ".hpp", ".hh", ".hxx": + return "cpp" + case ".m", ".mm": + return "objc" + case ".cs": + return "csharp" + case ".php": + return "php" + case ".swift": + return "swift" + case ".kt", ".kts": + return "kotlin" + case ".cbl", ".cob", ".cpy": + return "cobol" + case ".jcl": + return "jcl" + } + return "" +} + +// Node classes. A node type is looked up in these sets; anything else is +// walked into. +var ( + // literalTypes: every byte is literal text, except interpolation children. + literalTypes = map[string]bool{ + "string": true, "string_literal": true, "raw_string_literal": true, + "char_literal": true, "character_literal": true, + "interpreted_string_literal": true, "rune_literal": true, + "template_string": true, "regex": true, + "encapsed_string": true, "heredoc": true, "nowdoc": true, "heredoc_body": true, + "shell_command_expression": true, + "verbatim_string_literal": true, "interpolated_string_expression": true, + "interpolated_verbatim_string_text": true, "interpolated_raw_string_text": true, + "raw_string_start": true, "raw_string_end": true, "raw_string_content": true, + "text_block": true, "line_string_literal": true, + "multi_line_string_literal": true, "raw_string_literal_content": true, + "string_content": true, + "text": true, // PHP inline HTML + "template_literal_type": true, "hash_bang_line": true, + } + // holeTypes: interpolation children inside literals; they are code. + holeTypes = map[string]bool{ + "interpolation": true, "template_substitution": true, + "string_interpolation": true, "interpolated_expression": true, + "interpolated_identifier": true, "raw_str_interpolation": true, + "variable_name": true, "member_access_expression": true, + "subscript_expression": true, "dynamic_variable_name": true, + "simple_expression": true, "template_type": true, + } + // dontCareTypes: either masked or kept is acceptable. + dontCareTypes = map[string]bool{ + "format_specifier": true, "interpolation_format_clause": true, + "preproc_arg": true, + // Swift `#error("...")` and `#if ... // c`: tree-sitter swallows the + // string and the trailing comment into the directive token. + "diagnostic": true, "directive": true, + } +) + +// phpLitParents are PHP literal nodes whose named children are either text +// parts or interpolated expressions (holes) of any expression type. +var phpLitParents = map[string]bool{ + "encapsed_string": true, "heredoc_body": true, "shell_command_expression": true, +} + +var textParts = map[string]bool{ + "string_content": true, "string_value": true, "escape_sequence": true, + "heredoc_start": true, "heredoc_end": true, "nowdoc_body": true, + "heredoc_body": true, "string": true, "text": true, +} + +// isHole reports whether child ch of a literal node of type parent is an +// interpolation hole. +func isHole(parent string, ch *sitter.Node) bool { + if holeTypes[ch.Type()] { + return true + } + return phpLitParents[parent] && ch.IsNamed() && !textParts[ch.Type()] +} + +// expectations builds the per-byte oracle: 0 code, 1 masked, 2 don't care. +// Nodes only reachable through a hole of PHP's grammar (variable_name etc.) +// are holes only when they are children of a literal. +func expectations(root *sitter.Node, n int) []byte { + exp := make([]byte, n) + fill := func(nd *sitter.Node, v byte) { + a, b := int(nd.StartByte()), int(nd.EndByte()) + if b > n { + b = n + } + for k := a; k < b; k++ { + exp[k] = v + } + } + var walk func(nd *sitter.Node, inLit bool) + // walkLit visits a literal's children: holes are code (walked afresh), + // nested literal parts stay masked. + walkLit := func(nd *sitter.Node) { + for k := 0; k < int(nd.ChildCount()); k++ { + ch := nd.Child(k) + if isHole(nd.Type(), ch) { + fill(ch, 0) + for j := 0; j < int(ch.ChildCount()); j++ { + walk(ch.Child(j), false) + } + continue + } + walk(ch, true) + } + } + walk = func(nd *sitter.Node, inLit bool) { + t := nd.Type() + switch { + case strings.Contains(t, "comment") && nd.IsNamed(): + fill(nd, 1) + return + case dontCareTypes[t]: + fill(nd, 2) + return + case literalTypes[t] && nd.IsNamed(): + fill(nd, 1) + walkLit(nd) + return + } + for k := 0; k < int(nd.ChildCount()); k++ { + walk(nd.Child(k), inLit) + } + } + walk(root, false) + return exp +} + +// holePunct relaxes the oracle for interpolation delimiters: grammars +// disagree on whether `${`, `\(`, `{`, `}` and `)` belong to the hole. +func relaxHolePunct(src string, exp []byte, root *sitter.Node) { + var walk func(nd *sitter.Node, inLit bool) + walk = func(nd *sitter.Node, inLit bool) { + t := nd.Type() + hole := false + if p := nd.Parent(); inLit && p != nil && isHole(p.Type(), nd) { + hole = true + } + if hole { + a, b := int(nd.StartByte()), int(nd.EndByte()) + relaxAround(src, exp, a, -1) + relaxAround(src, exp, a, +1) + relaxAround(src, exp, b, -1) + relaxAround(src, exp, b, +1) + } + lit := inLit || literalTypes[t] && nd.IsNamed() + if hole { + lit = false + } + for k := 0; k < int(nd.ChildCount()); k++ { + walk(nd.Child(k), lit) + } + } + walk(root, false) +} + +// relaxAround marks up to three hole-punctuation bytes next to offset k +// (scanning in direction dir, skipping whitespace) as don't-care. +func relaxAround(src string, exp []byte, k, dir int) { + if dir < 0 { + k-- + } + marked := 0 + for k >= 0 && k < len(src) && marked < 3 { + c := src[k] + switch { + case c == ' ' || c == '\t' || c == '\n' || c == '\r': + case strings.IndexByte("${}()\\#", c) >= 0: + exp[k] = 2 + marked++ + default: + return + } + k += dir + } +} + +type mismatch struct { + line int + want byte + snippet string + nodeKind string +} + +// checkOracle returns mismatches between Mask and tree-sitter for one source, +// or ok=false when the source does not parse cleanly. +func checkOracle(t testing.TB, lang, src string) ([]mismatch, bool) { + g := grammar(lang) + if g == nil { + return nil, false + } + p := sitter.NewParser() + defer p.Close() + p.SetLanguage(g) + tree, err := p.ParseCtx(context.Background(), nil, []byte(src)) + if err != nil || tree == nil { + return nil, false + } + defer tree.Close() + root := tree.RootNode() + if root.HasError() { + return nil, false + } + exp := expectations(root, len(src)) + relaxHolePunct(src, exp, root) + got := MaskFile(lang, src) + var out []mismatch + lastLine := -1 + for k := 0; k < len(src); k++ { + c := src[k] + if c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v' || exp[k] == 2 { + continue + } + masked := got[k] == ' ' + if masked == (exp[k] == 1) { + continue + } + line := strings.Count(src[:k], "\n") + 1 + if line == lastLine { + continue + } + lastLine = line + ls := strings.LastIndexByte(src[:k], '\n') + 1 + le := strings.IndexByte(src[k:], '\n') + if le < 0 { + le = len(src) + } else { + le += k + } + snip := src[ls:le] + if len(snip) > 160 { + snip = snip[:160] + } + kind := "" + if nd := root.NamedDescendantForPointRange(pointAt(src, k), pointAt(src, k)); nd != nil { + kind = nd.Type() + if p := nd.Parent(); p != nil { + kind = p.Type() + ">" + kind + } + } + out = append(out, mismatch{line: line, want: exp[k], snippet: snip, nodeKind: kind}) + } + return out, true +} + +func pointAt(src string, k int) sitter.Point { + row := strings.Count(src[:k], "\n") + col := k - (strings.LastIndexByte(src[:k], '\n') + 1) + return sitter.Point{Row: uint32(row), Column: uint32(col)} +} + +func TestMaskMatchesTreeSitter(t *testing.T) { + files, err := filepath.Glob("testdata/*") + if err != nil { + t.Fatal(err) + } + checked := 0 + for _, f := range files { + lang := langOfPath(f) + if grammar(lang) == nil { + continue + } + b, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + ms, ok := checkOracle(t, lang, string(b)) + if !ok { + if os.Getenv("TEXTMASK_DUMP") != "" { + dumpErrors(t, lang, string(b)) + } + t.Errorf("%s: tree-sitter reports a parse error; fix the snippet", f) + continue + } + checked++ + for _, m := range ms { + t.Errorf("%s:%d: want %s [%s]: %s", f, m.line, wantName(m.want), m.nodeKind, m.snippet) + } + } + if checked == 0 { + t.Fatal("no testdata checked") + } +} + +func wantName(w byte) string { + if w == 1 { + return "masked" + } + return "code" +} + +func dumpErrors(t testing.TB, lang, src string) { + p := sitter.NewParser() + defer p.Close() + p.SetLanguage(grammar(lang)) + tree, _ := p.ParseCtx(context.Background(), nil, []byte(src)) + defer tree.Close() + var walk func(nd *sitter.Node) + walk = func(nd *sitter.Node) { + if nd.IsError() || nd.IsMissing() { + t.Logf(" %s at %d:%d %q", nd.Type(), nd.StartPoint().Row+1, nd.StartPoint().Column, clip(src[nd.StartByte():nd.EndByte()])) + return + } + for k := 0; k < int(nd.ChildCount()); k++ { + walk(nd.Child(k)) + } + } + walk(tree.RootNode()) +} + +func clip(s string) string { + if len(s) > 60 { + return s[:60] + } + return s +} + +// TestMaskCorpus runs the oracle over a directory tree named by +// TEXTMASK_CORPUS. TEXTMASK_CAP bounds files per language (default 400). +func TestMaskCorpus(t *testing.T) { + root := os.Getenv("TEXTMASK_CORPUS") + if root == "" { + t.Skip("TEXTMASK_CORPUS not set") + } + limit := 400 + if v, err := strconv.Atoi(os.Getenv("TEXTMASK_CAP")); err == nil && v > 0 { + limit = v + } + verbose := os.Getenv("TEXTMASK_VERBOSE") != "" + type stat struct{ seen, checked, parseErr, bad, badLines int } + stats := map[string]*stat{} + for _, dir := range strings.Split(root, string(os.PathListSeparator)) { + _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + if d.IsDir() { + switch d.Name() { + case "node_modules", ".git", "vendor", ".grove", "dist", "build": + return filepath.SkipDir + } + return nil + } + lang := langOfPath(p) + if lang == "" || grammar(lang) == nil { + return nil + } + st := stats[lang] + if st == nil { + st = &stat{} + stats[lang] = st + } + if st.seen >= limit { + return nil + } + info, err := d.Info() + if err != nil || info.Size() > 512<<10 || info.Size() == 0 { + return nil + } + b, err := os.ReadFile(p) + if err != nil { + return nil + } + st.seen++ + ms, ok := checkOracle(t, lang, string(b)) + if !ok { + st.parseErr++ + return nil + } + st.checked++ + if len(ms) > 0 { + st.bad++ + st.badLines += len(ms) + shown := ms + if !verbose && len(shown) > 3 { + shown = shown[:3] + } + for _, m := range shown { + t.Logf("MISMATCH %s:%d want %s [%s]: %s", p, m.line, wantName(m.want), m.nodeKind, strings.TrimSpace(m.snippet)) + } + } + return nil + }) + } + langs := make([]string, 0, len(stats)) + for l := range stats { + langs = append(langs, l) + } + sort.Strings(langs) + total := 0 + for _, l := range langs { + st := stats[l] + t.Logf("%-10s files=%d checked=%d parse-errors=%d mismatched-files=%d mismatched-lines=%d", + l, st.seen, st.checked, st.parseErr, st.bad, st.badLines) + total += st.bad + } + fmt.Fprintf(os.Stderr, "textmask corpus: %d mismatched files\n", total) +} diff --git a/textmask/preproc.go b/textmask/preproc.go new file mode 100644 index 0000000..97f488f --- /dev/null +++ b/textmask/preproc.go @@ -0,0 +1,91 @@ +package textmask + +import "strings" + +// MaskInactivePreprocessor masks the lines of `#if 0` branches (C, C++, +// Objective-C; `#if false` for C#), including any nested conditionals inside +// them, up to the matching `#else`, `#elif` or `#endif`. The branch after +// `#else`/`#elif` stays live. The directive lines of the disabled +// conditional itself are kept; directives nested inside it are masked. +// Comments and strings are not masked (compose with Mask for that), but they +// are respected: a `#if 0` inside a block comment is not a directive. Other +// languages return src unchanged. +func MaskInactivePreprocessor(lang, src string) string { + var off []string + switch lang { + case "c", "cpp", "objc": + off = []string{"0", "(0)"} + case "csharp": + off = []string{"false", "(false)"} + default: + return src + } + if !strings.Contains(src, "#") { + return src + } + view := MaskComments(lang, src) + out := []byte(src) + changed := false + depth := 0 // > 0 while inside a disabled branch: conditional nesting depth + for ls := 0; ls < len(view); { + le := strings.IndexByte(view[ls:], '\n') + if le < 0 { + le = len(view) + } else { + le += ls + } + dir, arg := directive(view[ls:le]) + switch { + case depth == 0: + if dir == "if" && contains(off, arg) { + depth = 1 + } + case dir == "if" || dir == "ifdef" || dir == "ifndef": + depth++ + fallthrough + default: + if depth == 1 && (dir == "else" || dir == "elif" || dir == "endif") { + depth = 0 + break + } + if dir == "endif" { + depth-- + } + for k := ls; k < le; k++ { + if c := out[k]; c != '\r' && c != '\n' { + out[k] = ' ' + changed = true + } + } + } + ls = le + 1 + } + if !changed { + return src + } + return string(out) +} + +// directive splits a preprocessor line into its directive name and its +// argument text (trimmed). Non-directive lines return "", "". +func directive(line string) (string, string) { + t := strings.TrimLeft(line, " \t") + if !strings.HasPrefix(t, "#") { + return "", "" + } + t = strings.TrimLeft(t[1:], " \t") + k := 0 + for k < len(t) && isIdentByte(t[k]) { + k++ + } + return t[:k], strings.TrimSpace(t[k:]) +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} diff --git a/textmask/testdata/c_basic.c b/textmask/testdata/c_basic.c new file mode 100644 index 0000000..305b762 --- /dev/null +++ b/textmask/testdata/c_basic.c @@ -0,0 +1,19 @@ +#include +#include "local/header.h" +#define GREETING "hello" +#define MULTI(a) \ + do { (a)++; } while (0) + +// line comment \ + continued: int phantom(int x); +/* block with 'quote and "dq" */ +int main(void) { + char c = '\''; + char d = '"'; + const char *s = "esc \" quote // not comment"; + const char *w = L"wide"; + const char *cont = "line \ +continued"; + printf("%s %c %c %s %s\n", s, c, d, w, cont); + return 0; +} diff --git a/textmask/testdata/cobol_fixed.cbl b/textmask/testdata/cobol_fixed.cbl new file mode 100644 index 0000000..07e92b6 --- /dev/null +++ b/textmask/testdata/cobol_fixed.cbl @@ -0,0 +1,8 @@ +000100 IDENTIFICATION DIVISION. PROG0001 +000200* THIS IS A COMMENT LINE WITH 'QUOTE PROG0002 +000300 PROGRAM-ID. HELLO. PROG0003 +000400/ PAGE EJECT COMMENT PROG0004 +000500 PROCEDURE DIVISION. PROG0005 +000600 DISPLAY 'IT''S A STRING'. PROG0006 +000700 DISPLAY "DOUBLE ""Q""". *> INLINE COMMENT PROG0007 +000800 STOP RUN. PROG0008 diff --git a/textmask/testdata/cobol_free.cbl b/textmask/testdata/cobol_free.cbl new file mode 100644 index 0000000..bd7c163 --- /dev/null +++ b/textmask/testdata/cobol_free.cbl @@ -0,0 +1,8 @@ + >>SOURCE FORMAT FREE +IDENTIFICATION DIVISION. +*> free-format comment with 'quote +PROGRAM-ID. HELLO. +PROCEDURE DIVISION. +* not a comment in free format + DISPLAY 'FREE ''TEXT'''. *> trailing + STOP RUN. diff --git a/textmask/testdata/cpp_basic.cpp b/textmask/testdata/cpp_basic.cpp new file mode 100644 index 0000000..20c7216 --- /dev/null +++ b/textmask/testdata/cpp_basic.cpp @@ -0,0 +1,25 @@ +#include +#include + +namespace foo { +constexpr int k = 1'000; +} +using namespace foo; +constexpr long big = 1'000'000; +constexpr unsigned hex = 0xFF'FF; + +const char *raw = R"( +int phantom(int x) { +)"; +const char *raw2 = R"xy(has )" inside)xy"; +const char *raw3 = u8R"(utf8 raw)"; +auto raw4 = LR"(wide raw)"; +auto raw5 = uR"(u16)"; +auto raw6 = UR"delim(u32 "q")delim"; +const char *u8s = u8"utf8"; +char16_t ch = u'x'; + +class Real { +public: + int value() const { return k; } // comment don't +}; diff --git a/textmask/testdata/cpp_error.cpp b/textmask/testdata/cpp_error.cpp new file mode 100644 index 0000000..af21ef7 --- /dev/null +++ b/textmask/testdata/cpp_error.cpp @@ -0,0 +1,4 @@ +#if 0 +#error don't build this +#endif +int after_error = 1; diff --git a/textmask/testdata/csharp_basic.cs b/textmask/testdata/csharp_basic.cs new file mode 100644 index 0000000..d12747a --- /dev/null +++ b/textmask/testdata/csharp_basic.cs @@ -0,0 +1,30 @@ +#region Don't parse this region name +using System; + +namespace Demo +{ + ///

Doc comment with 'apostrophe + [Description("class Y : Fake")] + public class X + { + string path = @"C:\x\"; + string verb = @"line one +line ""two"""; + string interp = $"{a}"; + string fmt = $"{price:N2} and {{literal}} {Call(1)}"; + string vi = $@"C:\{dir}\file"; + string iv = @$"{dir}\x"; + string raw = """raw "quoted" text"""; + string raw2 = """ + class Phantom { } + """; + string raw3 = $$"""{{Name}} and {literal}"""; + char c = '\''; + char q = '"'; + int a = 1, dir = 2, price = 3; + string Name = "n"; + int Call(int x) => x; + string nested = $"{(a > 1 ? "yes" : "no")}"; + } +} +#endregion diff --git a/textmask/testdata/go_basic.go b/textmask/testdata/go_basic.go new file mode 100644 index 0000000..9c7bd56 --- /dev/null +++ b/textmask/testdata/go_basic.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "strings" +) + +// Comment with 'apostrophe and "quote" +/* block comment + func phantom() {} +*/ +func fix(p string) string { + p = strings.ReplaceAll(p, `\`, "/") + r := '\'' + q := '"' + u := 'é' + raw := ` +func phantom2() { + return "x" +} +` + s := "escaped \" quote // not comment" + fmt.Println(r, q, u, raw, s) + return p +} diff --git a/textmask/testdata/java_basic.java b/textmask/testdata/java_basic.java new file mode 100644 index 0000000..229573d --- /dev/null +++ b/textmask/testdata/java_basic.java @@ -0,0 +1,27 @@ +package x; + +import java.util.List; + +/** + * Javadoc: class Phantom { void m() {} } + */ +@SuppressWarnings("unchecked") +public class Basic { + // line comment with 'apostrophe + private String tb = """ + He said "hi" and "" + then \""" escaped + class Inner {} + """; + private char c = '\''; + private char d = '"'; + private String s = "a \"quoted\" // not comment"; + int n = 1_000_000; + + void run() { + String odd = """ + one " two "" three \""" + """; + System.out.println(odd + s + c + d + tb); + } +} diff --git a/textmask/testdata/jcl_basic.jcl b/textmask/testdata/jcl_basic.jcl new file mode 100644 index 0000000..31e2ab8 --- /dev/null +++ b/textmask/testdata/jcl_basic.jcl @@ -0,0 +1,4 @@ +//MYJOB JOB (ACCT),'PROGRAMMER NAME',CLASS=A +//* THIS IS A COMMENT WITH 'QUOTE +//STEP1 EXEC PGM=IEFBR14,PARM='IT''S' +//DD1 DD DSN=MY.DATA.SET,DISP=SHR diff --git a/textmask/testdata/js_basic.js b/textmask/testdata/js_basic.js new file mode 100644 index 0000000..7f36820 --- /dev/null +++ b/textmask/testdata/js_basic.js @@ -0,0 +1,26 @@ +#!/usr/bin/env node +'use strict'; +// line comment with 'apostrophe +/* block: function phantom() {} */ +class Widget { + #priv = 1; + #tpl; + constructor(b) { + this.#tpl = `a ${b} c`; + this.re = /["']/g; + this.re2 = /[/]\/x/; + this.ratio = this.#priv / 2 / 3; + const nested = `outer ${cond ? `inner ${b + 1}` : 'no'} end`; + const multi = `line one +function phantom2() {} +line ${b}`; + const s = 'it\'s' + "say \"hi\""; + const cont = 'line \ +continued'; + if (/^x/.test(s)) { + return typeof /y/; + } + } +} +const obj = { re: /a\/b/i, n: (1) / 2 }; +export default Widget; diff --git a/textmask/testdata/kotlin_basic.kt b/textmask/testdata/kotlin_basic.kt new file mode 100644 index 0000000..e7bf0f3 --- /dev/null +++ b/textmask/testdata/kotlin_basic.kt @@ -0,0 +1,18 @@ +package demo + +/* outer /* nested */ fun phantom() {} */ +// line comment with 'apostrophe +class Greeter(val name: String) { + fun label(n: Int): String = "n$n" + + fun greet(): String { + val v = "v=${label(3)} and $name and \"esc\"" + val raw = """ + fun phantom2() {} + ${label(4)} $name + """ + val c = '\'' + val q = '"' + return v + raw + c + q + } +} diff --git a/textmask/testdata/objc_basic.m b/textmask/testdata/objc_basic.m new file mode 100644 index 0000000..92c6a1a --- /dev/null +++ b/textmask/testdata/objc_basic.m @@ -0,0 +1,15 @@ +#import + +// comment with 'apostrophe +@interface Greeter : NSObject +- (NSString *)greet; +@end + +@implementation Greeter +- (NSString *)greet { + /* block: - (void)phantom; */ + NSString *s = @"hello \"world\" // not comment"; + char c = '\''; + return s; +} +@end diff --git a/textmask/testdata/php_basic.php b/textmask/testdata/php_basic.php new file mode 100644 index 0000000..0ec4730 --- /dev/null +++ b/textmask/testdata/php_basic.php @@ -0,0 +1,32 @@ +act(2)} and $n and {$n} and $this->name"; + $arr = "v=$items[0] w=$obj->prop"; + $here = <<act(3)} with $n +function phantom2() {} +EOT; + $now = <<<'NOW' +Nowdoc {$this->act(4)} raw $n +NOW; + $indented = <<<"IND" + text $n + IND; + $cmd = `ls $dir`; + return $s . $d . $arr . $here . $now . $indented . $cmd; + } +} +?> +inline HTML don't parse + diff --git a/textmask/testdata/python_312.py b/textmask/testdata/python_312.py new file mode 100644 index 0000000..0426575 --- /dev/null +++ b/textmask/testdata/python_312.py @@ -0,0 +1,6 @@ +songs = ["a", "b"] +s = f"{", ".join(songs)}" +v = f"{f"{inner(1)}"}" +d = f"{x=}" +e = f"{'\n'.join(rows)}" +k = f"{a!r:>{w}}" diff --git a/textmask/testdata/python_basic.py b/textmask/testdata/python_basic.py new file mode 100644 index 0000000..67136a6 --- /dev/null +++ b/textmask/testdata/python_basic.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import logging + +log = logging.getLogger(__name__) + + +class Svc: + '''Don't do this: self.cache = Redis()''' + + def start(self): + log.info('starting up') + self.client = Foo() + x = 'a' + y = "it's # not a comment" + z = r"\"" # raw string with escaped quote + w = rb'\x00' + Rb"bytes" + BR'more' + u'uni' + doc = """ + def phantom(self): + return 'nope' + """ + msg = f"v={label(3)} {{literal}} {x!r} {y:>10} {z:{width}}" + nested = f"{d['key']} and {', '.join(items)}" + multi = f''' + {compute(1, + 2)} + ''' + cont = 'line one \ +line two' + named = f"\N{EM DASH} {value}" + rawf = rf"\d+{pattern}" + return self.client # trailing comment with 'quote diff --git a/textmask/testdata/rust_basic.rs b/textmask/testdata/rust_basic.rs new file mode 100644 index 0000000..7db5a10 --- /dev/null +++ b/textmask/testdata/rust_basic.rs @@ -0,0 +1,46 @@ +#![allow(dead_code)] +//! Crate docs with 'quote +use std::fmt; + +/// Docs: don't parse this +#[derive(Debug, Clone)] +struct Holder<'a> { + name: &'a str, +} + +/* outer /* nested */ still comment fn phantom() {} */ +fn f<'a>(x: &'a str) -> char { + '"' +} + +fn g() -> &'static str { + let f = r#" +let client = Bar::new(); +"#; + let h = r##"has "# inside"##; + let b = b"bytes\""; + let br = br#"raw "bytes""#; + let c = c"cstr"; + let ch = 'a'; + let nl = '\n'; + let uni = '\u{1F600}'; + let e = 'é'; + let by = b'x'; + let bq = b'\''; + let r#type = 1; + let multi = "line one +line two"; + 'outer: loop { + break 'outer; + } + let _ = (f, h, b, br, c, ch, nl, uni, e, by, bq, r#type, multi); + "done" +} + +fn lifetimes<'b, '_>(x: &'b u8) {} + +impl<'a> fmt::Display for Holder<'a> { + fn fmt(&self, w: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(w, "{}", self.name) + } +} diff --git a/textmask/testdata/swift_basic.swift b/textmask/testdata/swift_basic.swift new file mode 100644 index 0000000..c454733 --- /dev/null +++ b/textmask/testdata/swift_basic.swift @@ -0,0 +1,21 @@ +import Foundation + +/* outer /* nested */ func phantom() {} */ +// line comment with 'apostrophe +struct Greeter { + let name: String + + func label(_ n: Int) -> String { return "n" } + + func greet() -> String { + let v = "v=\(label(3)) and \"esc\"" + let raw = #"raw "quoted""# + let rawHole = #"hole \#(label(4)) and \(notHole)"# + let multi = """ + func phantom2() {} + \(label(5)) + """ + let deeper = ##"two "# hashes"## + return v + raw + rawHole + multi + deeper + name + } +} diff --git a/textmask/testdata/ts_basic.ts b/textmask/testdata/ts_basic.ts new file mode 100644 index 0000000..c20674b --- /dev/null +++ b/textmask/testdata/ts_basic.ts @@ -0,0 +1,19 @@ +// TypeScript sample with 'apostrophe +interface Shape { + name: string; // don't +} + +type Tpl = `prefix-${string}`; + +export function area(s: T, w: number, h: number): number { + const label = `shape ${s.name}: ${w * h}`; + const parts = s.name.split(/[,;]/); + const half = w / 2; + return half * h + parts.length + label.length; +} + +/** Doc: class Phantom {} */ +export class Box implements Shape { + #secret = 'x'; + name = "box"; +} diff --git a/textmask/testdata/tsx_basic.tsx b/textmask/testdata/tsx_basic.tsx new file mode 100644 index 0000000..889ef5e --- /dev/null +++ b/textmask/testdata/tsx_basic.tsx @@ -0,0 +1,12 @@ +import React from 'react'; + +// component comment with 'apostrophe +export function App(props: { title: string }) { + const cls = `app ${props.title}`; + return ( +
+ {props.title} + {/* jsx comment */} +
+ ); +} diff --git a/textmask/textmask.go b/textmask/textmask.go new file mode 100644 index 0000000..724a7bd --- /dev/null +++ b/textmask/textmask.go @@ -0,0 +1,1563 @@ +// Package textmask blanks comments and string literals out of source text so +// that callers scanning for code (identifiers, calls, declarations) never +// read prose or literal text as code. +// +// Every function preserves byte offsets: a masked byte becomes ' ' except +// '\n' and '\r', which are kept, so line and column numbers are unchanged +// and a match in the masked text can be sliced out of the original. +// +// The lexer is a single forward pass per language, linear in the input, with +// no regular expressions. It is tolerant: unterminated comments and literals +// run to the end of the input (or, for single-line literals, the end of the +// line) instead of failing. +// +// Interpolation holes are code. `${...}` (JavaScript, TypeScript, Kotlin), +// `{...}` (Python f/t-strings, C# interpolated strings), `\(...)` (Swift) and +// `{$...}`, `${...}`, `$var`, `$var->prop`, `$var[...]` (PHP double quotes, +// heredocs and backticks) are lexed recursively as code, so their identifiers +// stay visible and literals inside them are masked in turn. The hole +// punctuation itself (`${`, `}`, `\(`, `)`) is kept. Kotlin `$name` is kept +// too. Format specifiers — the text after a top-level `:` in a Python or C# +// hole (`{x:>10}`, `{d:yyyy-MM-dd}`) — are literal text and are masked, except +// for nested holes inside them. A Python `!r`/`!s`/`!a` conversion is kept. +// +// Known gaps, all heuristic boundaries a lexer cannot settle without a +// parser: +// - JavaScript/TypeScript regex literals are told from division by the +// previous token. `)` and `]` mean division (so `if (x) /re/.test(s)` is +// misread), `}` means regex, except `{x} />` (a JSX self-closing tag). +// A TypeScript non-null `x! / 2` is division. +// - JSX/TSX text between tags is lexed as code: an apostrophe in +// `

Don't

` opens a quote that runs to the end of that line, and +// `//` in `http://x` starts a comment. A quoted string right +// after `=` may span lines (a JSX attribute value). +// - Swift regex literals (`/.../`, `#/.../#`) are not recognized. +// - C/C++ strings inside `#define` bodies are masked although tree-sitter +// treats a macro body as opaque text; `#error`/`#warning` text is kept. +// - COBOL continuation lines (`-` in column 7) are not joined; each line's +// literal ends at that line's end. +// - JCL: only `//*` comment lines and quoted strings are handled. +// +// Correctness is checked against tree-sitter (see oracle_test.go): set +// TEXTMASK_CORPUS to a list of directories to run the comparison over real +// code. +package textmask + +import ( + "strings" + "unicode/utf8" +) + +// Languages lists the language keys Mask understands (astkit LanguageKey +// strings), plus the alias "jsx" for JavaScript. +var Languages = []string{ + "go", "typescript", "tsx", "javascript", "jsx", "python", "java", "rust", + "c", "cpp", "csharp", "php", "swift", "kotlin", "objc", "cobol", "jcl", +} + +// Supported reports whether lang is a key Mask understands. Mask returns the +// input unchanged for any other key. +func Supported(lang string) bool { + return familyOf(lang) != famNone +} + +// Mask returns src with comment text and string/char literal text replaced by +// spaces. Every byte keeps its offset: masked bytes become ' ', except '\n' +// and '\r', which are kept, so line and column numbers are unchanged. Code +// inside interpolation holes is kept and itself masked recursively. String +// delimiters and prefixes (quotes, r#, @, $, R"delim() are masked too. src may +// be a whole file or a symbol snippet; it is lexed as code from byte 0. +func Mask(lang, src string) string { + return run(lang, src, true, false) +} + +// MaskComments masks only comments; string literals are kept verbatim (they +// are still lexed, so a comment marker inside a string is not a comment). +// Useful for scans that read literal text, such as `#include "x.h"`. +func MaskComments(lang, src string) string { + return run(lang, src, false, false) +} + +// MaskFile is Mask for a whole file. It is identical to Mask except for PHP, +// where the file starts in inline-HTML mode: text outside `` and +// `` is masked. +func MaskFile(lang, src string) string { + return run(lang, src, true, true) +} + +func run(lang, src string, maskStrings, phpHTML bool) string { + fam := familyOf(lang) + if fam == famNone || src == "" { + return src + } + s := &scanner{ + src: src, + out: []byte(src), + fam: fam, + lang: lang, + maskStrings: maskStrings, + } + switch fam { + case famCOBOL: + s.cobol() + case famJCL: + s.jcl() + case famPHP: + i := 0 + if phpHTML { + i = s.phpHTML(0) + } + for i < len(src) { + i = s.code(i, 0, false) + if i < len(src) { + // code stopped at `?>`; the tag is code, the rest is HTML. + i = s.phpHTML(i + 2) + } + } + default: + s.code(0, 0, false) + } + return string(s.out) +} + +type family int + +const ( + famNone family = iota + famC // C, C++, Objective-C + famJava + famCSharp + famGo + famRust + famJS + famPython + famPHP + famKotlin + famSwift + famCOBOL + famJCL +) + +func familyOf(lang string) family { + switch lang { + case "c", "cpp", "objc": + return famC + case "java": + return famJava + case "csharp": + return famCSharp + case "go": + return famGo + case "rust": + return famRust + case "javascript", "jsx", "typescript", "tsx": + return famJS + case "python": + return famPython + case "php": + return famPHP + case "kotlin": + return famKotlin + case "swift": + return famSwift + case "cobol": + return famCOBOL + case "jcl": + return famJCL + } + return famNone +} + +type scanner struct { + src string + out []byte + fam family + lang string + maskStrings bool + // regexOK is the JavaScript previous-token state: true when a `/` at + // this point starts a regex literal rather than a division. + regexOK bool +} + +// blank masks src[a:b], keeping line breaks. +func (s *scanner) blank(a, b int) { + if b > len(s.out) { + b = len(s.out) + } + for k := a; k < b; k++ { + if c := s.out[k]; c != '\n' && c != '\r' { + s.out[k] = ' ' + } + } +} + +// lit masks literal text src[a:b] when strings are being masked. +func (s *scanner) lit(a, b int) { + if s.maskStrings { + s.blank(a, b) + } +} + +func isIdentByte(c byte) bool { + return c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c >= 0x80 +} + +func isIdentStart(c byte) bool { + return c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= 0x80 +} + +func isDigit(c byte) bool { return c >= '0' && c <= '9' } + +func isAlnum(c byte) bool { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' +} + +// at reports whether src has lit at offset i. +func (s *scanner) at(i int, lit string) bool { + return i >= 0 && i+len(lit) <= len(s.src) && s.src[i:i+len(lit)] == lit +} + +func (s *scanner) byteAt(i int) byte { + if i >= 0 && i < len(s.src) { + return s.src[i] + } + return 0 +} + +// eol returns the offset of the next '\n' at or after i, or len(src). +func (s *scanner) eol(i int) int { + if k := strings.IndexByte(s.src[i:], '\n'); k >= 0 { + return i + k + } + return len(s.src) +} + +// prevByte returns the last non-blank byte before i, or 0. +func (s *scanner) prevByte(i int) byte { + for k := i - 1; k >= 0; k-- { + switch c := s.src[k]; c { + case ' ', '\t', '\n', '\r': + default: + return c + } + } + return 0 +} + +// backtickName skips a backtick-quoted identifier (Kotlin, Swift), which is +// code even when it holds spaces or apostrophes: fun `it doesn't fail`(). +func (s *scanner) backtickName(i int) int { + for k := i + 1; k < len(s.src); k++ { + switch s.src[k] { + case '`': + return k + 1 + case '\n': + return i + 1 + } + } + return i + 1 +} + +// atLineStart reports whether only spaces and tabs precede i on its line. +func (s *scanner) atLineStart(i int) bool { + for k := i - 1; k >= 0; k-- { + switch s.src[k] { + case ' ', '\t': + continue + case '\n', '\r': + return true + default: + return false + } + } + return true +} + +// lineComment masks a comment from i to the end of its line and returns the +// offset of the '\n'. With cont, a backslash before the newline continues the +// comment onto the next line (C preprocessor semantics). +func (s *scanner) lineComment(i int, cont bool) int { + k := s.eol(i) + for cont && k < len(s.src) { + j := k - 1 + if j > i && s.src[j] == '\r' { + j-- + } + if j < i || s.src[j] != '\\' { + break + } + k = s.eol(k + 1) + } + s.blank(i, k) + return k +} + +// blockComment masks a `/* ... */` comment starting at i and returns the +// offset after it. With nested, inner `/*` pairs nest (Rust, Kotlin, Swift). +func (s *scanner) blockComment(i int, nested bool) int { + n := len(s.src) + k := i + 2 + depth := 1 + for k < n { + if s.src[k] == '*' && k+1 < n && s.src[k+1] == '/' { + k += 2 + depth-- + if depth == 0 { + break + } + continue + } + if nested && s.src[k] == '/' && k+1 < n && s.src[k+1] == '*' { + depth++ + k += 2 + continue + } + k++ + } + if k > n { + k = n + } + s.blank(i, k) + return k +} + +// quoted masks a literal opening with quote q at i (prefix bytes from start +// are masked with it) and returns the offset after it. With esc, a backslash +// skips the next byte. Without multiline, an unescaped newline ends it. +func (s *scanner) quoted(start, i int, q byte, esc, multiline bool) int { + n := len(s.src) + k := i + 1 + for k < n { + c := s.src[k] + if c == '\\' && esc { + k = s.esc(k) + continue + } + if c == q { + k++ + break + } + if c == '\n' && !multiline { + break + } + k++ + } + if k > n { + k = n + } + s.lit(start, k) + return k +} + +// doubled masks a literal opening with quote q at i where a doubled quote is +// an escaped quote (C# verbatim strings, COBOL, JCL). Multiline unless +// oneLine. +func (s *scanner) doubled(start, i int, q byte, oneLine bool) int { + n := len(s.src) + k := i + 1 + for k < n { + c := s.src[k] + if c == q { + if k+1 < n && s.src[k+1] == q { + k += 2 + continue + } + k++ + break + } + if c == '\n' && oneLine { + break + } + k++ + } + s.lit(start, k) + return k +} + +// number skips a numeric literal starting at i. Digit separators: `_` in +// every language, and `'` between alphanumerics in the C family (C++14, +// C23), so `1'000` and `0xFF'FF` are not character literals. +func (s *scanner) number(i int) int { + n := len(s.src) + k := i + 1 + for k < n { + c := s.src[k] + switch { + case isIdentByte(c) || c == '.': + k++ + case (c == '+' || c == '-') && (s.src[k-1] == 'e' || s.src[k-1] == 'E' || s.src[k-1] == 'p' || s.src[k-1] == 'P'): + k++ + case c == '\'' && s.fam == famC && k+1 < n && isAlnum(s.src[k+1]): + k++ + default: + return k + } + } + return k +} + +// esc returns the offset after the escape sequence whose backslash is at k: +// two bytes, or three for a backslash before CRLF (a line continuation). +func (s *scanner) esc(k int) int { + if s.at(k+1, "\r\n") { + return k + 3 + } + return k + 2 +} + +// code lexes src from i as code. It returns the offset of the first +// unmatched closer (')' or '}', when closer is non-zero) or len(src). With +// fmtStop, a top-level ':' (and, for Python, a '!' conversion) also stops it: +// the start of a format specifier inside an interpolation hole. PHP stops at +// `?>` as well. +func (s *scanner) code(i int, closer byte, fmtStop bool) int { + n := len(s.src) + depth := 0 + if s.fam == famJS { + s.regexOK = true + } + for i < n { + c := s.src[i] + switch c { + case ' ', '\t', '\n', '\r', '\f', '\v': + i++ + continue + case '(', '[', '{': + depth++ + i++ + s.regexOK = true + continue + case ')', ']', '}': + if depth == 0 && closer != 0 && c == closer { + return i + } + if depth > 0 { + depth-- + } + i++ + s.regexOK = c == '}' + continue + case ':': + if fmtStop && depth == 0 && !s.at(i+1, ":") && !s.at(i-1, ":") { + return i + } + case '!': + if fmtStop && depth == 0 && s.fam == famPython && !s.at(i+1, "=") { + return i + } + } + if isIdentStart(c) || c == '$' && s.fam == famJS { + k := i + 1 + for k < n && (isIdentByte(s.src[k]) || s.src[k] == '$' && s.fam == famJS) { + k++ + } + if j, ok := s.prefixed(i, k); ok { + i = j + s.regexOK = false + continue + } + if s.fam == famJS { + s.regexOK = jsRegexKeyword(s.src[i:k]) + } + i = k + continue + } + if isDigit(c) { + i = s.number(i) + s.regexOK = false + continue + } + j := s.token(i) + if j < 0 { + return i // PHP `?>` + } + if j > i { + i = j + s.regexOK = false + continue + } + // Plain punctuation. + if s.fam == famJS { + s.regexOK = jsRegexAfter(c) + if c == '!' && i > 0 && (isIdentByte(s.src[i-1]) || s.src[i-1] == ')' || s.src[i-1] == ']') { + s.regexOK = false // TypeScript non-null assertion `x! / 2` + } + } + i++ + } + return n +} + +func jsRegexAfter(c byte) bool { + switch c { + case ',', '=', ':', '!', '&', '|', '?', ';', '+', '-', '*', '%', '>', '~', '^': + return true + } + return false +} + +func jsRegexKeyword(w string) bool { + switch w { + case "return", "typeof", "instanceof", "in", "of", "new", "delete", "void", + "throw", "case", "do", "else", "yield", "await": + return true + } + return false +} + +// prefixed handles an identifier src[i:k] that may be a string prefix. It +// returns the offset after the literal and true when one was consumed. +func (s *scanner) prefixed(i, k int) (int, bool) { + if k >= len(s.src) { + return 0, false + } + next := s.src[k] + w := s.src[i:k] + switch s.fam { + case famC: + if next == '"' || next == '\'' { + switch w { + case "L", "u", "U", "u8": + return s.quoted(i, k, next, true, false), true + case "R", "LR", "uR", "UR", "u8R": + if next == '"' && s.lang != "c" { + if j, ok := s.cppRaw(i, k); ok { + return j, true + } + } + } + } + case famRust: + switch w { + case "b": + if next == '\'' { + return s.quoted(i, k, '\'', true, false), true + } + if next == '"' { + return s.quoted(i, k, '"', true, true), true + } + case "c": + if next == '"' { + return s.quoted(i, k, '"', true, true), true + } + case "r", "br", "cr": + if next == '"' || next == '#' { + return s.rustRaw(i, k) + } + } + case famPython: + if (next == '"' || next == '\'') && len(w) <= 2 { + raw, fmt := false, false + for j := 0; j < len(w); j++ { + switch w[j] | 0x20 { + case 'r': + raw = true + case 'f', 't': + fmt = true + case 'b', 'u': + default: + return 0, false + } + } + if len(w) == 2 && w[0]|0x20 == w[1]|0x20 { + return 0, false + } + return s.pyString(i, k, raw, fmt), true + } + } + return 0, false +} + +// token lexes a comment or literal starting at a non-identifier byte i and +// returns the offset after it, i when src[i] is plain punctuation, or -1 for +// PHP's `?>`. +func (s *scanner) token(i int) int { + c := s.src[i] + next := s.byteAt(i + 1) + if c == '/' && s.fam != famPython && s.fam != famCOBOL && s.fam != famJCL { + if next == '/' { + if s.fam == famPHP { + return s.phpLineComment(i) + } + return s.lineComment(i, s.fam == famC) + } + if next == '*' { + nested := s.fam == famRust || s.fam == famKotlin || s.fam == famSwift + return s.blockComment(i, nested) + } + } + switch s.fam { + case famC: + switch c { + case '"', '\'': + return s.quoted(i, i, c, true, false) + case '@': + if next == '"' && s.lang == "objc" { + return s.quoted(i, i+1, '"', true, false) + } + case '#': + if s.atLineStart(i) { + return s.cDirective(i) + } + } + case famJava: + switch c { + case '"': + if s.at(i, `"""`) { + return s.javaTextBlock(i) + } + return s.quoted(i, i, c, true, false) + case '\'': + return s.quoted(i, i, c, true, false) + } + case famGo: + switch c { + case '"', '\'': + return s.quoted(i, i, c, true, false) + case '`': + return s.quoted(i, i, c, false, true) + } + case famRust: + switch c { + case '"': + return s.quoted(i, i, c, true, true) + case '\'': + return s.rustQuote(i) + } + case famJS: + switch c { + case '"', '\'': + // A JSX attribute value (`title="..."`) may span lines; a + // plain JavaScript string may not. + return s.quoted(i, i, c, true, s.prevByte(i) == '=') + case '`': + return s.jsTemplate(i) + case '#': + if i == 0 && next == '!' { + return s.lineComment(i, false) + } + case '/': + // `{expr} />` closes a JSX tag; `(/>/g)` is a regex. + if s.regexOK && !(next == '>' && s.prevByte(i) == '}') { + return s.jsRegex(i) + } + } + case famPython: + switch c { + case '#': + return s.lineComment(i, false) + case '"', '\'': + return s.pyString(i, i, false, false) + } + case famPHP: + switch c { + case '#': + if next != '[' { + return s.phpLineComment(i) + } + case '\'': + return s.quoted(i, i, c, true, true) + case '"', '`': + return s.phpInterp(i, i+1, c) + case '<': + if s.at(i, "<<<") { + if j, ok := s.phpHeredoc(i); ok { + return j + } + return i + 3 + } + case '?': + if next == '>' { + return -1 + } + } + case famCSharp: + return s.csharpToken(i) + case famKotlin: + switch c { + case '`': + return s.backtickName(i) + case '"': + return s.kotlinString(i) + case '\'': + return s.quoted(i, i, c, true, false) + } + case famSwift: + switch c { + case '`': + return s.backtickName(i) + case '"': + return s.swiftString(i, i, 0) + case '#': + h := i + for h < len(s.src) && s.src[h] == '#' { + h++ + } + if h < len(s.src) && s.src[h] == '"' { + return s.swiftString(i, h, h-i) + } + return h + } + } + return i +} + +// ---- C family ---- + +// cDirective handles a preprocessor line starting at i. `#error` and +// `#warning` take free text (apostrophes included), so their line is skipped +// as code apart from comments. Other directives are lexed as code. +func (s *scanner) cDirective(i int) int { + k := i + 1 + for k < len(s.src) && (s.src[k] == ' ' || s.src[k] == '\t') { + k++ + } + w := k + for w < len(s.src) && isIdentByte(s.src[w]) { + w++ + } + switch s.src[k:w] { + case "error", "warning": + return s.freeTextLine(w, true) + } + return i + 1 +} + +// freeTextLine skips text from i to the end of the line (with backslash +// continuation when cont) treating only `//` and `/*` as special. +func (s *scanner) freeTextLine(i int, cont bool) int { + n := len(s.src) + for i < n { + c := s.src[i] + if c == '\n' { + j := i - 1 + if j >= 0 && s.src[j] == '\r' { + j-- + } + if cont && j >= 0 && s.src[j] == '\\' { + i++ + continue + } + return i + } + if c == '/' && i+1 < n && s.src[i+1] == '/' { + return s.lineComment(i, cont) + } + if c == '/' && i+1 < n && s.src[i+1] == '*' { + i = s.blockComment(i, false) + continue + } + i++ + } + return n +} + +// cppRaw lexes a C++ raw string `R"delim( ... )delim"` whose quote is at q. +func (s *scanner) cppRaw(start, q int) (int, bool) { + n := len(s.src) + k := q + 1 + for k < n && k-q <= 17 { + c := s.src[k] + if c == '(' { + break + } + if c == ' ' || c == ')' || c == '\\' || c == '\t' || c == '\n' || c == '"' { + return 0, false + } + k++ + } + if k >= n || s.src[k] != '(' { + return 0, false + } + delim := s.src[q+1 : k] + end := n + for p := k + 1; p < n; p++ { + if s.src[p] == ')' && s.at(p+1, delim) && s.byteAt(p+1+len(delim)) == '"' { + end = p + 2 + len(delim) + break + } + } + s.lit(start, end) + return end, true +} + +// ---- Java ---- + +func (s *scanner) javaTextBlock(i int) int { + n := len(s.src) + k := i + 3 + for k < n { + if s.src[k] == '\\' { + k = s.esc(k) + continue + } + if s.at(k, `"""`) { + k += 3 + break + } + k++ + } + if k > n { + k = n + } + s.lit(i, k) + return k +} + +// ---- Rust ---- + +// rustQuote tells a char literal (`'a'`, `'\n'`, `'é'`, `'"'`) from a +// lifetime or label (`'a`, `'static`, `'_`), which is code. +func (s *scanner) rustQuote(i int) int { + n := len(s.src) + if i+1 >= n { + return i + 1 + } + if s.src[i+1] == '\\' { + return s.quoted(i, i, '\'', true, false) + } + _, size := utf8.DecodeRuneInString(s.src[i+1:]) + if k := i + 1 + size; k < n && s.src[k] == '\'' { + s.lit(i, k+1) + return k + 1 + } + return i + 1 // lifetime: the name after it lexes as an identifier +} + +// rustRaw lexes `r"..."`, `r#"..."#`, `br##"..."##`; the prefix spans +// src[start:k]. `r#ident` (a raw identifier) is not a string. +func (s *scanner) rustRaw(start, k int) (int, bool) { + h := k + for h < len(s.src) && s.src[h] == '#' { + h++ + } + if h >= len(s.src) || s.src[h] != '"' { + return 0, false + } + hashes := h - k + end := len(s.src) + for q := h + 1; q < len(s.src); q++ { + if s.src[q] != '"' { + continue + } + e := q + 1 + for e < len(s.src) && e-q-1 < hashes && s.src[e] == '#' { + e++ + } + if e-q-1 == hashes { + end = e + break + } + } + s.lit(start, end) + return end, true +} + +// ---- JavaScript / TypeScript ---- + +func (s *scanner) jsTemplate(i int) int { + n := len(s.src) + seg := i + k := i + 1 + for k < n { + c := s.src[k] + if c == '\\' { + k = s.esc(k) + continue + } + if c == '`' { + k++ + s.lit(seg, k) + return k + } + if c == '$' && k+1 < n && s.src[k+1] == '{' { + s.lit(seg, k) + k = s.hole(k+2, '}') + seg = k + continue + } + k++ + } + s.lit(seg, n) + return n +} + +// hole lexes an interpolation hole's code starting at i up to closer and +// returns the offset after the closer. +func (s *scanner) hole(i int, closer byte) int { + saved := s.regexOK + k := s.code(i, closer, false) + s.regexOK = saved + if k < len(s.src) { + k++ + } + return k +} + +// jsRegex lexes a regex literal at i, or returns i when the line ends before +// a closing slash (then the '/' was a division after all). +func (s *scanner) jsRegex(i int) int { + n := len(s.src) + k := i + 1 + inClass := false + for k < n { + c := s.src[k] + switch { + case c == '\\': + k = s.esc(k) + continue + case c == '\n' || c == '\r': + return i + case c == '[': + inClass = true + case c == ']': + inClass = false + case c == '/' && !inClass: + k++ + for k < n && isIdentByte(s.src[k]) { + k++ + } + s.lit(i, k) + return k + } + k++ + } + return i +} + +// ---- Python ---- + +// pyString lexes a Python string whose prefix starts at start and whose +// opening quote is at q. +func (s *scanner) pyString(start, q int, raw, fmtStr bool) int { + n := len(s.src) + qc := s.src[q] + triple := q+2 < n && s.src[q+1] == qc && s.src[q+2] == qc + k := q + 1 + if triple { + k = q + 3 + } + seg := start + for k < n { + c := s.src[k] + switch { + case c == '\\': + if fmtStr && s.byteAt(k+1) == '{' && raw { + k++ // rf"\{x}": the brace still opens a hole + continue + } + if fmtStr && !raw && s.at(k+1, "N{") { + // \N{NAME} is an escape, not a hole. + if e := strings.IndexByte(s.src[k:], '}'); e >= 0 { + k += e + 1 + continue + } + } + k = s.esc(k) + continue + case c == qc: + if !triple { + k++ + s.lit(seg, k) + return k + } + if k+2 < n && s.src[k+1] == qc && s.src[k+2] == qc { + k += 3 + s.lit(seg, k) + return k + } + case c == '\n' && !triple: + s.lit(seg, k) + return k + case fmtStr && c == '{': + if k+1 < n && s.src[k+1] == '{' { + k += 2 + continue + } + s.lit(seg, k) + k = s.fmtHole(k + 1) + seg = k + continue + } + k++ + } + if k > n { + k = n + } + s.lit(seg, k) + return k +} + +// fmtHole lexes a Python f-string or C# interpolation hole whose code starts +// at i: the expression, an optional Python `!r` conversion, and an optional +// format specifier (masked as literal text, nested holes kept). It returns +// the offset after the closing '}'. +func (s *scanner) fmtHole(i int) int { + n := len(s.src) + k := s.code(i, '}', true) + if k < n && s.src[k] == '!' { + k++ + for k < n && isIdentByte(s.src[k]) { + k++ + } + } + if k < n && s.src[k] == ':' { + k = s.fmtSpec(k) + } + if k < n && s.src[k] == '}' { + k++ + } + return k +} + +// fmtSpec masks a format specifier starting at its ':' up to the hole's +// closing '}', which it returns the offset of. +func (s *scanner) fmtSpec(i int) int { + n := len(s.src) + seg := i + k := i + for k < n { + c := s.src[k] + if c == '}' { + break + } + if c == '{' && s.fam == famPython { + s.lit(seg, k) + k = s.fmtHole(k + 1) + seg = k + continue + } + if c == '\n' && s.fam == famCSharp { + break + } + k++ + } + s.lit(seg, k) + return k +} + +// ---- PHP ---- + +// phpHTML masks inline HTML from i up to the next open tag (`` close tag. +func (s *scanner) phpLineComment(i int) int { + e := s.eol(i) + if t := strings.Index(s.src[i:e], "?>"); t >= 0 { + e = i + t + } + s.blank(i, e) + return e +} + +// phpInterp lexes a double-quoted or backtick PHP string whose quote is at +// q-1 and whose text starts at q, keeping `{$...}`, `${...}` and simple +// variable interpolations as code. +func (s *scanner) phpInterp(start, k int, q byte) int { + n := len(s.src) + seg := start + for k < n { + c := s.src[k] + if c == '\\' { + k = s.esc(k) + continue + } + if c == q { + k++ + s.lit(seg, k) + return k + } + if j := s.phpVar(k); j > k { + s.lit(seg, k) + seg, k = j, j + continue + } + k++ + } + s.lit(seg, n) + return n +} + +// phpVar recognises an interpolation at k inside a PHP string: `{$expr}`, +// `${expr}`, `$name`, `$name->prop`, `$name?->prop` or `$name[key]`. It +// returns the offset after it, or k when there is none. +func (s *scanner) phpVar(k int) int { + n := len(s.src) + if s.at(k, "{$") { + return s.hole(k+1, '}') + } + if s.at(k, "${") { + return s.hole(k+2, '}') + } + if s.src[k] != '$' || k+1 >= n || !isIdentStart(s.src[k+1]) { + return k + } + j := k + 2 + for j < n && isIdentByte(s.src[j]) { + j++ + } + switch { + case s.at(j, "->") && j+2 < n && isIdentStart(s.src[j+2]): + j += 3 + for j < n && isIdentByte(s.src[j]) { + j++ + } + case s.at(j, "?->") && j+3 < n && isIdentStart(s.src[j+3]): + j += 4 + for j < n && isIdentByte(s.src[j]) { + j++ + } + case s.at(j, "["): + e := j + 1 + for e < n && s.src[e] != ']' && s.src[e] != '\n' && s.src[e] != '"' { + e++ + } + if e < n && s.src[e] == ']' { + j = e + 1 + } + } + return j +} + +// phpHeredoc lexes `<<= n || s.src[k] != quote { + return 0, false + } + k++ + } + if k < n && s.src[k] == '\r' { + k++ + } + if k >= n || s.src[k] != '\n' { + return 0, false + } + interp := quote != '\'' + seg := i + k++ + lineStart := true + for k < n { + if lineStart { + j := k + for j < n && (s.src[j] == ' ' || s.src[j] == '\t') { + j++ + } + if s.at(j, id) && !isIdentByte(s.byteAt(j+len(id))) { + end := j + len(id) + s.lit(seg, end) + return end, true + } + lineStart = false + } + c := s.src[k] + if c == '\n' { + lineStart = true + k++ + continue + } + if interp { + if c == '\\' { + k = s.esc(k) + continue + } + if j := s.phpVar(k); j > k { + s.lit(seg, k) + seg, k = j, j + continue + } + } + k++ + } + s.lit(seg, n) + return n, true +} + +// ---- C# ---- + +func (s *scanner) csharpToken(i int) int { + n := len(s.src) + c := s.src[i] + switch c { + case '\'': + return s.quoted(i, i, c, true, false) + case '#': + if s.atLineStart(i) { + k := i + 1 + for k < n && (s.src[k] == ' ' || s.src[k] == '\t') { + k++ + } + w := k + for w < n && isIdentByte(s.src[w]) { + w++ + } + switch s.src[k:w] { + case "region", "endregion", "error", "warning", "pragma", "line", "nullable": + return s.freeTextLine(w, false) + } + } + return i + case '"', '@', '$': + default: + return i + } + // String prefixes: `$`* and `@` in either order, then quotes. + k := i + dollars, verbatim := 0, false + for k < n { + if s.src[k] == '$' { + dollars++ + } else if s.src[k] == '@' && !verbatim { + verbatim = true + } else { + break + } + k++ + } + if k >= n || s.src[k] != '"' { + return i + } + q := k + for q < n && s.src[q] == '"' { + q++ + } + quotes := q - k + if quotes >= 3 && !verbatim { + return s.csharpRaw(i, k, quotes, dollars) + } + if verbatim { + if dollars == 0 { + return s.doubled(i, k, '"', false) + } + return s.csharpInterp(i, k+1, true) + } + if dollars > 0 { + return s.csharpInterp(i, k+1, false) + } + return s.quoted(i, k, '"', true, false) +} + +// csharpInterp lexes `$"..."` (and `$@"..."` when verbatim) from the text +// start k, keeping `{expr}` holes as code. +func (s *scanner) csharpInterp(start, k int, verbatim bool) int { + n := len(s.src) + seg := start + for k < n { + c := s.src[k] + switch { + case c == '\\' && !verbatim: + k = s.esc(k) + continue + case c == '"': + if verbatim && k+1 < n && s.src[k+1] == '"' { + k += 2 + continue + } + k++ + s.lit(seg, k) + return k + case c == '\n' && !verbatim: + s.lit(seg, k) + return k + case c == '{' || c == '}': + if k+1 < n && s.src[k+1] == c { + k += 2 + continue + } + if c == '{' { + s.lit(seg, k) + k = s.fmtHole(k + 1) + seg = k + continue + } + } + k++ + } + s.lit(seg, n) + return n +} + +// csharpRaw lexes a raw string literal of quotes quote characters starting +// at k (prefix from start). With dollars > 0 it is interpolated: a run of +// `dollars` braces opens a hole, fewer are literal text. +func (s *scanner) csharpRaw(start, k, quotes, dollars int) int { + n := len(s.src) + seg := start + k += quotes + for k < n { + c := s.src[k] + if c == '"' { + r := k + for r < n && s.src[r] == '"' { + r++ + } + if r-k >= quotes { + s.lit(seg, r) + return r + } + k = r + continue + } + if c == '{' && dollars > 0 { + r := k + for r < n && s.src[r] == '{' { + r++ + } + if r-k < dollars { + k = r + continue + } + open := r - dollars + s.lit(seg, open) + e := s.code(r, '}', true) + if e < n && s.src[e] == ':' { + e = s.fmtSpec(e) + } + for j := 0; j < dollars && e < n && s.src[e] == '}'; j++ { + e++ + } + seg, k = e, e + continue + } + k++ + } + s.lit(seg, n) + return n +} + +// ---- Kotlin ---- + +func (s *scanner) kotlinString(i int) int { + n := len(s.src) + raw := s.at(i, `"""`) + k := i + 1 + if raw { + k = i + 3 + } + seg := i + for k < n { + c := s.src[k] + switch { + case c == '\\' && !raw: + k = s.esc(k) + continue + case c == '"': + if !raw { + k++ + s.lit(seg, k) + return k + } + if s.at(k, `"""`) { + k += 3 + for k < n && s.src[k] == '"' { + k++ + } + s.lit(seg, k) + return k + } + case c == '\n' && !raw: + s.lit(seg, k) + return k + case c == '$' && k+1 < n: + if s.src[k+1] == '{' { + s.lit(seg, k) + k = s.hole(k+2, '}') + seg = k + continue + } + if isIdentStart(s.src[k+1]) || s.src[k+1] == '`' { + s.lit(seg, k) + k += 2 + for k < n && isIdentByte(s.src[k]) { + k++ + } + seg = k + continue + } + } + k++ + } + s.lit(seg, n) + return n +} + +// ---- Swift ---- + +// swiftString lexes a Swift string whose prefix starts at start, whose +// first quote is at q and which has hashes extended delimiters. +func (s *scanner) swiftString(start, q, hashes int) int { + n := len(s.src) + multi := s.at(q, `"""`) + k := q + 1 + if multi { + k = q + 3 + } + seg := start + for k < n { + c := s.src[k] + switch { + case c == '\\': + j := k + 1 + h := 0 + for j < n && s.src[j] == '#' && h < hashes { + j++ + h++ + } + if h < hashes { + k++ + continue + } + if j < n && s.src[j] == '(' { + s.lit(seg, k) + k = s.hole(j+1, ')') + seg = k + continue + } + k = j + 1 + continue + case c == '"': + if multi && !s.at(k, `"""`) { + k++ + continue + } + e := k + 1 + if multi { + e = k + 3 + } + h := 0 + for e < n && s.src[e] == '#' && h < hashes { + e++ + h++ + } + if h == hashes { + s.lit(seg, e) + return e + } + k++ + continue + case c == '\n' && !multi: + s.lit(seg, k) + return k + } + k++ + } + s.lit(seg, n) + return n +} + +// ---- COBOL ---- + +// cobol masks COBOL source line by line. Fixed format (the default): a `*` +// or `/` in column 7 makes the line a comment (columns 7-72 masked; the +// sequence areas are left as-is). `*>` starts an inline comment in either +// format. `>>SOURCE FORMAT FREE` switches to free format. +func (s *scanner) cobol() { + n := len(s.src) + free := false + for ls := 0; ls < n; { + le := s.eol(ls) + line := s.src[ls:le] + if t := strings.TrimSpace(line); strings.HasPrefix(t, ">>") || strings.HasPrefix(t, "$SET") { + u := strings.ToUpper(t) + if strings.Contains(u, "FREE") { + free = true + } else if strings.Contains(u, "FIXED") { + free = false + } + } + start := ls + if !free { + if len(line) > 6 && (line[6] == '*' || line[6] == '/') { + end := le + if end > ls+72 { + end = ls + 72 + } + s.blank(ls+6, end) + ls = le + 1 + continue + } + start = ls + 7 + if start > le { + start = le + } + } + for k := start; k < le; { + c := s.src[k] + switch { + case c == '*' && k+1 < le && s.src[k+1] == '>': + end := le + if !free && end > ls+72 { + end = ls + 72 // leave the sequence area + } + s.blank(k, end) + k = le + case c == '\'' || c == '"': + k = s.doubled(k, k, c, true) + default: + k++ + } + } + ls = le + 1 + } +} + +// ---- JCL ---- + +// jcl masks `//*` comment lines and quoted strings (`”` escapes a quote). +func (s *scanner) jcl() { + n := len(s.src) + for ls := 0; ls < n; { + le := s.eol(ls) + if s.at(ls, "//*") { + s.blank(ls, le) + ls = le + 1 + continue + } + for k := ls; k < le; { + if s.src[k] == '\'' { + k = s.doubled(k, k, '\'', true) + } else { + k++ + } + } + ls = le + 1 + } +} diff --git a/textmask/textmask_test.go b/textmask/textmask_test.go new file mode 100644 index 0000000..492bad8 --- /dev/null +++ b/textmask/textmask_test.go @@ -0,0 +1,491 @@ +package textmask + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// maskCase lists substrings of src that Mask must keep verbatim (code) and +// substrings it must blank entirely (comment or literal text). Each +// substring is located by its first occurrence in src. +type maskCase struct { + name string + lang string + src string + keep []string + gone []string +} + +func checkCase(t *testing.T, fn func(lang, src string) string, tc maskCase) { + t.Helper() + got := fn(tc.lang, tc.src) + if len(got) != len(tc.src) { + t.Fatalf("%s: length %d, want %d", tc.name, len(got), len(tc.src)) + } + for _, k := range tc.keep { + i := strings.Index(tc.src, k) + if i < 0 { + t.Fatalf("%s: keep %q not in src", tc.name, k) + } + if got[i:i+len(k)] != k { + t.Errorf("%s: want %q kept, got %q\n%s", tc.name, k, got[i:i+len(k)], got) + } + } + for _, g := range tc.gone { + i := strings.Index(tc.src, g) + if i < 0 { + t.Fatalf("%s: gone %q not in src", tc.name, g) + } + if strings.Trim(got[i:i+len(g)], " \r\n") != "" { + t.Errorf("%s: want %q masked, got %q\n%s", tc.name, g, got[i:i+len(g)], got) + } + } +} + +var maskCases = []maskCase{ + // ---- Python ---- + {name: "py single-quoted string is not a lifetime", lang: "python", + src: "log.info('starting up')\nself.client = Foo()\nx = 'a'\n", + keep: []string{"log.info(", "self.client = Foo()", "x = "}, + gone: []string{"'starting up'", "'a'"}}, + {name: "py docstring with apostrophe", lang: "python", + src: "def f():\n '''Don't do this: self.cache = Redis()'''\n return g()\n", + keep: []string{"def f():", "return g()"}, + gone: []string{"'''Don't do this: self.cache = Redis()'''"}}, + {name: "py hash in string is not a comment", lang: "python", + src: "a = \"x # y\"; b = c # real\n", + keep: []string{"a = ", "; b = c"}, + gone: []string{"\"x # y\"", "# real"}}, + {name: "py prefixes", lang: "python", + src: "a = rb'\\x' + BR\"y\" + u'z' + Rb'''w''' + F'{v}'\n", + keep: []string{"a = ", " + ", "{v}"}, + gone: []string{"rb'\\x'", "BR\"y\"", "u'z'", "Rb'''w'''", "F'"}}, + {name: "py raw string escaped quote", lang: "python", + src: "z = r\"\\\"\" + tail()\n", + keep: []string{"+ tail()"}, + gone: []string{"r\"\\\"\""}}, + {name: "py f-string holes", lang: "python", + src: "m = f\"v={label(3)} {{lit}} {x!r} {y:>10} {z:{w}}\"\n", + keep: []string{"{label(3)}", "{x!r", "{w}"}, + gone: []string{"v=", "{{lit}}", ">10"}}, + {name: "py f-string nested quotes (3.12)", lang: "python", + src: "s = f\"{\", \".join(songs)}\"; t = f'{d['k']}'\n", + keep: []string{".join(songs)}", "; t = ", "{d[", "]}"}, + gone: []string{"\", \"", "'k'"}}, + {name: "py t-string", lang: "python", + src: "t = t\"hi {name}\"\n", + keep: []string{"{name}"}, + gone: []string{"t\"hi "}}, + {name: "py backslash continuation in single-quoted string", lang: "python", + src: "c = 'one \\\ntwo'\nd = 1\n", + keep: []string{"d = 1"}, + gone: []string{"'one \\", "two'"}}, + {name: "py \\N{} is an escape not a hole", lang: "python", + src: "n = f\"\\N{EM DASH}{v}\"\n", + keep: []string{"{v}"}, + gone: []string{"\\N{EM DASH}"}}, + {name: "py triple-quoted f-string hole across lines", lang: "python", + src: "m = f'''\n{compute(1,\n 2)}\n'''\n", + keep: []string{"{compute(1,", "2)}"}}, + + // ---- Rust ---- + {name: "rust raw string with hashes", lang: "rust", + src: "let f = r#\"\nlet client = Bar::new();\n\"#;\nlet g = 1;\n", + keep: []string{"let f = ", "let g = 1;"}, + gone: []string{"let client = Bar::new();", "r#\"", "\"#"}}, + {name: "rust lifetimes vs char literals", lang: "rust", + src: "fn f<'a>(x: &'a str) -> char { '\"' }\nfn g() -> &'static str { \"s\" }\n", + keep: []string{"fn f<'a>(x: &'a str) -> char {", "fn g() -> &'static str {"}, + gone: []string{"'\"'", "\"s\""}}, + {name: "rust char forms", lang: "rust", + src: "let a = ['a', '\\n', '\\u{1F600}', 'é', b'x', b'\\''];\nlet r#type = 1; 'l: loop { break 'l; }\n", + keep: []string{"let a = [", "let r#type = 1; 'l: loop { break 'l; }"}, + gone: []string{"'a'", "'\\n'", "'\\u{1F600}'", "'é'", "b'x'", "b'\\''"}}, + {name: "rust nested block comment", lang: "rust", + src: "/* a /* b */ fn phantom() {} */ fn real() {}\n", + keep: []string{"fn real() {}"}, + gone: []string{"fn phantom() {}"}}, + {name: "rust byte, raw byte, c strings", lang: "rust", + src: "let x = (b\"by\\\"tes\", br##\"r\"#b\"##, c\"cs\", \"multi\nline\");\n", + keep: []string{"let x = ("}, + gone: []string{"b\"by\\\"tes\"", "br##\"r\"#b\"##", "c\"cs\"", "\"multi", "line\""}}, + {name: "rust attributes are code", lang: "rust", + src: "#![allow(x)]\n#[derive(Debug)]\nstruct S;\n", + keep: []string{"#![allow(x)]", "#[derive(Debug)]"}}, + + // ---- Go ---- + {name: "go raw string has no escapes", lang: "go", + src: "p = strings.ReplaceAll(p, `\\`, \"/\")\nq := 1\n", + keep: []string{"p = strings.ReplaceAll(p, ", "q := 1"}, + gone: []string{"`\\`", "\"/\""}}, + {name: "go runes and multi-line raw", lang: "go", + src: "r := '\\''\nraw := `\nfunc phantom() {}\n`\nx := r\n", + keep: []string{"r := ", "x := r"}, + gone: []string{"'\\''", "func phantom() {}"}}, + + // ---- C / C++ / ObjC ---- + {name: "cpp digit separators", lang: "cpp", + src: "constexpr int k = 1'000;\nusing namespace foo;\nauto h = 0xFF'FF;\n", + keep: []string{"constexpr int k = 1'000;", "using namespace foo;", "auto h = 0xFF'FF;"}}, + {name: "cpp raw string spanning lines", lang: "cpp", + src: "auto s = R\"(\nint phantom(int x) {\n)\";\nint real();\n", + keep: []string{"auto s = ", "int real();"}, + gone: []string{"int phantom(int x) {", "R\"("}}, + {name: "cpp raw string custom delimiter and prefixes", lang: "cpp", + src: "auto a = R\"xy(has )\" inside)xy\"; auto b = u8R\"(u)\"; auto c = LR\"(l)\"; auto d = uR\"(x)\"; auto e = UR\"d(y)d\";\n", + keep: []string{"auto a = ", "; auto b = ", "; auto c = ", "; auto d = ", "; auto e = "}, + gone: []string{"R\"xy(has )\" inside)xy\"", "u8R\"(u)\"", "LR\"(l)\"", "uR\"(x)\"", "UR\"d(y)d\""}}, + {name: "c error directive apostrophe", lang: "c", + src: "#error don't build\nint after = 1;\n", + keep: []string{"#error don't build", "int after = 1;"}}, + {name: "c line comment continuation", lang: "c", + src: "// comment \\\n int phantom(int x);\nint real;\n", + keep: []string{"int real;"}, + gone: []string{"int phantom(int x);"}}, + {name: "c apostrophe in comment", lang: "c", + src: "/* don't */ int a; // isn't\nint b;\n", + keep: []string{"int a;", "int b;"}, + gone: []string{"don't", "isn't"}}, + {name: "c include and define", lang: "c", + src: "#include \n#include \"x.h\"\n#define N 'n'\n", + keep: []string{"#include ", "#include ", "#define N "}, + gone: []string{"\"x.h\"", "'n'"}}, + {name: "c string continuation", lang: "c", + src: "const char *s = \"a \\\nb\"; int z;\n", + keep: []string{"int z;"}, + gone: []string{"\"a \\", "b\""}}, + {name: "objc at-string", lang: "objc", + src: "NSString *s = @\"hi \\\" // no\"; int x;\n", + keep: []string{"NSString *s = ", "; int x;"}, + gone: []string{"@\"hi \\\" // no\""}}, + + // ---- Java ---- + {name: "java text block", lang: "java", + src: "String s = \"\"\"\n one \" two \"\" three \\\"\"\"\n class Phantom {}\n \"\"\";\nint x;\n", + keep: []string{"String s = ", "int x;"}, + gone: []string{"class Phantom {}", "one \" two \"\" three \\\"\"\""}}, + {name: "java chars", lang: "java", + src: "char c = '\\''; char d = '\"'; int n = 1;\n", + keep: []string{"char c = ", "; char d = ", "; int n = 1;"}, + gone: []string{"'\\''", "'\"'"}}, + + // ---- C# ---- + {name: "cs attribute string", lang: "csharp", + src: "[Description(\"class Y : Fake\")]\nclass X {}\n", + keep: []string{"[Description(", ")]", "class X {}"}, + gone: []string{"class Y : Fake"}}, + {name: "cs verbatim string ending in backslash", lang: "csharp", + src: "string p = @\"C:\\x\\\"; int after;\n", + keep: []string{"string p = ", "; int after;"}, + gone: []string{"@\"C:\\x\\\""}}, + {name: "cs verbatim doubled quotes multi-line", lang: "csharp", + src: "string v = @\"one\n\"\"two\"\"\"; int k;\n", + keep: []string{"string v = ", "; int k;"}, + gone: []string{"@\"one", "\"\"two\"\"\""}}, + {name: "cs interpolation", lang: "csharp", + src: "string s = $\"{a} and {{lit}} {price:N2} {Call(\"x\")}\";\n", + keep: []string{"{a}", "{price", "{Call(", ")}"}, + gone: []string{" and {{lit}} ", "N2", "\"x\""}}, + {name: "cs verbatim interpolation both orders", lang: "csharp", + src: "var a = $@\"C:\\{dir}\\f\"; var b = @$\"{dir}\\x\";\n", + keep: []string{"{dir}", "var b = "}, + gone: []string{"$@\"C:\\", "\\f\"", "\\x\""}}, + {name: "cs raw strings", lang: "csharp", + src: "var r = \"\"\"raw \"q\" text\"\"\"; var m = \"\"\"\"\n has \"\"\" inside\n \"\"\"\"; var i = $$\"\"\"{{Name}} {lit}\"\"\";\n", + keep: []string{"var r = ", "; var m = ", "; var i = ", "{{Name}}"}, + gone: []string{"raw \"q\" text", "has \"\"\" inside", "{lit}"}}, + {name: "cs region line is code", lang: "csharp", + src: "#region Don't parse\nclass X {}\n#endregion\n", + keep: []string{"#region Don't parse", "class X {}"}}, + + // ---- JavaScript / TypeScript ---- + {name: "js private field and template", lang: "javascript", + src: "this.#tpl = `a ${b} c`;\n", + keep: []string{"this.#tpl = ", "${b}"}, + gone: []string{"`a ", " c`"}}, + {name: "js regex with quotes", lang: "javascript", + src: "const re = /[\"']/g; const x = y;\n", + keep: []string{"const re = ", "; const x = y;"}, + gone: []string{"/[\"']/g"}}, + {name: "js division is not a regex", lang: "javascript", + src: "const r = a / b / c; const s = 'x';\n", + keep: []string{"const r = a / b / c; const s = "}, + gone: []string{"'x'"}}, + {name: "js nested templates", lang: "javascript", + src: "const n = `o ${c ? `i ${b + 1}` : 'no'} e`;\n", + keep: []string{"${c ? ", "${b + 1}", " : "}, + gone: []string{"`o ", "`i ", "'no'", " e`"}}, + {name: "js hashbang", lang: "javascript", + src: "#!/usr/bin/env node\nrun();\n", + keep: []string{"run();"}, + gone: []string{"#!/usr/bin/env node"}}, + {name: "js regex after keyword", lang: "javascript", + src: "return /'/.test(s) && typeof /\"/;\n", + keep: []string{"return ", ".test(s) && typeof "}, + gone: []string{"/'/", "/\"/"}}, + {name: "ts non-null assertion before division", lang: "typescript", + src: "const w = `${h! / 2}px`; const q = 'x';\n", + keep: []string{"${h! / 2}", "const q = "}, + gone: []string{"px`", "'x'"}}, + {name: "tsx self-closing tag after expression", lang: "tsx", + src: "const e = ; const s = 'x';\n", + keep: []string{"; const s = "}, + gone: []string{"'x'"}}, + {name: "jsx alias", lang: "jsx", + src: "const s = 'x'; // c\n", + keep: []string{"const s = "}, + gone: []string{"'x'", "// c"}}, + + // ---- PHP ---- + {name: "php attribute is not a comment", lang: "php", + src: "#[Route('/x')] public function act() {} # real comment\n", + keep: []string{"#[Route(", ")] public function act() {}"}, + gone: []string{"'/x'", "# real comment"}}, + {name: "php interpolation holes", lang: "php", + src: "$d = \"r={$this->act(2)} v=$n w=$o->p x=$a[0] y=${z}\";\n", + keep: []string{"$d = ", "$this->act(2)", "$n", "$o->p", "$a[0]", "z"}, + gone: []string{"r=", " v=", " w=", " x=", " y="}}, + {name: "php single quotes do not interpolate", lang: "php", + src: "$s = 'it\\'s {$no} #x'; $t = 1;\n", + keep: []string{"$s = ", "; $t = 1;"}, + gone: []string{"'it\\'s {$no} #x'"}}, + {name: "php heredoc", lang: "php", + src: "$h = <<act(3)} $n\nfunction phantom() {}\nEOT;\n$after = 1;\n", + keep: []string{"$h = ", "$this->act(3)", "$n\n", ";\n$after = 1;"}, + gone: []string{"<<\n

don't parse

\n", "don't parse

"}}, + {name: "php line comment ends at close tag", lang: "php", + src: "echo 1; // c ?>x", "x"}}, + + // ---- Kotlin ---- + {name: "kotlin templates", lang: "kotlin", + src: "val v = \"v=${label(3)} and $name \\\"q\\\"\"\n", + keep: []string{"val v = ", "${label(3)}", "$name"}, + gone: []string{"\"v=", " and ", "\\\"q\\\"\""}}, + {name: "kotlin raw string with templates", lang: "kotlin", + src: "val r = \"\"\"\nfun phantom() {}\n${label(4)}\n\"\"\"\nval c = '\\''\n", + keep: []string{"val r = ", "${label(4)}", "val c = "}, + gone: []string{"fun phantom() {}", "'\\''"}}, + {name: "kotlin backtick names", lang: "kotlin", + src: "fun `it doesn't fail`() { val s = \"x\" }\n", + keep: []string{"fun `it doesn't fail`() { val s = "}, + gone: []string{"\"x\""}}, + {name: "kotlin nested comments", lang: "kotlin", + src: "/* a /* b */ fun phantom() */ fun real() {}\n", + keep: []string{"fun real() {}"}, + gone: []string{"fun phantom()"}}, + + // ---- Swift ---- + {name: "swift interpolation", lang: "swift", + src: "let v = \"v=\\(label(3)) \\\"q\\\"\"\n", + keep: []string{"let v = ", "label(3)"}, + gone: []string{"\"v=", "\\\"q\\\"\""}}, + {name: "swift extended delimiters", lang: "swift", + src: "let r = #\"raw \"quoted\"\"#; let h = #\"\\#(label(4)) \\(no)\"#; let d = ##\"a \"# b\"##\n", + keep: []string{"let r = ", "; let h = ", "label(4)", "; let d = "}, + gone: []string{"#\"raw \"quoted\"\"#", "\\(no)\"#", "##\"a \"# b\"##"}}, + {name: "swift multi-line string", lang: "swift", + src: "let m = \"\"\"\n func phantom() {}\n \\(label(5))\n \"\"\"\nlet x = 1\n", + keep: []string{"let m = ", "label(5)", "let x = 1"}, + gone: []string{"func phantom() {}"}}, + {name: "swift apostrophe is code", lang: "swift", + src: "let a = b' + 1\n", + keep: []string{"let a = b' + 1"}}, + + // ---- COBOL / JCL ---- + {name: "cobol fixed comment and strings", lang: "cobol", + src: "000100* COMMENT 'Q\n000200 DISPLAY 'IT''S'. *> INLINE\n000300 STOP RUN.\n", + keep: []string{"000100", "000200 DISPLAY ", ".", "000300 STOP RUN."}, + gone: []string{"* COMMENT 'Q", "'IT''S'", "*> INLINE"}}, + {name: "cobol free format", lang: "cobol", + src: ">>SOURCE FORMAT FREE\n* not a comment\nDISPLAY \"A\"\"B\". *> c\n", + keep: []string{"* not a comment", "DISPLAY ", "."}, + gone: []string{"\"A\"\"B\"", "*> c"}}, + {name: "jcl", lang: "jcl", + src: "//* COMMENT 'Q\n//S1 EXEC PGM=X,PARM='IT''S'\n", + keep: []string{"//S1 EXEC PGM=X,PARM="}, + gone: []string{"//* COMMENT 'Q", "'IT''S'"}}, +} + +func TestMaskRules(t *testing.T) { + for _, tc := range maskCases { + t.Run(tc.name, func(t *testing.T) { checkCase(t, Mask, tc) }) + } +} + +func TestMaskComments(t *testing.T) { + cases := []maskCase{ + {name: "c include kept", lang: "c", + src: "#include \"x.h\" // c\n/* b */ int a;\n", + keep: []string{"#include \"x.h\" ", "int a;"}, + gone: []string{"// c", "/* b */"}}, + {name: "comment marker inside string", lang: "python", + src: "s = 'a # b' # c\n", + keep: []string{"s = 'a # b'"}, + gone: []string{"# c"}}, + {name: "js url string", lang: "javascript", + src: "const u = \"http://x\"; // c\n", + keep: []string{"const u = \"http://x\";"}, + gone: []string{"// c"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { checkCase(t, MaskComments, tc) }) + } +} + +func TestMaskFilePHP(t *testing.T) { + src := "don't\n\nt\n" + checkCase(t, MaskFile, maskCase{name: "php file", lang: "php", src: src, + keep: []string{"", ""}, + gone: []string{"don't", "'x'", "t"}}) + // Mask starts in code mode: the same text before ` %q", fn.name, name, k, src[k], got[k]) + break + } + } + if fn.name != "MaskInactivePreprocessor" { + if again := fn.f(lang, got); again != got { + t.Errorf("%s %s: not idempotent", fn.name, name) + } + } + } + } +} + +// TestMaskTruncated: every prefix of every testdata file masks without +// panicking (symbol snippets and unterminated constructs). +func TestMaskTruncated(t *testing.T) { + files, _ := filepath.Glob("testdata/*") + for _, f := range files { + b, _ := os.ReadFile(f) + src, lang := string(b), langOfPath(f) + for n := 0; n <= len(src); n++ { + if got := MaskFile(lang, src[:n]); len(got) != n { + t.Fatalf("%s[:%d]: length %d", f, n, len(got)) + } + if got := Mask(lang, src[n:]); len(got) != len(src)-n { + t.Fatalf("%s[%d:]: length %d", f, n, len(got)) + } + } + } +} + +func BenchmarkMask(b *testing.B) { + files, _ := filepath.Glob("testdata/*") + byLang := map[string]string{} + for _, f := range files { + data, _ := os.ReadFile(f) + byLang[langOfPath(f)] += string(data) + "\n" + } + for _, lang := range []string{"go", "python", "javascript", "cpp", "csharp", "rust", "php"} { + unit := byLang[lang] + var sb strings.Builder + for sb.Len() < 1<<20 { + sb.WriteString(unit) + } + src := sb.String() + b.Run(lang, func(b *testing.B) { + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + Mask(lang, src) + } + }) + } +} diff --git a/treeedit.go b/treeedit.go new file mode 100644 index 0000000..50e5186 --- /dev/null +++ b/treeedit.go @@ -0,0 +1,60 @@ +package astkit + +/* +#include + +// Layouts of tree-sitter's api.h TSPoint and TSInputEdit; ts_tree_edit is +// linked from go-tree-sitter's runtime. +typedef struct { uint32_t row; uint32_t column; } astkitPoint; +typedef struct { + uint32_t start_byte, old_end_byte, new_end_byte; + astkitPoint start_point, old_end_point, new_end_point; +} astkitInputEdit; + +void ts_tree_edit(void *self, const astkitInputEdit *edit); + +static void astkit_tree_edit(void *tree, astkitInputEdit edit) { ts_tree_edit(tree, &edit); } +*/ +import "C" + +import ( + "reflect" + "unsafe" + + sitter "github.com/smacker/go-tree-sitter" +) + +// treeEdit applies e to tree like sitter.Tree.Edit, which passes +// OldEndPoint as the new end point (go-tree-sitter bug), so an edit that +// adds lines never moves a row. It reaches the native tree through +// go-tree-sitter's unexported BaseTree.c and reports false when that +// layout is not there. +func treeEdit(tree *sitter.Tree, e sitter.EditInput) bool { + ptr := nativeTree(tree) + if ptr == nil { + return false + } + pt := func(p sitter.Point) C.astkitPoint { + return C.astkitPoint{row: C.uint32_t(p.Row), column: C.uint32_t(p.Column)} + } + C.astkit_tree_edit(ptr, C.astkitInputEdit{ + start_byte: C.uint32_t(e.StartIndex), + old_end_byte: C.uint32_t(e.OldEndIndex), + new_end_byte: C.uint32_t(e.NewEndIndex), + start_point: pt(e.StartPoint), + old_end_point: pt(e.OldEndPoint), + new_end_point: pt(e.NewEndPoint), + }) + return true +} + +func nativeTree(tree *sitter.Tree) unsafe.Pointer { + if tree == nil || tree.BaseTree == nil { + return nil + } + f := reflect.ValueOf(tree.BaseTree).Elem().FieldByName("c") + if !f.IsValid() || f.Kind() != reflect.Pointer || f.IsNil() { + return nil + } + return f.UnsafePointer() +} diff --git a/treeedit_test.go b/treeedit_test.go new file mode 100644 index 0000000..cb4be95 --- /dev/null +++ b/treeedit_test.go @@ -0,0 +1,20 @@ +package astkit + +import ( + "context" + "testing" +) + +// treeEdit reaches the native tree through go-tree-sitter's unexported +// layout; without it Kotlin falls back to a parse whose recovery comments +// can move. +func TestTreeEditReachesNativeTree(t *testing.T) { + tree, err := NewEngine().Parse(context.Background(), LangGo, []byte("package p\n")) + if err != nil || tree == nil { + t.Fatalf("parse: %v", err) + } + defer tree.Close() + if nativeTree(tree) == nil { + t.Fatal("go-tree-sitter's Tree layout changed: nativeTree found no native tree") + } +}