Repository navigation
Expand file tree
/
Copy pathengine.go
More file actions
288 lines (264 loc) · 8.79 KB
/
Copy pathengine.go
File metadata and controls
288 lines (264 loc) · 8.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
package astkit
import (
"context"
"errors"
"fmt"
"sync"
"time"
sitter "github.com/smacker/go-tree-sitter"
"github.com/smacker/go-tree-sitter/c"
"github.com/smacker/go-tree-sitter/cpp"
"github.com/smacker/go-tree-sitter/csharp"
"github.com/smacker/go-tree-sitter/golang"
"github.com/smacker/go-tree-sitter/java"
"github.com/smacker/go-tree-sitter/javascript"
"github.com/smacker/go-tree-sitter/kotlin"
"github.com/smacker/go-tree-sitter/php"
"github.com/smacker/go-tree-sitter/python"
"github.com/smacker/go-tree-sitter/rust"
"github.com/smacker/go-tree-sitter/swift"
tstsx "github.com/smacker/go-tree-sitter/typescript/tsx"
tstype "github.com/smacker/go-tree-sitter/typescript/typescript"
"github.com/provasign/astkit/thirdparty/tsobjc"
)
// DefaultParseTimeout caps tree-sitter parse calls. Callers can override per
// Engine via WithTimeout.
const DefaultParseTimeout = 2 * time.Second
// Engine wraps tree-sitter parsing for every language astkit knows about.
// Engines are safe for concurrent use.
type Engine struct {
timeout time.Duration
}
// NewEngine returns an Engine using DefaultParseTimeout.
func NewEngine() *Engine { return &Engine{timeout: DefaultParseTimeout} }
// WithTimeout returns a copy of e with the parse timeout overridden.
func (e *Engine) WithTimeout(d time.Duration) *Engine {
cp := *e
cp.timeout = d
return &cp
}
// Parse parses src under lang and returns a tree-sitter tree. Caller is
// responsible for tree.Close().
//
// For non-AST languages (config formats, unknown) Parse returns (nil, nil)
// rather than an error — letting callers branch on `tree == nil` without
// inspecting error types.
func (e *Engine) Parse(ctx context.Context, lang LanguageKey, src []byte) (*sitter.Tree, error) {
tsLang, ok := treeSitterLanguage(lang)
if !ok {
return nil, nil
}
p := sitter.NewParser()
// Release the native parser eagerly: the returned tree does not depend on
// it, and waiting for the finalizer lets large indexes accumulate
// significant native memory before GC notices.
defer p.Close()
p.SetLanguage(tsLang)
cctx, cancel := context.WithTimeout(ctx, e.timeout)
defer cancel()
var tree *sitter.Tree
var err error
if lang == LangKotlin {
tree, err = parseKotlinWithoutComments(cctx, p, src)
} else {
tree, err = p.ParseCtx(cctx, nil, src)
}
if err != nil {
return nil, fmt.Errorf("astkit: parse %s: %w", lang, err)
}
return tree, nil
}
// ErrSyntax is wrapped by the error Validate returns for source whose
// tree contains syntax errors (ERROR or MISSING nodes).
var ErrSyntax = errors.New("syntax error")
// Validate parses src and discards the tree. It returns the parse error, an
// error wrapping ErrSyntax (naming the first error's 1-based line and
// column) when the tree contains syntax errors, or nil. Languages without a
// grammar (config/data formats) parse to no tree and always validate.
func (e *Engine) Validate(ctx context.Context, lang LanguageKey, src []byte) error {
tree, err := e.Parse(ctx, lang, src)
if err != nil || tree == nil {
return err
}
defer tree.Close()
root := tree.RootNode()
if !root.HasError() {
return nil
}
p := firstSyntaxError(root).StartPoint()
return fmt.Errorf("astkit: %s %w at %d:%d", lang, ErrSyntax, p.Row+1, p.Column+1)
}
// firstSyntaxError returns the first ERROR or MISSING node under n in
// source order, or n itself when none is found below it.
func firstSyntaxError(n *sitter.Node) *sitter.Node {
for n != nil {
if n.IsError() || n.IsMissing() {
return n
}
var next *sitter.Node
for i := 0; i < int(n.ChildCount()); i++ {
if c := n.Child(i); c != nil && (c.HasError() || c.IsMissing()) {
next = c
break
}
}
if next == nil {
return n
}
n = next
}
return n
}
// treeSitterLanguage maps a LanguageKey to its tree-sitter grammar.
func treeSitterLanguage(lang LanguageKey) (*sitter.Language, bool) {
switch lang {
case LangGo:
return golang.GetLanguage(), true
case LangTypeScript:
return tstype.GetLanguage(), true
case LangTSX:
return tstsx.GetLanguage(), true
case LangJavaScript:
return javascript.GetLanguage(), true
case LangPython:
return python.GetLanguage(), true
case LangJava:
return java.GetLanguage(), true
case LangRust:
return rust.GetLanguage(), true
case LangC:
return c.GetLanguage(), true
case LangCPP:
return cpp.GetLanguage(), true
case LangCSharp:
return csharp.GetLanguage(), true
case LangPHP:
return php.GetLanguage(), true
case LangSwift:
return swift.GetLanguage(), true
case LangKotlin:
return kotlin.GetLanguage(), true
case LangObjC:
return tsobjc.GetLanguage(), true
default:
return nil, false
}
}
// Strategy is implemented by every supported language. A Strategy converts a
// parsed tree-sitter tree into the canonical astkit types.
type Strategy interface {
Language() LanguageKey
Extensions() []string
Extract(tree *sitter.Tree, src []byte) ([]Symbol, error)
ExtractImports(tree *sitter.Tree, src []byte) ([]ImportStatement, error)
}
// TextStrategy is an optional capability a Strategy may implement when it
// extracts from source text alone — no tree-sitter grammar exists for its
// language, and its Extract/ExtractImports accept a nil tree. Intended for
// line-structured artifact types (job control, data-division extracts,
// sort control cards); no current language implements it.
type TextStrategy interface {
Strategy
// ExtractsFromText reports that Extract tolerates a nil tree.
ExtractsFromText() bool
}
// Registry is a thread-safe map of LanguageKey → Strategy.
type Registry struct {
mu sync.RWMutex
m map[LanguageKey]Strategy
}
// NewRegistry returns an empty registry.
func NewRegistry() *Registry { return &Registry{m: make(map[LanguageKey]Strategy)} }
// Register adds s under its declared language.
func (r *Registry) Register(s Strategy) {
r.mu.Lock()
defer r.mu.Unlock()
r.m[s.Language()] = s
}
// RegisterAs adds s under an additional key (used for JS/JSX, TS/TSX aliasing).
func (r *Registry) RegisterAs(key LanguageKey, s Strategy) {
r.mu.Lock()
defer r.mu.Unlock()
r.m[key] = s
}
// Get returns the strategy for lang, or nil if none registered.
func (r *Registry) Get(lang LanguageKey) Strategy {
r.mu.RLock()
defer r.mu.RUnlock()
return r.m[lang]
}
// Extract is a convenience that looks up the strategy and runs Extract.
// Returns (nil, nil) when no strategy is registered.
// TextCapable reports whether the strategy registered under lang extracts
// from source text alone (Extract accepts a nil tree). Callers use it to
// decide whether a language with no tree-sitter grammar still yields
// symbols; it is false for every current language.
func (r *Registry) TextCapable(lang LanguageKey) bool {
ts, ok := r.Get(lang).(TextStrategy)
return ok && ts.ExtractsFromText()
}
func (r *Registry) Extract(lang LanguageKey, tree *sitter.Tree, src []byte) ([]Symbol, error) {
s := r.Get(lang)
if s == nil {
return nil, nil
}
if err := guardDepth(tree); err != nil {
return nil, err
}
syms, err := s.Extract(tree, src)
if err == nil {
fillCallColumns(string(lang), syms, src)
}
return syms, err
}
// ExtractImports is a convenience analogous to Extract.
func (r *Registry) ExtractImports(lang LanguageKey, tree *sitter.Tree, src []byte) ([]ImportStatement, error) {
s := r.Get(lang)
if s == nil {
return nil, nil
}
if err := guardDepth(tree); err != nil {
return nil, err
}
return s.ExtractImports(tree, src)
}
// MaxASTDepth bounds tree nesting before extraction. Every per-language
// extractor walks the tree by direct Go recursion (one call frame per level);
// a crafted file nested a few hundred thousand levels deep ("((((((…))))))")
// parses fine but overflows the goroutine stack during extraction — a FATAL,
// non-recoverable runtime error that takes down the whole host process and
// every concurrent request Grove/Fuse is serving. A recover() cannot catch a
// stack overflow, so the depth must be rejected BEFORE the recursive walk.
// 4000 is far past any real source file (deeply-generated code rarely exceeds
// a few hundred) while leaving ample headroom below the stack limit.
const MaxASTDepth = 4000
// guardDepth rejects a tree whose nesting exceeds MaxASTDepth. It walks with a
// TreeCursor (iterative, O(1) stack — it cannot itself overflow) rather than
// recursion, so it is safe to run on the very input it is protecting against.
func guardDepth(tree *sitter.Tree) error {
if tree == nil {
return nil
}
root := tree.RootNode()
if root == nil {
return nil
}
c := sitter.NewTreeCursor(root)
defer c.Close()
depth := 0
for {
if c.GoToFirstChild() {
depth++
if depth > MaxASTDepth {
return fmt.Errorf("astkit: AST nesting exceeds %d levels — refusing extraction (possible denial-of-service input)", MaxASTDepth)
}
continue
}
for !c.GoToNextSibling() {
if !c.GoToParent() {
return nil // walked the whole tree; within bounds
}
depth--
}
}
}