diff --git a/eval/go.mod b/eval/go.mod index 7f278e0..2ae6546 100644 --- a/eval/go.mod +++ b/eval/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/provasign/astkit v0.16.9 // indirect + github.com/provasign/astkit v0.16.10 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 // indirect github.com/sourcegraph/beaut v0.0.0-20240611013027-627e4c25335a // indirect diff --git a/eval/go.sum b/eval/go.sum index 98adab7..f9b848c 100644 --- a/eval/go.sum +++ b/eval/go.sum @@ -18,8 +18,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/provasign/astkit v0.16.9 h1:R8QeXlGhnahkyfxMKTfqfijth29BxP6dTBhar4036Mw= -github.com/provasign/astkit v0.16.9/go.mod h1:uSQxIxw3dVpK/6KmHvqIE8+DK/nhxDuPklKv7g96Jr4= +github.com/provasign/astkit v0.16.10 h1:KqLxccJByRASIBm+wOHTnd3F9Xs0+qnbUfJns37xqUE= +github.com/provasign/astkit v0.16.10/go.mod h1:uSQxIxw3dVpK/6KmHvqIE8+DK/nhxDuPklKv7g96Jr4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/scip-code/scip/bindings/go/scip v0.8.1 h1:ZkyT9RDgkbd6ffvMbi9nYdwh7+gLoz8Z9ftFMxgUiVs= diff --git a/go.mod b/go.mod index 6bb6357..dfd646d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26 require modernc.org/sqlite v1.51.0 require ( - github.com/provasign/astkit v0.16.9 + github.com/provasign/astkit v0.16.10 github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 golang.org/x/sys v0.46.0 google.golang.org/protobuf v1.36.11 diff --git a/go.sum b/go.sum index 56c2684..025a28e 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/provasign/astkit v0.16.9 h1:R8QeXlGhnahkyfxMKTfqfijth29BxP6dTBhar4036Mw= -github.com/provasign/astkit v0.16.9/go.mod h1:uSQxIxw3dVpK/6KmHvqIE8+DK/nhxDuPklKv7g96Jr4= +github.com/provasign/astkit v0.16.10 h1:KqLxccJByRASIBm+wOHTnd3F9Xs0+qnbUfJns37xqUE= +github.com/provasign/astkit v0.16.10/go.mod h1:uSQxIxw3dVpK/6KmHvqIE8+DK/nhxDuPklKv7g96Jr4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4= diff --git a/internal/core/types.go b/internal/core/types.go index c7ecf77..e98d8d7 100644 --- a/internal/core/types.go +++ b/internal/core/types.go @@ -199,6 +199,11 @@ const ( // MetaNativeComplete: JSON list of native analyzers that have completed // a full pass on this index (later passes may be incremental). MetaNativeComplete = "native-complete" + // MetaNativeLastGood: JSON object analyzer -> the diagnostic lines of + // its last pass that produced edges. A run that carries an analyzer's + // stored edges forward reports these lines once, followed by the + // current attempt's lines (see index.carryForwardVerdict). + MetaNativeLastGood = "native-last-good" ) type IndexResult struct { diff --git a/internal/index/compilerinputs_test.go b/internal/index/compilerinputs_test.go new file mode 100644 index 0000000..3ce949b --- /dev/null +++ b/internal/index/compilerinputs_test.go @@ -0,0 +1,147 @@ +package index + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/provasign/grove/internal/core" + "github.com/provasign/grove/internal/native" + "github.com/provasign/grove/internal/parser" + "github.com/provasign/grove/internal/store" +) + +func writeFiles(t *testing.T, root string, files map[string]string) { + t.Helper() + for rel, body := range files { + p := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } +} + +// Stack audit 2026-10-20 #2: editing only tsconfig.json after a completed +// TypeScript pass skipped the analyzer ("no changed files in its +// languages"): config files are plaintext and native scope used only the +// changed source languages. +func TestConfigOnlyChangeInvalidatesNative(t *testing.T) { + root := t.TempDir() + ctx := context.Background() + writeFiles(t, root, map[string]string{ + "a.ts": "export function target() {}\nexport function caller() { target(); }\n", + "tsconfig.json": `{"compilerOptions":{"strict":false},"include":["*.ts"]}`, + }) + st, err := store.Open(root) + if err != nil { + t.Fatal(err) + } + defer st.Close() + idx := NewWithNativeConfig(parser.NewEngine(), st, native.Config{Enabled: false}) + if _, initial, err := idx.Index(ctx, root); err != nil || initial.EdgeCount == 0 { + t.Fatalf("seed: %v %#v", err, initial) + } + if err := st.SetMeta(ctx, core.MetaNativeComplete, `["go","js-ts"]`); err != nil { + t.Fatal(err) + } + writeFiles(t, root, map[string]string{"tsconfig.json": `{"compilerOptions":{"strict":true},"include":["*.ts"]}`}) + idx.SetNativeConfig(native.Config{Enabled: true}) + _, r, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(r.Native, "\n") + if strings.Contains(joined, "js-ts: skipped: no changed files") { + t.Fatalf("tsconfig changed but TypeScript analysis was skipped: %v", r.Native) + } + if !strings.Contains(joined, "js-ts: compiler inputs changed (tsconfig.json)") { + t.Fatalf("invalidation not reported: %v", r.Native) + } + raw, _, _ := st.GetMeta(ctx, core.MetaNativeComplete) + if strings.Contains(raw, "js-ts") || !strings.Contains(raw, "go") { + t.Fatalf("native-complete = %s, want js-ts dropped and go kept", raw) + } +} + +// go.mod is not even an indexed file: a module-only edit was a no-op index +// that kept the old go list/type-check results. +func TestGoModOnlyChangeRerunsGoAnalyzer(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go not found") + } + root := t.TempDir() + ctx := context.Background() + writeFiles(t, root, map[string]string{ + "go.mod": "module inputs\n\ngo 1.22\n", + "main.go": "package main\n\nfunc main() { run() }\n\nfunc run() {}\n", + }) + st, err := store.Open(root) + if err != nil { + t.Fatal(err) + } + defer st.Close() + idx := New(parser.NewEngine(), st) + _, first, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + skipUnless(t, first.Native, "go: resolved ") + + // Touched, identical content: still a no-op. + later := time.Now().Add(time.Minute) + if err := os.Chtimes(filepath.Join(root, "go.mod"), later, later); err != nil { + t.Fatal(err) + } + _, touched, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(touched.Native, "\n"), "no file changes") { + t.Fatalf("touch-only go.mod should stay a no-op: %v", touched.Native) + } + + writeFiles(t, root, map[string]string{"go.mod": "module inputs\n\ngo 1.23\n"}) + _, r, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(r.Native, "\n") + if strings.Contains(joined, "no file changes") || !strings.Contains(joined, "go: compiler inputs changed (go.mod)") || + !strings.Contains(joined, "go: full pass") { + t.Fatalf("go.mod change did not re-run the Go analyzer: %v", r.Native) + } + if r.EdgeCount == 0 { + t.Fatalf("graph lost its edges: %#v", r) + } + // Settled: the next plain index is a no-op again. + _, again, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(again.Native, "\n"), "no file changes") { + t.Fatalf("after the re-run the index should settle: %v", again.Native) + } +} + +func TestCompilerInputAnalyzers(t *testing.T) { + cases := map[string]string{ + "go.mod": "go", "sub/go.sum": "go", "go.work": "go", + "tsconfig.json": "js-ts", "packages/x/tsconfig.build.json": "js-ts", "jsconfig.json": "js-ts", "web/package.json": "js-ts", + "pom.xml": "java", "app/build.gradle.kts": "java", "Cargo.toml": "rust", "compile_commands.json": "cfamily", + "src/App.csproj": "csharp", "composer.json": "php", "pyproject.toml": "python", + "README.md": "", "config.json": "", "main.go": "", + } + for rel, want := range cases { + got := strings.Join(native.CompilerInputAnalyzers(rel), ",") + if got != want { + t.Errorf("%s: got %q want %q", rel, got, want) + } + } +} diff --git a/internal/index/indexer.go b/internal/index/indexer.go index 8a85515..3ebbb3a 100644 --- a/internal/index/indexer.go +++ b/internal/index/indexer.go @@ -2,6 +2,8 @@ package index import ( "context" + "crypto/sha1" + "encoding/hex" "encoding/json" "fmt" "os" @@ -25,6 +27,8 @@ type Indexer struct { parser *parser.Engine store *store.Store nativeConfig native.Config + // now is the clock for native retry scheduling (tests); nil = time.Now. + now func() time.Time // pendingSpliceCheck is the store fingerprint check of the last splice // write when Options.DeferSpliceCheck was set; see TakeSpliceCheck. @@ -416,8 +420,9 @@ func (i *Indexer) indexWithOptions(ctx context.Context, root string, opts Option return nil, result, err } forceExtract := opts.Force || storedExtractor != parser.ExtractorVersion || storedResolver != graph.ResolverVersion - statRefresh := map[string][2]int64{} // touched but content-identical - var walkFailed []string // relPaths whose subtree could not be read this run + statRefresh := map[string][2]int64{} // touched but content-identical + var walkFailed []string // relPaths whose subtree could not be read this run + compilerInputs := map[string]os.FileInfo{} // build-configuration files, see native.CompilerInputAnalyzers err = filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { // A failed subtree contributes nothing to currentFiles, so the @@ -453,7 +458,18 @@ func (i *Indexer) indexWithOptions(ctx context.Context, root string, opts Option } return nil } - if isSensitivePath(relPath) || ignore.Ignored(relPath, false) { + if isSensitivePath(relPath) { + return nil + } + // Compiler inputs are fingerprinted even when gitignored (a + // generated compile_commands.json usually is): the toolchain reads + // them either way. Only their stat is recorded here. + if native.CompilerInputAnalyzers(relPath) != nil { + if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() { + compilerInputs[relPath] = info + } + } + if ignore.Ignored(relPath, false) { return nil } // A symlinked FILE entry inside root can point anywhere; WalkDir @@ -497,6 +513,14 @@ func (i *Indexer) indexWithOptions(ctx context.Context, root string, opts Option blobSHA, err := parser.FileBlobSHA(path) if err != nil { result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", relPath, err)) + // The stat cache above did not match, so the bytes on disk are + // not the ones indexed (or cannot be read at all: over the size + // cap, no longer a regular file, permission denied). As with a + // parse failure, the stored symbols would describe content that + // no longer exists, presented as current — drop the file from + // currentFiles so the prune pass removes them. The error stays + // in result.Errors on every run until the file indexes again. + delete(currentFiles, relPath) return nil } var size, mtime int64 = -1, -1 @@ -522,6 +546,38 @@ func (i *Indexer) indexWithOptions(ctx context.Context, root string, opts Option return nil, result, err } tick("walk+sha-scan") + // A changed compiler input (tsconfig.json, go.mod, pom.xml, ...) + // invalidates its analyzer's completed pass: it runs in full on this + // run even when none of its source files changed, and its resident + // worker state is dropped. + invalidated, err := i.syncCompilerInputs(ctx, root, compilerInputs, walkFailed) + if err != nil { + return nil, result, err + } + if len(invalidated) > 0 { + names := make([]string, 0, len(invalidated)) + for name := range invalidated { + names = append(names, name) + } + sort.Strings(names) + if err := i.invalidateNativeComplete(ctx, names); err != nil { + return nil, result, err + } + native.InvalidateResident(root, names) + // Due now, and durable: a run interrupted before the analyzer + // completes leaves it retried by the next index, changes or not. + retryState := i.loadNativeRetry(ctx) + for _, name := range names { + retryState[name] = nativeRetry{Reason: "compiler inputs changed"} + } + if err := i.saveNativeRetry(ctx, retryState); err != nil { + return nil, result, err + } + for _, name := range names { + result.Native = append(result.Native, fmt.Sprintf("%s: compiler inputs changed (%s); running a full pass", + name, strings.Join(invalidated[name], ", "))) + } + } prog.phase("parsing", fmt.Sprintf("0/%d changed files parsed", len(tasks))) // Phase 2 (parallel): parse changed files. Tree-sitter parsing is the @@ -662,7 +718,26 @@ func (i *Indexer) indexWithOptions(ctx context.Context, root string, opts Option // The check runs BEFORE AllSymbols so a no-op never pays the full symbol // load; counts come from COUNT(*) queries instead of loaded slices. recoverMissingEdges := false - if !forceExtract && result.FilesUpdated == 0 && result.FilesPruned == 0 { + completed := i.loadNativeComplete(ctx) + retryState := i.loadNativeRetry(ctx) + var retryNow map[string]bool + if !forceExtract && result.FilesUpdated == 0 && result.FilesPruned == 0 && len(invalidated) == 0 { + var pending []string + retryNow, pending, err = i.noopRetries(ctx, root, completed, retryState) + if err != nil { + return nil, result, err + } + if len(retryNow) == 0 { + msg := "skipped: no file changes since last index" + if len(pending) > 0 { + msg += "; " + strings.Join(pending, "; ") + } else { + msg += " (use --force to re-run analyzers)" + } + result.Native = append(result.Native, msg) + } + } + if len(retryNow) == 0 && !forceExtract && result.FilesUpdated == 0 && result.FilesPruned == 0 && len(invalidated) == 0 { status, err := i.store.Status(ctx) if err != nil { return nil, result, err @@ -673,7 +748,6 @@ func (i *Indexer) indexWithOptions(ctx context.Context, root string, opts Option if status.EdgeCount > 0 || status.SymbolCount == 0 { result.SymbolCount = status.SymbolCount result.EdgeCount = status.EdgeCount - result.Native = append(result.Native, "skipped: no file changes since last index (use --force to re-run analyzers)") if opts.SkipNoopGraph { return nil, result, nil } @@ -753,20 +827,29 @@ func (i *Indexer) indexWithOptions(ctx context.Context, root string, opts Option } sort.Strings(nativeCfg.CallerFiles) } - completed := map[string]bool{} - if raw, ok, err := i.store.GetMeta(ctx, core.MetaNativeComplete); err == nil && ok && raw != "" { - var names []string - if json.Unmarshal([]byte(raw), &names) == nil { - for _, n := range names { - completed[n] = true - } - } - } + // An analyzer without a completed pass gets a full pass when its + // languages changed, when its retry is due, or when the no-change + // check above chose to retry it. Otherwise it is skipped like a + // completed one, its stored edges carried, until the retry is due — + // a failing 30-minute compiler pass is not re-launched by every edit + // in another language. nativeCfg.RunToCompletion = map[string]bool{} for _, a := range native.PriorityAnalyzers() { - if !completed[a.Name()] { - nativeCfg.RunToCompletion[a.Name()] = true + name := a.Name() + if completed[name] { + continue + } + touched := false + for _, lang := range a.Languages() { + touched = touched || changedLanguages[lang] } + if retryNow[name] || touched || (retryNow == nil && i.retryDue(retryState, name)) { + nativeCfg.RunToCompletion[name] = true + } + } + for _, name := range sortedKeys(retryNow) { + r := retryState[name] + result.Native = append(result.Native, fmt.Sprintf("%s: retrying its incomplete full pass (last attempt: %s)", name, r.Reason)) } nativeResult := native.AnalyzeChangedFiles(ctx, root, symbols, nativeCfg, scope, changedRel, allFiles) if len(nativeResult.Completed) > 0 { @@ -783,6 +866,9 @@ func (i *Indexer) indexWithOptions(ctx context.Context, root string, opts Option } } result.Native = append(result.Native, nativeResult.Diagnostics...) + if err := i.recordNativeAttempts(ctx, retryState, nativeResult); err != nil { + return nil, result, err + } if len(nativeResult.Symbols) > 0 { // Compiler-declared symbols (native.Result.Symbols) join the parsed // ones in the store and in this run's symbol set, in store order, so @@ -1182,36 +1268,271 @@ func remapEdgeEndpoints(edges []core.Edge, remap map[string]string) { } } -// carryForwardVerdict keeps the stored per-analyzer verdict for every -// analyzer this run skipped with "previous edges carried forward". The -// persisted verdict describes the native edges in the database, and those -// edges did not change. Without this, editing a README after a full Java -// index rewrote the verdict to "java: skipped: no changed files", and prism -// then marked every change-impact answer on the untouched Java graph as -// unconfirmed (all compiled-language cells in the 2026-10-11 bench). +// carryForwardVerdict builds the per-analyzer verdict this run persists. +// The persisted verdict describes the native edges in the database. An +// analyzer that ran and produced edges reports its current lines. An +// analyzer whose stored edges were carried forward (skipped because nothing +// of its changed, unavailable, failed, timed out) reports the lines of its +// last pass that produced edges — the pass those carried edges come from — +// once, followed by this run's lines saying why nothing replaced them. +// Without the carry, editing a README after a full Java index rewrote the +// verdict to "java: skipped: no changed files", and prism then marked every +// change-impact answer on the untouched Java graph as unconfirmed (all +// compiled-language cells in the 2026-10-11 bench). +// +// The last-good lines live in their own meta key, so a carry never re-reads +// its own output: re-carrying the previous merged verdict appended the +// whole history again each run, and one two-line diagnostic grew to 4, 6, +// 8, 10 lines (stack audit 2026-10-20 #10). func carryForwardVerdict(ctx context.Context, st *store.Store, current []string) []string { + lastGood := loadLastGood(ctx, st) + type group struct { + name string + lines []string + } + var groups []*group + byName := map[string]*group{} + for _, line := range current { + name, _, found := strings.Cut(line, ": ") + if !found { + name = "" // "native analyzers disabled": not one analyzer's line + } + g := byName[name] + if g == nil || name == "" { + g = &group{name: name} + groups = append(groups, g) + if name != "" { + byName[name] = g + } + } + g.lines = append(g.lines, line) + } + var merged []string + for _, g := range groups { + if g.name == "" { + merged = append(merged, g.lines...) + continue + } + carried, disabled := false, false + for _, line := range g.lines { + _, rest, _ := strings.Cut(line, ": ") + carried = carried || carriedVerdictLine(rest) + disabled = disabled || rest == "skipped: disabled by config" + } + switch { + case carried: + inCurrent := map[string]bool{} + for _, line := range g.lines { + inCurrent[line] = true + } + for _, line := range lastGood[g.name] { + if !inCurrent[line] { + merged = append(merged, line) + } + } + case disabled: + delete(lastGood, g.name) // its edges are not carried + default: + lastGood[g.name] = append([]string(nil), g.lines...) + } + merged = append(merged, g.lines...) + } + // An analyzer with no lines at all had no files of its languages: it + // has no edges left to carry. + for name := range lastGood { + if byName[name] == nil { + delete(lastGood, name) + } + } + if raw, err := json.Marshal(lastGood); err == nil { + _ = st.SetMeta(ctx, core.MetaNativeLastGood, string(raw)) + } + return merged +} + +// carriedVerdictLine reports whether an analyzer diagnostic (without its +// "name: " prefix) says the analyzer's stored edges were carried forward +// instead of replaced this run. +func carriedVerdictLine(rest string) bool { + return strings.Contains(rest, "carried forward") && + (strings.HasPrefix(rest, "skipped:") || strings.Contains(rest, "previous native edges carried forward")) +} + +// loadLastGood returns the stored last-good lines per analyzer. An index +// written before the key existed derives them from its stored verdict: each +// analyzer's lines minus skip/carry lines and pass timings (which may +// belong to a failed attempt), deduplicated — the earlier carry format +// repeated them. +func loadLastGood(ctx context.Context, st *store.Store) map[string][]string { + out := map[string][]string{} + if raw, ok, err := st.GetMeta(ctx, core.MetaNativeLastGood); err == nil && ok && raw != "" { + if json.Unmarshal([]byte(raw), &out) == nil { + if out == nil { + out = map[string][]string{} + } + return out + } + out = map[string][]string{} + } raw, ok, err := st.GetMeta(ctx, core.MetaIndexNative) if err != nil || !ok || raw == "" { - return current + return out } var previous []string - if json.Unmarshal([]byte(raw), &previous) != nil || len(previous) == 0 { - return current + if json.Unmarshal([]byte(raw), &previous) != nil { + return out } - var merged []string - for _, line := range current { + seen := map[string]bool{} + for _, line := range previous { name, rest, found := strings.Cut(line, ": ") - carried := found && strings.Contains(rest, "carried forward") && - (strings.HasPrefix(rest, "skipped:") || strings.Contains(rest, "previous native edges carried forward")) - if carried { - for _, prev := range previous { - prevName, prevRest, prevFound := strings.Cut(prev, ": ") - if prevFound && prevName == name && !strings.HasPrefix(prevRest, "skipped:") { - merged = append(merged, prev) - } + if !found || seen[line] || strings.HasPrefix(rest, "skipped:") || carriedVerdictLine(rest) || + (strings.Contains(rest, " pass over ") && strings.Contains(rest, " took ")) { + continue + } + seen[line] = true + out[name] = append(out[name], line) + } + return out +} + +// compilerInputState is the stored fingerprint of one compiler input. +type compilerInputState struct { + Size int64 `json:"size"` + Mtime int64 `json:"mtime"` + SHA string `json:"sha"` +} + +// metaNativeInputs: JSON object relPath -> compilerInputState. +const metaNativeInputs = "native-inputs" + +// syncCompilerInputs compares the walked compiler inputs with the stored +// fingerprints and returns, per analyzer, the inputs that were added, +// removed or changed in content (sorted). A file whose size and mtime match +// keeps its stored hash; a touched but identical file is not a change. The +// first run that records fingerprints reports nothing: there is no earlier +// state to differ from, and a cold index runs every analyzer anyway. Inputs +// under a subtree the walk could not read keep their stored state. +func (i *Indexer) syncCompilerInputs(ctx context.Context, root string, inputs map[string]os.FileInfo, walkFailed []string) (map[string][]string, error) { + raw, haveStored, err := i.store.GetMeta(ctx, metaNativeInputs) + if err != nil { + return nil, err + } + stored := map[string]compilerInputState{} + if haveStored && raw != "" && json.Unmarshal([]byte(raw), &stored) != nil { + stored = map[string]compilerInputState{} + haveStored = false + } + current := make(map[string]compilerInputState, len(inputs)) + changed := map[string]bool{} + for rel, info := range inputs { + state := compilerInputState{Size: info.Size(), Mtime: info.ModTime().UnixNano()} + prev, ok := stored[rel] + if ok && prev.Size == state.Size && prev.Mtime == state.Mtime { + state.SHA = prev.SHA + } else { + state.SHA = compilerInputHash(filepath.Join(root, filepath.FromSlash(rel)), state) + } + current[rel] = state + if !ok || prev.SHA != state.SHA { + changed[rel] = true + } + } + for rel, prev := range stored { + if _, ok := current[rel]; ok { + continue + } + shielded := false + for _, failed := range walkFailed { + if rel == failed || strings.HasPrefix(rel, failed+"/") { + shielded = true + break } } - merged = append(merged, line) + if shielded { + current[rel] = prev + continue + } + changed[rel] = true } - return merged + same := len(current) == len(stored) + if same { + for rel, state := range current { + if stored[rel] != state { + same = false + break + } + } + } + if !same || !haveStored { + encoded, err := json.Marshal(current) + if err != nil { + return nil, err + } + if err := i.store.SetMeta(ctx, metaNativeInputs, string(encoded)); err != nil { + return nil, err + } + } + if !haveStored || len(changed) == 0 { + return nil, nil + } + out := map[string][]string{} + for rel := range changed { + for _, name := range native.CompilerInputAnalyzers(rel) { + out[name] = append(out[name], rel) + } + } + for name := range out { + sort.Strings(out[name]) + } + return out, nil +} + +// compilerInputHash hashes a compiler input's content. Inputs are small; +// one past the indexer's size cap (or unreadable) is fingerprinted by its +// stat alone. +func compilerInputHash(path string, state compilerInputState) string { + if state.Size <= parser.MaxFileSizeBytes { + if data, err := os.ReadFile(path); err == nil && int64(len(data)) <= parser.MaxFileSizeBytes { + sum := sha1.Sum(data) + return hex.EncodeToString(sum[:]) + } + } + return fmt.Sprintf("stat:%d:%d", state.Size, state.Mtime) +} + +// invalidateNativeComplete removes analyzers from the completed-pass list, +// so the next native run gives them a full pass (RunToCompletion). +func (i *Indexer) invalidateNativeComplete(ctx context.Context, names []string) error { + raw, ok, err := i.store.GetMeta(ctx, core.MetaNativeComplete) + if err != nil || !ok || raw == "" { + return err + } + var completed []string + if json.Unmarshal([]byte(raw), &completed) != nil { + return nil + } + drop := map[string]bool{} + for _, n := range names { + drop[n] = true + } + kept := completed[:0] + for _, n := range completed { + if !drop[n] { + kept = append(kept, n) + } + } + encoded, err := json.Marshal(kept) + if err != nil { + return err + } + return i.store.SetMeta(ctx, core.MetaNativeComplete, string(encoded)) +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out } diff --git a/internal/index/native_cgo_test.go b/internal/index/native_cgo_test.go new file mode 100644 index 0000000..d11a44c --- /dev/null +++ b/internal/index/native_cgo_test.go @@ -0,0 +1,54 @@ +package index + +import ( + "context" + "os/exec" + "strings" + "testing" + + "github.com/provasign/grove/internal/core" +) + +// Stack audit 2026-10-20 #4: a cgo-only package (GoFiles=[], CgoFiles=[...]) +// lost its Go declarations to importers, and its own calls were never +// type-checked. Both sides now resolve, positioned in the user's source. +func TestGoNativeAnalyzesCgoOnlyPackage(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go not found") + } + if out, err := exec.Command("go", "env", "CGO_ENABLED").Output(); err != nil || strings.TrimSpace(string(out)) != "1" { + t.Skip("cgo disabled") + } + got, diags, st := compilerFixture(t, map[string]string{ + "go.mod": "module audit\n\ngo 1.22\n", + "binding/binding.go": "package binding\n\n// #include \nimport \"C\"\n\n" + + "type Thing struct{ n C.int }\n\nfunc (t Thing) Do() int { return int(t.n) }\n\n" + + "func helper() {}\n\nfunc New() Thing {\n\thelper()\n\treturn Thing{}\n}\n\n" + + "type Doer interface{ Do() int }\n", + "main.go": "package main\n\nimport \"audit/binding\"\n\nfunc main() {\n\tvar d binding.Doer = binding.New()\n\td.Do()\n}\n", + }) + skipUnless(t, diags, "go: resolved ") + for _, d := range diags { + if strings.Contains(d, "invalid package name") || strings.Contains(d, "project import audit/binding") { + t.Errorf("cgo package failed to load: %s", d) + } + } + wantEdges(t, got, diags, "main→New calls", "main→Do calls", "New→helper calls") + edges, err := st.AllEdges(context.Background()) + if err != nil { + t.Fatal(err) + } + positioned := false + for _, e := range edges { + if e.Source == core.EvidenceSourceNative && e.Type == core.EdgeCalls && strings.Contains(e.To, "helper") && e.Lines != "" { + // helper() is on line 13 of binding.go as written. + positioned = true + if e.Lines != "13:2" { + t.Errorf("cgo call position not in user source coordinates: %q", e.Lines) + } + } + } + if !positioned { + t.Logf("no positioned New→helper edge to check (positions: %v)", diags) + } +} diff --git a/internal/index/nativeretry.go b/internal/index/nativeretry.go new file mode 100644 index 0000000..57c1d5e --- /dev/null +++ b/internal/index/nativeretry.go @@ -0,0 +1,186 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "time" + + "github.com/provasign/grove/internal/core" + "github.com/provasign/grove/internal/native" +) + +// Retrying analyzers without a completed pass. +// +// An analyzer that has never completed a full pass on this index (compiler +// not installed, dependencies missing, a timeout, invalidated by a compiler +// input change) used to retry only when some file changed: a no-change +// index returned "no file changes since last index" before reaching the +// retry logic, so installing the missing compiler and running a plain +// index changed nothing until --force (stack audit 2026-10-20 #5). +// +// Retries are bounded so that the no-change index every query runs stays +// cheap: each analyzer's attempts are recorded with the time the next one +// is due. An analyzer that never started (unavailable) is rechecked with +// its cheap availability probe, and re-run only once it reports available; +// one that ran and failed is re-run on an exponential schedule, since a +// failed compiler pass can cost minutes. + +// metaNativeRetry: JSON object analyzer -> nativeRetry. +const metaNativeRetry = "native-retry" + +type nativeRetry struct { + Failures int `json:"failures"` + Next int64 `json:"next"` // unix nanoseconds; 0 = due now + Unavailable bool `json:"unavailable,omitempty"` + Reason string `json:"reason,omitempty"` +} + +const ( + unavailableRecheckBase = 15 * time.Second + unavailableRecheckMax = 10 * time.Minute + failedRetryBase = time.Minute + failedRetryMax = 6 * time.Hour +) + +// retryDelay is the wait after the r.Failures-th failed attempt. +func retryDelay(r nativeRetry) time.Duration { + base, max := failedRetryBase, failedRetryMax + if r.Unavailable { + base, max = unavailableRecheckBase, unavailableRecheckMax + } + d := base + for n := 1; n < r.Failures && d < max; n++ { + d *= 2 + } + if d > max { + d = max + } + return d +} + +func (i *Indexer) clock() time.Time { + if i.now != nil { + return i.now() + } + return time.Now() +} + +func (i *Indexer) loadNativeRetry(ctx context.Context) map[string]nativeRetry { + out := map[string]nativeRetry{} + if raw, ok, err := i.store.GetMeta(ctx, metaNativeRetry); err == nil && ok && raw != "" { + if json.Unmarshal([]byte(raw), &out) != nil || out == nil { + out = map[string]nativeRetry{} + } + } + return out +} + +func (i *Indexer) saveNativeRetry(ctx context.Context, state map[string]nativeRetry) error { + raw, err := json.Marshal(state) + if err != nil { + return err + } + return i.store.SetMeta(ctx, metaNativeRetry, string(raw)) +} + +// loadNativeComplete returns the analyzers with a completed full pass. +func (i *Indexer) loadNativeComplete(ctx context.Context) map[string]bool { + completed := map[string]bool{} + if raw, ok, err := i.store.GetMeta(ctx, core.MetaNativeComplete); err == nil && ok && raw != "" { + var names []string + if json.Unmarshal([]byte(raw), &names) == nil { + for _, n := range names { + completed[n] = true + } + } + } + return completed +} + +// noopRetries decides, for a run with no file changes, which incomplete +// analyzers to retry now. It returns them with notes for the analyzers +// still waiting, and persists any availability recheck it made. Analyzers +// with no recorded attempt are not retried here (no files of their +// languages, or an index from before attempts were recorded): the next run +// with changes attempts them as before. +func (i *Indexer) noopRetries(ctx context.Context, root string, completed map[string]bool, state map[string]nativeRetry) (map[string]bool, []string, error) { + if !i.nativeConfig.Enabled || len(state) == 0 { + return nil, nil, nil + } + now := i.clock() + names := make([]string, 0, len(state)) + for n := range state { + names = append(names, n) + } + sort.Strings(names) + retry := map[string]bool{} + var notes []string + dirty := false + for _, name := range names { + r := state[name] + if completed[name] { + continue + } + if now.UnixNano() < r.Next { + wait := time.Duration(r.Next - now.UnixNano()).Round(time.Second) + notes = append(notes, fmt.Sprintf("%s has no completed pass (%s); next retry in %s, or use --force to retry now", name, r.Reason, wait)) + continue + } + if !r.Unavailable { + retry[name] = true + continue + } + avail, ok := native.AnalyzerAvailable(ctx, root, name, i.nativeConfig) + if !ok { + continue // disabled by config: nothing to retry + } + if avail.Available { + retry[name] = true + continue + } + r.Failures++ + r.Reason = avail.Reason + r.Next = now.Add(retryDelay(r)).UnixNano() + state[name] = r + dirty = true + notes = append(notes, fmt.Sprintf("%s still unavailable (%s); rechecked now, next check in %s", name, avail.Reason, retryDelay(r))) + } + if dirty { + if err := i.saveNativeRetry(ctx, state); err != nil { + return nil, nil, err + } + } + return retry, notes, nil +} + +// retryDue reports whether an incomplete analyzer's next full attempt is due +// on a run where none of its languages changed. +func (i *Indexer) retryDue(state map[string]nativeRetry, name string) bool { + r, ok := state[name] + return !ok || i.clock().UnixNano() >= r.Next +} + +// recordNativeAttempts updates the retry schedule from a native run. +func (i *Indexer) recordNativeAttempts(ctx context.Context, state map[string]nativeRetry, res native.Result) error { + if len(res.Completed) == 0 && len(res.Incomplete) == 0 { + return nil + } + now := i.clock() + for _, name := range res.Completed { + delete(state, name) + } + for name, why := range res.Incomplete { + r := state[name] + if r.Unavailable != why.Unavailable { + r.Failures = 0 // a different failure mode starts its own schedule + } + r.Failures++ + r.Unavailable = why.Unavailable + r.Reason = why.Reason + r.Next = now.Add(retryDelay(r)).UnixNano() + state[name] = r + } + return i.saveNativeRetry(ctx, state) +} diff --git a/internal/index/nativeretry_test.go b/internal/index/nativeretry_test.go new file mode 100644 index 0000000..c2b1742 --- /dev/null +++ b/internal/index/nativeretry_test.go @@ -0,0 +1,154 @@ +package index + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/provasign/grove/internal/core" + "github.com/provasign/grove/internal/native" + "github.com/provasign/grove/internal/parser" + "github.com/provasign/grove/internal/store" +) + +type fakeClock struct{ t time.Time } + +func (c *fakeClock) now() time.Time { return c.t } + +// Stack audit 2026-10-20 #5: after a failed first pass (compiler +// unavailable), a plain index with no file changes returned "no file +// changes" and never retried. Now the analyzer is rechecked once its +// bounded wait has passed, and the no-change message says what is pending. +func TestNoopRechecksUnavailableNativePass(t *testing.T) { + root := t.TempDir() + ctx := context.Background() + writeFiles(t, root, map[string]string{ + "a.ts": "export function target() {}\nexport function caller() { target(); }\n", + "tsconfig.json": `{"include":["*.ts"]}`, + }) + st, err := store.Open(root) + if err != nil { + t.Fatal(err) + } + defer st.Close() + clock := &fakeClock{t: time.Unix(1_800_000_000, 0)} + idx := NewWithNativeConfig(parser.NewEngine(), st, native.Config{Enabled: true}) + idx.now = clock.now + _, initial, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(initial.Native, "\n"), "typescript not resolvable") { + t.Skipf("fixture needs an unavailable TypeScript compiler: %v", initial.Native) + } + + // Immediately again: within the recheck wait, a cheap no-op that says so. + _, next, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(next.Native, "\n") + if !strings.Contains(joined, "js-ts has no completed pass (typescript not resolvable in the project); next retry in 15s") || + strings.Contains(joined, "use --force to re-run analyzers") { + t.Fatalf("pending retry not reported accurately: %v", next.Native) + } + + // After the wait: rechecked (still unavailable), next wait doubles. + clock.t = clock.t.Add(16 * time.Second) + _, rechecked, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + if joined := strings.Join(rechecked.Native, "\n"); !strings.Contains(joined, "js-ts still unavailable (typescript not resolvable in the project); rechecked now, next check in 30s") { + t.Fatalf("recheck not reported: %v", rechecked.Native) + } +} + +// The full recovery path with a real toolchain: the Go analyzer is +// unavailable (go not on PATH) on the first index; once go is back and the +// wait has passed, a plain index with no file changes runs and completes it. +func TestNoopRetryRecoversNativePass(t *testing.T) { + goBin, err := exec.LookPath("go") + if err != nil { + t.Skip("go not found") + } + root := t.TempDir() + ctx := context.Background() + writeFiles(t, root, map[string]string{ + "go.mod": "module retry\n\ngo 1.22\n", + "main.go": "package main\n\nfunc main() { run() }\n\nfunc run() {}\n", + }) + st, err := store.Open(root) + if err != nil { + t.Fatal(err) + } + defer st.Close() + clock := &fakeClock{t: time.Unix(1_800_000_000, 0)} + idx := New(parser.NewEngine(), st) + idx.now = clock.now + + origPath := os.Getenv("PATH") + t.Setenv("PATH", t.TempDir()) + _, first, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(first.Native, "\n"), "go: skipped: go executable not found") { + t.Fatalf("expected go unavailable: %v", first.Native) + } + + t.Setenv("PATH", filepath.Dir(goBin)+string(os.PathListSeparator)+origPath) + // Not yet due: still a no-op. + _, early, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(early.Native, "\n"), "go has no completed pass") { + t.Fatalf("expected a pending retry: %v", early.Native) + } + clock.t = clock.t.Add(time.Minute) + _, retried, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(retried.Native, "\n") + if strings.Contains(joined, "no file changes") || !strings.Contains(joined, "go: retrying its incomplete full pass") || + !strings.Contains(joined, "go: resolved ") { + t.Fatalf("plain index did not recover the Go pass: %v", retried.Native) + } + raw, _, _ := st.GetMeta(ctx, core.MetaNativeComplete) + if !strings.Contains(raw, `"go"`) { + t.Fatalf("go not recorded complete after recovery: %s", raw) + } + // Recovered: back to a plain no-op. + _, settled, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(settled.Native, "\n"), "no file changes since last index (use --force") { + t.Fatalf("expected a settled no-op: %v", settled.Native) + } +} + +func TestRetryDelayIsBounded(t *testing.T) { + cases := []struct { + r nativeRetry + want time.Duration + }{ + {nativeRetry{Failures: 1, Unavailable: true}, 15 * time.Second}, + {nativeRetry{Failures: 2, Unavailable: true}, 30 * time.Second}, + {nativeRetry{Failures: 50, Unavailable: true}, 10 * time.Minute}, + {nativeRetry{Failures: 1}, time.Minute}, + {nativeRetry{Failures: 4}, 8 * time.Minute}, + {nativeRetry{Failures: 50}, 6 * time.Hour}, + } + for _, c := range cases { + if got := retryDelay(c.r); got != c.want { + t.Errorf("%+v: got %s want %s", c.r, got, c.want) + } + } +} diff --git a/internal/index/verdict_test.go b/internal/index/verdict_test.go new file mode 100644 index 0000000..d8d90c3 --- /dev/null +++ b/internal/index/verdict_test.go @@ -0,0 +1,103 @@ +package index + +import ( + "context" + "encoding/json" + "reflect" + "testing" + + "github.com/provasign/grove/internal/core" + "github.com/provasign/grove/internal/store" +) + +func verdictStore(t *testing.T, stored []string) *store.Store { + t.Helper() + st, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + if stored != nil { + persistVerdict(t, st, stored) + } + return st +} + +func persistVerdict(t *testing.T, st *store.Store, lines []string) { + t.Helper() + raw, _ := json.Marshal(lines) + if err := st.SetMeta(context.Background(), core.MetaIndexNative, string(raw)); err != nil { + t.Fatal(err) + } +} + +// Stack audit 2026-10-20 #10: re-carrying an unchanged diagnostic must not +// grow the verdict (it went 2, 4, 6, 8, 10 lines). +func TestCarryForwardVerdictDoesNotGrow(t *testing.T) { + ctx := context.Background() + initial := []string{"csharp: full pass over 1 file(s) took 30ms", "csharp: no edges produced — previous native edges carried forward"} + st := verdictStore(t, initial) + for round := 1; round <= 4; round++ { + next := carryForwardVerdict(ctx, st, initial) + persistVerdict(t, st, next) + if !reflect.DeepEqual(next, initial) { + t.Fatalf("round %d: verdict = %q, want the current attempt only %q", round, next, initial) + } + } +} + +// A good pass, then repeated carries (unchanged, then unavailable): the +// last good pass is reported once, followed by the current reason only; +// earlier attempts are superseded, not accumulated. +func TestCarryForwardVerdictKeepsLastGoodOnce(t *testing.T) { + ctx := context.Background() + st := verdictStore(t, nil) + good := []string{"java: full pass over 3 file(s) took 2s", "java: javac attributed 3 file(s)", "go: full pass over 1 file(s) took 1s", "go: resolved 4 native call edge(s)"} + persistVerdict(t, st, carryForwardVerdict(ctx, st, good)) + + unchanged := []string{"java: skipped: no changed files in its languages (previous edges carried forward)", "go: incremental pass over 1 file(s) took 1s", "go: resolved 5 native call edge(s)"} + got := carryForwardVerdict(ctx, st, unchanged) + persistVerdict(t, st, got) + want := []string{good[0], good[1], unchanged[0], unchanged[1], unchanged[2]} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unchanged carry:\n got %q\nwant %q", got, want) + } + + for round := 0; round < 3; round++ { + unavailable := []string{"java: skipped: no JDK 11+ found (previous native edges carried forward)", "go: skipped: no changed files in its languages (previous edges carried forward)"} + got = carryForwardVerdict(ctx, st, unavailable) + persistVerdict(t, st, got) + want = []string{good[0], good[1], unavailable[0], unchanged[1], unchanged[2], unavailable[1]} + if !reflect.DeepEqual(got, want) { + t.Fatalf("round %d unavailable carry:\n got %q\nwant %q", round, got, want) + } + } + + // Disabled: its edges are dropped, so nothing of the old pass is claimed. + got = carryForwardVerdict(ctx, st, []string{"java: skipped: disabled by config", "go: skipped: no changed files in its languages (previous edges carried forward)"}) + persistVerdict(t, st, got) + got = carryForwardVerdict(ctx, st, []string{"java: skipped: no JDK 11+ found (previous native edges carried forward)"}) + if want := []string{"java: skipped: no JDK 11+ found (previous native edges carried forward)"}; !reflect.DeepEqual(got, want) { + t.Fatalf("after disable: got %q want %q", got, want) + } +} + +// An index written before the last-good key existed: its verdict holds the +// old accumulated history. The success lines are recovered once, the +// repeated attempt lines are dropped. +func TestCarryForwardVerdictMigratesAccumulatedHistory(t *testing.T) { + ctx := context.Background() + old := []string{ + "csharp: full pass over 9 file(s) took 1s", "csharp: resolved 12 native call edge(s)", + "csharp: full pass over 9 file(s) took 1s", "csharp: resolved 12 native call edge(s)", + "csharp: no edges produced — previous native edges carried forward", + "csharp: no edges produced — previous native edges carried forward", + } + st := verdictStore(t, old) + current := []string{"csharp: skipped: no changed files in its languages (previous edges carried forward)"} + got := carryForwardVerdict(ctx, st, current) + want := []string{"csharp: resolved 12 native call edge(s)", current[0]} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %q want %q", got, want) + } +} diff --git a/internal/index/walkerror_test.go b/internal/index/walkerror_test.go index cd6ffff..7a4016a 100644 --- a/internal/index/walkerror_test.go +++ b/internal/index/walkerror_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "github.com/provasign/grove/internal/parser" @@ -101,3 +102,59 @@ func TestIndexerShieldsUnreadableSubtreeFromPruning(t *testing.T) { t.Fatalf("sub/sub.go missing from store after unreadable walk: %v", files) } } + +// TestIndexerDropsSymbolsOfUnreadableChangedFile (stack audit 2026-10-20 #1): +// a file whose new content cannot be read (here: over the size cap) must +// not keep serving the symbols of its last indexed version as current. Same +// contract as a parse failure: the stored facts are pruned, and the error is +// reported in result.Errors on every run until the file indexes again. +func TestIndexerDropsSymbolsOfUnreadableChangedFile(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "a.go") + if err := os.WriteFile(path, []byte("package a\n\nfunc Target() {}\n"), 0o644); err != nil { + t.Fatal(err) + } + st, err := store.Open(root) + if err != nil { + t.Fatal(err) + } + defer st.Close() + idx := New(parser.NewEngine(), st) + ctx := context.Background() + if _, res, err := idx.Index(ctx, root); err != nil || res.FilesUpdated != 1 { + t.Fatalf("seed index: %v %#v", err, res) + } + big := make([]byte, parser.MaxFileSizeBytes+1) + copy(big, "package a\n\n// no Target here\n") + for i := len("package a\n\n// no Target here\n"); i < len(big); i++ { + big[i] = '\n' + } + if err := os.WriteFile(path, big, 0o644); err != nil { + t.Fatal(err) + } + for run := 1; run <= 2; run++ { + _, res, err := idx.Index(ctx, root) + if err != nil { + t.Fatal(err) + } + if len(res.Errors) == 0 || !strings.Contains(strings.Join(res.Errors, "\n"), "a.go") { + t.Fatalf("run %d: oversize file not reported: %#v", run, res.Errors) + } + syms, err := st.SymbolsForFiles(ctx, []string{"a.go"}) + if err != nil { + t.Fatal(err) + } + for _, s := range syms { + if s.Name == "Target" { + t.Fatalf("run %d: removed Target still indexed from the last readable version", run) + } + } + } + // Readable again: indexed again. + if err := os.WriteFile(path, []byte("package a\n\nfunc Target() {}\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, res, err := idx.Index(ctx, root); err != nil || len(res.Errors) != 0 || res.FilesUpdated != 1 { + t.Fatalf("recovery index: %v %#v", err, res) + } +} diff --git a/internal/native/go.go b/internal/native/go.go index 4c4f57b..12874c2 100644 --- a/internal/native/go.go +++ b/internal/native/go.go @@ -49,9 +49,15 @@ func (goAnalyzer) Available(_ context.Context, root string) Availability { } type goListPackage struct { - Dir string - ImportPath string - GoFiles []string + Dir string + ImportPath string + GoFiles []string + // CgoFiles are the package's files that import "C". They are ordinary + // Go source for everything but C.x references: a package whose files + // all use cgo has GoFiles=[] and only CgoFiles (astkit's + // thirdparty/tsobjc), and loading GoFiles alone made it an empty + // package named "" (stack audit 2026-10-20 #4). + CgoFiles []string TestGoFiles []string Imports []string // Test-file imports are reported separately by go list; the scoped-delta @@ -357,7 +363,7 @@ func CleanupLegacyCaches(root string) { } func packageFiles(root string, pkg goListPackage) []string { - names := append(append([]string{}, pkg.GoFiles...), pkg.TestGoFiles...) + names := append(append(append([]string{}, pkg.GoFiles...), pkg.CgoFiles...), pkg.TestGoFiles...) out := make([]string, 0, len(names)) for _, name := range names { rel, ok := relFile(root, filepath.Join(pkg.Dir, name)) @@ -543,6 +549,11 @@ func goSemanticPackageEdges(root, pkgPath, dir string, files []string, symbolIdx conf := types.Config{ Importer: projectImporter, Error: func(error) {}, + // cgo files are checked as written, so positions stay those of the + // user's source (no cgo-generated files): `import "C"` declares an + // empty package and C.x references are untyped, as the dependency + // source importer already does. Everything else type-checks. + FakeImportC: true, } pkg, _ := conf.Check(pkgPath, fset, parsed, info) dispatch := newGoInterfaceDispatch(pkg, root, dir, fset, symbolIdx, pkgDirsByImport) diff --git a/internal/native/go_cgo_test.go b/internal/native/go_cgo_test.go new file mode 100644 index 0000000..3127727 --- /dev/null +++ b/internal/native/go_cgo_test.go @@ -0,0 +1,67 @@ +package native + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// writeCgoModule writes a module whose binding package has only cgo files +// (GoFiles=[], CgoFiles=[binding.go]) and returns go list's JSON for it, or +// skips when go or cgo is unavailable. +func writeCgoModule(t *testing.T) (string, []byte) { + t.Helper() + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go not found") + } + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "binding"), 0o755); err != nil { + t.Fatal(err) + } + files := map[string]string{ + "go.mod": "module audit\n\ngo 1.22\n", + "binding/binding.go": "package binding\n\n// #include \nimport \"C\"\n\n" + + "type Thing struct{ n C.int }\n\nfunc (t Thing) Do() int { return int(t.n) }\n\nfunc New() Thing { return Thing{} }\n", + } + for rel, body := range files { + if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(rel)), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + cmd := exec.Command("go", "list", "-json", "./binding") + cmd.Dir = root + raw, err := cmd.Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + var listed struct{ GoFiles, CgoFiles []string } + if err := json.Unmarshal(raw, &listed); err != nil { + t.Fatal(err) + } + if len(listed.CgoFiles) == 0 { + t.Skip("cgo unavailable (CGO_ENABLED=0 or no C compiler)") + } + if len(listed.GoFiles) != 0 { + t.Fatalf("fixture expects a cgo-only package, go list GoFiles=%v", listed.GoFiles) + } + return root, raw +} + +// Stack audit 2026-10-20 #4: a local package with only cgo files loaded as an +// empty package named "" because the project importer read GoFiles only. +func TestGoProjectImporterIncludesCgoDeclarations(t *testing.T) { + _, raw := writeCgoModule(t) + var meta goListPackage + if err := json.Unmarshal(raw, &meta); err != nil { + t.Fatal(err) + } + pkg, err := newGoProjectImporter([]goListPackage{meta}, goDeps{}).Import(meta.ImportPath) + if err != nil { + t.Fatal(err) + } + if pkg.Name() != "binding" || pkg.Scope().Lookup("Thing") == nil || pkg.Scope().Lookup("New") == nil { + t.Fatalf("cgo package lost its Go declarations: name=%q scope=%v", pkg.Name(), pkg.Scope().Names()) + } +} diff --git a/internal/native/go_importer.go b/internal/native/go_importer.go index 8bd2d38..34e2a24 100644 --- a/internal/native/go_importer.go +++ b/internal/native/go_importer.go @@ -128,8 +128,11 @@ func (p *goProjectImporter) load(path string) (loaded *types.Package, err error) } } fset := token.NewFileSet() - files := make([]*ast.File, 0, len(meta.GoFiles)) - for _, name := range meta.GoFiles { + // cgo files join the ordinary Go files (the dependency source importer + // below does the same); FakeImportC handles their `import "C"`. + names := append(append([]string(nil), meta.GoFiles...), meta.CgoFiles...) + files := make([]*ast.File, 0, len(names)) + for _, name := range names { file, err := parser.ParseFile(fset, filepath.Join(meta.Dir, name), nil, parser.SkipObjectResolution) if file == nil { return nil, err @@ -139,11 +142,28 @@ func (p *goProjectImporter) load(path string) (loaded *types.Package, err error) } files = append(files, file) } - conf := types.Config{Importer: p, Error: func(error) {}} + // Under FakeImportC every C.x is untyped, so expressions using one + // (int(t.n) for a C.int field) report type errors that only say cgo + // was not run. Those are expected and do not make the package partial; + // the first error elsewhere does. + cgoFiles := map[string]bool{} + for _, name := range meta.CgoFiles { + cgoFiles[filepath.Join(meta.Dir, name)] = true + } + var firstErr error + conf := types.Config{Importer: p, FakeImportC: true, Error: func(e error) { + if te, ok := e.(types.Error); ok && cgoFiles[te.Fset.Position(te.Pos).Filename] { + return + } + if firstErr == nil { + firstErr = e + } + }} pkg, err := conf.Check(path, fset, files, nil) if pkg == nil { return nil, err } + err = firstErr // A type error (typically one unresolvable third-party import: gin's // binding imports go.mongodb.org/mongo-driver, absent from a fresh // module cache) does not void the package. go/types checked everything diff --git a/internal/native/inputs.go b/internal/native/inputs.go new file mode 100644 index 0000000..16f3038 --- /dev/null +++ b/internal/native/inputs.go @@ -0,0 +1,74 @@ +package native + +import ( + "path" + "strings" +) + +// Compiler inputs: build-configuration files an analyzer's result depends on +// besides the source files themselves. Grove indexes most of them as +// plaintext (tsconfig.json, pom.xml) or not at all (go.mod, go.sum), so a +// change to one never touched the analyzer's languages and the analyzer was +// skipped with its previous edges carried forward — after a tsconfig "paths" +// or module-resolution edit the graph kept the old import resolution (stack +// audit 2026-10-20 #2). The indexer fingerprints these files and treats a +// change as invalidating the analyzer's completed pass. +// +// Matching is by base name, anywhere in the walked tree (nested modules and +// workspace packages have their own). Each list names what the analyzer or +// its toolchain reads: module/workspace files for go list and go/types, +// tsconfig/jsconfig and package.json ("types", "exports", "imports", +// workspaces) for the TypeScript program, the build files javac's classpath +// is resolved from (the same list as the Java classpath cache key), Cargo +// manifests, compile_commands.json, project files, and the files whose +// presence decides availability (pyproject.toml, composer.json). + +// CompilerInputAnalyzers returns the analyzers (by Analyzer.Name) whose +// compiler inputs include the slash-separated relPath, or nil. +func CompilerInputAnalyzers(relPath string) []string { + base := path.Base(relPath) + switch base { + case "go.mod", "go.sum", "go.work", "go.work.sum": + return []string{"go"} + case "package.json": + return []string{"js-ts"} + case "pyproject.toml", "setup.cfg", "requirements.txt": + return []string{"python"} + case "pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts", + "gradle.properties", "libs.versions.toml": + return []string{"java"} + case "Cargo.toml", "Cargo.lock": + return []string{"rust"} + case "compile_commands.json": + return []string{"cfamily"} + case "Directory.Build.props": + return []string{"csharp"} + case "composer.json": + return []string{"php"} + } + if strings.HasSuffix(base, ".json") && (strings.HasPrefix(base, "tsconfig") || strings.HasPrefix(base, "jsconfig")) { + return []string{"js-ts"} // tsconfig.base.json, tsconfig.build.json, ... + } + if strings.HasSuffix(base, ".csproj") { + return []string{"csharp"} + } + return nil +} + +// InvalidateResident stops the resident worker processes that hold compiler +// state (parsed programs, a loaded workspace) of the named analyzers for +// root, so the next run starts from the changed configuration instead of +// reusing state built under the old one. Analyzers without resident state +// are ignored. +func InvalidateResident(root string, analyzers []string) { + for _, name := range analyzers { + switch name { + case "js-ts": + stopResidentWorkers(root, "ts\x00") + case "java": + stopResidentWorkers(root, "java\x00") + case "rust": + stopRustLSP(root) + } + } +} diff --git a/internal/native/inputs_test.go b/internal/native/inputs_test.go new file mode 100644 index 0000000..b38cb5f --- /dev/null +++ b/internal/native/inputs_test.go @@ -0,0 +1,27 @@ +package native + +import ( + "os/exec" + "testing" +) + +// A compiler-input change drops the resident state of that analyzer only. +func TestInvalidateResidentStopsOnlyNamedAnalyzers(t *testing.T) { + start := func() *exec.Cmd { return exec.Command("true") } + residentWorkerFor("ts\x00/r\x00/r", "/r", "m", start) + residentWorkerFor("java\x00/r\x00java", "/r", "m", start) + residentWorkerFor("ts\x00/other\x00/other", "/other", "m", start) + defer StopTSWorkers("") + InvalidateResident("/r", []string{"js-ts"}) + residentWorkers.Lock() + defer residentWorkers.Unlock() + if _, ok := residentWorkers.m["ts\x00/r\x00/r"]; ok { + t.Error("TypeScript worker of the root survived") + } + if _, ok := residentWorkers.m["java\x00/r\x00java"]; !ok { + t.Error("Java worker stopped by a TypeScript config change") + } + if _, ok := residentWorkers.m["ts\x00/other\x00/other"]; !ok { + t.Error("another root's worker stopped") + } +} diff --git a/internal/native/native.go b/internal/native/native.go index be0448e..b5695af 100644 --- a/internal/native/native.go +++ b/internal/native/native.go @@ -159,6 +159,44 @@ type Result struct { Partial map[string][]string // Completed names analyzers that finished a full pass this run. Completed []string + // Incomplete names analyzers that were due a full pass this run, had + // files to analyze, and did not complete it: unavailable, failed, timed + // out, or produced no edges. The indexer schedules their retry. + Incomplete map[string]Incomplete +} + +// Incomplete describes why an analyzer's full pass did not complete. +type Incomplete struct { + // Unavailable: the analyzer never started (toolchain or project + // configuration missing), so a cheap availability check decides + // whether a retry can succeed. + Unavailable bool + Reason string +} + +func (r *Result) incomplete(name string, why Incomplete) { + if r.Incomplete == nil { + r.Incomplete = map[string]Incomplete{} + } + r.Incomplete[name] = why +} + +// AnalyzerAvailable runs the named analyzer's availability check for root. +// ok is false when no such analyzer exists or cfg disables it (natively +// disabled analyzers are never retried). +func AnalyzerAvailable(ctx context.Context, root, name string, cfg Config) (avail Availability, ok bool) { + if !cfg.Enabled { + return Availability{}, false + } + for _, a := range PriorityAnalyzers() { + if a.Name() == name { + if !analyzerEnabled(a, cfg) { + return Availability{}, false + } + return a.Available(ctx, root), true + } + } + return Availability{}, false } func PriorityAnalyzers() []Analyzer { @@ -227,6 +265,9 @@ func AnalyzeChangedFiles(ctx context.Context, root string, symbols []core.Symbol combined.Diagnostics = append(combined.Diagnostics, analyzer.Name()+": skipped: "+avail.Reason+" (previous native edges carried forward)") combined.SkippedLanguages = append(combined.SkippedLanguages, analyzer.Languages()...) + if full { + combined.incomplete(analyzer.Name(), Incomplete{Unavailable: true, Reason: avail.Reason}) + } continue } // Budget scales with input size unless the user pinned a timeout: @@ -288,12 +329,21 @@ func AnalyzeChangedFiles(ctx context.Context, root string, symbols []core.Symbol combined.Diagnostics = append(combined.Diagnostics, analyzer.Name()+": "+reason+" — previous native edges carried forward") combined.SkippedLanguages = append(combined.SkippedLanguages, analyzer.Languages()...) + if full { + combined.incomplete(analyzer.Name(), Incomplete{Reason: reason}) + } continue } combined.Edges = append(combined.Edges, result.Edges...) combined.Symbols = append(combined.Symbols, result.Symbols...) if full && !timedOut && len(result.Partial) == 0 { combined.Completed = append(combined.Completed, analyzer.Name()) + } else if full { + reason := "partial result" + if timedOut { + reason = "timed out after " + timeout.String() + } + combined.incomplete(analyzer.Name(), Incomplete{Reason: reason}) } for lang, dirs := range result.Partial { if combined.Partial == nil { diff --git a/internal/native/rust_worker.go b/internal/native/rust_worker.go index d5b7e05..3f17e01 100644 --- a/internal/native/rust_worker.go +++ b/internal/native/rust_worker.go @@ -80,6 +80,18 @@ func rustLSPFor(root, bin string) (*rustLSP, error) { return w, nil } +// stopRustLSP stops root's resident rust-analyzer server, if any; the next +// incremental run that needs one starts it fresh. +func stopRustLSP(root string) { + rustLSPs.Lock() + w := rustLSPs.m[root] + delete(rustLSPs.m, root) + rustLSPs.Unlock() + if w != nil && w.alive() { + w.stop() + } +} + func (w *rustLSP) alive() bool { select { case <-w.done: diff --git a/internal/native/tsworker.go b/internal/native/tsworker.go index 442baca..779b6c6 100644 --- a/internal/native/tsworker.go +++ b/internal/native/tsworker.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "strconv" + "strings" "time" "fmt" @@ -90,10 +91,16 @@ func tsWorkerFor(root, dir string) *residentWorker { // repository root, or of every root when root is "". Engines call it on // Close. func StopTSWorkers(root string) { + stopResidentWorkers(root, "") +} + +// stopResidentWorkers stops the resident workers of root ("" = every root) +// whose registry key starts with prefix ("" = every kind). +func stopResidentWorkers(root, prefix string) { residentWorkers.Lock() defer residentWorkers.Unlock() for key, w := range residentWorkers.m { - if root == "" || w.root == root { + if (root == "" || w.root == root) && strings.HasPrefix(key, prefix) { w.mu.Lock() w.stopLocked() w.mu.Unlock()