Fix/cgo line directives

internal/document/document.go

@@ -38,11 +38,14 @@ func IsDocDeprecated(docs []string) bool {
 
 func NewDocument(
 	relative string,
+	originAbs string,
 	pkg *packages.Package,
 	pkgSymbols *lookup.Package,
 ) *Document {
 	return &Document{
 		RelativePath: relative,
+		originAbs:    originAbs,
+		lineLen:      loadLineLengths(originAbs),
 		pkg:          pkg,
 		pkgSymbols:   pkgSymbols,
 
@@ -70,6 +73,90 @@ type Document struct {
 	// pkgSymbols maps positions to symbol names within
 	// this document.
 	pkgSymbols *lookup.Package
+
+	// originAbs is the cleaned, symlink-resolved absolute path of the source file
+	// this document represents; lineLen is the byte length of each of its lines
+	// (0-indexed), or nil if it couldn't be read. Together they let AppendOccurrence
+	// callers reject occurrences whose //line-adjusted range escapes the real file.
+	originAbs string
+	lineLen   []int
+
+	// occurrences accumulates every occurrence routed to this document. Usually
+	// that is only its own file's, but a generated file may attribute occurrences
+	// here via //line directives (e.g. cgo's rewritten source). extraSymbols
+	// accumulates SymbolInformation from the file(s) that map here.
+	occurrences  []*scip.Occurrence
+	extraSymbols []*scip.SymbolInformation
+}
+
+// InBounds reports whether r is a well-formed range that fits within this
+// document's source file. It rejects:
+//   - negative line/column (a `//line file:N` directive with no column collapses
+//     positions to column 0 -> scip -1, which is malformed);
+//   - lines past EOF or columns past the line (cgo's `defer C.f(x)` rewrites and
+//     inserted `_cgoCheckPointer` thunks land here);
+//   - reversed ranges (end before start).
+//
+// Such occurrences cannot be faithfully represented and are dropped rather than
+// emitted with a bogus location (which downstream SCIP consumers reject). A
+// document whose source could not be read admits everything (no over-dropping).
+func (d *Document) InBounds(r scip.Range) bool {
+	sl, sc, el, ec := int(r.Start.Line), int(r.Start.Character), int(r.End.Line), int(r.End.Character)
+	if sl < 0 || sc < 0 || el < 0 || ec < 0 {
+		return false
+	}
+	if el < sl || (el == sl && ec < sc) {
+		return false
+	}
+	if d.lineLen == nil {
+		return true
+	}
+	if sl >= len(d.lineLen) || el >= len(d.lineLen) {
+		return false
+	}
+	return sc <= d.lineLen[sl] && ec <= d.lineLen[el]
+}
+
+// AppendOccurrence records occ against this document. Called from the single
+// file-walking goroutine, so no synchronization is required.
+func (d *Document) AppendOccurrence(occ *scip.Occurrence) {
+	d.occurrences = append(d.occurrences, occ)
+}
+
+// AddSymbols records SymbolInformation contributed by a file that maps here.
+func (d *Document) AddSymbols(syms []*scip.SymbolInformation) {
+	d.extraSymbols = append(d.extraSymbols, syms...)
+}
+
+// ToScip renders the accumulated occurrences and symbols as a scip.Document.
+func (d *Document) ToScip() *scip.Document {
+	occurrences := d.occurrences
+	if d.PackageOccurrence != nil {
+		occurrences = append([]*scip.Occurrence{d.PackageOccurrence}, occurrences...)
+	}
+	return &scip.Document{
+		Language:     "go",
+		RelativePath: d.RelativePath,
+		Occurrences:  occurrences,
+		Symbols:      d.extraSymbols,
+	}
+}
+
+// loadLineLengths returns the byte length of each line of path (0-indexed), or
+// nil if it can't be read. Used to bounds-check occurrence ranges against the
+// real source (cgo rewrites some constructs to positions past the original
+// line/EOF; those can't be faithfully represented and are dropped).
+func loadLineLengths(path string) []int {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	lines := strings.Split(string(b), "\n")
+	lengths := make([]int, len(lines))
+	for i, line := range lines {
+		lengths[i] = len(strings.TrimSuffix(line, "\r"))
+	}
+	return lengths
 }
 
 func (d *Document) GetSymbol(pos token.Pos) (string, bool) {

internal/document/inbounds_test.go

@@ -0,0 +1,56 @@
+package document
+
+import (
+	"testing"
+
+	"github.com/scip-code/scip/bindings/go/scip"
+)
+
+func rng(sl, sc, el, ec int32) scip.Range {
+	return scip.Range{
+		Start: scip.Position{Line: sl, Character: sc},
+		End:   scip.Position{Line: el, Character: ec},
+	}
+}
+
+func TestInBounds(t *testing.T) {
+	// Two lines, byte lengths 5 and 10.
+	d := &Document{lineLen: []int{5, 10}}
+
+	cases := []struct {
+		name string
+		r    scip.Range
+		want bool
+	}{
+		{"in bounds, single line", rng(0, 0, 0, 5), true},
+		{"end column at EOL", rng(1, 0, 1, 10), true},
+		{"spans two lines", rng(0, 1, 1, 2), true},
+		{"start column past EOL", rng(0, 6, 0, 6), false},
+		{"end column past EOL", rng(1, 0, 1, 11), false},
+		{"line past EOF", rng(2, 0, 2, 0), false},
+		{"negative start column", rng(0, -1, 0, -1), false},
+		{"negative line", rng(-1, 0, -1, 0), false},
+		{"reversed same line", rng(0, 4, 0, 1), false},
+		{"reversed across lines", rng(1, 0, 0, 0), false},
+	}
+	for _, tc := range cases {
+		if got := d.InBounds(tc.r); got != tc.want {
+			t.Errorf("%s: InBounds(%v) = %v, want %v", tc.name, tc.r, got, tc.want)
+		}
+	}
+}
+
+func TestInBoundsUnknownSourceAdmitsWellFormed(t *testing.T) {
+	// A document whose source couldn't be read (lineLen == nil) must not
+	// over-drop: it admits any well-formed range but still rejects malformed ones.
+	d := &Document{}
+	if !d.InBounds(rng(999, 0, 999, 3)) {
+		t.Error("nil lineLen should admit a well-formed range")
+	}
+	if d.InBounds(rng(0, -1, 0, 0)) {
+		t.Error("nil lineLen should still reject a negative column")
+	}
+	if d.InBounds(rng(2, 0, 1, 0)) {
+		t.Error("nil lineLen should still reject a reversed range")
+	}
+}

internal/index/scip.go

@@ -70,10 +70,15 @@ func ListMissing(opts config.IndexOpts) (missing []string, err error) {
 	}
 
 	for _, pkg := range projectPackages {
+		goFiles := visitors.RealGoFiles(pkg)
 		for _, f := range pkg.Syntax {
-			docName := pkg.Fset.File(f.Package).Name()
-			if _, ok := pathToDocuments[docName]; !ok {
-				missing = append(missing, docName)
+			origin := visitors.OriginFile(pkg, f.Package)
+			if _, isReal := goFiles[origin]; !isReal {
+				// Generated file with no real-source origin (e.g. cgo glue).
+				continue
+			}
+			if _, ok := pathToDocuments[origin]; !ok {
+				missing = append(missing, origin)
 			}
 		}
 	}
@@ -120,7 +125,6 @@ func Index(writer func(proto.Message) error, opts config.IndexOpts) error {
 
 	var count uint64
 	var wg sync.WaitGroup
-	var writeErr error
 	wg.Add(1)
 
 	go func() {
@@ -131,30 +135,31 @@ func Index(writer func(proto.Message) error, opts config.IndexOpts) error {
 			pkgSymbols := globalSymbols.GetPackage(pkg)
 
 			for _, file := range pkg.Syntax {
-				doc := pathToDocument[pkg.Fset.File(file.Package).Name()]
+				origin := visitors.OriginFile(pkg, file.Package)
+				doc := pathToDocument[origin]
 				if doc == nil {
+					// No document: a generated file (e.g. cgo's
+					// _cgo_gotypes.go) whose occurrences are compiler glue.
 					continue
 				}
 
-				// If possible, any state required for created a scip document
-				// should be contained in the visitor. This makes sure that we can
-				// garbage collect everything that's there after each loop,
-				// rather than holding on to every occurrence and piece of data
+				// The visitor routes each occurrence to the document of its
+				// //line-adjusted origin (usually this file, but a generated file
+				// can attribute some occurrences to another real source file), so
+				// it needs the whole document map, not just this file's document.
 				visitor := visitors.NewFileVisitor(
 					doc,
 					pkg,
 					file,
 					pkgSymbols,
 					globalSymbols,
+					pathToDocument,
 				)
 
-				// Traverse the file
+				// Traverse the file (routing occurrences), then attach this
+				// file's symbols to its document.
 				ast.Walk(visitor, file)
-
-				// Write the document
-				if writeErr = writer(visitor.ToScipDocument()); writeErr != nil {
-					return
-				}
+				visitor.Finish()
 			}
 
 			atomic.AddUint64(&count, 1)
@@ -163,8 +168,12 @@ func Index(writer func(proto.Message) error, opts config.IndexOpts) error {
 
 	output.WithProgressParallel(&wg, "Visiting Project Files", &count, uint64(pkgLen))
 
-	if writeErr != nil {
-		return writeErr
+	// Emit one document per source file -- occurrences routed from every file
+	// that maps here are now accumulated -- in a stable, path-sorted order.
+	for _, origin := range slices.Sorted(maps.Keys(pathToDocument)) {
+		if err := writer(pathToDocument[origin].ToScip()); err != nil {
+			return err
+		}
 	}
 
 	// Emit external symbols for remote types that implement local interfaces
@@ -227,14 +236,20 @@ func indexVisitPackages(
 					Text:     "package " + pkg.Name,
 				},
 			}
-			firstFile := pkg.Syntax[0]
-			firstDoc := pathToDocuments[pkg.Fset.File(firstFile.Package).Name()]
-			firstDoc.SetSymbolInformation(firstFile.Name.NamePos, symInfo)
-
+			// Attach the package symbol to the first real document and a
+			// package occurrence to each. Generated files (e.g. cgo's
+			// _cgo_gotypes.go) have no document, so skip them.
+			pkgDeclared := false
 			for _, f := range pkg.Syntax {
-				doc := pathToDocuments[pkg.Fset.File(f.Package).Name()]
+				doc := pathToDocuments[visitors.OriginFile(pkg, f.Package)]
+				if doc == nil {
+					continue
+				}
+				if !pkgDeclared {
+					doc.SetSymbolInformation(f.Name.NamePos, symInfo)
+					pkgDeclared = true
+				}
 				position := pkg.Fset.Position(f.Name.NamePos)
-
 				doc.PackageOccurrence = &scip.Occurrence{
 					TypedRange:  symbols.RangeFromName(position, f.Name.Name, false).AsTypedRange(),
 					Symbol:      pkgSymbol,

internal/loader/loader.go

@@ -22,6 +22,7 @@ import (
 type PackageLookup map[newtypes.PackageID]*packages.Package
 
 var loadMode = packages.NeedExportFile |
+	packages.NeedFiles |
 	packages.NeedImports |
 	packages.NeedSyntax |
 	packages.NeedTypes |