gopls/internal/lsp/source/typerefs: new package to analyze syntax deps

Add a new typerefs package that analyzes type-affecting references
between declarations. This can be used to compute the set of packages
that may influence eachothers types, for the purpose of precise pruning.

This CL include both an implementation of the references analysis, and a
standalone implementation of the whole-graph reachability analysis. The
latter is intended to be temporary, as for various reasons it is more
convenient for the memoization and computation of graph data to live in
the gopls/internal/lsp/cache package.

For golang/go#57987

Change-Id: Ia1355ff4dfe476319e6f4f8f06b2a79dd7ba554f
Reviewed-on: https://go-review.googlesource.com/c/tools/+/479296
Run-TryBot: Robert Findley <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Alan Donovan <[email protected]>
gopls-CI: kokoro <[email protected]>

Author: findleyr

gopls/internal/lsp/cache/parse.go

@@ -51,12 +51,14 @@ func parseGoImpl(ctx context.Context, fset *token.FileSet, fh source.FileHandle,
 	if ctx.Err() != nil {
 		return nil, ctx.Err()
 	}
-	pgf, _ := parseGoSrc(ctx, fset, fh.URI(), content, mode)
+	pgf, _ := ParseGoSrc(ctx, fset, fh.URI(), content, mode)
 	return pgf, nil
 }
 
-// parseGoSrc parses a buffer of Go source, repairing the tree if necessary.
-func parseGoSrc(ctx context.Context, fset *token.FileSet, uri span.URI, src []byte, mode parser.Mode) (res *source.ParsedGoFile, fixes []fixType) {
+// ParseGoSrc parses a buffer of Go source, repairing the tree if necessary.
+//
+// The provided ctx is used only for logging.
+func ParseGoSrc(ctx context.Context, fset *token.FileSet, uri span.URI, src []byte, mode parser.Mode) (res *source.ParsedGoFile, fixes []fixType) {
 	file, err := parser.ParseFile(fset, uri.Filename(), src, mode)
 	var parseErr scanner.ErrorList
 	if err != nil {

gopls/internal/lsp/cache/parse_cache.go

@@ -166,7 +166,7 @@ func (c *parseCache) startParse(mode parser.Mode, fhs ...source.FileHandle) ([]*
 			// inside of parseGoSrc without exceeding the allocated space.
 			base, nextBase := c.allocateSpace(2*len(content) + parsePadding)
 
-			pgf, fixes1 := parseGoSrc(ctx, fileSetWithBase(base), uri, content, mode)
+			pgf, fixes1 := ParseGoSrc(ctx, fileSetWithBase(base), uri, content, mode)
 			file := pgf.Tok
 			if file.Base()+file.Size()+1 > nextBase {
 				// The parsed file exceeds its allocated space, likely due to multiple
@@ -178,7 +178,7 @@ func (c *parseCache) startParse(mode parser.Mode, fhs ...source.FileHandle) ([]*
 				// there, as parseGoSrc will repeat them.
 				actual := file.Base() + file.Size() - base // actual size consumed, after re-parsing
 				base2, nextBase2 := c.allocateSpace(actual)
-				pgf2, fixes2 := parseGoSrc(ctx, fileSetWithBase(base2), uri, content, mode)
+				pgf2, fixes2 := ParseGoSrc(ctx, fileSetWithBase(base2), uri, content, mode)
 
 				// In golang/go#59097 we observed that this panic condition was hit.
 				// One bug was found and fixed, but record more information here in

gopls/internal/lsp/source/typerefs/doc.go

@@ -0,0 +1,111 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package typerefs extracts from Go syntax a graph of symbol-level
+// dependencies, for the purpose of precise invalidation of package data.
+//
+// # Background
+//
+// The goal of this analysis is to determine, for each package P, a nearly
+// minimal set of packages that could affect the type checking of P. This set
+// may contain false positives, but the smaller this set the better we can
+// invalidate and prune packages in gopls.
+//
+// More precisely, for each package P we define the set of "reachable" packages
+// from P as the set of packages that may affect the (deep) export data of the
+// direct dependencies of P. By this definition, the complement of this set
+// cannot affect any information derived from type checking P (e.g.
+// diagnostics, cross references, or method sets). Therefore we need not
+// invalidate any results for P when a package in the complement of this set
+// changes.
+//
+// # Computing references
+//
+// For a given declaration D, references are computed based on identifiers or
+// dotted identifiers referenced in the declaration of D, that may affect
+// the type of D. However, these references reflect only local knowledge of the
+// package and its dependency metadata, and do not depend on any analysis of
+// the dependencies themselves.
+//
+// Specifically, if a referring identifier I appears in the declaration, we
+// record an edge from D to each object possibly referenced by I. We search for
+// references within type syntax, but do not actual type-check, so we can't
+// reliably determine whether an expression is a type or a term, or whether a
+// function is a builtin or generic. For example, the type of x in var x =
+// p.F(W) only depends on W if p.F is a builtin or generic function, which we
+// cannot know without type-checking package p. So we may over-approximate in
+// this way.
+//
+//   - If I is declared in the current package, record a reference to its
+//     declaration.
+//   - Else, if there are any dot-imported imports in the current file and I is
+//     exported, record a (possibly dangling) edge to the corresponding
+//     declaration in each dot-imported package.
+//
+// If a dotted identifier q.I appears in the declaration, we
+// perform a similar operation:
+//   - If q is declared in the current package, we record a reference to that
+//     object. It may be a var or const that has a field or method I.
+//   - Else, if q is a valid import name based on imports in the current file
+//     and the provided metadata for dependency package names, record a
+//     reference to the object I in that package.
+//   - Additionally, handle the case where Q is exported, and Q.I may refer to
+//     a field or method in a dot-imported package.
+//
+// That is essentially the entire algorithm, though there is some subtlety to
+// visiting the set of identifiers or dotted identifiers that may affect the
+// declaration type. See the visitDeclOrSpec function for the details of this
+// analysis. Notably, we also skip identifiers that refer to type parameters in
+// generic declarations.
+//
+// # API
+//
+// The main entry point for this analysis is the [Refs] function, which
+// implements the aforementioned syntactic analysis for a set of files
+// constituting a package.
+//
+// These references use shared state to efficiently represent references, by
+// way of the [PackageIndex] and [PackageSet] types.
+//
+// The [BuildPackageGraph] constructor implements a whole-graph analysis similar
+// to that which will be implemented by gopls, but for various reasons the
+// logic for this analysis will eventually live in the
+// [golang.org/x/tools/gopls/internal/lsp/cache] package. Nevertheless,
+// BuildPackageGraph and its test serve to verify the syntactic analysis, and
+// may serve as a proving ground for new optimizations of the whole-graph analysis.
+//
+// # Comparison with export data
+//
+// At first it may seem that the simplest way to implement this analysis would
+// be to consider the types.Packages of the dependencies of P, for example
+// during export. After all, it makes sense that the type checked packages
+// themselves could describe their dependencies. However, this does not work as
+// type information does not describe certain syntactic relationships.
+//
+// For example, the following scenarios cause type information to miss
+// syntactic relationships:
+//
+// Named type forwarding:
+//
+//	package a; type A b.B
+//	package b; type B int
+//
+// Aliases:
+//
+//	package a; func A(f b.B)
+//	package b; type B = func()
+//
+// Initializers:
+//
+//	package a; var A = b.B()
+//	package b; func B() string { return "hi" }
+//
+// Use of the unsafe package:
+//
+//	package a; type A [unsafe.Sizeof(B{})]int
+//	package b; type B struct { f1, f2, f3 int }
+//
+// In all of these examples, types do not contain information about the edge
+// between the a.A and b.B declarations.
+package typerefs

gopls/internal/lsp/source/typerefs/packageset.go

@@ -0,0 +1,148 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package typerefs
+
+import (
+	"fmt"
+	"math/bits"
+	"sort"
+	"strings"
+	"sync"
+
+	"golang.org/x/tools/gopls/internal/lsp/source"
+)
+
+// PackageIndex stores common data to enable efficient representation of
+// references and package sets.
+type PackageIndex struct {
+	// For now, PackageIndex just indexes package ids, to save space and allow for
+	// faster unions via sparse int vectors.
+	mu  sync.Mutex
+	ids []source.PackageID
+	m   map[source.PackageID]packageIdx
+}
+
+type packageIdx int // for additional type safety: an index in PackageIndex.ids
+
+// NewPackageIndex creates a new PackageIndex instance for use in building
+// reference and package sets.
+func NewPackageIndex() *PackageIndex {
+	return &PackageIndex{
+		m: make(map[source.PackageID]packageIdx),
+	}
+}
+
+// idx returns the packageIdx referencing id, creating one if id is not yet
+// tracked by the receiver.
+func (r *PackageIndex) idx(id source.PackageID) packageIdx {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if i, ok := r.m[id]; ok {
+		return i
+	}
+	i := packageIdx(len(r.ids))
+	r.m[id] = i
+	r.ids = append(r.ids, id)
+	return i
+}
+
+// id returns the PackageID for idx.
+//
+// idx must have been created by this PackageIndex instance.
+func (r *PackageIndex) id(idx packageIdx) source.PackageID {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.ids[idx]
+}
+
+// A PackageSet is a set of source.PackageIDs, optimized for inuse memory
+// footprint and efficient union operations.
+type PackageSet struct {
+	// PackageSet is a sparse int vector of package indexes from parent.
+	parent *PackageIndex
+	sparse map[int]blockType // high bits in key, set of low bits in value
+}
+
+type blockType = uint // type of each sparse vector element
+const blockSize = bits.UintSize
+
+// New creates a new PackageSet bound to this PackageIndex instance.
+//
+// PackageSets may only be combined with other PackageSets from the same
+// instance.
+func (s *PackageIndex) New() *PackageSet {
+	return &PackageSet{
+		parent: s,
+		sparse: make(map[int]blockType),
+	}
+}
+
+// add records a new element in the package set.
+//
+// For internal use, since it adds by index rather than ID, to avoid lookups.
+func (s *PackageSet) add(idx packageIdx) {
+	i := int(idx)
+	s.sparse[i/blockSize] |= 1 << (i % blockSize)
+}
+
+// Union records all elements from other into the receiver, mutating the
+// receiver set but not the argument set. The receiver must not be nil, but the
+// argument set may be nil.
+//
+// Precondition: both package sets were created with the same PackageIndex.
+func (s *PackageSet) Union(other *PackageSet) {
+	if other == nil {
+		return // e.g. unsafe
+	}
+	if other.parent != s.parent {
+		panic("other set is from a different PackageIndex instance")
+	}
+	for k, v := range other.sparse {
+		if v0 := s.sparse[k]; v0 != v {
+			s.sparse[k] = v0 | v
+		}
+	}
+}
+
+// Contains reports whether id is contained in the receiver set.
+func (s *PackageSet) Contains(id source.PackageID) bool {
+	i := int(s.parent.idx(id))
+	return s.sparse[i/blockSize]&(1<<(i%blockSize)) != 0
+}
+
+// Elems calls f for each element of the set in ascending order.
+func (s *PackageSet) Elems(f func(source.PackageID)) {
+	blockIndexes := make([]int, 0, len(s.sparse))
+	for k := range s.sparse {
+		blockIndexes = append(blockIndexes, k)
+	}
+	sort.Ints(blockIndexes)
+	for _, i := range blockIndexes {
+		v := s.sparse[i]
+		for b := 0; b < blockSize; b++ {
+			if (v & (1 << b)) != 0 {
+				f(s.parent.id(packageIdx(i*blockSize + b)))
+			}
+		}
+	}
+}
+
+// Len reports the length of the receiver set.
+func (s *PackageSet) Len() int { // could be optimized
+	l := 0
+	s.Elems(func(source.PackageID) {
+		l++
+	})
+	return l
+}
+
+// String returns a human-readable representation of the set: {A, B, ...}.
+func (s *PackageSet) String() string {
+	var ids []string
+	s.Elems(func(id source.PackageID) {
+		ids = append(ids, string(id))
+	})
+	return fmt.Sprintf("{%s}", strings.Join(ids, ", "))
+}