From 6d4096937ead82e37399301a8eb7881e871670f5 Mon Sep 17 00:00:00 2001 From: Henry Wong Date: Mon, 19 Aug 2019 22:59:45 +0800 Subject: [PATCH] [elastic] Cache the results of 'invokeGo()' call 'invokeGo' is used to get the import outline of the specified folder(package), so there are duplications that will slow down the go langserver. Since go langserver serves several repos at the same time, use LRU cache in case of blow up the memory. 'determineRootDirsCached' serves single view, i.e. single workspace folder, 'golistDriverLRUCached' serves single subfolder, i.e. package. Given that we will skip 'vendor' folder, for now, the number of the repos are simultaneously indexing is very low, that's why set the 'determineRootDirsCached' entry number to 16. Future plan: skip the 'vendor' file download the deps during the initialize request eliminate the 'invokeGo' call find a better approach to handle the 'vendor' folder --- .ci/Dockerfile | 3 ++- .travis.yml | 1 + appveyor.yml | 1 + go.mod | 1 + go.sum | 2 ++ go/packages/golist.go | 36 +++++++++++++++++++++++++++++++++-- go/packages/golist_overlay.go | 27 ++++++++++++++++++++++++++ 7 files changed, 68 insertions(+), 3 deletions(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 6676568d916..3311f6585a4 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -7,7 +7,8 @@ ENV GO111MODULE=off RUN apk add --no-cache --quiet make curl git jq unzip tree && \ apk add bash && \ go get golang.org/x/sync/errgroup && \ - go get golang.org/x/xerrors + go get golang.org/x/xerrors && \ + go get github.com/hashicorp/golang-lru VOLUME /go/src/golang.org/x/tools WORKDIR /go/src/golang.org/x/tools diff --git a/.travis.yml b/.travis.yml index 9693d38545f..1ffc78ffd88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ env: before_install: - go get golang.org/x/sync/errgroup - go get golang.org/x/xerrors + - go get github.com/hashicorp/golang-lru matrix: fast_finish: true diff --git a/appveyor.yml b/appveyor.yml index a365cf389e9..ee3fa5d0aae 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,6 +26,7 @@ environment: build_script: - go get golang.org/x/sync/errgroup - go get golang.org/x/xerrors + - go get github.com/hashicorp/golang-lru # TODO(henrywong) For now, there are problems about the windows test. # - go test ./internal/lsp -v - mkdir go-langserver-windows diff --git a/go.mod b/go.mod index 026a2638844..28a70df3b20 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module golang.org/x/tools go 1.11 require ( + github.com/hashicorp/golang-lru v0.5.3 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 diff --git a/go.sum b/go.sum index c4cc4a6d321..6d3e7f7e802 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/go/packages/golist.go b/go/packages/golist.go index 9f8d4ced77c..8c3594e3796 100644 --- a/go/packages/golist.go +++ b/go/packages/golist.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "go/types" + "hash/fnv" "io/ioutil" "log" "os" @@ -22,6 +23,7 @@ import ( "sync" "time" + "github.com/hashicorp/golang-lru" "golang.org/x/tools/go/internal/packagesdriver" "golang.org/x/tools/internal/gopathwalk" "golang.org/x/tools/internal/semver" @@ -91,7 +93,7 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) { var rootDirs map[string]string var rootDirsReady = make(chan struct{}) go func() { - rootDirs = determineRootDirs(cfg) + rootDirs = determineRootDirsLRUCached(cfg) close(rootDirsReady) }() getRootDirs := func() map[string]string { @@ -101,7 +103,7 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) { // always pass getRootDirs to golistDriver golistDriver := func(cfg *Config, patterns ...string) (*driverResponse, error) { - return golistDriver(cfg, getRootDirs, patterns...) + return golistDriverLRUCached(cfg, getRootDirs, patterns...) } // Determine files requested in contains patterns @@ -597,6 +599,36 @@ func otherFiles(p *jsonPackage) [][]string { return [][]string{p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles, p.SwigCXXFiles, p.SysoFiles} } +type goListResult struct { + response *driverResponse + err error +} + +var ( + goListLRUCache *lru.Cache + createGoListLRUCache sync.Once + goListLRUEntries = 16 * 32 +) + +func golistDriverLRUCached(cfg *Config, rootsDirs func() map[string]string, words ...string) (*driverResponse, error) { + createGoListLRUCache.Do(func() { + goListLRUCache, _ = lru.New(goListLRUEntries) + }) + + h := fnv.New32a() + h.Write([]byte(cfg.Dir)) + h.Write([]byte(words[len(words)-1])) + hashKey := h.Sum32() + if val, ok := goListLRUCache.Get(hashKey); ok { + res := val.(goListResult) + return res.response, res.err + } else { + res, err := golistDriver(cfg, rootsDirs, words...) + goListLRUCache.Add(hashKey, goListResult{res, err}) + return res, err + } +} + // golistDriver uses the "go list" command to expand the pattern // words and return metadata for the specified packages. dir may be // "" and env may be nil, as per os/exec.Command. diff --git a/go/packages/golist_overlay.go b/go/packages/golist_overlay.go index b051327df66..45075d36492 100644 --- a/go/packages/golist_overlay.go +++ b/go/packages/golist_overlay.go @@ -10,6 +10,9 @@ import ( "path/filepath" "strconv" "strings" + "sync" + + "github.com/hashicorp/golang-lru" ) // processGolistOverlay provides rudimentary support for adding @@ -194,6 +197,30 @@ func hasTestFiles(p *Package) bool { return false } +type determineRootDirsResult struct { + roots map[string]string +} + +var ( + determineRootDirsLRUCache *lru.Cache + createRootLRUCache sync.Once + rootLRUEntries = 16 +) + +func determineRootDirsLRUCached(cfg *Config) map[string]string { + createRootLRUCache.Do(func() { + determineRootDirsLRUCache, _ = lru.New(rootLRUEntries) + }) + if val, ok := determineRootDirsLRUCache.Get(cfg.Dir); ok { + res := val.(determineRootDirsResult) + return res.roots + } else { + roots := determineRootDirs(cfg) + determineRootDirsLRUCache.Add(cfg.Dir, determineRootDirsResult{roots}) + return roots + } +} + // determineRootDirs returns a mapping from directories code can be contained in to the // corresponding import path prefixes of those directories. // Its result is used to try to determine the import path for a package containing