From 859fbf559dbc1eca7b525e3092270a6a8372254c Mon Sep 17 00:00:00 2001 From: romangurevitch Date: Thu, 16 Nov 2017 11:59:46 +0200 Subject: [PATCH 1/5] Registry support --- cmd/dep/integration_test.go | 3 +- cmd/dep/main.go | 10 + .../ensure/registry/basic/final/Gopkg.lock | 21 ++ .../ensure/registry/basic/final/Gopkg.toml | 8 + .../ensure/registry/basic/initial/main.go | 13 + .../ensure/registry/basic/testcase.json | 14 + .../init/registry/basic/final/Gopkg.lock | 21 ++ .../init/registry/basic/final/Gopkg.toml | 8 + .../init/registry/basic/initial/foo/bar.go | 13 + .../init/registry/basic/initial/main.go | 19 ++ .../init/registry/basic/testcase.json | 14 + context.go | 2 + gps/deduce.go | 137 ++++++++- gps/registry.go | 286 ++++++++++++++++++ gps/source_manager.go | 32 +- internal/test/integration/testcase.go | 1 + internal/test/integration/testproj.go | 5 +- internal/test/registry/registry.go | 231 ++++++++++++++ .../github.com%2Fsdboyer%2Fdeptest.tar.gz | Bin 0 -> 152 bytes .../github.com%2Fsdboyer%2Fdeptest.tar.gz | Bin 0 -> 213 bytes .../github.com%2Fsdboyer%2Fdeptest.tar.gz | Bin 0 -> 152 bytes .../github.com%2Fsdboyer%2Fdeptestdos.tar.gz | Bin 0 -> 253 bytes .../github.com%2Fsdboyer%2Fdeptesttres.tar.gz | Bin 0 -> 366 bytes internal/test/test.go | 11 + 24 files changed, 834 insertions(+), 15 deletions(-) create mode 100644 cmd/dep/testdata/harness_tests/ensure/registry/basic/final/Gopkg.lock create mode 100644 cmd/dep/testdata/harness_tests/ensure/registry/basic/final/Gopkg.toml create mode 100644 cmd/dep/testdata/harness_tests/ensure/registry/basic/initial/main.go create mode 100644 cmd/dep/testdata/harness_tests/ensure/registry/basic/testcase.json create mode 100644 cmd/dep/testdata/harness_tests/init/registry/basic/final/Gopkg.lock create mode 100644 cmd/dep/testdata/harness_tests/init/registry/basic/final/Gopkg.toml create mode 100644 cmd/dep/testdata/harness_tests/init/registry/basic/initial/foo/bar.go create mode 100644 cmd/dep/testdata/harness_tests/init/registry/basic/initial/main.go create mode 100644 cmd/dep/testdata/harness_tests/init/registry/basic/testcase.json create mode 100644 gps/registry.go create mode 100644 internal/test/registry/registry.go create mode 100644 internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptest/v0.8.0/github.com%2Fsdboyer%2Fdeptest.tar.gz create mode 100644 internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptest/v0.8.1/github.com%2Fsdboyer%2Fdeptest.tar.gz create mode 100644 internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptest/v1.0.0/github.com%2Fsdboyer%2Fdeptest.tar.gz create mode 100644 internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptestdos/v2.0.0/github.com%2Fsdboyer%2Fdeptestdos.tar.gz create mode 100644 internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptesttres/v1.0.0/github.com%2Fsdboyer%2Fdeptesttres.tar.gz diff --git a/cmd/dep/integration_test.go b/cmd/dep/integration_test.go index cc9f959cc9..96ccf02a08 100644 --- a/cmd/dep/integration_test.go +++ b/cmd/dep/integration_test.go @@ -23,6 +23,7 @@ func TestIntegration(t *testing.T) { test.NeedsExternalNetwork(t) test.NeedsGit(t) + test.SetupRegistry(t) wd, err := os.Getwd() if err != nil { @@ -94,7 +95,7 @@ func testIntegration(name, relPath, wd string, run integration.RunFunc) func(t * // Set up environment testCase := integration.NewTestCase(t, filepath.Join(wd, relPath), name) - testProj := integration.NewTestProject(t, testCase.InitialPath(), wd, run) + testProj := integration.NewTestProject(t, testCase.InitialPath(), wd, testCase.Env, run) defer testProj.Cleanup() // Create and checkout the vendor revisions diff --git a/cmd/dep/main.go b/cmd/dep/main.go index aa29e0a589..8e6acdca96 100644 --- a/cmd/dep/main.go +++ b/cmd/dep/main.go @@ -18,6 +18,8 @@ import ( "text/tabwriter" "github.com/golang/dep" + "github.com/golang/dep/gps" + "net/url" ) var ( @@ -188,6 +190,14 @@ func (c *Config) Run() int { DisableLocking: getEnv(c.Env, "DEPNOLOCK") != "", } + registryUrl, err := url.Parse(getEnv(c.Env, "DEPREGISTRYURL")) + if err != nil { + errLogger.Printf("%v\n", err) + return errorExitCode + } + token := getEnv(c.Env, "DEPREGISTRYTOKEN") + ctx.Registry = gps.NewRegistryConfig(registryUrl, token) + GOPATHS := filepath.SplitList(getEnv(c.Env, "GOPATH")) ctx.SetPaths(c.WorkingDir, GOPATHS...) diff --git a/cmd/dep/testdata/harness_tests/ensure/registry/basic/final/Gopkg.lock b/cmd/dep/testdata/harness_tests/ensure/registry/basic/final/Gopkg.lock new file mode 100644 index 0000000000..dccbe32a00 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/registry/basic/final/Gopkg.lock @@ -0,0 +1,21 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/sdboyer/deptest" + packages = ["."] + revision = "v1.0.0" + version = "v1.0.0" + +[[projects]] + name = "github.com/sdboyer/deptesttres" + packages = ["."] + revision = "v1.0.0" + version = "v1.0.0" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "55634e168d751f76d7e6385cbeca6d347beca79f1436812697438477d18b4b2c" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/cmd/dep/testdata/harness_tests/ensure/registry/basic/final/Gopkg.toml b/cmd/dep/testdata/harness_tests/ensure/registry/basic/final/Gopkg.toml new file mode 100644 index 0000000000..fb78927ac7 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/registry/basic/final/Gopkg.toml @@ -0,0 +1,8 @@ + +[[constraint]] + name = "github.com/sdboyer/deptesttres" + version = "1.0.0" + +[[constraint]] + name = "github.com/sdboyer/deptest" + version = "1.0.0" diff --git a/cmd/dep/testdata/harness_tests/ensure/registry/basic/initial/main.go b/cmd/dep/testdata/harness_tests/ensure/registry/basic/initial/main.go new file mode 100644 index 0000000000..8049e4ec2c --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/registry/basic/initial/main.go @@ -0,0 +1,13 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "github.com/sdboyer/deptesttres" +) + +func main() { + type a deptesttres.Bar +} diff --git a/cmd/dep/testdata/harness_tests/ensure/registry/basic/testcase.json b/cmd/dep/testdata/harness_tests/ensure/registry/basic/testcase.json new file mode 100644 index 0000000000..d86a9483e0 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/registry/basic/testcase.json @@ -0,0 +1,14 @@ +{ + "env": { + "DEPREGISTRYURL": "http://localhost:9090", + "DEPREGISTRYTOKEN": "2erygdasE45rty5JKwewrr75cb15rdeE" + }, + "commands": [ + ["init", "-no-examples"], + ["ensure", "-add", "github.com/sdboyer/deptest"] + ], + "vendor-final": [ + "github.com/sdboyer/deptest", + "github.com/sdboyer/deptesttres" + ] +} diff --git a/cmd/dep/testdata/harness_tests/init/registry/basic/final/Gopkg.lock b/cmd/dep/testdata/harness_tests/init/registry/basic/final/Gopkg.lock new file mode 100644 index 0000000000..95850ea86a --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/registry/basic/final/Gopkg.lock @@ -0,0 +1,21 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/sdboyer/deptest" + packages = ["."] + revision = "v1.0.0" + version = "v1.0.0" + +[[projects]] + name = "github.com/sdboyer/deptestdos" + packages = ["."] + revision = "v2.0.0" + version = "v2.0.0" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "a6ba2237d28d125b55fc6c86e94e33363f1dfd880d471118d36d7587398c30b4" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/cmd/dep/testdata/harness_tests/init/registry/basic/final/Gopkg.toml b/cmd/dep/testdata/harness_tests/init/registry/basic/final/Gopkg.toml new file mode 100644 index 0000000000..d57fc4dad0 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/registry/basic/final/Gopkg.toml @@ -0,0 +1,8 @@ + +[[constraint]] + name = "github.com/sdboyer/deptest" + version = "1.0.0" + +[[constraint]] + name = "github.com/sdboyer/deptestdos" + version = "2.0.0" diff --git a/cmd/dep/testdata/harness_tests/init/registry/basic/initial/foo/bar.go b/cmd/dep/testdata/harness_tests/init/registry/basic/initial/foo/bar.go new file mode 100644 index 0000000000..c1ed69fc65 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/registry/basic/initial/foo/bar.go @@ -0,0 +1,13 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo + +import "github.com/sdboyer/deptest" + +func Foo() deptest.Foo { + var y deptest.Foo + + return y +} diff --git a/cmd/dep/testdata/harness_tests/init/registry/basic/initial/main.go b/cmd/dep/testdata/harness_tests/init/registry/basic/initial/main.go new file mode 100644 index 0000000000..83a4dfcd57 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/registry/basic/initial/main.go @@ -0,0 +1,19 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + + "github.com/golang/notexist/foo" + "github.com/sdboyer/deptestdos" +) + +func main() { + var x deptestdos.Bar + y := foo.FooFunc() + + fmt.Println(x, y) +} diff --git a/cmd/dep/testdata/harness_tests/init/registry/basic/testcase.json b/cmd/dep/testdata/harness_tests/init/registry/basic/testcase.json new file mode 100644 index 0000000000..55f40192e6 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/registry/basic/testcase.json @@ -0,0 +1,14 @@ +{ + "env": { + "DEPREGISTRYURL": "http://localhost:9090", + "DEPREGISTRYTOKEN": "2erygdasE45rty5JKwewrr75cb15rdeE" + }, + "commands": [ + ["init", "-no-examples"] + ], + "error-expected": "", + "vendor-final": [ + "github.com/sdboyer/deptest", + "github.com/sdboyer/deptestdos" + ] +} diff --git a/context.go b/context.go index bf2502ddb2..6be4abb85b 100644 --- a/context.go +++ b/context.go @@ -40,6 +40,7 @@ type Ctx struct { Out, Err *log.Logger // Required loggers. Verbose bool // Enables more verbose logging. DisableLocking bool // When set, no lock file will be created to protect against simultaneous dep processes. + Registry gps.Registry } // SetPaths sets the WorkingDir and GOPATHs fields. If GOPATHs is empty, then @@ -91,6 +92,7 @@ func (c *Ctx) SourceManager() (*gps.SourceMgr, error) { Cachedir: filepath.Join(c.GOPATH, "pkg", "dep"), Logger: c.Out, DisableLocking: c.DisableLocking, + Registry: c.Registry, }) } diff --git a/gps/deduce.go b/gps/deduce.go index b40bfb4f09..ca4094db45 100644 --- a/gps/deduce.go +++ b/gps/deduce.go @@ -16,8 +16,10 @@ import ( "strings" "sync" - radix "github.com/armon/go-radix" + "encoding/json" + "github.com/armon/go-radix" "github.com/pkg/errors" + "io/ioutil" ) var ( @@ -561,7 +563,7 @@ type deductionCoordinator struct { deducext *deducerTrie } -func newDeductionCoordinator(superv *supervisor) *deductionCoordinator { +func newDeductionCoordinator(superv *supervisor) deducer { dc := &deductionCoordinator{ suprvsr: superv, rootxt: radix.New(), @@ -571,6 +573,71 @@ func newDeductionCoordinator(superv *supervisor) *deductionCoordinator { return dc } +func newRegistryDeductionCoordinator(superv *supervisor, registry Registry) deducer { + dc := ®istryDeductionCoordinator{ + suprvsr: superv, + rootxt: radix.New(), + deducext: pathDeducerTrie(), + registry: registry, + } + + return dc +} + +type registryDeductionCoordinator struct { + suprvsr *supervisor + mut sync.RWMutex + rootxt *radix.Tree + deducext *deducerTrie + registry Registry +} + +func (dc *registryDeductionCoordinator) deduceRootPath(ctx context.Context, path string) (pathDeduction, error) { + if err := dc.suprvsr.ctx.Err(); err != nil { + return pathDeduction{}, err + } + // First, check the rootxt to see if there's a prefix match - if so, we + // can return that and move on. + dc.mut.RLock() + prefix, data, has := dc.rootxt.LongestPrefix(path) + dc.mut.RUnlock() + if has && isPathPrefixOrEqual(prefix, path) { + switch d := data.(type) { + case maybeSource: + return pathDeduction{root: prefix, mb: d}, nil + case *registryDeducer: + return d.deduce(ctx, path) + } + panic(fmt.Sprintf("unexpected %T in deductionCoordinator.rootxt: %v", data, data)) + } + + // The err indicates no known path matched. It's still possible that + // retrieving go get metadata might do the trick. + rmd := ®istryDeducer{ + basePath: path, + registry: dc.registry, + suprvsr: dc.suprvsr, + // The vanity deducer will call this func with a completed + // pathDeduction if it succeeds in finding one. We process it + // back through the action channel to ensure serialized + // access to the rootxt map. + returnFunc: func(pd pathDeduction) { + dc.mut.Lock() + dc.rootxt.Insert(pd.root, pd.mb) + dc.mut.Unlock() + }, + } + + // Save the hmd in the rootxt so that calls checking on similar + // paths made while the request is in flight can be folded together. + dc.mut.Lock() + dc.rootxt.Insert(path, rmd) + dc.mut.Unlock() + + // Trigger the HTTP-backed deduction process for this requestor. + return rmd.deduce(ctx, path) +} + // deduceRootPath takes an import path and attempts to deduce various // metadata about it - what type of source should handle it, and where its // "root" is (for vcs repositories, the repository root). @@ -697,6 +764,72 @@ func (dc *deductionCoordinator) deduceKnownPaths(path string) (pathDeduction, er return pathDeduction{}, errNoKnownPathMatch } +type registryDeducer struct { + once sync.Once + deduced pathDeduction + deduceErr error + basePath string + registry Registry + returnFunc func(pathDeduction) + suprvsr *supervisor +} + +func (rd *registryDeducer) getProjectName(importPath string) (string, error) { + u, err := url.Parse(rd.registry.URL()) + if err != nil { + return "", err + } + u.Path = path.Join(u.Path, "api/v1/projects/root", url.PathEscape(importPath)) + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return "", err + } + + req.Header.Set("Authorization", "BEARER "+rd.registry.Token()) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return "", errNotFound + } + return "", errors.Errorf("%s %s", u, http.StatusText(resp.StatusCode)) + } + + var projectName struct { + ProjectName string `json:"project_name"` + } + var bytes []byte + bytes, err = ioutil.ReadAll(resp.Body) + err = json.Unmarshal(bytes, &projectName) + return projectName.ProjectName, nil +} + +func (rd *registryDeducer) deduce(ctx context.Context, path string) (pathDeduction, error) { + rd.once.Do(func() { + projectName, err := rd.getProjectName(path) + if err != nil { + rd.deduceErr = err + return + } + var u *url.URL + u, err = url.Parse(rd.registry.URL()) + if err != nil { + return + } + + rd.deduced = pathDeduction{mb: maybeRegistrySource{path: projectName, url: u, token: rd.registry.Token()}, root: projectName} + rd.returnFunc(rd.deduced) + return + + if rd.deduced.mb == nil { + rd.deduceErr = errors.Errorf("unable to find root path for: %s", path) + } + }) + return rd.deduced, rd.deduceErr +} + type httpMetadataDeducer struct { once sync.Once deduced pathDeduction diff --git a/gps/registry.go b/gps/registry.go new file mode 100644 index 0000000000..05c27d4d6c --- /dev/null +++ b/gps/registry.go @@ -0,0 +1,286 @@ +package gps + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "github.com/golang/dep/gps/pkgtree" + "github.com/golang/dep/internal/fs" + "github.com/pkg/errors" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path" + "path/filepath" +) + +var errNotFound = errors.New(http.StatusText(http.StatusNotFound)) + +// Registry configuration interface +// Set env vars: DEPREGISTRYURL, DEPREGISTRYTOKEN +type Registry interface { + URL() string + Token() string +} + +func NewRegistryConfig(url *url.URL, token string) Registry { + return &RegitryConfig{url: url.String(), token: token} +} + +type RegitryConfig struct { + url string + token string +} + +func (s *RegitryConfig) URL() string { + return s.url +} + +func (s *RegitryConfig) Token() string { + return s.token +} + +type registrySource struct { + path string + url string + token string + sourceCachePath string +} + +func (s *registrySource) URL() string { + return s.url +} + +func (s *registrySource) Token() string { + return s.token +} + +func (s *registrySource) existsLocally(ctx context.Context) bool { + return false +} + +func (s *registrySource) existsUpstream(ctx context.Context) bool { + return true +} + +func (s *registrySource) upstreamURL() string { + return path.Join(s.url + s.path) +} + +func (s *registrySource) initLocal(ctx context.Context) error { + return nil +} + +func (s *registrySource) updateLocal(ctx context.Context) error { + return nil +} + +// Get version from registry +func (s *registrySource) execGetVersions() (*rawVersions, error) { + u, err := url.Parse(s.url) + if err != nil { + return nil, err + } + u.Path = path.Join(u.Path, "api/v1/versions", url.PathEscape(s.path)) + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", "BEARER "+s.token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, errNotFound + } + return nil, errors.Errorf("%s %s", u.String(), http.StatusText(resp.StatusCode)) + } + + var bytes []byte + bytes, err = ioutil.ReadAll(resp.Body) + var versionsResp rawVersions + err = json.Unmarshal(bytes, &versionsResp) + return &versionsResp, err +} + +func (s *registrySource) execDownloadDependency(ctx context.Context, pr ProjectRoot, r Revision) (*http.Response, error) { + u, err := url.Parse(s.url) + if err != nil { + return nil, err + } + u.Path = path.Join(u.Path, "api/v1/projects", url.PathEscape(s.path), r.String()) + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", "BEARER "+s.token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, errNotFound + } + return nil, errors.Errorf("%s %s", u, http.StatusText(resp.StatusCode)) + } + return resp, nil +} + +type rawVersions struct { + Versions map[string]rawPublished `json:"versions"` +} +type rawPublished struct { + Published string `json:"published"` +} + +func (s *registrySource) listVersions(ctx context.Context) (vlist []PairedVersion, err error) { + vers := []PairedVersion{} + rawResp, err := s.execGetVersions() + if err != nil { + return vers, err + } + + for k := range rawResp.Versions { + vers = append(vers, NewVersion(k).Pair(Revision(k))) + } + return vers, nil +} + +func (s *registrySource) getManifestAndLock(ctx context.Context, pr ProjectRoot, r Revision, an ProjectAnalyzer) (Manifest, Lock, error) { + m, l, err := an.DeriveManifestAndLock(s.sourceCachePath, pr) + if err != nil { + return nil, nil, err + } + + if l != nil && l != Lock(nil) { + l = prepLock(l) + } + + return prepManifest(m), l, nil +} + +func (s *registrySource) listPackages(ctx context.Context, pr ProjectRoot, r Revision) (ptree pkgtree.PackageTree, err error) { + resp, err := s.execDownloadDependency(ctx, pr, r) + if err != nil { + return pkgtree.PackageTree{}, err + } + defer resp.Body.Close() + + h := sha256.New() + tee := io.TeeReader(resp.Body, h) + + err = extractDependency(tee, s.sourceCachePath) + if err != nil { + return pkgtree.PackageTree{}, err + } + + if hex.EncodeToString(h.Sum(nil)) != resp.Header.Get("X-Checksum-Sha256") { + return pkgtree.PackageTree{}, errors.Errorf("sha256 checksum validation failed for %s %s", s.path, r) + } + + return pkgtree.ListPackages(s.sourceCachePath, string(pr)) +} + +func extractDependency(r io.Reader, target string) error { + gzr, err := gzip.NewReader(r) + defer gzr.Close() + if err != nil { + return err + } + + // Remove other versions of the same dependency. + if err = os.RemoveAll(target); err != nil { + return err + } + if err = os.MkdirAll(target, 0755); err != nil { + return err + } + + tr := tar.NewReader(gzr) + for { + header, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + path := filepath.Join(target, header.Name) + info := header.FileInfo() + if info.IsDir() { + if err = os.MkdirAll(path, info.Mode()); err != nil { + return err + } + continue + } + + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(file, tr) + if err != nil { + return err + } + } + return nil +} + +func (s *registrySource) revisionPresentIn(r Revision) (bool, error) { + return false, nil +} + +func (s *registrySource) exportRevisionTo(ctx context.Context, r Revision, to string) error { + if err := os.MkdirAll(s.sourceCachePath, 0755); err != nil { + return err + } + return fs.CopyDir(s.sourceCachePath, to) +} + +func (s *registrySource) sourceType() string { + return "registry" +} + +type maybeRegistrySource struct { + path string + url *url.URL + token string +} + +func (m maybeRegistrySource) possibleURLs() []*url.URL { + return []*url.URL{m.url} +} + +func (s *registrySource) disambiguateRevision(ctx context.Context, r Revision) (Revision, error) { + return r, nil +} + +func (m maybeRegistrySource) try(ctx context.Context, cachedir string, c singleSourceCache, superv *supervisor) (source, sourceState, error) { + registry, err := NewRegistrySource(m.url.String(), m.token, m.path, cachedir) + if err != nil { + return nil, 0, err + } + return registry, sourceIsSetUp | sourceExistsUpstream, nil +} + +// NewRegistrySource creates new registry source +func NewRegistrySource(rURL, token, rPath, cachedir string) (source, error) { + return ®istrySource{ + path: rPath, + url: rURL, + token: token, + sourceCachePath: sourceCachePath(cachedir, rURL), + }, nil +} diff --git a/gps/source_manager.go b/gps/source_manager.go index bb12d492ad..d8eaf01712 100644 --- a/gps/source_manager.go +++ b/gps/source_manager.go @@ -155,16 +155,16 @@ func (p ProjectAnalyzerInfo) String() string { // There's no (planned) reason why it would need to be reimplemented by other // tools; control via dependency injection is intended to be sufficient. type SourceMgr struct { - cachedir string // path to root of cache dir - lf locker // handle for the sm lock file on disk - suprvsr *supervisor // subsystem that supervises running calls/io - cancelAll context.CancelFunc // cancel func to kill all running work - deduceCoord *deductionCoordinator // subsystem that manages import path deduction - srcCoord *sourceCoordinator // subsystem that manages sources - sigmut sync.Mutex // mutex protecting signal handling setup/teardown - qch chan struct{} // quit chan for signal handler - relonce sync.Once // once-er to ensure we only release once - releasing int32 // flag indicating release of sm has begun + cachedir string // path to root of cache dir + lf locker // handle for the sm lock file on disk + suprvsr *supervisor // subsystem that supervises running calls/io + cancelAll context.CancelFunc // cancel func to kill all running work + deduceCoord deducer // subsystem that manages import path deduction + srcCoord *sourceCoordinator // subsystem that manages sources + sigmut sync.Mutex // mutex protecting signal handling setup/teardown + qch chan struct{} // quit chan for signal handler + relonce sync.Once // once-er to ensure we only release once + releasing int32 // flag indicating release of sm has begun } var _ SourceManager = &SourceMgr{} @@ -179,6 +179,11 @@ type SourceManagerConfig struct { Cachedir string // Where to store local instances of upstream sources. Logger *log.Logger // Optional info/warn logger. Discards if nil. DisableLocking bool // True if the SourceManager should NOT use a lock file to protect the Cachedir from multiple processes. + Registry Registry +} + +func (smc *SourceManagerConfig) usingRegistry() bool { + return smc.Registry != nil && smc.Registry.URL() != "" && smc.Registry.Token() != "" } // NewSourceManager produces an instance of gps's built-in SourceManager. @@ -276,7 +281,12 @@ func NewSourceManager(c SourceManagerConfig) (*SourceMgr, error) { ctx, cf := context.WithCancel(context.TODO()) superv := newSupervisor(ctx) - deducer := newDeductionCoordinator(superv) + var deducer deducer + if c.usingRegistry() { + deducer = newRegistryDeductionCoordinator(superv, c.Registry) + } else { + deducer = newDeductionCoordinator(superv) + } sm := &SourceMgr{ cachedir: c.Cachedir, diff --git a/internal/test/integration/testcase.go b/internal/test/integration/testcase.go index bd772da7d8..06929c76a3 100644 --- a/internal/test/integration/testcase.go +++ b/internal/test/integration/testcase.go @@ -23,6 +23,7 @@ type TestCase struct { rootPath string initialPath string finalPath string + Env map[string]string `json:"env"` Commands [][]string `json:"commands"` ErrorExpected string `json:"error-expected"` GopathInitial map[string]string `json:"gopath-initial"` diff --git a/internal/test/integration/testproj.go b/internal/test/integration/testproj.go index 760e1cc52d..b22410c7aa 100644 --- a/internal/test/integration/testproj.go +++ b/internal/test/integration/testproj.go @@ -41,7 +41,7 @@ type TestProject struct { } // NewTestProject initializes a new test's project directory. -func NewTestProject(t *testing.T, initPath, wd string, run RunFunc) *TestProject { +func NewTestProject(t *testing.T, initPath, wd string, env map[string]string, run RunFunc) *TestProject { new := &TestProject{ t: t, origWd: wd, @@ -53,6 +53,9 @@ func NewTestProject(t *testing.T, initPath, wd string, run RunFunc) *TestProject new.CopyTree(initPath) new.Setenv("GOPATH", new.tempdir) + for k, v := range env { + new.Setenv(k, v) + } return new } diff --git a/internal/test/registry/registry.go b/internal/test/registry/registry.go new file mode 100644 index 0000000000..b21ef8fb97 --- /dev/null +++ b/internal/test/registry/registry.go @@ -0,0 +1,231 @@ +package registry + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strings" +) + +const TOKEN_AUTH = "2erygdasE45rty5JKwewrr75cb15rdeE" + +var tokenResp = "{\"token\": \"" + TOKEN_AUTH + "\"}" + +type rawVersions struct { + Versions map[string]rawPublished `json:"versions"` +} +type rawPublished struct { + Published string `json:"published"` +} + +type rawProjectName struct { + ProjectName string `json:"project_name"` +} + +func verifyToken(r *http.Request) error { + tokens, ok := r.Header["Authorization"] + if ok && len(tokens) >= 1 { + token := tokens[0] + token = strings.TrimPrefix(token, "BEARER ") + if token == TOKEN_AUTH { + return nil + } + } + + return errors.New(http.StatusText(http.StatusUnauthorized)) +} + +func getSourcesPath() (string, error) { + wd, err := os.Getwd() + if err != nil { + return "", err + } + return filepath.Join(wd, "..", "..", "internal", "test", "registry", "sources"), nil +} + +func getVersions(dependency string) rawVersions { + versions := rawVersions{Versions: map[string]rawPublished{}} + sourcesPath, err := getSourcesPath() + if err != nil { + return versions + } + + files, err := ioutil.ReadDir(filepath.Join(sourcesPath, dependency)) + if err != nil { + return versions + } + for _, v := range files { + if v.IsDir() { + versions.Versions[v.Name()] = rawPublished{Published: v.ModTime().String()} + } + } + return versions +} + +func token(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.NotFound(w, r) + return + } + + fmt.Fprint(w, tokenResp) // send data to client side +} + +func versions(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.NotFound(w, r) + return + } + + if err := verifyToken(r); err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + depName := strings.TrimPrefix(r.URL.Path, "/api/v1/versions/") + versions := getVersions(depName) + if len(versions.Versions) == 0 { + http.NotFound(w, r) + return + } + requestContent, err := json.Marshal(versions) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(requestContent) +} + +func getDependency(w http.ResponseWriter, r *http.Request) { + if err := verifyToken(r); err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + dep := strings.TrimPrefix(r.URL.Path, "/api/v1/projects/") + nameVer := strings.Split(dep, "/") + if len(nameVer) != 2 { + http.NotFound(w, r) + return + } + + sourcesPath, err := getSourcesPath() + if err != nil { + http.NotFound(w, r) + return + } + + f, err := os.OpenFile(filepath.Join(sourcesPath, dep, nameVer[0]+".tar.gz"), os.O_RDONLY, 0666) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer f.Close() + b, err := ioutil.ReadAll(f) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + h := sha256.New() + h.Write(b) + + w.Header().Add("X-Go-Project-Name", strings.Replace(nameVer[0], "%2F", "/", -1)) + w.Header().Add("X-Go-Project-Version", nameVer[1]) + w.Header().Add("X-Checksum-Sha256", hex.EncodeToString(h.Sum(nil))) + w.Write(b) +} + +func putDependency(w http.ResponseWriter, r *http.Request) { + if err := verifyToken(r); err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + content, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + // validate content with sha256 header + h := sha256.New() + h.Write(content) + + if hex.EncodeToString(h.Sum(nil)) != r.Header.Get("X-Checksum-Sha256") { + http.Error(w, "sha256 checksum validation failed", http.StatusForbidden) + } + // Do nothing with the received dependency + return +} + +func dependency(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + getDependency(w, r) + case http.MethodPut: + putDependency(w, r) + default: + http.NotFound(w, r) + } +} + +// Simple impl to get project name from import path +func projectName(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.NotFound(w, r) + return + } + + importPath := strings.TrimPrefix(r.URL.Path, "/api/v1/projects/root/") + sourcePath, err := getSourcesPath() + if err != nil { + http.NotFound(w, r) + return + } + + files, err := ioutil.ReadDir(sourcePath) + if err != nil { + log.Fatal(err) + } + + for _, v := range files { + if isRootProject(v.Name(), importPath) { + projectName := &rawProjectName{ProjectName: strings.Replace(v.Name(), "%2F", "/", -1)} + requestContent, err := json.Marshal(projectName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Write(requestContent) + return + } + } + + http.Error(w, "could not find project name", http.StatusInternalServerError) + return +} + +func isRootProject(projectName, importPath string) bool { + suffix := strings.TrimPrefix(projectName, importPath) + return suffix == "" || strings.HasPrefix(suffix, "%2F") +} + +func notFound(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) +} + +func SetupAndRun(addr string) error { + http.HandleFunc("/api/v1/auth/token", token) + http.HandleFunc("/api/v1/versions/", versions) + http.HandleFunc("/api/v1/projects/root/", projectName) + http.HandleFunc("/api/v1/projects/", dependency) + http.HandleFunc("/", notFound) + return http.ListenAndServe(addr, nil) +} diff --git a/internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptest/v0.8.0/github.com%2Fsdboyer%2Fdeptest.tar.gz b/internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptest/v0.8.0/github.com%2Fsdboyer%2Fdeptest.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..55bc32882ddb0a7b57458148960af8d39aa205eb GIT binary patch literal 152 zcmb2|=3sa}b6q3@^V>^~T!#zG*s~6 z|AdO}nMb3~2zd7v+b+o|Q|5l(zdq-epI!0NdnQ_Qw=KG~B~9|}Rg0VJFWs(PmsoVp z=cm}QEw$fs|1bBlhuj4QNJ)~uA#dYIEIQ@KLa zd@D^($LxENqE83k#_#7I_5Z2P?>>DDQjhPTEc2xc@|u$D+g$$kc9`4S=J{p2Ssv;c z&v=-BYwz-}ovq0KPgpFb=)dpJ`P2u=Nj^f;NleXf<)@)9zDaH5I{*Lx0000000000 P0Qh^ZY8|nb04M+e7N=^_ literal 0 HcmV?d00001 diff --git a/internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptest/v1.0.0/github.com%2Fsdboyer%2Fdeptest.tar.gz b/internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptest/v1.0.0/github.com%2Fsdboyer%2Fdeptest.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..fd0ba90b3842196d389f1af774462fe7f792b701 GIT binary patch literal 152 zcmb2|=3vm6wJwr@`Rye~u0sX_tq<1}cg=BFCOiAyfoc}%rVihX$A@l5zjQJ`8Y=kk ze?mp~%%jm~1iX8TZI|SfDRaN?U!U{K&#rjsJrk|D+ZJ8gk|z1~s>RLqmu}atODsC) z^Hc2DmfG*R|Cf7N#5;H8Mf^W->zBsGxkY8uH{Och&dh)eaw?hr6ufF>&|qKy0J?oe AwEzGB literal 0 HcmV?d00001 diff --git a/internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptestdos/v2.0.0/github.com%2Fsdboyer%2Fdeptestdos.tar.gz b/internal/test/registry/sources/github.com%2Fsdboyer%2Fdeptestdos/v2.0.0/github.com%2Fsdboyer%2Fdeptestdos.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f32c6f7e6d5f905ee13d7af7d4b5ce4f75d0208a GIT binary patch literal 253 zcmV7LCRqWm))%@cg)UTC$nucsGn`Xqn*axm+De|2-W&pNx{9&`Cu<=6b{s>+f7pRitE z;w$At^OVNuC^6Np4dl6NVUX^=(VU)Oj3j|%JjOO;B!kF#i`nQ(sooH;h~pTcSWX%J^-qz1l6Z~m^= z$=`8oZcuCBaeV2Y=bs1ReGp07Bu_}3=Bm(ip~?jzyDAHh$5(6mL@rcm`iEGr*qS_( z-ySv$WGizmOFewif6`n3-0jrA?O3S)NwoFIM#ntIOgAMUi*Mi`oRAFk9XI9^_mdh|0%TnUo!>Xo4-die*gdg000000049F M0&iUf1OO-i06$T+J^%m! literal 0 HcmV?d00001 diff --git a/internal/test/test.go b/internal/test/test.go index 9a7fbc8d98..a260b515e3 100644 --- a/internal/test/test.go +++ b/internal/test/test.go @@ -20,6 +20,7 @@ import ( "sync" "testing" + "github.com/golang/dep/internal/test/registry" "github.com/pkg/errors" ) @@ -247,6 +248,16 @@ func NeedsExternalNetwork(t *testing.T) { } } +// SetupRegistry sets and runs mock registry for the tests. +func SetupRegistry(t *testing.T) { + go func(t *testing.T) { + if err := registry.SetupAndRun(":9090"); err != nil { + t.Fatal(err) + t.Skip(err) + } + }(t) +} + // NeedsGit will make sure the tests that require git will be skipped if the // git binary is not available. func NeedsGit(t *testing.T) { From fd875f44ec29b71da1258ff843917f9908aef3c6 Mon Sep 17 00:00:00 2001 From: romangurevitch Date: Thu, 16 Nov 2017 18:03:54 +0200 Subject: [PATCH 2/5] Registry support --- cmd/dep/main.go | 17 +++++++---- gps/deduce.go | 28 +++++-------------- gps/registry.go | 8 ------ .../{registry.go => registry_mock.go} | 15 ++++------ 4 files changed, 23 insertions(+), 45 deletions(-) rename internal/test/registry/{registry.go => registry_mock.go} (91%) diff --git a/cmd/dep/main.go b/cmd/dep/main.go index 8e6acdca96..8f0e7f90f2 100644 --- a/cmd/dep/main.go +++ b/cmd/dep/main.go @@ -190,13 +190,18 @@ func (c *Config) Run() int { DisableLocking: getEnv(c.Env, "DEPNOLOCK") != "", } - registryUrl, err := url.Parse(getEnv(c.Env, "DEPREGISTRYURL")) - if err != nil { - errLogger.Printf("%v\n", err) - return errorExitCode - } + registryUrl := getEnv(c.Env, "DEPREGISTRYURL") token := getEnv(c.Env, "DEPREGISTRYTOKEN") - ctx.Registry = gps.NewRegistryConfig(registryUrl, token) + + // check if url is not define do not create registry + if registryUrl != "" && token != "" { + registryUrl, err := url.Parse(registryUrl) + if err != nil { + errLogger.Printf("%v\n", err) + return errorExitCode + } + ctx.Registry = gps.NewRegistryConfig(registryUrl, token) + } GOPATHS := filepath.SplitList(getEnv(c.Env, "GOPATH")) ctx.SetPaths(c.WorkingDir, GOPATHS...) diff --git a/gps/deduce.go b/gps/deduce.go index ca4094db45..9177ce162d 100644 --- a/gps/deduce.go +++ b/gps/deduce.go @@ -16,10 +16,8 @@ import ( "strings" "sync" - "encoding/json" "github.com/armon/go-radix" "github.com/pkg/errors" - "io/ioutil" ) var ( @@ -580,7 +578,6 @@ func newRegistryDeductionCoordinator(superv *supervisor, registry Registry) dedu deducext: pathDeducerTrie(), registry: registry, } - return dc } @@ -611,13 +608,11 @@ func (dc *registryDeductionCoordinator) deduceRootPath(ctx context.Context, path panic(fmt.Sprintf("unexpected %T in deductionCoordinator.rootxt: %v", data, data)) } - // The err indicates no known path matched. It's still possible that - // retrieving go get metadata might do the trick. - rmd := ®istryDeducer{ + rd := ®istryDeducer{ basePath: path, registry: dc.registry, suprvsr: dc.suprvsr, - // The vanity deducer will call this func with a completed + // The registry deducer will call this func with a completed // pathDeduction if it succeeds in finding one. We process it // back through the action channel to ensure serialized // access to the rootxt map. @@ -628,14 +623,14 @@ func (dc *registryDeductionCoordinator) deduceRootPath(ctx context.Context, path }, } - // Save the hmd in the rootxt so that calls checking on similar + // Save the rd in the rootxt so that calls checking on similar // paths made while the request is in flight can be folded together. dc.mut.Lock() - dc.rootxt.Insert(path, rmd) + dc.rootxt.Insert(path, rd) dc.mut.Unlock() // Trigger the HTTP-backed deduction process for this requestor. - return rmd.deduce(ctx, path) + return rd.deduce(ctx, path) } // deduceRootPath takes an import path and attempts to deduce various @@ -780,7 +775,7 @@ func (rd *registryDeducer) getProjectName(importPath string) (string, error) { return "", err } u.Path = path.Join(u.Path, "api/v1/projects/root", url.PathEscape(importPath)) - req, err := http.NewRequest("GET", u.String(), nil) + req, err := http.NewRequest("HEAD", u.String(), nil) if err != nil { return "", err } @@ -791,19 +786,10 @@ func (rd *registryDeducer) getProjectName(importPath string) (string, error) { return "", err } if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return "", errNotFound - } return "", errors.Errorf("%s %s", u, http.StatusText(resp.StatusCode)) } - var projectName struct { - ProjectName string `json:"project_name"` - } - var bytes []byte - bytes, err = ioutil.ReadAll(resp.Body) - err = json.Unmarshal(bytes, &projectName) - return projectName.ProjectName, nil + return resp.Header.Get("X-Go-Project-Name"), nil } func (rd *registryDeducer) deduce(ctx context.Context, path string) (pathDeduction, error) { diff --git a/gps/registry.go b/gps/registry.go index 05c27d4d6c..e237f9d742 100644 --- a/gps/registry.go +++ b/gps/registry.go @@ -19,8 +19,6 @@ import ( "path/filepath" ) -var errNotFound = errors.New(http.StatusText(http.StatusNotFound)) - // Registry configuration interface // Set env vars: DEPREGISTRYURL, DEPREGISTRYTOKEN type Registry interface { @@ -99,9 +97,6 @@ func (s *registrySource) execGetVersions() (*rawVersions, error) { } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return nil, errNotFound - } return nil, errors.Errorf("%s %s", u.String(), http.StatusText(resp.StatusCode)) } @@ -129,9 +124,6 @@ func (s *registrySource) execDownloadDependency(ctx context.Context, pr ProjectR return nil, err } if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return nil, errNotFound - } return nil, errors.Errorf("%s %s", u, http.StatusText(resp.StatusCode)) } return resp, nil diff --git a/internal/test/registry/registry.go b/internal/test/registry/registry_mock.go similarity index 91% rename from internal/test/registry/registry.go rename to internal/test/registry/registry_mock.go index b21ef8fb97..d39019aa10 100644 --- a/internal/test/registry/registry.go +++ b/internal/test/registry/registry_mock.go @@ -171,6 +171,8 @@ func dependency(w http.ResponseWriter, r *http.Request) { getDependency(w, r) case http.MethodPut: putDependency(w, r) + case http.MethodHead: + projectName(w, r) default: http.NotFound(w, r) } @@ -178,7 +180,7 @@ func dependency(w http.ResponseWriter, r *http.Request) { // Simple impl to get project name from import path func projectName(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { + if r.Method != http.MethodHead { http.NotFound(w, r) return } @@ -197,18 +199,12 @@ func projectName(w http.ResponseWriter, r *http.Request) { for _, v := range files { if isRootProject(v.Name(), importPath) { - projectName := &rawProjectName{ProjectName: strings.Replace(v.Name(), "%2F", "/", -1)} - requestContent, err := json.Marshal(projectName) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.Write(requestContent) + w.Header().Set("X-Go-Project-Name", strings.Replace(v.Name(), "%2F", "/", -1)) return } } - http.Error(w, "could not find project name", http.StatusInternalServerError) + http.Error(w, "could not find project name", http.StatusNotFound) return } @@ -224,7 +220,6 @@ func notFound(w http.ResponseWriter, r *http.Request) { func SetupAndRun(addr string) error { http.HandleFunc("/api/v1/auth/token", token) http.HandleFunc("/api/v1/versions/", versions) - http.HandleFunc("/api/v1/projects/root/", projectName) http.HandleFunc("/api/v1/projects/", dependency) http.HandleFunc("/", notFound) return http.ListenAndServe(addr, nil) From 779a8f7092c5ba2e8b89abd81ffa2aec37a329a4 Mon Sep 17 00:00:00 2001 From: romangurevitch Date: Tue, 12 Dec 2017 16:32:09 +0200 Subject: [PATCH 3/5] Registry support - REST API modifications --- cmd/dep/integration_test.go | 4 +- cmd/dep/main.go | 6 +-- gps/deduce.go | 6 +-- gps/registry.go | 14 ++++-- internal/test/registry/registry_mock.go | 61 ++++++++++++++----------- 5 files changed, 49 insertions(+), 42 deletions(-) diff --git a/cmd/dep/integration_test.go b/cmd/dep/integration_test.go index 1eb37d1d88..bb3cfc5735 100644 --- a/cmd/dep/integration_test.go +++ b/cmd/dep/integration_test.go @@ -77,7 +77,7 @@ func TestDepCachedir(t *testing.T) { t.Run("env-cachedir", func(t *testing.T) { t.Parallel() - testProj := integration.NewTestProject(t, initPath, wd, runMain) + testProj := integration.NewTestProject(t, initPath, wd, nil, runMain) defer testProj.Cleanup() testProj.TempDir("cachedir") @@ -105,7 +105,7 @@ func TestDepCachedir(t *testing.T) { }) t.Run("env-invalid-cachedir", func(t *testing.T) { t.Parallel() - testProj := integration.NewTestProject(t, initPath, wd, runMain) + testProj := integration.NewTestProject(t, initPath, wd, nil, runMain) defer testProj.Cleanup() var d []byte diff --git a/cmd/dep/main.go b/cmd/dep/main.go index 7fc15d49a4..93aff208cf 100644 --- a/cmd/dep/main.go +++ b/cmd/dep/main.go @@ -205,12 +205,12 @@ func (c *Config) Run() int { Cachedir: cachedir, } - registryUrl := getEnv(c.Env, "DEPREGISTRYURL") + registryURL := getEnv(c.Env, "DEPREGISTRYURL") token := getEnv(c.Env, "DEPREGISTRYTOKEN") // check if url is not define do not create registry - if registryUrl != "" && token != "" { - registryUrl, err := url.Parse(registryUrl) + if registryURL != "" && token != "" { + registryUrl, err := url.Parse(registryURL) if err != nil { errLogger.Printf("%v\n", err) return errorExitCode diff --git a/gps/deduce.go b/gps/deduce.go index 9177ce162d..2aa6d7ea3e 100644 --- a/gps/deduce.go +++ b/gps/deduce.go @@ -774,7 +774,7 @@ func (rd *registryDeducer) getProjectName(importPath string) (string, error) { if err != nil { return "", err } - u.Path = path.Join(u.Path, "api/v1/projects/root", url.PathEscape(importPath)) + u.Path = path.Join(u.Path, "api/v1/projects/", url.PathEscape(importPath)) req, err := http.NewRequest("HEAD", u.String(), nil) if err != nil { return "", err @@ -808,10 +808,6 @@ func (rd *registryDeducer) deduce(ctx context.Context, path string) (pathDeducti rd.deduced = pathDeduction{mb: maybeRegistrySource{path: projectName, url: u, token: rd.registry.Token()}, root: projectName} rd.returnFunc(rd.deduced) return - - if rd.deduced.mb == nil { - rd.deduceErr = errors.Errorf("unable to find root path for: %s", path) - } }) return rd.deduced, rd.deduceErr } diff --git a/gps/registry.go b/gps/registry.go index e237f9d742..4610bdddf9 100644 --- a/gps/registry.go +++ b/gps/registry.go @@ -26,19 +26,23 @@ type Registry interface { Token() string } +// NewRegistryConfig creates new registry config using provided url and token func NewRegistryConfig(url *url.URL, token string) Registry { return &RegitryConfig{url: url.String(), token: token} } +// RegistryConfig holds url and token of Golang registry type RegitryConfig struct { url string token string } +// URL returns registry URL func (s *RegitryConfig) URL() string { return s.url } +// Token returns registry token func (s *RegitryConfig) Token() string { return s.token } @@ -84,7 +88,7 @@ func (s *registrySource) execGetVersions() (*rawVersions, error) { if err != nil { return nil, err } - u.Path = path.Join(u.Path, "api/v1/versions", url.PathEscape(s.path)) + u.Path = path.Join(u.Path, "api/v1/projects", url.PathEscape(s.path), "info") req, err := http.NewRequest("GET", u.String(), nil) if err != nil { return nil, err @@ -112,7 +116,7 @@ func (s *registrySource) execDownloadDependency(ctx context.Context, pr ProjectR if err != nil { return nil, err } - u.Path = path.Join(u.Path, "api/v1/projects", url.PathEscape(s.path), r.String()) + u.Path = path.Join(u.Path, "api/v1/projects", url.PathEscape(s.path), "versions", r.String()) req, err := http.NewRequest("GET", u.String(), nil) if err != nil { return nil, err @@ -208,16 +212,16 @@ func extractDependency(r io.Reader, target string) error { return err } - path := filepath.Join(target, header.Name) + filePath := filepath.Join(target, header.Name) info := header.FileInfo() if info.IsDir() { - if err = os.MkdirAll(path, info.Mode()); err != nil { + if err = os.MkdirAll(filePath, info.Mode()); err != nil { return err } continue } - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) + file, err := os.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) if err != nil { return err } diff --git a/internal/test/registry/registry_mock.go b/internal/test/registry/registry_mock.go index d39019aa10..97e8254b78 100644 --- a/internal/test/registry/registry_mock.go +++ b/internal/test/registry/registry_mock.go @@ -14,9 +14,9 @@ import ( "strings" ) -const TOKEN_AUTH = "2erygdasE45rty5JKwewrr75cb15rdeE" +const TokenAuth = "2erygdasE45rty5JKwewrr75cb15rdeE" -var tokenResp = "{\"token\": \"" + TOKEN_AUTH + "\"}" +var tokenResp = "{\"token\": \"" + TokenAuth + "\"}" type rawVersions struct { Versions map[string]rawPublished `json:"versions"` @@ -34,7 +34,7 @@ func verifyToken(r *http.Request) error { if ok && len(tokens) >= 1 { token := tokens[0] token = strings.TrimPrefix(token, "BEARER ") - if token == TOKEN_AUTH { + if token == TokenAuth { return nil } } @@ -78,7 +78,7 @@ func token(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, tokenResp) // send data to client side } -func versions(w http.ResponseWriter, r *http.Request) { +func getInfo(w http.ResponseWriter, r *http.Request, projectName string) { if r.Method != http.MethodGet { http.NotFound(w, r) return @@ -89,8 +89,7 @@ func versions(w http.ResponseWriter, r *http.Request) { return } - depName := strings.TrimPrefix(r.URL.Path, "/api/v1/versions/") - versions := getVersions(depName) + versions := getVersions(projectName) if len(versions.Versions) == 0 { http.NotFound(w, r) return @@ -104,26 +103,14 @@ func versions(w http.ResponseWriter, r *http.Request) { w.Write(requestContent) } -func getDependency(w http.ResponseWriter, r *http.Request) { - if err := verifyToken(r); err != nil { - http.Error(w, err.Error(), http.StatusUnauthorized) - return - } - - dep := strings.TrimPrefix(r.URL.Path, "/api/v1/projects/") - nameVer := strings.Split(dep, "/") - if len(nameVer) != 2 { - http.NotFound(w, r) - return - } - +func getDependency(w http.ResponseWriter, r *http.Request, projectName, version string) { sourcesPath, err := getSourcesPath() if err != nil { http.NotFound(w, r) return } - f, err := os.OpenFile(filepath.Join(sourcesPath, dep, nameVer[0]+".tar.gz"), os.O_RDONLY, 0666) + f, err := os.OpenFile(filepath.Join(sourcesPath, projectName, version, projectName+".tar.gz"), os.O_RDONLY, 0666) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -138,13 +125,33 @@ func getDependency(w http.ResponseWriter, r *http.Request) { h := sha256.New() h.Write(b) - w.Header().Add("X-Go-Project-Name", strings.Replace(nameVer[0], "%2F", "/", -1)) - w.Header().Add("X-Go-Project-Version", nameVer[1]) + w.Header().Add("X-Go-Project-Name", strings.Replace(projectName, "%2F", "/", -1)) + w.Header().Add("X-Go-Project-Version", version) w.Header().Add("X-Checksum-Sha256", hex.EncodeToString(h.Sum(nil))) w.Write(b) } -func putDependency(w http.ResponseWriter, r *http.Request) { +func getProject(w http.ResponseWriter, r *http.Request) { + if err := verifyToken(r); err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + dep := strings.TrimPrefix(r.URL.Path, "/api/v1/projects/") + pathComponents := strings.Split(dep, "/") + + switch pathComponents[1] { + case "info": + getInfo(w, r, pathComponents[0]) + return + case "versions": + getDependency(w, r, pathComponents[0], pathComponents[2]) + return + } + http.NotFound(w, r) +} + +func putProject(w http.ResponseWriter, r *http.Request) { if err := verifyToken(r); err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return @@ -168,9 +175,9 @@ func putDependency(w http.ResponseWriter, r *http.Request) { func dependency(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: - getDependency(w, r) + getProject(w, r) case http.MethodPut: - putDependency(w, r) + putProject(w, r) case http.MethodHead: projectName(w, r) default: @@ -185,7 +192,7 @@ func projectName(w http.ResponseWriter, r *http.Request) { return } - importPath := strings.TrimPrefix(r.URL.Path, "/api/v1/projects/root/") + importPath := strings.TrimPrefix(r.URL.Path, "/api/v1/projects/") sourcePath, err := getSourcesPath() if err != nil { http.NotFound(w, r) @@ -217,9 +224,9 @@ func notFound(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) } +// SetupAndRun starts the registry mock for testing purposes func SetupAndRun(addr string) error { http.HandleFunc("/api/v1/auth/token", token) - http.HandleFunc("/api/v1/versions/", versions) http.HandleFunc("/api/v1/projects/", dependency) http.HandleFunc("/", notFound) return http.ListenAndServe(addr, nil) From f0792afceb3e8c0fbfef06023c0ef6edf23698c9 Mon Sep 17 00:00:00 2001 From: romangurevitch Date: Thu, 28 Dec 2017 15:06:51 +0200 Subject: [PATCH 4/5] Registry support - Bearer spelling fix --- gps/deduce.go | 2 +- gps/registry.go | 4 ++-- internal/test/registry/registry_mock.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gps/deduce.go b/gps/deduce.go index 2aa6d7ea3e..210f323a9e 100644 --- a/gps/deduce.go +++ b/gps/deduce.go @@ -780,7 +780,7 @@ func (rd *registryDeducer) getProjectName(importPath string) (string, error) { return "", err } - req.Header.Set("Authorization", "BEARER "+rd.registry.Token()) + req.Header.Set("Authorization", "Bearer "+rd.registry.Token()) resp, err := http.DefaultClient.Do(req) if err != nil { return "", err diff --git a/gps/registry.go b/gps/registry.go index 4610bdddf9..5cacbc74d3 100644 --- a/gps/registry.go +++ b/gps/registry.go @@ -94,7 +94,7 @@ func (s *registrySource) execGetVersions() (*rawVersions, error) { return nil, err } - req.Header.Set("Authorization", "BEARER "+s.token) + req.Header.Set("Authorization", "Bearer "+s.token) resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err @@ -122,7 +122,7 @@ func (s *registrySource) execDownloadDependency(ctx context.Context, pr ProjectR return nil, err } - req.Header.Set("Authorization", "BEARER "+s.token) + req.Header.Set("Authorization", "Bearer "+s.token) resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err diff --git a/internal/test/registry/registry_mock.go b/internal/test/registry/registry_mock.go index 97e8254b78..f64e7f872d 100644 --- a/internal/test/registry/registry_mock.go +++ b/internal/test/registry/registry_mock.go @@ -33,7 +33,7 @@ func verifyToken(r *http.Request) error { tokens, ok := r.Header["Authorization"] if ok && len(tokens) >= 1 { token := tokens[0] - token = strings.TrimPrefix(token, "BEARER ") + token = strings.TrimPrefix(token, "Bearer ") if token == TokenAuth { return nil } From 716976db00556cb14e42c3ae7d7936b807b6fdbf Mon Sep 17 00:00:00 2001 From: romangurevitch Date: Wed, 3 Jan 2018 14:43:15 +0200 Subject: [PATCH 5/5] Registry support - deduct known paths --- cmd/dep/main.go | 4 ++-- gps/deduce.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cmd/dep/main.go b/cmd/dep/main.go index 93aff208cf..e7dfd51cc3 100644 --- a/cmd/dep/main.go +++ b/cmd/dep/main.go @@ -18,8 +18,8 @@ import ( "text/tabwriter" "github.com/golang/dep" - "github.com/golang/dep/internal/fs" "github.com/golang/dep/gps" + "github.com/golang/dep/internal/fs" "net/url" ) @@ -208,7 +208,7 @@ func (c *Config) Run() int { registryURL := getEnv(c.Env, "DEPREGISTRYURL") token := getEnv(c.Env, "DEPREGISTRYTOKEN") - // check if url is not define do not create registry + // If the url or the token is not defined do not create registry config if registryURL != "" && token != "" { registryUrl, err := url.Parse(registryURL) if err != nil { diff --git a/gps/deduce.go b/gps/deduce.go index 210f323a9e..e68d8fa12b 100644 --- a/gps/deduce.go +++ b/gps/deduce.go @@ -608,6 +608,23 @@ func (dc *registryDeductionCoordinator) deduceRootPath(ctx context.Context, path panic(fmt.Sprintf("unexpected %T in deductionCoordinator.rootxt: %v", data, data)) } + if _, mtch, has := dc.deducext.LongestPrefix(path); has { + root, err := mtch.deduceRoot(path) + if err == nil { + var u *url.URL + u, err = url.Parse(dc.registry.URL()) + if err != nil { + return pathDeduction{}, err + } + mb := maybeRegistrySource{path: root, url: u, token: dc.registry.Token()} + + dc.mut.Lock() + dc.rootxt.Insert(root, mb) + dc.mut.Unlock() + return pathDeduction{root: root, mb: mb}, nil + } + } + rd := ®istryDeducer{ basePath: path, registry: dc.registry,