diff --git a/CHANGELOG.md b/CHANGELOG.md index dbbd3ef2fe..f461742cea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ NEW FEATURES: +* Wildcard ignore support. (#1156) * Disable SourceManager lock by setting `DEPNOLOCK` environment variable. (#1206) diff --git a/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/final/Gopkg.lock b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/final/Gopkg.lock new file mode 100644 index 0000000000..6dffa9af2d --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/final/Gopkg.lock @@ -0,0 +1,15 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/sdboyer/deptest" + packages = ["."] + revision = "3f4c3bea144e112a69bbe5d8d01c1b09a544253f" + version = "v0.8.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "2381e3b3c6973c22f589e8f7cdf4fa63d009cfb815f86c7987a57b6b3661ebc3" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/final/Gopkg.toml b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/final/Gopkg.toml new file mode 100644 index 0000000000..5e109dd48f --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/final/Gopkg.toml @@ -0,0 +1,5 @@ +ignored = ["github.com/golang/notexist/samples*"] + +[[constraint]] + branch = "master" + name = "github.com/sdboyer/deptest" diff --git a/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/Gopkg.lock b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/Gopkg.lock new file mode 100644 index 0000000000..bef2d0092e --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/Gopkg.lock @@ -0,0 +1,9 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "ab4fef131ee828e96ba67d31a7d690bd5f2f42040c6766b1b12fe856f87e0ff7" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/Gopkg.toml b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/Gopkg.toml new file mode 100644 index 0000000000..5e109dd48f --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/Gopkg.toml @@ -0,0 +1,5 @@ +ignored = ["github.com/golang/notexist/samples*"] + +[[constraint]] + branch = "master" + name = "github.com/sdboyer/deptest" diff --git a/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/main.go b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/main.go new file mode 100644 index 0000000000..e23fcf34c5 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/main.go @@ -0,0 +1,12 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + _ "github.com/sdboyer/deptest" +) + +func main() { +} diff --git a/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/samples/samples.go b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/samples/samples.go new file mode 100644 index 0000000000..d07de170f7 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/samples/samples.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package samples + +import _ "github.com/sdboyer/deptestdos" diff --git a/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/samples/subsamples/subsamples.go b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/samples/subsamples/subsamples.go new file mode 100644 index 0000000000..5136538e6d --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/initial/samples/subsamples/subsamples.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package subsamples + +import _ "github.com/sdboyer/dep-test" diff --git a/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/testcase.json b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/testcase.json new file mode 100644 index 0000000000..729de9d0f4 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/ensure/pkg-ignored/wildcard-ignore/testcase.json @@ -0,0 +1,9 @@ +{ + "commands": [ + ["ensure"] + ], + "error-expected": "", + "vendor-final": [ + "github.com/sdboyer/deptest" + ] +} diff --git a/docs/Gopkg.toml.md b/docs/Gopkg.toml.md index 30d538977c..c17e14fe65 100644 --- a/docs/Gopkg.toml.md +++ b/docs/Gopkg.toml.md @@ -36,6 +36,12 @@ You might also try [virtualgo](https://github.com/GetStream/vg), which installs ignored = ["github.com/user/project/badpkg"] ``` +Use wildcard character (*) to define a package prefix to be ignored. Use this +to ignore any package and their subpackages. +```toml +ignored = ["github.com/user/project/badpkg*"] +``` + **Use this for:** preventing a package and any of that package's unique dependencies from being installed. diff --git a/internal/gps/hash.go b/internal/gps/hash.go index 43e84c711d..be35a17a2d 100644 --- a/internal/gps/hash.go +++ b/internal/gps/hash.go @@ -13,6 +13,8 @@ import ( "strings" ) +const wcIgnoreSuffix = "*" + // string headers used to demarcate sections in hash input creation const ( hhConstraints = "-CONSTRAINTS-" @@ -81,10 +83,24 @@ func (s *solver) writeHashingInputs(w io.Writer) { writeString(hhIgnores) ig := make([]string, 0, len(s.rd.ig)) for pkg := range s.rd.ig { + // Skip wildcard ignore. They are handled separately below. + if strings.HasSuffix(pkg, wcIgnoreSuffix) { + continue + } + if !strings.HasPrefix(pkg, s.rd.rpt.ImportRoot) || !isPathPrefixOrEqual(s.rd.rpt.ImportRoot, pkg) { ig = append(ig, pkg) } } + + // Add wildcard ignores to ignore list. + if s.rd.igpfx != nil { + s.rd.igpfx.Walk(func(s string, v interface{}) bool { + ig = append(ig, s+"*") + return false + }) + } + sort.Strings(ig) for _, igp := range ig { diff --git a/internal/gps/hash_test.go b/internal/gps/hash_test.go index ac2300a6df..aa5113962f 100644 --- a/internal/gps/hash_test.go +++ b/internal/gps/hash_test.go @@ -594,3 +594,94 @@ func diffHashingInputs(s Solver, wnt []string) string { tw.Flush() return buf.String() } + +func TestHashInputsIneffectualWildcardIgs(t *testing.T) { + fix := basicFixtures["shared dependency with overlapping constraints"] + + rm := fix.rootmanifest().(simpleRootManifest).dup() + + params := SolveParameters{ + RootDir: string(fix.ds[0].n), + RootPackageTree: fix.rootTree(), + Manifest: rm, + ProjectAnalyzer: naiveAnalyzer{}, + stdLibFn: func(string) bool { return false }, + mkBridgeFn: overrideMkBridge, + } + + cases := []struct { + name string + ignoreMap map[string]bool + elems []string + }{ + { + name: "no wildcard ignores", + elems: []string{ + hhConstraints, + "a", + "sv-1.0.0", + "b", + "sv-1.0.0", + hhImportsReqs, + "a", + "b", + hhIgnores, + hhOverrides, + hhAnalyzer, + "naive-analyzer", + "1", + }, + }, + { + name: "different wildcard ignores", + ignoreMap: map[string]bool{ + "foobar*": true, + "foobarbaz*": true, + "foozapbar*": true, + }, + elems: []string{ + hhConstraints, + "a", + "sv-1.0.0", + "b", + "sv-1.0.0", + hhImportsReqs, + "a", + "b", + hhIgnores, + "foobar*", + "foozapbar*", + hhOverrides, + hhAnalyzer, + "naive-analyzer", + "1", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + + rm.ig = c.ignoreMap + + params.Manifest = rm + + s, err := Prepare(params, newdepspecSM(fix.ds, nil)) + if err != nil { + t.Fatalf("Unexpected error while prepping solver: %s", err) + } + + dig := s.HashInputs() + h := sha256.New() + + for _, v := range c.elems { + h.Write([]byte(v)) + } + correct := h.Sum(nil) + + if !bytes.Equal(dig, correct) { + t.Errorf("Hashes are not equal. Inputs:\n%s", diffHashingInputs(s, c.elems)) + } + }) + } +} diff --git a/internal/gps/pkgtree/pkgtree.go b/internal/gps/pkgtree/pkgtree.go index 76049e6f28..fd4d10b7bc 100644 --- a/internal/gps/pkgtree/pkgtree.go +++ b/internal/gps/pkgtree/pkgtree.go @@ -18,8 +18,13 @@ import ( "strconv" "strings" "unicode" + + "github.com/armon/go-radix" ) +// wildcard ignore suffix +const wcIgnoreSuffix = "*" + // Package represents a Go package. It contains a subset of the information // go/build.Package does. type Package struct { @@ -550,6 +555,9 @@ func (t PackageTree) ToReachMap(main, tests, backprop bool, ignore map[string]bo ignore = make(map[string]bool) } + // Radix tree for ignore prefixes. + xt := CreateIgnorePrefixTree(ignore) + // world's simplest adjacency list workmap := make(map[string]wm) @@ -572,6 +580,14 @@ func (t PackageTree) ToReachMap(main, tests, backprop bool, ignore map[string]bo continue } + if xt != nil { + // Skip ignored packages prefix matches + _, _, ok := xt.LongestPrefix(ip) + if ok { + continue + } + } + // TODO (kris-nova) Disable to get staticcheck passing //imps = imps[:0] @@ -1026,3 +1042,42 @@ func uniq(a []string) []string { } return a[:i] } + +// CreateIgnorePrefixTree takes a set of strings to be ignored and returns a +// trie consisting of strings prefixed with wildcard ignore suffix (*). +func CreateIgnorePrefixTree(ig map[string]bool) *radix.Tree { + var xt *radix.Tree + + // Create a sorted list of all the ignores to have a proper order in + // ignores parsing. + sortedIgnores := make([]string, len(ig)) + for k := range ig { + sortedIgnores = append(sortedIgnores, k) + } + sort.Strings(sortedIgnores) + + for _, i := range sortedIgnores { + // Skip global ignore. + if i == "*" { + continue + } + + // Check if it's a recursive ignore. + if strings.HasSuffix(i, wcIgnoreSuffix) { + // Create trie if it doesn't exists. + if xt == nil { + xt = radix.New() + } + // Check if it is ineffectual. + _, _, ok := xt.LongestPrefix(i) + if ok { + // Skip ineffectual wildcard ignore. + continue + } + // Create the ignore prefix and insert in the radix tree. + xt.Insert(i[:len(i)-len(wcIgnoreSuffix)], true) + } + } + + return xt +} diff --git a/internal/gps/pkgtree/pkgtree_test.go b/internal/gps/pkgtree/pkgtree_test.go index fb1c0e8814..554d9cac0e 100644 --- a/internal/gps/pkgtree/pkgtree_test.go +++ b/internal/gps/pkgtree/pkgtree_test.go @@ -2038,3 +2038,92 @@ func getTestdataRootDir(t *testing.T) string { } return filepath.Join(cwd, "..", "_testdata") } + +func TestCreateIgnorePrefixTree(t *testing.T) { + cases := []struct { + name string + ignoreMap map[string]bool + wantInTree []string + notWantInTree []string + wantNilTree bool + }{ + { + name: "empty ignore", + wantNilTree: true, + }, + { + name: "ignores without ignore suffix", + ignoreMap: map[string]bool{ + "x/y/z": true, + "*a/b/c": true, + "gophers": true, + }, + wantNilTree: true, + }, + { + name: "ignores with ignore suffix", + ignoreMap: map[string]bool{ + "x/y/z*": true, + "a/b/c": true, + "gophers": true, + }, + wantInTree: []string{"x/y/z"}, + notWantInTree: []string{"a/b/c", "gophers"}, + }, + { + name: "only skip global ignore", + ignoreMap: map[string]bool{"*": true}, + wantNilTree: true, + }, + { + name: "global ignore with other strings", + ignoreMap: map[string]bool{ + "*": true, + "gophers*": true, + "x/y/z*": true, + "a/b/c": true, + }, + wantInTree: []string{"gophers", "x/y/z"}, + notWantInTree: []string{"*", "a/b/c"}, + }, + { + name: "ineffectual ignore with wildcard", + ignoreMap: map[string]bool{ + "a/b*": true, + "a/b/c*": true, + "a/b/x/y": true, + "a/c*": true, + }, + wantInTree: []string{"a/b", "a/c"}, + notWantInTree: []string{"a/b/c", "a/b/x/y"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotTree := CreateIgnorePrefixTree(c.ignoreMap) + + if c.wantNilTree { + if gotTree != nil { + t.Fatalf("expected nil tree, got non-nil tree") + } + } + + // Check if the wildcard suffix ignores are in the tree. + for _, p := range c.wantInTree { + _, has := gotTree.Get(p) + if !has { + t.Fatalf("expected %q to be in the tree", p) + } + } + + // Check if non-wildcard suffix ignores are not in the tree. + for _, p := range c.notWantInTree { + _, has := gotTree.Get(p) + if has { + t.Fatalf("expected %q to not be in the tree", p) + } + } + }) + } +} diff --git a/internal/gps/rootdata.go b/internal/gps/rootdata.go index d783174cb4..eeae1c04f8 100644 --- a/internal/gps/rootdata.go +++ b/internal/gps/rootdata.go @@ -20,6 +20,9 @@ type rootdata struct { // Map of packages to ignore. ig map[string]bool + // Radix tree of ignore prefixes. + igpfx *radix.Tree + // Map of packages to require. req map[string]bool @@ -202,3 +205,20 @@ func (rd rootdata) rootAtom() atomWithPackages { pl: list, } } + +// isIgnored checks if a given path is ignored, comparing with the list of +// ignore packages and ignore prefixes. +func (rd rootdata) isIgnored(path string) bool { + // Check if the path is present in ignore packages. + if rd.ig[path] { + return true + } + + if rd.igpfx != nil { + // Check if the path matches any of the ignore prefixes. + _, _, ok := rd.igpfx.LongestPrefix(path) + return ok + } + + return false +} diff --git a/internal/gps/rootdata_test.go b/internal/gps/rootdata_test.go index 484c2cfcb9..8578c12394 100644 --- a/internal/gps/rootdata_test.go +++ b/internal/gps/rootdata_test.go @@ -7,6 +7,8 @@ package gps import ( "reflect" "testing" + + "github.com/golang/dep/internal/gps/pkgtree" ) func TestRootdataExternalImports(t *testing.T) { @@ -222,3 +224,58 @@ func TestGetApplicableConstraints(t *testing.T) { }) } } + +func TestIsIgnored(t *testing.T) { + cases := []struct { + name string + ignorePkgs map[string]bool + wantIgnored []string + wantNotIgnored []string + }{ + { + name: "no ignore", + }, + { + name: "ignores without wildcard", + ignorePkgs: map[string]bool{ + "a/b/c": true, + "m/n": true, + "gophers": true, + }, + wantIgnored: []string{"a/b/c", "m/n", "gophers"}, + wantNotIgnored: []string{"somerandomstring"}, + }, + { + name: "ignores with wildcard", + ignorePkgs: map[string]bool{ + "a/b/c*": true, + "m/n*/o": true, + "*x/y/z": true, + "A/B*/C/D*": true, + }, + wantIgnored: []string{"a/b/c", "a/b/c/d", "a/b/c-d", "m/n*/o", "*x/y/z", "A/B*/C/D", "A/B*/C/D/E"}, + wantNotIgnored: []string{"m/n/o", "m/n*", "x/y/z", "*x/y/z/a", "*x", "A/B", "A/B*/C"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + rd := rootdata{ + ig: c.ignorePkgs, + igpfx: pkgtree.CreateIgnorePrefixTree(c.ignorePkgs), + } + + for _, p := range c.wantIgnored { + if !rd.isIgnored(p) { + t.Fatalf("expected %q to be ignored", p) + } + } + + for _, p := range c.wantNotIgnored { + if rd.isIgnored(p) { + t.Fatalf("expected %q to be not ignored", p) + } + } + }) + } +} diff --git a/internal/gps/solver.go b/internal/gps/solver.go index 3553b66344..106ab5351a 100644 --- a/internal/gps/solver.go +++ b/internal/gps/solver.go @@ -206,10 +206,13 @@ func (params SolveParameters) toRootdata() (rootdata, error) { rd.ovr = make(ProjectConstraints) } + // Create ignore prefix tree using the provided ignore packages + rd.igpfx = pkgtree.CreateIgnorePrefixTree(rd.ig) + if len(rd.ig) != 0 { var both []string for pkg := range params.Manifest.RequiredPackages() { - if rd.ig[pkg] { + if rd.isIgnored(pkg) { both = append(both, pkg) } } @@ -658,7 +661,7 @@ func (s *solver) getImportsAndConstraintsOf(a atomWithPackages) ([]string, []com // explicitly listed in the atom for _, pkg := range a.pl { // Skip ignored packages - if s.rd.ig[pkg] { + if s.rd.isIgnored(pkg) { continue } diff --git a/internal/gps/solver_inputs_test.go b/internal/gps/solver_inputs_test.go index c811e4864c..3784c2fdd0 100644 --- a/internal/gps/solver_inputs_test.go +++ b/internal/gps/solver_inputs_test.go @@ -114,6 +114,17 @@ func TestBadSolveOpts(t *testing.T) { } else if !strings.Contains(err.Error(), "multiple packages given as both required and ignored:") { t.Error("Prepare should have given error with multiple ignore/require conflict error, but gave:", err) } + + params.Manifest = simpleRootManifest{ + ig: map[string]bool{"foo*": true}, + req: map[string]bool{"foo/bar": true}, + } + _, err = Prepare(params, sm) + if err == nil { + t.Errorf("Should have errored on pkg both ignored (with wildcard) and required") + } else if !strings.Contains(err.Error(), "was given as both a required and ignored package") { + t.Error("Prepare should have given error with single ignore/require conflict error, but gave:", err) + } params.Manifest = nil params.ToChange = []ProjectRoot{"foo"}