Skip to content

Commit 028e5a3

Browse files
acabarbayeacabarbaye
authored
⭐ [filesystem] Expand APIs to enable negation patterns (#141)
<!-- Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors. All rights reserved. SPDX-License-Identifier: Apache-2.0 --> ### Description Expand APIs in the `filesystem` module to enable `negation patterns` ### Test Coverage <!-- Please put an `x` in the correct box e.g. `[x]` to indicate the testing coverage of this change. --> - [x] This change is covered by existing or additional automated tests. - [ ] Manual testing has been performed (and evidence provided) as automated testing was not feasible. - [ ] Additional tests are not required for this change (e.g. documentation update).
1 parent 9c73974 commit 028e5a3

File tree

9 files changed

+1767
-815
lines changed

9 files changed

+1767
-815
lines changed

‎changes/202210040057.feature‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
`[filesystem]` Expanded most recursive filesystem APIs so that exclusion patterns can be used to ignore some paths during the processing

‎utils/filesystem/exclusion.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package filesystem
2+
3+
import (
4+
"fmt"
5+
"regexp"
6+
7+
"github.com/ARM-software/golang-utils/utils/commonerrors"
8+
"github.com/ARM-software/golang-utils/utils/reflection"
9+
)
10+
11+
func ExcludeFiles(files []string, regexes []*regexp.Regexp) (cleansedList []string, err error) {
12+
cleansedList = []string{}
13+
for i := range files {
14+
f := files[i]
15+
if !IsPathExcluded(f, regexes...) {
16+
cleansedList = append(cleansedList, f)
17+
}
18+
}
19+
return
20+
}
21+
22+
func NewExclusionRegexList(pathSeparator rune, exclusionPatterns ...string) ([]*regexp.Regexp, error) {
23+
var regexes []*regexp.Regexp
24+
var patternsExtendedList []string
25+
for i := range exclusionPatterns {
26+
pattern := exclusionPatterns[i]
27+
28+
if !reflection.IsEmpty(pattern) {
29+
patternsExtendedList = append(patternsExtendedList, pattern, fmt.Sprintf(".*/%v/.*", pattern), fmt.Sprintf(".*%v%v%v.*", pathSeparator, pattern, pathSeparator))
30+
}
31+
}
32+
for i := range patternsExtendedList {
33+
r, err := regexp.Compile(patternsExtendedList[i])
34+
if err != nil {
35+
return nil, fmt.Errorf("%w: could not compile pattern [%v]: %v", commonerrors.ErrInvalid, patternsExtendedList[i], err.Error())
36+
}
37+
regexes = append(regexes, r)
38+
}
39+
return regexes, nil
40+
}
41+
42+
func IsPathExcludedFromPatterns(path string, pathSeparator rune, exclusionPatterns ...string) bool {
43+
regexes, err := NewExclusionRegexList(pathSeparator, exclusionPatterns...)
44+
if err != nil {
45+
return false
46+
}
47+
return IsPathExcluded(path, regexes...)
48+
}
49+
50+
func IsPathExcluded(path string, exclusionPatterns ...*regexp.Regexp) bool {
51+
for i := range exclusionPatterns {
52+
if exclusionPatterns[i].MatchString(path) {
53+
return true
54+
}
55+
}
56+
return false
57+
}
58+
59+
// ExcludeAll excludes files
60+
func ExcludeAll(files []string, exclusionPatterns ...string) ([]string, error) {
61+
return globalFileSystem.ExcludeAll(files, exclusionPatterns...)
62+
}
63+
64+
func (fs *VFS) ExcludeAll(files []string, exclusionPatterns ...string) ([]string, error) {
65+
regexes, err := NewExclusionRegexList(fs.PathSeparator(), exclusionPatterns...)
66+
if err != nil {
67+
return nil, err
68+
}
69+
return ExcludeFiles(files, regexes)
70+
}

‎utils/filesystem/exclusion_test.go‎

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
package filesystem
2+
3+
import (
4+
"fmt"
5+
"sort"
6+
"testing"
7+
8+
"github.com/bxcodec/faker/v3"
9+
"github.com/stretchr/testify/assert"
10+
"github.com/stretchr/testify/require"
11+
12+
"github.com/ARM-software/golang-utils/utils/commonerrors"
13+
)
14+
15+
func TestExcludeFiles(t *testing.T) {
16+
test0Entry := fmt.Sprintf("%v test0", faker.Sentence())
17+
test3Entry := fmt.Sprintf("%vtest3%v", faker.Name(), faker.Word())
18+
fileList := []string{
19+
test0Entry,
20+
fmt.Sprintf("test1&£&(%v", faker.URL()),
21+
fmt.Sprintf("%vtest2%v", faker.URL(), faker.Sentence()),
22+
test3Entry,
23+
fmt.Sprintf("%v()T$^test4%v", faker.IPv4(), faker.UUIDHyphenated()),
24+
}
25+
26+
tests := []struct {
27+
files []string
28+
exclusionPatterns []string
29+
expectedError error
30+
expectedResult []string
31+
}{
32+
{
33+
files: fileList,
34+
exclusionPatterns: nil,
35+
expectedError: nil,
36+
expectedResult: fileList,
37+
},
38+
{
39+
files: fileList,
40+
exclusionPatterns: []string{".*test0.*", ".*test1.*", ".*test2.*", ".*test3.*", ".*test4.*"},
41+
expectedError: nil,
42+
expectedResult: []string{},
43+
},
44+
{
45+
files: fileList,
46+
exclusionPatterns: []string{".*test1.*", ".*test2.*", ".*test4.*"},
47+
expectedError: nil,
48+
expectedResult: []string{test0Entry, test3Entry},
49+
},
50+
{
51+
files: fileList,
52+
exclusionPatterns: []string{".*test1.*", ".*test2.*", ".*test3.*", ".*test4.*"},
53+
expectedError: nil,
54+
expectedResult: []string{test0Entry},
55+
},
56+
{
57+
files: fileList,
58+
exclusionPatterns: []string{".*test0.*", ".*test1.*", ".*test2.*", ".*test4.*"},
59+
expectedError: nil,
60+
expectedResult: []string{test3Entry},
61+
},
62+
{
63+
files: fileList,
64+
exclusionPatterns: []string{"*test0**"},
65+
expectedError: commonerrors.ErrInvalid,
66+
expectedResult: []string{},
67+
},
68+
}
69+
70+
for i := range tests {
71+
test := tests[i]
72+
t.Run(fmt.Sprintf("exlusions [%v]", test.exclusionPatterns), func(t *testing.T) {
73+
regexes, err := NewExclusionRegexList(globalFileSystem.PathSeparator(), test.exclusionPatterns...)
74+
if test.expectedError != nil {
75+
require.Error(t, err)
76+
assert.True(t, commonerrors.Any(err, test.expectedError))
77+
} else {
78+
newList, err := ExcludeFiles(test.files, regexes)
79+
require.NoError(t, err)
80+
sort.Strings(newList)
81+
sort.Strings(test.expectedResult)
82+
assert.Equal(t, test.expectedResult, newList)
83+
for i := range newList {
84+
assert.False(t, IsPathExcludedFromPatterns(newList[i], globalFileSystem.PathSeparator(), test.exclusionPatterns...))
85+
}
86+
}
87+
88+
})
89+
}
90+
}
91+
92+
func TestExcludes(t *testing.T) {
93+
t.Parallel() // marks TLog as capable of running in parallel with other tests
94+
var listOfPaths = []string{
95+
"some/path", "somepath", ".snapshot", ".snapshot/path", "test/.snapshot/some/path", ".snapshot123", ".snapshot123/path", "test/.snapshot123/some-path", "test/.snapshot123/some/path",
96+
}
97+
tests := []struct {
98+
inputlist []string
99+
exclusions []string
100+
expectedResults []string
101+
}{
102+
{
103+
inputlist: listOfPaths,
104+
exclusions: []string{},
105+
expectedResults: listOfPaths,
106+
},
107+
{
108+
inputlist: listOfPaths,
109+
exclusions: []string{"noexclusion"},
110+
expectedResults: listOfPaths,
111+
},
112+
{
113+
inputlist: []string{},
114+
exclusions: []string{"any"},
115+
expectedResults: []string{},
116+
},
117+
{
118+
inputlist: listOfPaths,
119+
exclusions: []string{""},
120+
expectedResults: listOfPaths,
121+
},
122+
{
123+
inputlist: listOfPaths,
124+
exclusions: []string{"some.*"},
125+
expectedResults: []string{".snapshot", ".snapshot/path", ".snapshot123", ".snapshot123/path"},
126+
},
127+
{
128+
inputlist: listOfPaths,
129+
exclusions: []string{".*path"},
130+
expectedResults: []string{".snapshot", ".snapshot123"},
131+
},
132+
{
133+
inputlist: listOfPaths,
134+
exclusions: []string{"[.]snapshot.*"},
135+
expectedResults: []string{"some/path", "somepath"},
136+
},
137+
{
138+
inputlist: listOfPaths,
139+
exclusions: []string{"[.]snapshot.*/.*"},
140+
expectedResults: []string{"some/path", "somepath", ".snapshot", ".snapshot123"},
141+
},
142+
{
143+
inputlist: listOfPaths,
144+
exclusions: []string{"[.]snapshot.*", ".*path"},
145+
expectedResults: []string{},
146+
},
147+
}
148+
for i := range tests {
149+
test := tests[i] // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
150+
t.Run(fmt.Sprintf("%v: %v", i, test.exclusions), func(t *testing.T) {
151+
t.Parallel() // marks each test case as capable of running in parallel with each other
152+
actualList, err := ExcludeAll(test.inputlist, test.exclusions...)
153+
require.NoError(t, err)
154+
assert.Equal(t, test.expectedResults, actualList)
155+
})
156+
}
157+
158+
}
159+
160+
func TestIsPathExcludedFromPatterns(t *testing.T) {
161+
tests := []struct {
162+
path string
163+
pathSeparator rune
164+
exclusionPatterns []string
165+
exclude bool
166+
}{
167+
{
168+
path: "",
169+
pathSeparator: '\\',
170+
exclusionPatterns: []string{},
171+
exclude: false,
172+
},
173+
{
174+
path: "",
175+
pathSeparator: '\\',
176+
exclusionPatterns: []string{".*[.]test[^13]"},
177+
exclude: false,
178+
},
179+
{
180+
path: "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test2",
181+
pathSeparator: '\\',
182+
exclusionPatterns: []string{},
183+
exclude: false,
184+
},
185+
{
186+
path: "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test2",
187+
pathSeparator: '\\',
188+
exclusionPatterns: []string{".*[.]test[^13]"},
189+
exclude: true,
190+
},
191+
{
192+
path: "C:/Users/adrcab01/AppData/Local/Temp/test-findall-929837903/level1/level2/test-findall-309750873.test2",
193+
pathSeparator: '/',
194+
exclusionPatterns: []string{".*[.]test[^13]"},
195+
exclude: true,
196+
},
197+
{
198+
path: "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
199+
pathSeparator: '\\',
200+
exclusionPatterns: []string{".*[.]test[^13]"},
201+
exclude: false,
202+
},
203+
{
204+
path: "C:/Users/adrcab01/AppData/Local/Temp/test-findall-929837903/level1/level2/test-findall-309750873.test3",
205+
pathSeparator: '/',
206+
exclusionPatterns: []string{".*[.]test[^13]"},
207+
exclude: false,
208+
},
209+
{
210+
path: "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test2",
211+
pathSeparator: '\\',
212+
exclusionPatterns: []string{".*[.]test2"},
213+
exclude: true,
214+
},
215+
{
216+
path: "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
217+
pathSeparator: '\\',
218+
exclusionPatterns: []string{".*[.]test2"},
219+
exclude: false,
220+
},
221+
{
222+
path: "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
223+
pathSeparator: '\\',
224+
exclusionPatterns: []string{".*"},
225+
exclude: true,
226+
},
227+
{
228+
path: "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
229+
pathSeparator: '\\',
230+
exclusionPatterns: []string{".*test3"},
231+
exclude: true,
232+
},
233+
{
234+
path: "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
235+
pathSeparator: '\\',
236+
exclusionPatterns: []string{".*test2"},
237+
exclude: false,
238+
},
239+
}
240+
for i := range tests {
241+
test := tests[i]
242+
t.Run(fmt.Sprintf("exlusions [%v]", test.exclusionPatterns), func(t *testing.T) {
243+
if test.exclude {
244+
assert.True(t, IsPathExcludedFromPatterns(test.path, test.pathSeparator, test.exclusionPatterns...))
245+
} else {
246+
assert.False(t, IsPathExcludedFromPatterns(test.path, test.pathSeparator, test.exclusionPatterns...))
247+
}
248+
249+
})
250+
}
251+
}

0 commit comments

Comments
 (0)