⭐ [filesystem] Expand APIs to enable negation patterns

changes/202210040057.feature

@@ -0,0 +1 @@
+`[filesystem]` Expanded most recursive filesystem APIs so that exclusion patterns can be used to ignore some paths during the processing

utils/filesystem/exclusion.go

@@ -0,0 +1,70 @@
+package filesystem
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors"
+	"github.com/ARM-software/golang-utils/utils/reflection"
+)
+
+func ExcludeFiles(files []string, regexes []*regexp.Regexp) (cleansedList []string, err error) {
+	cleansedList = []string{}
+	for i := range files {
+		f := files[i]
+		if !IsPathExcluded(f, regexes...) {
+			cleansedList = append(cleansedList, f)
+		}
+	}
+	return
+}
+
+func NewExclusionRegexList(pathSeparator rune, exclusionPatterns ...string) ([]*regexp.Regexp, error) {
+	var regexes []*regexp.Regexp
+	var patternsExtendedList []string
+	for i := range exclusionPatterns {
+		pattern := exclusionPatterns[i]
+
+		if !reflection.IsEmpty(pattern) {
+			patternsExtendedList = append(patternsExtendedList, pattern, fmt.Sprintf(".*/%v/.*", pattern), fmt.Sprintf(".*%v%v%v.*", pathSeparator, pattern, pathSeparator))
+		}
+	}
+	for i := range patternsExtendedList {
+		r, err := regexp.Compile(patternsExtendedList[i])
+		if err != nil {
+			return nil, fmt.Errorf("%w: could not compile pattern [%v]: %v", commonerrors.ErrInvalid, patternsExtendedList[i], err.Error())
+		}
+		regexes = append(regexes, r)
+	}
+	return regexes, nil
+}
+
+func IsPathExcludedFromPatterns(path string, pathSeparator rune, exclusionPatterns ...string) bool {
+	regexes, err := NewExclusionRegexList(pathSeparator, exclusionPatterns...)
+	if err != nil {
+		return false
+	}
+	return IsPathExcluded(path, regexes...)
+}
+
+func IsPathExcluded(path string, exclusionPatterns ...*regexp.Regexp) bool {
+	for i := range exclusionPatterns {
+		if exclusionPatterns[i].MatchString(path) {
+			return true
+		}
+	}
+	return false
+}
+
+// ExcludeAll excludes files
+func ExcludeAll(files []string, exclusionPatterns ...string) ([]string, error) {
+	return globalFileSystem.ExcludeAll(files, exclusionPatterns...)
+}
+
+func (fs *VFS) ExcludeAll(files []string, exclusionPatterns ...string) ([]string, error) {
+	regexes, err := NewExclusionRegexList(fs.PathSeparator(), exclusionPatterns...)
+	if err != nil {
+		return nil, err
+	}
+	return ExcludeFiles(files, regexes)
+}

utils/filesystem/exclusion_test.go

@@ -0,0 +1,251 @@
+package filesystem
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/bxcodec/faker/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors"
+)
+
+func TestExcludeFiles(t *testing.T) {
+	test0Entry := fmt.Sprintf("%v test0", faker.Sentence())
+	test3Entry := fmt.Sprintf("%vtest3%v", faker.Name(), faker.Word())
+	fileList := []string{
+		test0Entry,
+		fmt.Sprintf("test1&£&(%v", faker.URL()),
+		fmt.Sprintf("%vtest2%v", faker.URL(), faker.Sentence()),
+		test3Entry,
+		fmt.Sprintf("%v()T$^test4%v", faker.IPv4(), faker.UUIDHyphenated()),
+	}
+
+	tests := []struct {
+		files             []string
+		exclusionPatterns []string
+		expectedError     error
+		expectedResult    []string
+	}{
+		{
+			files:             fileList,
+			exclusionPatterns: nil,
+			expectedError:     nil,
+			expectedResult:    fileList,
+		},
+		{
+			files:             fileList,
+			exclusionPatterns: []string{".*test0.*", ".*test1.*", ".*test2.*", ".*test3.*", ".*test4.*"},
+			expectedError:     nil,
+			expectedResult:    []string{},
+		},
+		{
+			files:             fileList,
+			exclusionPatterns: []string{".*test1.*", ".*test2.*", ".*test4.*"},
+			expectedError:     nil,
+			expectedResult:    []string{test0Entry, test3Entry},
+		},
+		{
+			files:             fileList,
+			exclusionPatterns: []string{".*test1.*", ".*test2.*", ".*test3.*", ".*test4.*"},
+			expectedError:     nil,
+			expectedResult:    []string{test0Entry},
+		},
+		{
+			files:             fileList,
+			exclusionPatterns: []string{".*test0.*", ".*test1.*", ".*test2.*", ".*test4.*"},
+			expectedError:     nil,
+			expectedResult:    []string{test3Entry},
+		},
+		{
+			files:             fileList,
+			exclusionPatterns: []string{"*test0**"},
+			expectedError:     commonerrors.ErrInvalid,
+			expectedResult:    []string{},
+		},
+	}
+
+	for i := range tests {
+		test := tests[i]
+		t.Run(fmt.Sprintf("exlusions [%v]", test.exclusionPatterns), func(t *testing.T) {
+			regexes, err := NewExclusionRegexList(globalFileSystem.PathSeparator(), test.exclusionPatterns...)
+			if test.expectedError != nil {
+				require.Error(t, err)
+				assert.True(t, commonerrors.Any(err, test.expectedError))
+			} else {
+				newList, err := ExcludeFiles(test.files, regexes)
+				require.NoError(t, err)
+				sort.Strings(newList)
+				sort.Strings(test.expectedResult)
+				assert.Equal(t, test.expectedResult, newList)
+				for i := range newList {
+					assert.False(t, IsPathExcludedFromPatterns(newList[i], globalFileSystem.PathSeparator(), test.exclusionPatterns...))
+				}
+			}
+
+		})
+	}
+}
+
+func TestExcludes(t *testing.T) {
+	t.Parallel() // marks TLog as capable of running in parallel with other tests
+	var listOfPaths = []string{
+		"some/path", "somepath", ".snapshot", ".snapshot/path", "test/.snapshot/some/path", ".snapshot123", ".snapshot123/path", "test/.snapshot123/some-path", "test/.snapshot123/some/path",
+	}
+	tests := []struct {
+		inputlist       []string
+		exclusions      []string
+		expectedResults []string
+	}{
+		{
+			inputlist:       listOfPaths,
+			exclusions:      []string{},
+			expectedResults: listOfPaths,
+		},
+		{
+			inputlist:       listOfPaths,
+			exclusions:      []string{"noexclusion"},
+			expectedResults: listOfPaths,
+		},
+		{
+			inputlist:       []string{},
+			exclusions:      []string{"any"},
+			expectedResults: []string{},
+		},
+		{
+			inputlist:       listOfPaths,
+			exclusions:      []string{""},
+			expectedResults: listOfPaths,
+		},
+		{
+			inputlist:       listOfPaths,
+			exclusions:      []string{"some.*"},
+			expectedResults: []string{".snapshot", ".snapshot/path", ".snapshot123", ".snapshot123/path"},
+		},
+		{
+			inputlist:       listOfPaths,
+			exclusions:      []string{".*path"},
+			expectedResults: []string{".snapshot", ".snapshot123"},
+		},
+		{
+			inputlist:       listOfPaths,
+			exclusions:      []string{"[.]snapshot.*"},
+			expectedResults: []string{"some/path", "somepath"},
+		},
+		{
+			inputlist:       listOfPaths,
+			exclusions:      []string{"[.]snapshot.*/.*"},
+			expectedResults: []string{"some/path", "somepath", ".snapshot", ".snapshot123"},
+		},
+		{
+			inputlist:       listOfPaths,
+			exclusions:      []string{"[.]snapshot.*", ".*path"},
+			expectedResults: []string{},
+		},
+	}
+	for i := range tests {
+		test := tests[i] // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
+		t.Run(fmt.Sprintf("%v: %v", i, test.exclusions), func(t *testing.T) {
+			t.Parallel() // marks each test case as capable of running in parallel with each other
+			actualList, err := ExcludeAll(test.inputlist, test.exclusions...)
+			require.NoError(t, err)
+			assert.Equal(t, test.expectedResults, actualList)
+		})
+	}
+
+}
+
+func TestIsPathExcludedFromPatterns(t *testing.T) {
+	tests := []struct {
+		path              string
+		pathSeparator     rune
+		exclusionPatterns []string
+		exclude           bool
+	}{
+		{
+			path:              "",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{},
+			exclude:           false,
+		},
+		{
+			path:              "",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{".*[.]test[^13]"},
+			exclude:           false,
+		},
+		{
+			path:              "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test2",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{},
+			exclude:           false,
+		},
+		{
+			path:              "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test2",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{".*[.]test[^13]"},
+			exclude:           true,
+		},
+		{
+			path:              "C:/Users/adrcab01/AppData/Local/Temp/test-findall-929837903/level1/level2/test-findall-309750873.test2",
+			pathSeparator:     '/',
+			exclusionPatterns: []string{".*[.]test[^13]"},
+			exclude:           true,
+		},
+		{
+			path:              "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{".*[.]test[^13]"},
+			exclude:           false,
+		},
+		{
+			path:              "C:/Users/adrcab01/AppData/Local/Temp/test-findall-929837903/level1/level2/test-findall-309750873.test3",
+			pathSeparator:     '/',
+			exclusionPatterns: []string{".*[.]test[^13]"},
+			exclude:           false,
+		},
+		{
+			path:              "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test2",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{".*[.]test2"},
+			exclude:           true,
+		},
+		{
+			path:              "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{".*[.]test2"},
+			exclude:           false,
+		},
+		{
+			path:              "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{".*"},
+			exclude:           true,
+		},
+		{
+			path:              "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{".*test3"},
+			exclude:           true,
+		},
+		{
+			path:              "C:\\Users\\adrcab01\\AppData\\Local\\Temp\\test-findall-929837903\\level1\\level2\\test-findall-309750873.test3",
+			pathSeparator:     '\\',
+			exclusionPatterns: []string{".*test2"},
+			exclude:           false,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(fmt.Sprintf("exlusions [%v]", test.exclusionPatterns), func(t *testing.T) {
+			if test.exclude {
+				assert.True(t, IsPathExcludedFromPatterns(test.path, test.pathSeparator, test.exclusionPatterns...))
+			} else {
+				assert.False(t, IsPathExcludedFromPatterns(test.path, test.pathSeparator, test.exclusionPatterns...))
+			}
+
+		})
+	}
+}