Add wildcard support to REPO_INDEXER_EXTENSIONS (no make vendor)

Author: Guillermo Prandi

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -177,8 +177,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 
 - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
 - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
-- `REPO_INDEXER_EXTENSIONS`: **empty**: A comma separated list of file extensions to exclude from the index; a \`.' matches files with no extension. An empty list means do not exclude any files.
-- `REPO_EXTENSIONS_LIST_INCLUDE`: **false**: If true, `REPO_INDEXER_EXTENSIONS` are the file extensions to include rather than exclude from the index.
+- `REPO_INDEXER_PATTERNS`: **empty**: A comma separated list of file name patterns (see https://github.com/gobwas/glob) to **exclude** from the index. An empty list means do not exclude any files. Use `**.txt` to match any files with .txt extension.
+- `REPO_PATTERNS_INCLUDE`: **false**: If true, `REPO_INDEXER_PATTERNS` are the file extensions to **include** rather than exclude from the index.
 - `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
 - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
 

models/repo_indexer.go

@@ -232,14 +232,16 @@ func addDelete(filename string, repo *Repository, batch rupture.FlushingBatch) e
 }
 
 func isIndexable(entry *git.TreeEntry) bool {
-	if setting.Indexer.FileExtensions != nil {
-		var ext string
-		parts := strings.Split(entry.Name(), ".")
-		cnt := len(parts)
-		if cnt > 1 {
-			ext = strings.ToLower(parts[cnt-1])
+	if setting.Indexer.FilePatterns != nil {
+		var found bool
+		name := strings.ToLower(entry.Name())
+		for _, g := range setting.Indexer.FilePatterns {
+			if g.Match(name) {
+				found = true
+				break
+			}
 		}
-		if setting.Indexer.FileExtensions[ext] != setting.Indexer.IncludeExtensions {
+		if found != setting.Indexer.IncludePatterns {
 			return false
 		}
 	}

modules/setting/indexer.go

@@ -8,6 +8,10 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/gobwas/glob"
 )
 
 // enumerates all the indexer queue types
@@ -30,8 +34,8 @@ var (
 		IssueQueueDir         string
 		IssueQueueConnStr     string
 		IssueQueueBatchNumber int
-		FileExtensions        map[string]bool
-		IncludeExtensions     bool
+		FilePatterns          []glob.Glob
+		IncludePatterns       bool
 	}{
 		IssueType:             "bleve",
 		IssuePath:             "indexers/issues.bleve",
@@ -54,8 +58,8 @@ func newIndexerService() {
 	if !filepath.IsAbs(Indexer.RepoPath) {
 		Indexer.RepoPath = path.Join(AppWorkPath, Indexer.RepoPath)
 	}
-	Indexer.FileExtensions = extensionsFromString(sec.Key("REPO_INDEXER_EXTENSIONS").MustString(""))
-	Indexer.IncludeExtensions = sec.Key("REPO_EXTENSIONS_LIST_INCLUDE").MustBool(false)
+	Indexer.FilePatterns = extensionsFromString(sec.Key("REPO_INDEXER_PATTERNS").MustString(""))
+	Indexer.IncludePatterns = sec.Key("REPO_PATTERNS_INCLUDE").MustBool(false)
 
 	Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
 	Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024)
@@ -65,25 +69,20 @@ func newIndexerService() {
 	Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20)
 }
 
-func extensionsFromString(from string) map[string]bool {
-	extmap := make(map[string]bool)
-	for _, ext := range strings.Split(strings.ToLower(from), ",") {
-		ext = strings.TrimSpace(ext)
-		// Accept *.txt, .txt and txt. Also use . to mean no ext
-		if strings.HasPrefix(ext, "*.") {
-			ext = ext[1:]
-		}
-		if ext == "." {
-			extmap[""] = true
-		} else {
-			ext = strings.TrimPrefix(ext, ".")
-			if ext != "" {
-				extmap[ext] = true
+func extensionsFromString(from string) []glob.Glob {
+	extarr := make([]glob.Glob, 0, 10)
+	for _, expr := range strings.Split(strings.ToLower(from), ",") {
+		expr = strings.TrimSpace(expr)
+		if expr != "" {
+			if g, err := glob.Compile(expr, '.', '/'); err != nil {
+				log.Trace("Index file extensions: '%s': bad pattern: %v", expr, err)
+			} else {
+				extarr = append(extarr, g)
 			}
 		}
 	}
-	if len(extmap) == 0 {
+	if len(extarr) == 0 {
 		return nil
 	}
-	return extmap
+	return extarr
 }