Rebuild-indexes cmd

cmd/rebuild_indexes.go

@@ -0,0 +1,97 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"code.gitea.io/gitea/modules/indexer"
+	"code.gitea.io/gitea/modules/indexer/issues"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/private"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/urfave/cli"
+)
+
+// CmdRebuildIndexes represents the available rebuild-indexes sub-command
+var CmdRebuildIndexes = cli.Command{
+	Name:        "rebuild-indexes",
+	Usage:       "Rebuild text indexes",
+	Description: "This command rebuilds text indexes for issues and repositories",
+	Action:      runRebuildIndexes,
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "repositories",
+			Usage: "Rebuild text indexes for repository content",
+		},
+		cli.BoolFlag{
+			Name:  "issues",
+			Usage: "Rebuild text indexes for issues",
+		},
+		cli.BoolFlag{
+			Name:  "all",
+			Usage: "Rebuild all text indexes",
+		},
+	},
+}
+
+func runRebuildIndexes(ctx *cli.Context) error {
+	setting.NewContext()
+	setting.NewServices()
+
+	var rebuildIssues bool
+	var rebuildRepositories bool
+
+	if ctx.IsSet("repositories") || ctx.IsSet("all") {
+		rebuildRepositories = true
+	}
+
+	if ctx.IsSet("issues") || ctx.IsSet("all") {
+		rebuildIssues = true
+	}
+
+	if !rebuildIssues && !rebuildRepositories {
+		fmt.Printf("At least one of --repositories, --issues or --all must be used\n")
+		return nil
+	}
+
+	if rebuildRepositories && !setting.Indexer.RepoIndexerEnabled {
+		fmt.Printf("Repository indexes are not enabled\n")
+		rebuildRepositories = false
+	}
+
+	if rebuildIssues && setting.Indexer.IssueType != "bleve" {
+		log.ColorFprintf(os.Stdout, "Issue index type '%s' does not support or does not require rebuilding\n", setting.Indexer.IssueType)
+		rebuildIssues = false
+	}
+
+	if rebuildRepositories {
+		attemptRebuild("Rebuild repository indexes", private.RebuildRepoIndex, indexer.DropRepoIndex)
+	}
+
+	if rebuildIssues {
+		attemptRebuild("Rebuild issue indexes", private.RebuildIssueIndex, issues.DropIssueIndex)
+	}
+
+	fmt.Println("Rebuild done or in process.")
+	return nil
+}
+
+func attemptRebuild(msg string, onlineRebuild func() error, offlineDrop func() error) {
+	log.Info(msg)
+	fmt.Printf("%s: attempting through Gitea API...\n", msg)
+	if err := onlineRebuild(); err != nil {
+		// FIXME: there's no good way of knowing if Gitea is running
+		log.ColorFprintf(os.Stdout, "Error (disregard if it's a connection error): %v\n", err)
+		// Attempt a direct delete
+		fmt.Printf("Gitea seems to be down; marking index files for recycling the next time Gitea runs.\n")
+		if err := offlineDrop(); err != nil {
+			log.ColorFprintf(os.Stdout, "Internal error: %v\n", err)
+			log.Fatal("Rebuild indexes: %v", err)
+		}
+	}
+}

main.go

@@ -68,6 +68,7 @@ arguments - which can alternatively be run by running the subcommand web.`
 		cmd.CmdMigrate,
 		cmd.CmdKeys,
 		cmd.CmdConvert,
+		cmd.CmdRebuildIndexes,
 	}
 	// Now adjust these commands to add our global configuration options
 

models/repo_indexer.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"sync"
 
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/charset"
@@ -27,6 +28,10 @@ type RepoIndexerStatus struct {
 	CommitSha string `xorm:"VARCHAR(40)"`
 }
 
+var (
+	rebuildLock sync.Mutex
+)
+
 func (repo *Repository) getIndexerStatus() error {
 	if repo.IndexerStatus != nil {
 		return nil
@@ -104,6 +109,8 @@ func populateRepoIndexerAsynchronously() error {
 // populateRepoIndexer populate the repo indexer with pre-existing data. This
 // should only be run when the indexer is created for the first time.
 func populateRepoIndexer(maxRepoID int64) {
+	rebuildLock.Lock()
+	defer rebuildLock.Unlock()
 	log.Info("Populating the repo indexer with existing repositories")
 	// start with the maximum existing repo ID and work backwards, so that we
 	// don't include repos that are created after gitea starts; such repos will
@@ -376,3 +383,26 @@ func addOperationToQueue(op repoIndexerOperation) {
 		}()
 	}
 }
+
+func rebuildRepoIndex() error {
+	// Make sure no other build is currently running
+	rebuildLock.Lock()
+	defer rebuildLock.Unlock()
+	if err := indexer.DropRepoIndex(); err != nil {
+		return err
+	}
+	// This could abort with Fatal()
+	indexer.InitRepoIndexer(nil)
+	return nil
+}
+
+// RebuildRepoIndex deletes and rebuilds text indexes for repositories
+func RebuildRepoIndex() error {
+	if !setting.Indexer.RepoIndexerEnabled {
+		return nil
+	}
+	if err := rebuildRepoIndex(); err != nil {
+		return err
+	}
+	return populateRepoIndexerAsynchronously()
+}

modules/indexer/indexer.go

@@ -8,8 +8,6 @@ import (
 	"os"
 	"strconv"
 
-	"code.gitea.io/gitea/modules/setting"
-
 	"github.com/blevesearch/bleve"
 	"github.com/blevesearch/bleve/analysis/token/unicodenorm"
 	"github.com/blevesearch/bleve/index/upsidedown"
@@ -47,7 +45,7 @@ const maxBatchSize = 16
 // updates and bleve version updates.  If index needs to be created (or
 // re-created), returns (nil, nil)
 func openIndexer(path string, latestVersion int) (bleve.Index, error) {
-	_, err := os.Stat(setting.Indexer.IssuePath)
+	_, err := os.Stat(path)
 	if err != nil && os.IsNotExist(err) {
 		return nil, nil
 	} else if err != nil {

modules/indexer/issues/bleve.go

@@ -248,3 +248,10 @@ func (b *BleveIndexer) Search(keyword string, repoID int64, limit, start int) (*
 	}
 	return &ret, nil
 }
+
+// Drop marks the index for rebuilding by invalidating its version number
+func (b *BleveIndexer) Drop(path string) (bool, error) {
+	return true, rupture.WriteIndexMetadata(path, &rupture.IndexMetadata{
+		Version: -1,
+	})
+}

modules/indexer/issues/db.go

@@ -43,3 +43,8 @@ func (db *DBIndexer) Search(kw string, repoID int64, limit, start int) (*SearchR
 	}
 	return &result, nil
 }
+
+// Drop dummy function
+func (db *DBIndexer) Drop(path string) (bool, error) {
+	return false, nil
+}

modules/indexer/issues/indexer.go

@@ -6,6 +6,7 @@ package issues
 
 import (
 	"fmt"
+	"sync"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
@@ -43,12 +44,14 @@ type Indexer interface {
 	Index(issue []*IndexerData) error
 	Delete(ids ...int64) error
 	Search(kw string, repoID int64, limit, start int) (*SearchResult, error)
+	Drop(path string) (bool, error)
 }
 
 var (
 	// issueIndexerQueue queue of issue ids to be updated
 	issueIndexerQueue Queue
 	issueIndexer      Indexer
+	issueRebuildLock  sync.Mutex
 )
 
 // InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until
@@ -121,6 +124,8 @@ func InitIssueIndexer(syncReindex bool) error {
 
 // populateIssueIndexer populate the issue indexer with issue data
 func populateIssueIndexer() {
+	issueRebuildLock.Lock()
+	defer issueRebuildLock.Unlock()
 	for page := 1; ; page++ {
 		repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
 			Page:        page,
@@ -138,26 +143,32 @@ func populateIssueIndexer() {
 		}
 
 		for _, repo := range repos {
-			is, err := models.Issues(&models.IssuesOptions{
-				RepoIDs:  []int64{repo.ID},
-				IsClosed: util.OptionalBoolNone,
-				IsPull:   util.OptionalBoolNone,
-			})
-			if err != nil {
-				log.Error("Issues: %v", err)
-				continue
-			}
-			if err = models.IssueList(is).LoadDiscussComments(); err != nil {
-				log.Error("LoadComments: %v", err)
-				continue
-			}
-			for _, issue := range is {
-				UpdateIssueIndexer(issue)
-			}
+			_ = populateIssueIndexerRepo(repo.ID)
 		}
 	}
 }
 
+// populateIssueIndexerRepo populate the issue indexer with issue data for one repo
+func populateIssueIndexerRepo(repoID int64) error {
+	is, err := models.Issues(&models.IssuesOptions{
+		RepoIDs:  []int64{repoID},
+		IsClosed: util.OptionalBoolNone,
+		IsPull:   util.OptionalBoolNone,
+	})
+	if err != nil {
+		log.Error("Issues: %v", err)
+		return err
+	}
+	if err = models.IssueList(is).LoadDiscussComments(); err != nil {
+		log.Error("LoadComments: %v", err)
+		return err
+	}
+	for _, issue := range is {
+		UpdateIssueIndexer(issue)
+	}
+	return nil
+}
+
 // UpdateIssueIndexer add/update an issue to the issue indexer
 func UpdateIssueIndexer(issue *models.Issue) {
 	var comments []string
@@ -206,3 +217,41 @@ func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
 	}
 	return issueIDs, nil
 }
+
+func rebuildIssueIndex() (bool, error) {
+	// Make sure no other build is currently running
+	issueRebuildLock.Lock()
+	defer issueRebuildLock.Unlock()
+
+	canrebuild, err := issueIndexer.Drop(setting.Indexer.IssuePath)
+	if !canrebuild || err != nil {
+		return false, err
+	}
+	_, err = issueIndexer.Init()
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// RebuildIssueIndex deletes and rebuilds text indexes for a repo
+func RebuildIssueIndex() error {
+	// Drop the index in a protected func
+	repopulate, err := rebuildIssueIndex()
+	if !repopulate || err != nil {
+		return err
+	}
+	// Launch repopulation
+	go populateIssueIndexer()
+
+	return nil
+}
+
+// DropIssueIndex deletes text indexes for issues; intended to be used from command line
+func DropIssueIndex() error {
+	if issueIndexer != nil {
+		_, err := issueIndexer.Drop(setting.Indexer.IssuePath)
+		return err
+	}
+	return nil
+}

modules/indexer/repo.go

@@ -85,6 +85,9 @@ func InitRepoIndexer(populateIndexer func() error) {
 	if err = createRepoIndexer(setting.Indexer.RepoPath, repoIndexerLatestVersion); err != nil {
 		log.Fatal("CreateRepoIndexer: %v", err)
 	}
+	if populateIndexer == nil {
+		return
+	}
 	if err = populateIndexer(); err != nil {
 		log.Fatal("PopulateRepoIndex: %v", err)
 	}
@@ -225,3 +228,10 @@ func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (i
 	}
 	return int64(result.Total), searchResults, nil
 }
+
+// DropRepoIndex marks the index for rebuilding by invalidating its version number
+func DropRepoIndex() error {
+	return rupture.WriteIndexMetadata(setting.Indexer.RepoPath, &rupture.IndexMetadata{
+		Version: -1,
+	})
+}

modules/private/rebuild_indexes.go

@@ -0,0 +1,47 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// RebuildRepoIndex rebuild a repository index
+func RebuildRepoIndex() error {
+	// Ask for running deliver hook and test pull request tasks.
+	reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-repo-index")
+	resp, err := newInternalRequest(reqURL, "GET").Response()
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	// All 2XX status codes are accepted and others will return an error
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("Failed to rebuild repository index: %s", decodeJSONError(resp).Err)
+	}
+	return nil
+}
+
+// RebuildIssueIndex rebuild issue index for a repo
+func RebuildIssueIndex() error {
+	// Ask for running deliver hook and test pull request tasks.
+	reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-issue-index")
+	resp, err := newInternalRequest(reqURL, "GET").Response()
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	// All 2XX status codes are accepted and others will return an error
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("Failed to rebuild issue index: %s", decodeJSONError(resp).Err)
+	}
+	return nil
+}

routers/private/internal.go

@@ -81,5 +81,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Get("/hook/post-receive/:owner/:repo", HookPostReceive)
 		m.Get("/serv/none/:keyid", ServNoCommand)
 		m.Get("/serv/command/:keyid/:owner/:repo", ServCommand)
+		m.Get("/maint/rebuild-repo-index", RebuildRepoIndex)
+		m.Get("/maint/rebuild-issue-index", RebuildIssueIndex)
 	}, CheckInternalToken)
 }