From 7ae24a1f9bfde824bbda7eea658a9d0021ae811b Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 16 Jun 2023 14:29:04 +0800 Subject: [PATCH 01/99] feat: IndexerMetadata --- modules/indexer/issues/bleve/bleve.go | 2 +- modules/indexer/issues/db/db.go | 2 +- .../issues/elasticsearch/elasticsearch.go | 2 +- modules/indexer/issues/indexer.go | 84 ++++++++++--------- modules/indexer/issues/internal/indexer.go | 4 +- modules/indexer/issues/internal/model.go | 14 ++-- .../indexer/issues/meilisearch/meilisearch.go | 2 +- modules/indexer/issues/util.go | 15 ++++ 8 files changed, 72 insertions(+), 53 deletions(-) create mode 100644 modules/indexer/issues/util.go diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index c368a67ab5863..05b024936b77e 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -115,7 +115,7 @@ func NewIndexer(indexDir string) *Indexer { } // Index will save the index data -func (b *Indexer) Index(_ context.Context, issues []*internal.IndexerData) error { +func (b *Indexer) Index(_ context.Context, issues ...*internal.IndexerData) error { batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize) for _, issue := range issues { if err := batch.Index(indexer_internal.Base36(issue.ID), (*IndexerData)(issue)); err != nil { diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index b054b9d800edb..b41147c3677c4 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -26,7 +26,7 @@ func NewIndexer() *Indexer { } // Index dummy function -func (i *Indexer) Index(_ context.Context, _ []*internal.IndexerData) error { +func (i *Indexer) Index(_ context.Context, _ ...*internal.IndexerData) error { return nil } diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index cfd3628c18507..6d586c6647fbb 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -68,7 +68,7 @@ const ( ) // Index will save the index data -func (b *Indexer) Index(ctx context.Context, issues []*internal.IndexerData) error { +func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) error { if len(issues) == 0 { return nil } else if len(issues) == 1 { diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index fe5c5d8f26d30..653ef04a9c63c 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -26,9 +26,24 @@ import ( "code.gitea.io/gitea/modules/util" ) +// IndexerMetadata is used to send data to the queue, so it contains only the ids. +// It may look weired, because it has to be compatible with the old queue data format. +// If the IsDelete flag is true, the IDs specify the issues to delete from the index without querying the database. +// If the IsDelete flag is false, the ID specify the issue to index, so Indexer will query the database to get the issue data. +// It should be noted that if the id is not existing in the database, it's index will be deleted too even if IsDelete is false. +// Valid values: +// - IsDelete = true, IDs = [1, 2, 3], and ID will be ignored +// - IsDelete = false, ID = 1, and IDs will be ignored +type IndexerMetadata struct { + ID int64 `json:"id"` + + IsDelete bool `json:"is_delete"` + IDs []int64 `json:"ids"` +} + var ( // issueIndexerQueue queue of issue ids to be updated - issueIndexerQueue *queue.WorkerPoolQueue[*internal.IndexerData] + issueIndexerQueue *queue.WorkerPoolQueue[*IndexerMetadata] // globalIndexer is the global indexer, it cannot be nil. // When the real indexer is not ready, it will be a dummy indexer which will return error to explain it's not ready. // So it's always safe use it as *globalIndexer.Load() and call its methods. @@ -52,24 +67,37 @@ func InitIssueIndexer(syncReindex bool) { // Create the Queue switch setting.Indexer.IssueType { case "bleve", "elasticsearch", "meilisearch": - handler := func(items ...*internal.IndexerData) (unhandled []*internal.IndexerData) { + handler := func(items ...*IndexerMetadata) (unhandled []*IndexerMetadata) { indexer := *globalIndexer.Load() - toIndex := make([]*internal.IndexerData, 0, len(items)) - for _, indexerData := range items { - log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete) - if indexerData.IsDelete { - if err := indexer.Delete(ctx, indexerData.IDs...); err != nil { - log.Error("Issue indexer handler: failed to from index: %v Error: %v", indexerData.IDs, err) - unhandled = append(unhandled, indexerData) + for _, item := range items { + log.Trace("IndexerMetadata Process: %d %v %t", item.ID, item.IDs, item.IsDelete) + if item.IsDelete { + if err := indexer.Delete(ctx, item.IDs...); err != nil { + log.Error("Issue indexer handler: failed to from index: %v Error: %v", item.IDs, err) + unhandled = append(unhandled, item) } continue } - toIndex = append(toIndex, indexerData) - } - if err := indexer.Index(ctx, toIndex); err != nil { - log.Error("Error whilst indexing: %v Error: %v", toIndex, err) - unhandled = append(unhandled, toIndex...) + data, existed, err := getIssueIndexerData(ctx, item.ID) + if err != nil { + log.Error("Issue indexer handler: failed to get issue data of %d: %v", item.ID, err) + unhandled = append(unhandled, item) + continue + } + if !existed { + if err := indexer.Delete(ctx, item.ID); err != nil { + log.Error("Issue indexer handler: failed to delete issue %d from index: %v", item.ID, err) + unhandled = append(unhandled, item) + } + continue + } + if err := indexer.Index(ctx, data); err != nil { + log.Error("Issue indexer handler: failed to index issue %d: %v", item.ID, err) + unhandled = append(unhandled, item) + continue + } } + return unhandled } @@ -79,7 +107,7 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unable to create issue indexer queue") } default: - issueIndexerQueue = queue.CreateSimpleQueue[*internal.IndexerData](ctx, "issue_indexer", nil) + issueIndexerQueue = queue.CreateSimpleQueue[*IndexerMetadata](ctx, "issue_indexer", nil) } graceful.GetManager().RunAtTerminate(finished) @@ -236,28 +264,8 @@ func UpdateRepoIndexer(ctx context.Context, repo *repo_model.Repository) { // UpdateIssueIndexer add/update an issue to the issue indexer func UpdateIssueIndexer(issue *issues_model.Issue) { - var comments []string - for _, comment := range issue.Comments { - if comment.Type == issues_model.CommentTypeComment { - comments = append(comments, comment.Content) - } - } - issueType := "issue" - if issue.IsPull { - issueType = "pull" - } - indexerData := &internal.IndexerData{ - ID: issue.ID, - RepoID: issue.RepoID, - State: string(issue.State()), - IssueType: issueType, - Title: issue.Title, - Content: issue.Content, - Comments: comments, - } - log.Debug("Adding to channel: %v", indexerData) - if err := issueIndexerQueue.Push(indexerData); err != nil { - log.Error("Unable to push to issue indexer: %v: Error: %v", indexerData, err) + if err := issueIndexerQueue.Push(&IndexerMetadata{ID: issue.ID}); err != nil { + log.Error("Unable to push to issue indexer: %v: Error: %v", issue.ID, err) } } @@ -273,7 +281,7 @@ func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) { if len(ids) == 0 { return } - indexerData := &internal.IndexerData{ + indexerData := &IndexerMetadata{ IDs: ids, IsDelete: true, } diff --git a/modules/indexer/issues/internal/indexer.go b/modules/indexer/issues/internal/indexer.go index b96517bb80db9..e7b1ee3eb0f42 100644 --- a/modules/indexer/issues/internal/indexer.go +++ b/modules/indexer/issues/internal/indexer.go @@ -13,7 +13,7 @@ import ( // Indexer defines an interface to indexer issues contents type Indexer interface { internal.Indexer - Index(ctx context.Context, issue []*IndexerData) error + Index(ctx context.Context, issue ...*IndexerData) error Delete(ctx context.Context, ids ...int64) error Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*SearchResult, error) } @@ -29,7 +29,7 @@ type dummyIndexer struct { internal.Indexer } -func (d *dummyIndexer) Index(ctx context.Context, issue []*IndexerData) error { +func (d *dummyIndexer) Index(ctx context.Context, issue ...*IndexerData) error { return fmt.Errorf("indexer is not ready") } diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 2b52d32302a06..71d4f95be38c8 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -5,15 +5,11 @@ package internal // IndexerData data stored in the issue indexer type IndexerData struct { - ID int64 `json:"id"` - RepoID int64 `json:"repo_id"` - State string `json:"state"` // open, closed, all - IssueType string `json:"type"` // issue or pull - Title string `json:"title"` - Content string `json:"content"` - Comments []string `json:"comments"` - IsDelete bool `json:"is_delete"` - IDs []int64 `json:"ids"` + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + Title string `json:"title"` + Content string `json:"content"` + Comments []string `json:"comments"` } // Match represents on search result diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 2ea06b576c0b8..677b6d7dc83f5 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -38,7 +38,7 @@ func NewIndexer(url, apiKey, indexerName string) *Indexer { } // Index will save the index data -func (b *Indexer) Index(_ context.Context, issues []*internal.IndexerData) error { +func (b *Indexer) Index(_ context.Context, issues ...*internal.IndexerData) error { if len(issues) == 0 { return nil } diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go new file mode 100644 index 0000000000000..ed9ca2834c646 --- /dev/null +++ b/modules/indexer/issues/util.go @@ -0,0 +1,15 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package issues + +import ( + "context" + + "code.gitea.io/gitea/modules/indexer/issues/internal" +) + +func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerData, bool, error) { + // TODO + return nil, false, nil +} From 1a13ce1d9e18bfd343351619ae19082e9b431308 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 16 Jun 2023 14:33:18 +0800 Subject: [PATCH 02/99] fix: use unique queue --- modules/indexer/issues/indexer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 653ef04a9c63c..96e9c4778f27c 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -101,13 +101,13 @@ func InitIssueIndexer(syncReindex bool) { return unhandled } - issueIndexerQueue = queue.CreateSimpleQueue(ctx, "issue_indexer", handler) + issueIndexerQueue = queue.CreateUniqueQueue(ctx, "issue_indexer", handler) if issueIndexerQueue == nil { log.Fatal("Unable to create issue indexer queue") } default: - issueIndexerQueue = queue.CreateSimpleQueue[*IndexerMetadata](ctx, "issue_indexer", nil) + issueIndexerQueue = queue.CreateUniqueQueue[*IndexerMetadata](ctx, "issue_indexer", nil) } graceful.GetManager().RunAtTerminate(finished) From 836ccf08b420bbedea2b90b0a3619fc2e87c005d Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 16 Jun 2023 14:51:00 +0800 Subject: [PATCH 03/99] feat: query IndexerData --- modules/indexer/issues/util.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index ed9ca2834c646..6284d9407030c 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -6,10 +6,39 @@ package issues import ( "context" + issue_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/modules/indexer/issues/internal" ) func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerData, bool, error) { - // TODO - return nil, false, nil + issue, err := issue_model.GetIssueByID(ctx, issueID) + if err != nil { + if issue_model.IsErrIssueNotExist(err) { + return nil, false, nil + } + return nil, false, err + } + + // FIXME: what if users want to search for a review comment of a pull request? + // The comment type is CommentTypeCode or CommentTypeReview. + // But LoadDiscussComments only loads CommentTypeComment. + if err := issue.LoadDiscussComments(ctx); err != nil { + return nil, false, err + } + + comments := make([]string, 0, len(issue.Comments)) + for _, comment := range issue.Comments { + if comment.Content != "" { + // what ever the comment type is, index the content if it is not empty. + comments = append(comments, comment.Content) + } + } + + return &internal.IndexerData{ + ID: issue.ID, + RepoID: issue.RepoID, + Title: issue.Title, + Content: issue.Content, + Comments: comments, + }, true, nil } From d5c91d33ab46108175238d0fb21a814027681288 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 16 Jun 2023 15:42:13 +0800 Subject: [PATCH 04/99] fix: new models --- modules/indexer/issues/internal/model.go | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 71d4f95be38c8..ff1785056f764 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -3,6 +3,8 @@ package internal +import "code.gitea.io/gitea/modules/util" + // IndexerData data stored in the issue indexer type IndexerData struct { ID int64 `json:"id"` @@ -10,6 +12,21 @@ type IndexerData struct { Title string `json:"title"` Content string `json:"content"` Comments []string `json:"comments"` + + IsPublicRepo bool `json:"is_public_repo"` // So if the availability of a repository has changed, we should reindex all issues of the repository + IsPull bool `json:"is_pull"` + IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. + Labels []string `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. + NoLabels bool `json:"no_labels"` // True if Labels is empty + Milestones []int64 `json:"milestones"` // So if the milestones of an issue have changed, we should reindex the issue. + NoMilestones bool `json:"no_milestones"` // True if Milestones is empty + Projects []int64 `json:"projects"` // So if the projects of an issue have changed, we should reindex the issue. + NoProjects bool `json:"no_projects"` // True if Projects is empty + Author int64 `json:"author"` // So if the author of an issue has changed, we should reindex the issue. + Assignee int64 `json:"assignee"` // So if the assignee of an issue has changed, we should reindex the issue. + Mentions []int64 `json:"mentions"` + Reviewers []int64 `json:"reviewers"` // So if the reviewers of an issue have changed, we should reindex the issue. + RequestedReviewers []int64 `json:"requested_reviewers"` // So if the requested reviewers of an issue have changed, we should reindex the issue. } // Match represents on search result @@ -23,3 +40,42 @@ type SearchResult struct { Total int64 Hits []Match } + +// SearchOptions represents search options +// So the search engine should support: +// - Filter by boolean/int value +// - Filter by "array contains any of specified elements" +// - Filter by "array doesn't contain any of specified elements" +type SearchOptions struct { + Keyword string // keyword to search + + Repos []int64 // repository IDs which the issues belong to + AllPublicRepos bool // search all public repositories + + IsPull util.OptionalBool // if the issues is a pull request + Closed util.OptionalBool // if the issues is closed + + Labels []string // labels the issues have + ExcludedLabels []string // labels the issues don't have + NoLabels bool // if the issues have no labels + + Milestones []int64 // milestones the issues have + NoMilestones bool // if the issues have no milestones + + Projects []int64 // projects the issues belong to + NoProjects bool // if the issues have no projects + + Authors []int64 // authors of the issues + + Assignees []int64 // assignees of the issues + NoAssignees bool // if the issues have no assignees + + Mentions []int64 // users mentioned in the issues + + Reviewers []int64 // reviewers of the issues + + RequestReviewers []int64 // users requested to review the issues + + Skip int // skip the first N results + Limit int // limit the number of results +} From 3a2650f8b3d5391ba9c5dcda0629952334bac75c Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 5 Jul 2023 19:06:08 +0800 Subject: [PATCH 05/99] feat: field for sorting --- modules/indexer/issues/internal/model.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index ff1785056f764..80882e2175d69 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -3,16 +3,22 @@ package internal -import "code.gitea.io/gitea/modules/util" +import ( + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) // IndexerData data stored in the issue indexer type IndexerData struct { - ID int64 `json:"id"` - RepoID int64 `json:"repo_id"` + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + + // Fields used for keyword searching Title string `json:"title"` Content string `json:"content"` Comments []string `json:"comments"` + // Fields used for filtering IsPublicRepo bool `json:"is_public_repo"` // So if the availability of a repository has changed, we should reindex all issues of the repository IsPull bool `json:"is_pull"` IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. @@ -27,6 +33,12 @@ type IndexerData struct { Mentions []int64 `json:"mentions"` Reviewers []int64 `json:"reviewers"` // So if the reviewers of an issue have changed, we should reindex the issue. RequestedReviewers []int64 `json:"requested_reviewers"` // So if the requested reviewers of an issue have changed, we should reindex the issue. + + // Fields used for sorting + CreatedAt timeutil.TimeStamp `json:"created_at"` + UpdatedAt timeutil.TimeStamp `json:"updated_at"` + CommentCount int64 `json:"comment_count"` + DueDate timeutil.TimeStamp `json:"due_date"` } // Match represents on search result @@ -39,6 +51,10 @@ type Match struct { type SearchResult struct { Total int64 Hits []Match + + // Imprecise indicates that the result is not accurate, and it needs second filtering and sorting by database. + // It could be removed when all engines support filtering and sorting. + Imprecise bool } // SearchOptions represents search options From 9f464ed351327ace589361425e834bb6a7ec9309 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 12 Jul 2023 15:42:15 +0800 Subject: [PATCH 06/99] fix: remove IsPublicRepo --- modules/indexer/issues/internal/model.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 80882e2175d69..c5b7577bf035b 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -19,7 +19,6 @@ type IndexerData struct { Comments []string `json:"comments"` // Fields used for filtering - IsPublicRepo bool `json:"is_public_repo"` // So if the availability of a repository has changed, we should reindex all issues of the repository IsPull bool `json:"is_pull"` IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. Labels []string `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. @@ -65,8 +64,7 @@ type SearchResult struct { type SearchOptions struct { Keyword string // keyword to search - Repos []int64 // repository IDs which the issues belong to - AllPublicRepos bool // search all public repositories + Repos []int64 // repository IDs which the issues belong to IsPull util.OptionalBool // if the issues is a pull request Closed util.OptionalBool // if the issues is closed From b1bcb88e14cb3aa8f94faf0c031686f30b4e1306 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 12 Jul 2023 15:43:33 +0800 Subject: [PATCH 07/99] fix: sortby --- modules/indexer/issues/internal/model.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index c5b7577bf035b..8d4154c715341 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -92,4 +92,6 @@ type SearchOptions struct { Skip int // skip the first N results Limit int // limit the number of results + + SortBy string // sort by field, could be "created", "updated", "comments", "due_date", add "-" prefix to sort in descending order } From ed16cd3216ec5f8f31b987166b3c4c353bb544e9 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 12 Jul 2023 15:50:16 +0800 Subject: [PATCH 08/99] feat: add mapping --- modules/indexer/issues/bleve/bleve.go | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 05b024936b77e..550ceecfc32de 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -74,10 +74,38 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) { textFieldMapping := bleve.NewTextFieldMapping() textFieldMapping.Store = false textFieldMapping.IncludeInAll = false + + boolFieldMapping := bleve.NewBooleanFieldMapping() + boolFieldMapping.Store = false + boolFieldMapping.IncludeInAll = false + + numberFieldMapping := bleve.NewNumericFieldMapping() + numberFieldMapping.Store = false + numberFieldMapping.IncludeInAll = false + docMapping.AddFieldMappingsAt("title", textFieldMapping) docMapping.AddFieldMappingsAt("content", textFieldMapping) docMapping.AddFieldMappingsAt("comments", textFieldMapping) + docMapping.AddFieldMappingsAt("is_pull", boolFieldMapping) + docMapping.AddFieldMappingsAt("is_closed", boolFieldMapping) + docMapping.AddFieldMappingsAt("labels", textFieldMapping) + docMapping.AddFieldMappingsAt("no_labels", boolFieldMapping) + docMapping.AddFieldMappingsAt("milestones", numberFieldMapping) + docMapping.AddFieldMappingsAt("no_milestones", boolFieldMapping) + docMapping.AddFieldMappingsAt("projects", numberFieldMapping) + docMapping.AddFieldMappingsAt("no_projects", boolFieldMapping) + docMapping.AddFieldMappingsAt("author", numberFieldMapping) + docMapping.AddFieldMappingsAt("assignee", numberFieldMapping) + docMapping.AddFieldMappingsAt("mentions", numberFieldMapping) + docMapping.AddFieldMappingsAt("reviewers", numberFieldMapping) + docMapping.AddFieldMappingsAt("requested_reviewers", numberFieldMapping) + + docMapping.AddFieldMappingsAt("created_at", numberFieldMapping) + docMapping.AddFieldMappingsAt("updated_at", numberFieldMapping) + docMapping.AddFieldMappingsAt("closed_at", numberFieldMapping) + docMapping.AddFieldMappingsAt("due_date", numberFieldMapping) + if err := addUnicodeNormalizeTokenFilter(mapping); err != nil { return nil, err } else if err = mapping.AddCustomAnalyzer(issueIndexerAnalyzer, map[string]any{ From 84d6c0b65b0efb27ea98b7fc83c4575a67ed6c32 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 13 Jul 2023 10:35:22 +0800 Subject: [PATCH 09/99] fix: build query --- modules/indexer/code/bleve/bleve.go | 13 ++----------- modules/indexer/internal/bleve/query.go | 26 +++++++++++++++++++++++++ modules/indexer/issues/bleve/bleve.go | 24 ++++------------------- 3 files changed, 32 insertions(+), 31 deletions(-) create mode 100644 modules/indexer/internal/bleve/query.go diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index 1e34226e8df14..0bfd85cb3f30d 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -41,15 +41,6 @@ const ( maxBatchSize = 16 ) -// numericEqualityQuery a numeric equality query for the given value and field -func numericEqualityQuery(value int64, field string) *query.NumericRangeQuery { - f := float64(value) - tru := true - q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru) - q.SetField(field) - return q -} - func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error { return m.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{ "type": unicodenorm.Name, @@ -225,7 +216,7 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st // Delete deletes indexes by ids func (b *Indexer) Delete(_ context.Context, repoID int64) error { - query := numericEqualityQuery(repoID, "RepoID") + query := inner_bleve.NumericEqualityQuery(repoID, "RepoID") searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false) result, err := b.inner.Indexer.Search(searchRequest) if err != nil { @@ -262,7 +253,7 @@ func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword if len(repoIDs) > 0 { repoQueries := make([]query.Query, 0, len(repoIDs)) for _, repoID := range repoIDs { - repoQueries = append(repoQueries, numericEqualityQuery(repoID, "RepoID")) + repoQueries = append(repoQueries, inner_bleve.NumericEqualityQuery(repoID, "RepoID")) } indexerQuery = bleve.NewConjunctionQuery( diff --git a/modules/indexer/internal/bleve/query.go b/modules/indexer/internal/bleve/query.go new file mode 100644 index 0000000000000..d2f6d13049bbd --- /dev/null +++ b/modules/indexer/internal/bleve/query.go @@ -0,0 +1,26 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package bleve + +import ( + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" +) + +// NumericEqualityQuery generates a numeric equality query for the given value and field +func NumericEqualityQuery(value int64, field string) *query.NumericRangeQuery { + f := float64(value) + tru := true + q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru) + q.SetField(field) + return q +} + +// MatchPhraseQuery generates a match phrase query for the given phrase, field and analyzer +func MatchPhraseQuery(matchPhrase, field, analyzer string) *query.MatchPhraseQuery { + q := bleve.NewMatchPhraseQuery(matchPhrase) + q.FieldVal = field + q.Analyzer = analyzer + return q +} diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 550ceecfc32de..eb3082370f900 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -26,22 +26,6 @@ const ( issueIndexerLatestVersion = 3 ) -// numericEqualityQuery a numeric equality query for the given value and field -func numericEqualityQuery(value int64, field string) *query.NumericRangeQuery { - f := float64(value) - tru := true - q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru) - q.SetField(field) - return q -} - -func newMatchPhraseQuery(matchPhrase, field, analyzer string) *query.MatchPhraseQuery { - q := bleve.NewMatchPhraseQuery(matchPhrase) - q.FieldVal = field - q.Analyzer = analyzer - return q -} - const unicodeNormalizeName = "unicodeNormalize" func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error { @@ -169,7 +153,7 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error { func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) { var repoQueriesP []*query.NumericRangeQuery for _, repoID := range repoIDs { - repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "repo_id")) + repoQueriesP = append(repoQueriesP, inner_bleve.NumericEqualityQuery(repoID, "repo_id")) } repoQueries := make([]query.Query, len(repoQueriesP)) for i, v := range repoQueriesP { @@ -179,9 +163,9 @@ func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, l indexerQuery := bleve.NewConjunctionQuery( bleve.NewDisjunctionQuery(repoQueries...), bleve.NewDisjunctionQuery( - newMatchPhraseQuery(keyword, "title", issueIndexerAnalyzer), - newMatchPhraseQuery(keyword, "content", issueIndexerAnalyzer), - newMatchPhraseQuery(keyword, "comments", issueIndexerAnalyzer), + inner_bleve.MatchPhraseQuery(keyword, "title", issueIndexerAnalyzer), + inner_bleve.MatchPhraseQuery(keyword, "content", issueIndexerAnalyzer), + inner_bleve.MatchPhraseQuery(keyword, "comments", issueIndexerAnalyzer), )) search := bleve.NewSearchRequestOptions(indexerQuery, limit, start, false) search.SortBy([]string{"-_score"}) From 29e3c7baa43cc480cddcfe88addabf0c9151f1f4 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 13 Jul 2023 10:42:40 +0800 Subject: [PATCH 10/99] fix: label int64 --- modules/indexer/issues/bleve/bleve.go | 2 +- modules/indexer/issues/internal/model.go | 32 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index eb3082370f900..a534434d09907 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -73,7 +73,7 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) { docMapping.AddFieldMappingsAt("is_pull", boolFieldMapping) docMapping.AddFieldMappingsAt("is_closed", boolFieldMapping) - docMapping.AddFieldMappingsAt("labels", textFieldMapping) + docMapping.AddFieldMappingsAt("labels", numberFieldMapping) docMapping.AddFieldMappingsAt("no_labels", boolFieldMapping) docMapping.AddFieldMappingsAt("milestones", numberFieldMapping) docMapping.AddFieldMappingsAt("no_milestones", boolFieldMapping) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 8d4154c715341..20a1117c6ad26 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -19,19 +19,19 @@ type IndexerData struct { Comments []string `json:"comments"` // Fields used for filtering - IsPull bool `json:"is_pull"` - IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. - Labels []string `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. - NoLabels bool `json:"no_labels"` // True if Labels is empty - Milestones []int64 `json:"milestones"` // So if the milestones of an issue have changed, we should reindex the issue. - NoMilestones bool `json:"no_milestones"` // True if Milestones is empty - Projects []int64 `json:"projects"` // So if the projects of an issue have changed, we should reindex the issue. - NoProjects bool `json:"no_projects"` // True if Projects is empty - Author int64 `json:"author"` // So if the author of an issue has changed, we should reindex the issue. - Assignee int64 `json:"assignee"` // So if the assignee of an issue has changed, we should reindex the issue. - Mentions []int64 `json:"mentions"` - Reviewers []int64 `json:"reviewers"` // So if the reviewers of an issue have changed, we should reindex the issue. - RequestedReviewers []int64 `json:"requested_reviewers"` // So if the requested reviewers of an issue have changed, we should reindex the issue. + IsPull bool `json:"is_pull"` + IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. + Labels []int64 `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. + NoLabels bool `json:"no_labels"` // True if Labels is empty + Milestones []int64 `json:"milestones"` // So if the milestones of an issue have changed, we should reindex the issue. + NoMilestones bool `json:"no_milestones"` // True if Milestones is empty + Projects []int64 `json:"projects"` // So if the projects of an issue have changed, we should reindex the issue. + NoProjects bool `json:"no_projects"` // True if Projects is empty + Author int64 `json:"author"` // So if the author of an issue has changed, we should reindex the issue. + Assignee int64 `json:"assignee"` // So if the assignee of an issue has changed, we should reindex the issue. + Mentions []int64 `json:"mentions"` + Reviewers []int64 `json:"reviewers"` // So if the reviewers of an issue have changed, we should reindex the issue. + RequestedReviewers []int64 `json:"requested_reviewers"` // So if the requested reviewers of an issue have changed, we should reindex the issue. // Fields used for sorting CreatedAt timeutil.TimeStamp `json:"created_at"` @@ -69,9 +69,9 @@ type SearchOptions struct { IsPull util.OptionalBool // if the issues is a pull request Closed util.OptionalBool // if the issues is closed - Labels []string // labels the issues have - ExcludedLabels []string // labels the issues don't have - NoLabels bool // if the issues have no labels + Labels []int64 // labels the issues have + ExcludedLabels []int64 // labels the issues don't have + NoLabels bool // if the issues have no labels Milestones []int64 // milestones the issues have NoMilestones bool // if the issues have no milestones From 7a83ee95d14a729dd1e1509100fd7f060cb31862 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 13 Jul 2023 11:38:18 +0800 Subject: [PATCH 11/99] feat: SearchIssues --- modules/indexer/issues/bleve/bleve.go | 20 ++++++------ modules/indexer/issues/db/db.go | 9 +++--- .../issues/elasticsearch/elasticsearch.go | 19 +++++------ modules/indexer/issues/indexer.go | 26 +++++++++++++-- modules/indexer/issues/internal/indexer.go | 8 ++--- .../indexer/issues/meilisearch/meilisearch.go | 32 +++++++++++-------- 6 files changed, 73 insertions(+), 41 deletions(-) diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index a534434d09907..309aa0c86fd38 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -150,9 +150,9 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs -func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) { +func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { var repoQueriesP []*query.NumericRangeQuery - for _, repoID := range repoIDs { + for _, repoID := range options.Repos { repoQueriesP = append(repoQueriesP, inner_bleve.NumericEqualityQuery(repoID, "repo_id")) } repoQueries := make([]query.Query, len(repoQueriesP)) @@ -163,11 +163,11 @@ func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, l indexerQuery := bleve.NewConjunctionQuery( bleve.NewDisjunctionQuery(repoQueries...), bleve.NewDisjunctionQuery( - inner_bleve.MatchPhraseQuery(keyword, "title", issueIndexerAnalyzer), - inner_bleve.MatchPhraseQuery(keyword, "content", issueIndexerAnalyzer), - inner_bleve.MatchPhraseQuery(keyword, "comments", issueIndexerAnalyzer), + inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer), + inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer), + inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer), )) - search := bleve.NewSearchRequestOptions(indexerQuery, limit, start, false) + search := bleve.NewSearchRequestOptions(indexerQuery, options.Limit, options.Skip, false) search.SortBy([]string{"-_score"}) result, err := b.inner.Indexer.SearchInContext(ctx, search) @@ -175,8 +175,10 @@ func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, l return nil, err } - ret := internal.SearchResult{ - Hits: make([]internal.Match, 0, len(result.Hits)), + ret := &internal.SearchResult{ + Total: int64(result.Total), + Hits: make([]internal.Match, 0, len(result.Hits)), + Imprecise: true, } for _, hit := range result.Hits { id, err := indexer_internal.ParseBase36(hit.ID) @@ -187,5 +189,5 @@ func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, l ID: id, }) } - return &ret, nil + return ret, nil } diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index b41147c3677c4..13bd2d90b0e18 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -36,14 +36,15 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error { } // Search searches for issues -func (i *Indexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) { - total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start) +func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { + total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, options.Keyword, options.Repos, options.Limit, options.Skip) if err != nil { return nil, err } result := internal.SearchResult{ - Total: total, - Hits: make([]internal.Match, 0, limit), + Total: total, + Hits: make([]internal.Match, 0, options.Limit), + Imprecise: true, } for _, id := range ids { result.Hits = append(result.Hits, internal.Match{ diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 6d586c6647fbb..b95bf3a3ead42 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -140,13 +140,13 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs -func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) { - kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments") +func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { + kwQuery := elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments") query := elastic.NewBoolQuery() query = query.Must(kwQuery) - if len(repoIDs) > 0 { - repoStrs := make([]any, 0, len(repoIDs)) - for _, repoID := range repoIDs { + if len(options.Repos) > 0 { + repoStrs := make([]any, 0, len(options.Repos)) + for _, repoID := range options.Repos { repoStrs = append(repoStrs, repoID) } repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) @@ -156,13 +156,13 @@ func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, l Index(b.inner.VersionedIndexName()). Query(query). Sort("_score", false). - From(start).Size(limit). + From(options.Skip).Size(options.Limit). Do(ctx) if err != nil { return nil, err } - hits := make([]internal.Match, 0, limit) + hits := make([]internal.Match, 0, options.Limit) for _, hit := range searchResult.Hits.Hits { id, _ := strconv.ParseInt(hit.Id, 10, 64) hits = append(hits, internal.Match{ @@ -171,7 +171,8 @@ func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, l } return &internal.SearchResult{ - Total: searchResult.TotalHits(), - Hits: hits, + Total: searchResult.TotalHits(), + Hits: hits, + Imprecise: true, }, nil } diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 96e9c4778f27c..b8611abee20bc 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -290,12 +290,18 @@ func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) { } } +// Deprecated: use SearchIssues instead // SearchIssuesByKeyword search issue ids by keywords and repo id // WARNNING: You have to ensure user have permission to visit repoIDs' issues -func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword, state string) ([]int64, error) { +func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) { var issueIDs []int64 indexer := *globalIndexer.Load() - res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0, state) + res, err := indexer.Search(ctx, &internal.SearchOptions{ + Keyword: keyword, + Repos: repoIDs, + Limit: 50, + Skip: 0, + }) if err != nil { return nil, err } @@ -309,3 +315,19 @@ func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword, state func IsAvailable(ctx context.Context) bool { return (*globalIndexer.Load()).Ping(ctx) == nil } + +// SearchOptions indicates the options for searching issues +type SearchOptions internal.SearchOptions + +// SearchResults indicates the search results +type SearchResults internal.SearchResult + +// SearchIssues search issues by options +func SearchIssues(ctx context.Context, opts *SearchOptions) (*SearchResults, error) { + indexer := *globalIndexer.Load() + res, err := indexer.Search(ctx, (*internal.SearchOptions)(opts)) + if err != nil { + return nil, err + } + return (*SearchResults)(res), nil +} diff --git a/modules/indexer/issues/internal/indexer.go b/modules/indexer/issues/internal/indexer.go index e7b1ee3eb0f42..95740bc598d6f 100644 --- a/modules/indexer/issues/internal/indexer.go +++ b/modules/indexer/issues/internal/indexer.go @@ -15,7 +15,7 @@ type Indexer interface { internal.Indexer Index(ctx context.Context, issue ...*IndexerData) error Delete(ctx context.Context, ids ...int64) error - Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*SearchResult, error) + Search(ctx context.Context, options *SearchOptions) (*SearchResult, error) } // NewDummyIndexer returns a dummy indexer @@ -29,14 +29,14 @@ type dummyIndexer struct { internal.Indexer } -func (d *dummyIndexer) Index(ctx context.Context, issue ...*IndexerData) error { +func (d *dummyIndexer) Index(_ context.Context, _ ...*IndexerData) error { return fmt.Errorf("indexer is not ready") } -func (d *dummyIndexer) Delete(ctx context.Context, ids ...int64) error { +func (d *dummyIndexer) Delete(_ context.Context, _ ...int64) error { return fmt.Errorf("indexer is not ready") } -func (d *dummyIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*SearchResult, error) { +func (d *dummyIndexer) Search(_ context.Context, _ *SearchOptions) (*SearchResult, error) { return nil, fmt.Errorf("indexer is not ready") } diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 677b6d7dc83f5..158b6bb558e04 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -70,23 +70,28 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs -func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) { - repoFilters := make([]string, 0, len(repoIDs)) - for _, repoID := range repoIDs { +func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { + repoFilters := make([]string, 0, len(options.Repos)) + for _, repoID := range options.Repos { repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10)) } filter := strings.Join(repoFilters, " OR ") + + // TBC: + /* if state == "open" || state == "closed" { - if filter != "" { - filter = "(" + filter + ") AND state = " + state - } else { - filter = "state = " + state + if filter != "" { + filter = "(" + filter + ") AND state = " + state + } else { + filter = "state = " + state + } } - } - searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(keyword, &meilisearch.SearchRequest{ + */ + + searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(options.Keyword, &meilisearch.SearchRequest{ Filter: filter, - Limit: int64(limit), - Offset: int64(start), + Limit: int64(options.Limit), + Offset: int64(options.Skip), }) if err != nil { return nil, err @@ -99,7 +104,8 @@ func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, l }) } return &internal.SearchResult{ - Total: searchRes.TotalHits, - Hits: hits, + Total: searchRes.TotalHits, + Hits: hits, + Imprecise: true, }, nil } From 39c07695612160536e3d0682481db989639df615 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 13 Jul 2023 11:52:54 +0800 Subject: [PATCH 12/99] test: use SearchIssues --- modules/indexer/issues/indexer.go | 20 +++--- modules/indexer/issues/indexer_test.go | 88 +++++++++++++++++++------- 2 files changed, 76 insertions(+), 32 deletions(-) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index b8611abee20bc..bc871fefb91d4 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -319,15 +319,19 @@ func IsAvailable(ctx context.Context) bool { // SearchOptions indicates the options for searching issues type SearchOptions internal.SearchOptions -// SearchResults indicates the search results -type SearchResults internal.SearchResult - -// SearchIssues search issues by options -func SearchIssues(ctx context.Context, opts *SearchOptions) (*SearchResults, error) { +// SearchIssues search issues by options. +// It returns issue ids and a bool value indicates if the result is imprecise. +func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, bool, error) { indexer := *globalIndexer.Load() - res, err := indexer.Search(ctx, (*internal.SearchOptions)(opts)) + result, err := indexer.Search(ctx, (*internal.SearchOptions)(opts)) if err != nil { - return nil, err + return nil, false, err } - return (*SearchResults)(res), nil + + ret := make([]int64, 0, len(result.Hits)) + for _, hit := range result.Hits { + ret = append(ret, hit.ID) + } + + return ret, result.Imprecise, nil } diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index 757eb2f3d9338..bd15d1e15707b 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -50,21 +50,41 @@ func TestBleveSearchIssues(t *testing.T) { time.Sleep(5 * time.Second) - ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2", "") - assert.NoError(t, err) - assert.EqualValues(t, []int64{2}, ids) + t.Run("issue2", func(t *testing.T) { + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + Keyword: "issue2", + Repos: []int64{1}, + }) + assert.NoError(t, err) + assert.EqualValues(t, []int64{2}, ids) + }) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first", "") - assert.NoError(t, err) - assert.EqualValues(t, []int64{1}, ids) + t.Run("first", func(t *testing.T) { + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + Keyword: "first", + Repos: []int64{1}, + }) + assert.NoError(t, err) + assert.EqualValues(t, []int64{1}, ids) + }) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for", "") - assert.NoError(t, err) - assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) + t.Run("for", func(t *testing.T) { + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + Keyword: "for", + Repos: []int64{1}, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) + }) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good", "") - assert.NoError(t, err) - assert.EqualValues(t, []int64{1}, ids) + t.Run("good", func(t *testing.T) { + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + Keyword: "good", + Repos: []int64{1}, + }) + assert.NoError(t, err) + assert.EqualValues(t, []int64{1}, ids) + }) } func TestDBSearchIssues(t *testing.T) { @@ -73,19 +93,39 @@ func TestDBSearchIssues(t *testing.T) { setting.Indexer.IssueType = "db" InitIssueIndexer(true) - ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2", "") - assert.NoError(t, err) - assert.EqualValues(t, []int64{2}, ids) + t.Run("issue2", func(t *testing.T) { + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + Keyword: "issue2", + Repos: []int64{1}, + }) + assert.NoError(t, err) + assert.EqualValues(t, []int64{2}, ids) + }) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first", "") - assert.NoError(t, err) - assert.EqualValues(t, []int64{1}, ids) + t.Run("first", func(t *testing.T) { + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + Keyword: "first", + Repos: []int64{1}, + }) + assert.NoError(t, err) + assert.EqualValues(t, []int64{1}, ids) + }) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for", "") - assert.NoError(t, err) - assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) + t.Run("for", func(t *testing.T) { + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + Keyword: "for", + Repos: []int64{1}, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) + }) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good", "") - assert.NoError(t, err) - assert.EqualValues(t, []int64{1}, ids) + t.Run("good", func(t *testing.T) { + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + Keyword: "good", + Repos: []int64{1}, + }) + assert.NoError(t, err) + assert.EqualValues(t, []int64{1}, ids) + }) } From abdbfbe5e95071841ebb871e736ec97e9c1e2e66 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 13 Jul 2023 11:57:07 +0800 Subject: [PATCH 13/99] fix: set default limit --- modules/indexer/issues/indexer.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index bc871fefb91d4..fdc50095d38e6 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -322,6 +322,12 @@ type SearchOptions internal.SearchOptions // SearchIssues search issues by options. // It returns issue ids and a bool value indicates if the result is imprecise. func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, bool, error) { + if opts.Limit <= 0 { + // It's meaningless to search with limit <= 0, probably the caller missed to set it. + // If the caller really wants to search all issues, set limit to a large number. + opts.Limit = 50 + } + indexer := *globalIndexer.Load() result, err := indexer.Search(ctx, (*internal.SearchOptions)(opts)) if err != nil { From c6114f206830bd1492a6815e3adec47d0d216fe1 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 13 Jul 2023 15:39:46 +0800 Subject: [PATCH 14/99] feat: RegisterFilterIssuesFunc --- modules/indexer/issues/dbfilter.go | 29 ++++++++++++++++++++++++++ modules/indexer/issues/indexer.go | 14 ++++++++++--- modules/indexer/issues/indexer_test.go | 16 +++++++------- 3 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 modules/indexer/issues/dbfilter.go diff --git a/modules/indexer/issues/dbfilter.go b/modules/indexer/issues/dbfilter.go new file mode 100644 index 0000000000000..58e1942fb009e --- /dev/null +++ b/modules/indexer/issues/dbfilter.go @@ -0,0 +1,29 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package issues + +import ( + "context" +) + +// filterIssuesByDB filters the given issuesIDs by the database. +// It is used to filter out issues coming from some indexers that are not supported fining filtering. +// Once all indexers support filtering, this function can be removed. +func filterIssuesByDB(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) { + if filterIssuesFunc != nil { + return filterIssuesFunc(ctx, issuesIDs, options) + } + return issuesIDs, nil +} + +var filterIssuesFunc func(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) + +// RegisterFilterIssuesFunc registers a function to filter issues by database. +// It's for issue_model to register its own filter function. +// Why not just put the function body here? +// Because modules can't depend on models by design. +// Although some packages have broken this rule, it's still a good practice to follow it. +func RegisterFilterIssuesFunc(f func(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error)) { + filterIssuesFunc = f +} diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index fdc50095d38e6..386330404f8bd 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -321,7 +321,7 @@ type SearchOptions internal.SearchOptions // SearchIssues search issues by options. // It returns issue ids and a bool value indicates if the result is imprecise. -func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, bool, error) { +func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, error) { if opts.Limit <= 0 { // It's meaningless to search with limit <= 0, probably the caller missed to set it. // If the caller really wants to search all issues, set limit to a large number. @@ -331,7 +331,7 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, bool, erro indexer := *globalIndexer.Load() result, err := indexer.Search(ctx, (*internal.SearchOptions)(opts)) if err != nil { - return nil, false, err + return nil, err } ret := make([]int64, 0, len(result.Hits)) @@ -339,5 +339,13 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, bool, erro ret = append(ret, hit.ID) } - return ret, result.Imprecise, nil + if result.Imprecise { + ret, err := filterIssuesByDB(ctx, ret, opts) + if err != nil { + return nil, err + } + return ret, nil + } + + return ret, nil } diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index bd15d1e15707b..186610e762d15 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -51,7 +51,7 @@ func TestBleveSearchIssues(t *testing.T) { time.Sleep(5 * time.Second) t.Run("issue2", func(t *testing.T) { - ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "issue2", Repos: []int64{1}, }) @@ -60,7 +60,7 @@ func TestBleveSearchIssues(t *testing.T) { }) t.Run("first", func(t *testing.T) { - ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "first", Repos: []int64{1}, }) @@ -69,7 +69,7 @@ func TestBleveSearchIssues(t *testing.T) { }) t.Run("for", func(t *testing.T) { - ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "for", Repos: []int64{1}, }) @@ -78,7 +78,7 @@ func TestBleveSearchIssues(t *testing.T) { }) t.Run("good", func(t *testing.T) { - ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "good", Repos: []int64{1}, }) @@ -94,7 +94,7 @@ func TestDBSearchIssues(t *testing.T) { InitIssueIndexer(true) t.Run("issue2", func(t *testing.T) { - ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "issue2", Repos: []int64{1}, }) @@ -103,7 +103,7 @@ func TestDBSearchIssues(t *testing.T) { }) t.Run("first", func(t *testing.T) { - ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "first", Repos: []int64{1}, }) @@ -112,7 +112,7 @@ func TestDBSearchIssues(t *testing.T) { }) t.Run("for", func(t *testing.T) { - ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "for", Repos: []int64{1}, }) @@ -121,7 +121,7 @@ func TestDBSearchIssues(t *testing.T) { }) t.Run("good", func(t *testing.T) { - ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "good", Repos: []int64{1}, }) From f8f0b72e8ac3e068e0869687fe702f265121f82c Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 13 Jul 2023 18:55:35 +0800 Subject: [PATCH 15/99] feat: WIP filterIssuesOfSearchResult --- models/issues/issue_search.go | 68 ++++++++++++++++++++++++ modules/indexer/issues/bleve/bleve.go | 1 + modules/indexer/issues/internal/model.go | 58 ++++++++++---------- 3 files changed, 99 insertions(+), 28 deletions(-) diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index 9fd13f09956af..8d6369968272e 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -13,6 +13,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" + indexer_issues "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/util" "xorm.io/builder" @@ -489,3 +490,70 @@ func SearchIssueIDsByKeyword(ctx context.Context, kw string, repoIDs []int64, li return total, ids, nil } + +func init() { + indexer_issues.RegisterFilterIssuesFunc(filterIssuesOfSearchResult) +} + +func filterIssuesOfSearchResult(ctx context.Context, issuesIDs []int64, options *indexer_issues.SearchOptions) ([]int64, error) { + convertID := func(id *int64) int64 { + if id == nil { + return db.NoConditionID + } + return *id + } + convertIDs := func(ids []int64, no bool) []int64 { + if no { + return []int64{db.NoConditionID} + } + return ids + } + convertBool := func(b *bool) util.OptionalBool { + if b == nil { + return util.OptionalBoolNone + } + return util.OptionalBoolOf(*b) + } + convertLabelIDs := func(includes, excludes []int64, no bool) []int64 { + if no { + return []int64{0} // It's zero, not db.NoConditionID, + } + ret := make([]int64, 0, len(includes)+len(excludes)) + ret = append(ret, includes...) + for _, id := range excludes { + ret = append(ret, -id) + } + } + + opts := &IssuesOptions{ + ListOptions: db.ListOptions{ + ListAll: true, + }, + RepoIDs: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + RepoCond: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + AssigneeID: convertID(options.AssigneeID), + PosterID: convertID(options.PosterID), + MentionedID: convertID(options.MentionID), + ReviewRequestedID: convertID(options.ReviewRequestedID), + ReviewedID: convertID(options.ReviewedID), + SubscriberID: convertID(options.SubscriberID), + MilestoneIDs: convertIDs(options.MilestoneIDs, options.NoMilestone), + ProjectID: convertID(options.ProjectID), + ProjectBoardID: convertID(options.ProjectBoardID), + IsClosed: convertBool(options.IsClosed), + IsPull: convertBool(options.IsPull), + LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.NoLabel), + IncludedLabelNames: nil, // use LabelIDs instead + ExcludedLabelNames: nil, // use LabelIDs instead + IncludeMilestones: nil, // use MilestoneIDs instead + SortType: "", // TBC + IssueIDs: issuesIDs, + UpdatedAfterUnix: 0, + UpdatedBeforeUnix: 0, + PriorityRepoID: 0, + IsArchived: 0, + Org: nil, + Team: nil, + User: nil, + } +} diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 309aa0c86fd38..73bc244129324 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -71,6 +71,7 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) { docMapping.AddFieldMappingsAt("content", textFieldMapping) docMapping.AddFieldMappingsAt("comments", textFieldMapping) + // TBC: IndexerData has been changed, but the mapping has not been updated docMapping.AddFieldMappingsAt("is_pull", boolFieldMapping) docMapping.AddFieldMappingsAt("is_closed", boolFieldMapping) docMapping.AddFieldMappingsAt("labels", numberFieldMapping) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 20a1117c6ad26..4f12cb754bce8 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -5,7 +5,6 @@ package internal import ( "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" ) // IndexerData data stored in the issue indexer @@ -20,18 +19,20 @@ type IndexerData struct { // Fields used for filtering IsPull bool `json:"is_pull"` - IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. - Labels []int64 `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. - NoLabels bool `json:"no_labels"` // True if Labels is empty - Milestones []int64 `json:"milestones"` // So if the milestones of an issue have changed, we should reindex the issue. - NoMilestones bool `json:"no_milestones"` // True if Milestones is empty - Projects []int64 `json:"projects"` // So if the projects of an issue have changed, we should reindex the issue. - NoProjects bool `json:"no_projects"` // True if Projects is empty - Author int64 `json:"author"` // So if the author of an issue has changed, we should reindex the issue. - Assignee int64 `json:"assignee"` // So if the assignee of an issue has changed, we should reindex the issue. - Mentions []int64 `json:"mentions"` - Reviewers []int64 `json:"reviewers"` // So if the reviewers of an issue have changed, we should reindex the issue. - RequestedReviewers []int64 `json:"requested_reviewers"` // So if the requested reviewers of an issue have changed, we should reindex the issue. + IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. + Labels []int64 `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. + NoLabels bool `json:"no_labels"` // True if Labels is empty + MilestoneIDs []int64 `json:"milestone_ids"` // So if the milestones of an issue have changed, we should reindex the issue. + NoMilestone bool `json:"no_milestone"` // True if Milestones is empty + ProjectIDs []int64 `json:"project_ids"` // So if the projects of an issue have changed, we should reindex the issue. + ProjectBoardIDs []int64 `json:"project_board_ids"` // So if the projects of an issue have changed, we should reindex the issue. + NoProject bool `json:"no_project"` // True if ProjectIDs is empty + PosterID int64 `json:"poster_id"` + AssigneeID int64 `json:"assignee_id"` // So if the assignee of an issue has changed, we should reindex the issue. + MentionIDs []int64 `json:"mention_ids"` + ReviewedIDs []int64 `json:"reviewed_ids"` // So if the reviewers of an issue have changed, we should reindex the issue. + ReviewRequestedIDs []int64 `json:"review_requested_ids"` // So if the requested reviewers of an issue have changed, we should reindex the issue. + SubscriberIDs []int64 `json:"subscriber_ids"` // So if the subscribers of an issue have changed, we should reindex the issue. // Fields used for sorting CreatedAt timeutil.TimeStamp `json:"created_at"` @@ -66,29 +67,30 @@ type SearchOptions struct { Repos []int64 // repository IDs which the issues belong to - IsPull util.OptionalBool // if the issues is a pull request - Closed util.OptionalBool // if the issues is closed + IsPull *bool // if the issues is a pull request + IsClosed *bool // if the issues is closed - Labels []int64 // labels the issues have - ExcludedLabels []int64 // labels the issues don't have - NoLabels bool // if the issues have no labels + IncludedLabelIDs []int64 // labels the issues have + ExcludedLabelIDs []int64 // labels the issues don't have + NoLabel bool // if the issues have no label, if true, IncludedLabelIDs and ExcludedLabelIDs will be ignored - Milestones []int64 // milestones the issues have - NoMilestones bool // if the issues have no milestones + MilestoneIDs []int64 // milestones the issues have + NoMilestone bool // if the issues have no milestones, if true, MilestoneIDs will be ignored - Projects []int64 // projects the issues belong to - NoProjects bool // if the issues have no projects + ProjectID *int64 // project the issues belong to + ProjectBoardID *int64 // project board the issues belong to - Authors []int64 // authors of the issues + PosterID *int64 // poster of the issues - Assignees []int64 // assignees of the issues - NoAssignees bool // if the issues have no assignees + AssigneeID *int64 // assignee of the issues, zero means no assignee - Mentions []int64 // users mentioned in the issues + MentionID *int64 // mentioned user of the issues - Reviewers []int64 // reviewers of the issues + ReviewedID *int64 // reviewer of the issues - RequestReviewers []int64 // users requested to review the issues + ReviewRequestedID *int64 // requested reviewer of the issues + + SubscriberID *int64 // subscriber of the issues Skip int // skip the first N results Limit int // limit the number of results From 668d831430457dddc1ba8428addcc6a3da9a1e14 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 14 Jul 2023 12:14:14 +0800 Subject: [PATCH 16/99] feat: register db searching --- models/issues/issue_search.go | 98 +++++++++++++++++-- models/issues/issue_test.go | 24 ----- modules/indexer/issues/bleve/bleve.go | 2 +- modules/indexer/issues/db/db.go | 21 ++-- modules/indexer/issues/dbfilter.go | 47 ++++++--- .../issues/elasticsearch/elasticsearch.go | 6 +- modules/indexer/issues/indexer.go | 15 ++- modules/indexer/issues/indexer_test.go | 16 +-- modules/indexer/issues/internal/model.go | 56 +++++++---- .../indexer/issues/meilisearch/meilisearch.go | 4 +- 10 files changed, 194 insertions(+), 95 deletions(-) diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index 8d6369968272e..525c00d39957f 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -448,6 +448,7 @@ func Issues(ctx context.Context, opts *IssuesOptions) ([]*Issue, error) { return issues, nil } +// Deprecated: use `searchIssues` or `indexer/issues.Search` instead // SearchIssueIDsByKeyword search issues on database func SearchIssueIDsByKeyword(ctx context.Context, kw string, repoIDs []int64, limit, start int) (int64, []int64, error) { repoCond := builder.In("repo_id", repoIDs) @@ -492,7 +493,50 @@ func SearchIssueIDsByKeyword(ctx context.Context, kw string, repoIDs []int64, li } func init() { - indexer_issues.RegisterFilterIssuesFunc(filterIssuesOfSearchResult) + indexer_issues.RegisterReFilterFunc(filterIssuesOfSearchResult) + indexer_issues.RegisterDBSearch(searchIssues) +} + +func searchIssues(ctx context.Context, options *indexer_issues.SearchOptions) ([]int64, int64, error) { + repoCond := builder.In("repo_id", options.RepoIDs) + subQuery := builder.Select("id").From("issue").Where(repoCond) + cond := builder.And( + repoCond, + builder.Or( + db.BuildCaseInsensitiveLike("name", options.Keyword), + db.BuildCaseInsensitiveLike("content", options.Keyword), + builder.In("id", builder.Select("issue_id"). + From("comment"). + Where(builder.And( + builder.Eq{"type": CommentTypeComment}, + builder.In("issue_id", subQuery), + db.BuildCaseInsensitiveLike("content", options.Keyword), + )), + ), + ), + ) + + ids := make([]int64, 0, options.Limit) + res := make([]struct { + ID int64 + UpdatedUnix int64 + }, 0, options.Limit) + err := db.GetEngine(ctx).Distinct("id", "updated_unix").Table("issue").Where(cond). + OrderBy("`updated_unix` DESC").Limit(options.Limit, options.Skip). + Find(&res) + if err != nil { + return nil, 0, err + } + for _, r := range res { + ids = append(ids, r.ID) + } + + total, err := db.GetEngine(ctx).Distinct("id").Table("issue").Where(cond).Count() + if err != nil { + return nil, 0, err + } + + return ids, total, nil } func filterIssuesOfSearchResult(ctx context.Context, issuesIDs []int64, options *indexer_issues.SearchOptions) ([]int64, error) { @@ -523,6 +567,32 @@ func filterIssuesOfSearchResult(ctx context.Context, issuesIDs []int64, options for _, id := range excludes { ret = append(ret, -id) } + return ret + } + convertInt64 := func(i *int64) int64 { + if i == nil { + return 0 + } + return *i + } + sortType := "" + switch options.SortBy { + case indexer_issues.SearchOptionsSortByCreatedAsc: + sortType = "oldest" + case indexer_issues.SearchOptionsSortByUpdatedAsc: + sortType = "leastupdate" + case indexer_issues.SearchOptionsSortByCommentsAsc: + sortType = "leastcomment" + case indexer_issues.SearchOptionsSortByDueAsc: + sortType = "farduedate" + case indexer_issues.SearchOptionsSortByCreatedDesc: + sortType = "" // default + case indexer_issues.SearchOptionsSortByUpdatedDesc: + sortType = "recentupdate" + case indexer_issues.SearchOptionsSortByCommentsDesc: + sortType = "mostcomment" + case indexer_issues.SearchOptionsSortByDueDesc: + sortType = "nearduedate" } opts := &IssuesOptions{ @@ -546,14 +616,24 @@ func filterIssuesOfSearchResult(ctx context.Context, issuesIDs []int64, options IncludedLabelNames: nil, // use LabelIDs instead ExcludedLabelNames: nil, // use LabelIDs instead IncludeMilestones: nil, // use MilestoneIDs instead - SortType: "", // TBC + SortType: sortType, IssueIDs: issuesIDs, - UpdatedAfterUnix: 0, - UpdatedBeforeUnix: 0, - PriorityRepoID: 0, - IsArchived: 0, - Org: nil, - Team: nil, - User: nil, + UpdatedAfterUnix: convertInt64(options.UpdatedAfterUnix), + UpdatedBeforeUnix: convertInt64(options.UpdatedBeforeUnix), + PriorityRepoID: 0, // don't use priority repo since it isn't supported by search to sort by priorityrepo + IsArchived: 0, // it's unnecessary since issuesIDs are already filtered by repoIDs + Org: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + Team: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + User: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + } + // FIXME: use a new function which returns ids only, to avoid unnecessary issues loading + issues, err := Issues(ctx, opts) + if err != nil { + return nil, err + } + issueIDs := make([]int64, 0, len(issues)) + for _, issue := range issues { + issueIDs = append(issueIDs, issue.ID) } + return issueIDs, nil } diff --git a/models/issues/issue_test.go b/models/issues/issue_test.go index 7f1eab1971378..8acfe33eb55b8 100644 --- a/models/issues/issue_test.go +++ b/models/issues/issue_test.go @@ -333,30 +333,6 @@ func TestIssue_loadTotalTimes(t *testing.T) { assert.Equal(t, int64(3682), ms.TotalTrackedTime) } -func TestIssue_SearchIssueIDsByKeyword(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - total, ids, err := issues_model.SearchIssueIDsByKeyword(context.TODO(), "issue2", []int64{1}, 10, 0) - assert.NoError(t, err) - assert.EqualValues(t, 1, total) - assert.EqualValues(t, []int64{2}, ids) - - total, ids, err = issues_model.SearchIssueIDsByKeyword(context.TODO(), "first", []int64{1}, 10, 0) - assert.NoError(t, err) - assert.EqualValues(t, 1, total) - assert.EqualValues(t, []int64{1}, ids) - - total, ids, err = issues_model.SearchIssueIDsByKeyword(context.TODO(), "for", []int64{1}, 10, 0) - assert.NoError(t, err) - assert.EqualValues(t, 5, total) - assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) - - // issue1's comment id 2 - total, ids, err = issues_model.SearchIssueIDsByKeyword(context.TODO(), "good", []int64{1}, 10, 0) - assert.NoError(t, err) - assert.EqualValues(t, 1, total) - assert.EqualValues(t, []int64{1}, ids) -} - func TestGetRepoIDsForIssuesOptions(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 73bc244129324..61c8878d561f1 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -153,7 +153,7 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error { // Returns the matching issue IDs func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { var repoQueriesP []*query.NumericRangeQuery - for _, repoID := range options.Repos { + for _, repoID := range options.RepoIDs { repoQueriesP = append(repoQueriesP, inner_bleve.NumericEqualityQuery(repoID, "repo_id")) } repoQueries := make([]query.Query, len(repoQueriesP)) diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 13bd2d90b0e18..5d0bd8cc29d25 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -5,8 +5,8 @@ package db import ( "context" + "fmt" - issues_model "code.gitea.io/gitea/models/issues" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_db "code.gitea.io/gitea/modules/indexer/internal/db" "code.gitea.io/gitea/modules/indexer/issues/internal" @@ -35,21 +35,12 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error { return nil } +var SearchFunc func(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) + // Search searches for issues func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { - total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, options.Keyword, options.Repos, options.Limit, options.Skip) - if err != nil { - return nil, err - } - result := internal.SearchResult{ - Total: total, - Hits: make([]internal.Match, 0, options.Limit), - Imprecise: true, - } - for _, id := range ids { - result.Hits = append(result.Hits, internal.Match{ - ID: id, - }) + if SearchFunc != nil { + return SearchFunc(ctx, options) } - return &result, nil + return nil, fmt.Errorf("SearchFunc is not registered") } diff --git a/modules/indexer/issues/dbfilter.go b/modules/indexer/issues/dbfilter.go index 58e1942fb009e..05dc05d10e3a4 100644 --- a/modules/indexer/issues/dbfilter.go +++ b/modules/indexer/issues/dbfilter.go @@ -5,25 +5,50 @@ package issues import ( "context" + "fmt" + + "code.gitea.io/gitea/modules/indexer/issues/db" + "code.gitea.io/gitea/modules/indexer/issues/internal" ) -// filterIssuesByDB filters the given issuesIDs by the database. +// reFilter filters the given issuesIDs by the database. // It is used to filter out issues coming from some indexers that are not supported fining filtering. // Once all indexers support filtering, this function can be removed. -func filterIssuesByDB(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) { - if filterIssuesFunc != nil { - return filterIssuesFunc(ctx, issuesIDs, options) +func reFilter(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) { + if reFilterFunc != nil { + return reFilterFunc(ctx, issuesIDs, options) } - return issuesIDs, nil + return nil, fmt.Errorf("reFilterFunc is not registered") } -var filterIssuesFunc func(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) +var reFilterFunc func(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) -// RegisterFilterIssuesFunc registers a function to filter issues by database. -// It's for issue_model to register its own filter function. -// Why not just put the function body here? +// Why not just put the function body here to avoid RegisterReFilterFunc and RegisterDBSearch? // Because modules can't depend on models by design. // Although some packages have broken this rule, it's still a good practice to follow it. -func RegisterFilterIssuesFunc(f func(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error)) { - filterIssuesFunc = f + +// RegisterReFilterFunc registers a function to filter issues by database. +// It's for issue_model to register its own filter function. +func RegisterReFilterFunc(f func(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error)) { + reFilterFunc = f +} + +func RegisterDBSearch(f func(ctx context.Context, options *SearchOptions) ([]int64, int64, error)) { + db.SearchFunc = func(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { + ids, total, err := f(ctx, (*SearchOptions)(options)) + if err != nil { + return nil, err + } + hits := make([]internal.Match, 0, len(ids)) + for _, id := range ids { + hits = append(hits, internal.Match{ + ID: id, + }) + } + return &internal.SearchResult{ + Hits: hits, + Total: total, + Imprecise: false, + }, nil + } } diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index b95bf3a3ead42..afcc6bcf6da77 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -144,9 +144,9 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( kwQuery := elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments") query := elastic.NewBoolQuery() query = query.Must(kwQuery) - if len(options.Repos) > 0 { - repoStrs := make([]any, 0, len(options.Repos)) - for _, repoID := range options.Repos { + if len(options.RepoIDs) > 0 { + repoStrs := make([]any, 0, len(options.RepoIDs)) + for _, repoID := range options.RepoIDs { repoStrs = append(repoStrs, repoID) } repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 386330404f8bd..297b693a00dee 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -298,7 +298,7 @@ func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) indexer := *globalIndexer.Load() res, err := indexer.Search(ctx, &internal.SearchOptions{ Keyword: keyword, - Repos: repoIDs, + RepoIDs: repoIDs, Limit: 50, Skip: 0, }) @@ -319,6 +319,17 @@ func IsAvailable(ctx context.Context) bool { // SearchOptions indicates the options for searching issues type SearchOptions internal.SearchOptions +const ( + SearchOptionsSortByCreatedAsc = internal.SearchOptionsSortByCreatedAsc + SearchOptionsSortByUpdatedAsc = internal.SearchOptionsSortByUpdatedAsc + SearchOptionsSortByCommentsAsc = internal.SearchOptionsSortByCommentsAsc + SearchOptionsSortByDueAsc = internal.SearchOptionsSortByDueAsc + SearchOptionsSortByCreatedDesc = internal.SearchOptionsSortByCreatedDesc + SearchOptionsSortByUpdatedDesc = internal.SearchOptionsSortByUpdatedDesc + SearchOptionsSortByCommentsDesc = internal.SearchOptionsSortByCommentsDesc + SearchOptionsSortByDueDesc = internal.SearchOptionsSortByDueDesc +) + // SearchIssues search issues by options. // It returns issue ids and a bool value indicates if the result is imprecise. func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, error) { @@ -340,7 +351,7 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, error) { } if result.Imprecise { - ret, err := filterIssuesByDB(ctx, ret, opts) + ret, err := reFilter(ctx, ret, opts) if err != nil { return nil, err } diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index 186610e762d15..762d6e2fbf746 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -53,7 +53,7 @@ func TestBleveSearchIssues(t *testing.T) { t.Run("issue2", func(t *testing.T) { ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "issue2", - Repos: []int64{1}, + RepoIDs: []int64{1}, }) assert.NoError(t, err) assert.EqualValues(t, []int64{2}, ids) @@ -62,7 +62,7 @@ func TestBleveSearchIssues(t *testing.T) { t.Run("first", func(t *testing.T) { ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "first", - Repos: []int64{1}, + RepoIDs: []int64{1}, }) assert.NoError(t, err) assert.EqualValues(t, []int64{1}, ids) @@ -71,7 +71,7 @@ func TestBleveSearchIssues(t *testing.T) { t.Run("for", func(t *testing.T) { ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "for", - Repos: []int64{1}, + RepoIDs: []int64{1}, }) assert.NoError(t, err) assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) @@ -80,7 +80,7 @@ func TestBleveSearchIssues(t *testing.T) { t.Run("good", func(t *testing.T) { ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "good", - Repos: []int64{1}, + RepoIDs: []int64{1}, }) assert.NoError(t, err) assert.EqualValues(t, []int64{1}, ids) @@ -96,7 +96,7 @@ func TestDBSearchIssues(t *testing.T) { t.Run("issue2", func(t *testing.T) { ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "issue2", - Repos: []int64{1}, + RepoIDs: []int64{1}, }) assert.NoError(t, err) assert.EqualValues(t, []int64{2}, ids) @@ -105,7 +105,7 @@ func TestDBSearchIssues(t *testing.T) { t.Run("first", func(t *testing.T) { ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "first", - Repos: []int64{1}, + RepoIDs: []int64{1}, }) assert.NoError(t, err) assert.EqualValues(t, []int64{1}, ids) @@ -114,7 +114,7 @@ func TestDBSearchIssues(t *testing.T) { t.Run("for", func(t *testing.T) { ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "for", - Repos: []int64{1}, + RepoIDs: []int64{1}, }) assert.NoError(t, err) assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) @@ -123,7 +123,7 @@ func TestDBSearchIssues(t *testing.T) { t.Run("good", func(t *testing.T) { ids, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "good", - Repos: []int64{1}, + RepoIDs: []int64{1}, }) assert.NoError(t, err) assert.EqualValues(t, []int64{1}, ids) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 4f12cb754bce8..9e57fd7dd272c 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -18,27 +18,27 @@ type IndexerData struct { Comments []string `json:"comments"` // Fields used for filtering - IsPull bool `json:"is_pull"` - IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. - Labels []int64 `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. - NoLabels bool `json:"no_labels"` // True if Labels is empty - MilestoneIDs []int64 `json:"milestone_ids"` // So if the milestones of an issue have changed, we should reindex the issue. - NoMilestone bool `json:"no_milestone"` // True if Milestones is empty - ProjectIDs []int64 `json:"project_ids"` // So if the projects of an issue have changed, we should reindex the issue. - ProjectBoardIDs []int64 `json:"project_board_ids"` // So if the projects of an issue have changed, we should reindex the issue. - NoProject bool `json:"no_project"` // True if ProjectIDs is empty - PosterID int64 `json:"poster_id"` - AssigneeID int64 `json:"assignee_id"` // So if the assignee of an issue has changed, we should reindex the issue. - MentionIDs []int64 `json:"mention_ids"` - ReviewedIDs []int64 `json:"reviewed_ids"` // So if the reviewers of an issue have changed, we should reindex the issue. - ReviewRequestedIDs []int64 `json:"review_requested_ids"` // So if the requested reviewers of an issue have changed, we should reindex the issue. - SubscriberIDs []int64 `json:"subscriber_ids"` // So if the subscribers of an issue have changed, we should reindex the issue. + IsPull bool `json:"is_pull"` + IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. + Labels []int64 `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. + NoLabels bool `json:"no_labels"` // True if Labels is empty + MilestoneIDs []int64 `json:"milestone_ids"` // So if the milestones of an issue have changed, we should reindex the issue. + NoMilestone bool `json:"no_milestone"` // True if Milestones is empty + ProjectIDs []int64 `json:"project_ids"` // So if the projects of an issue have changed, we should reindex the issue. + ProjectBoardIDs []int64 `json:"project_board_ids"` // So if the projects of an issue have changed, we should reindex the issue. + NoProject bool `json:"no_project"` // True if ProjectIDs is empty + PosterID int64 `json:"poster_id"` + AssigneeID int64 `json:"assignee_id"` // So if the assignee of an issue has changed, we should reindex the issue. + MentionIDs []int64 `json:"mention_ids"` + ReviewedIDs []int64 `json:"reviewed_ids"` // So if the reviewers of an issue have changed, we should reindex the issue. + ReviewRequestedIDs []int64 `json:"review_requested_ids"` // So if the requested reviewers of an issue have changed, we should reindex the issue. + SubscriberIDs []int64 `json:"subscriber_ids"` // So if the subscribers of an issue have changed, we should reindex the issue. + UpdatedUnix timeutil.TimeStamp `json:"updated_unix"` // Fields used for sorting - CreatedAt timeutil.TimeStamp `json:"created_at"` - UpdatedAt timeutil.TimeStamp `json:"updated_at"` + CreatedUnix timeutil.TimeStamp `json:"created_unix"` + DueUnix timeutil.TimeStamp `json:"due_unix"` CommentCount int64 `json:"comment_count"` - DueDate timeutil.TimeStamp `json:"due_date"` } // Match represents on search result @@ -65,7 +65,7 @@ type SearchResult struct { type SearchOptions struct { Keyword string // keyword to search - Repos []int64 // repository IDs which the issues belong to + RepoIDs []int64 // repository IDs which the issues belong to IsPull *bool // if the issues is a pull request IsClosed *bool // if the issues is closed @@ -92,8 +92,24 @@ type SearchOptions struct { SubscriberID *int64 // subscriber of the issues + UpdatedAfterUnix *int64 + UpdatedBeforeUnix *int64 + Skip int // skip the first N results Limit int // limit the number of results - SortBy string // sort by field, could be "created", "updated", "comments", "due_date", add "-" prefix to sort in descending order + SortBy SearchOptionsSortBy // sort by field } + +type SearchOptionsSortBy string + +const ( + SearchOptionsSortByCreatedAsc SearchOptionsSortBy = "created" + SearchOptionsSortByUpdatedAsc SearchOptionsSortBy = "updated" + SearchOptionsSortByCommentsAsc SearchOptionsSortBy = "comments" + SearchOptionsSortByDueAsc SearchOptionsSortBy = "due" + SearchOptionsSortByCreatedDesc SearchOptionsSortBy = "-created" + SearchOptionsSortByUpdatedDesc SearchOptionsSortBy = "-updated" + SearchOptionsSortByCommentsDesc SearchOptionsSortBy = "-comments" + SearchOptionsSortByDueDesc SearchOptionsSortBy = "-due" +) diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 158b6bb558e04..651f4fe24be82 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -71,8 +71,8 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { - repoFilters := make([]string, 0, len(options.Repos)) - for _, repoID := range options.Repos { + repoFilters := make([]string, 0, len(options.RepoIDs)) + for _, repoID := range options.RepoIDs { repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10)) } filter := strings.Join(repoFilters, " OR ") From d547195153e0eeaf0cbf5fe9778595ea714fb55f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 14 Jul 2023 15:52:09 +0800 Subject: [PATCH 17/99] fix: db search --- models/issues/issue_search.go | 191 ----------------------------- models/issues/issue_test.go | 14 ++- modules/indexer/issues/db/db.go | 67 +++++++++- modules/indexer/issues/dbfilter.go | 132 ++++++++++++++------ 4 files changed, 170 insertions(+), 234 deletions(-) diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index 525c00d39957f..fd4077ce3e03d 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -13,7 +13,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" - indexer_issues "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/util" "xorm.io/builder" @@ -447,193 +446,3 @@ func Issues(ctx context.Context, opts *IssuesOptions) ([]*Issue, error) { return issues, nil } - -// Deprecated: use `searchIssues` or `indexer/issues.Search` instead -// SearchIssueIDsByKeyword search issues on database -func SearchIssueIDsByKeyword(ctx context.Context, kw string, repoIDs []int64, limit, start int) (int64, []int64, error) { - repoCond := builder.In("repo_id", repoIDs) - subQuery := builder.Select("id").From("issue").Where(repoCond) - cond := builder.And( - repoCond, - builder.Or( - db.BuildCaseInsensitiveLike("name", kw), - db.BuildCaseInsensitiveLike("content", kw), - builder.In("id", builder.Select("issue_id"). - From("comment"). - Where(builder.And( - builder.Eq{"type": CommentTypeComment}, - builder.In("issue_id", subQuery), - db.BuildCaseInsensitiveLike("content", kw), - )), - ), - ), - ) - - ids := make([]int64, 0, limit) - res := make([]struct { - ID int64 - UpdatedUnix int64 - }, 0, limit) - err := db.GetEngine(ctx).Distinct("id", "updated_unix").Table("issue").Where(cond). - OrderBy("`updated_unix` DESC").Limit(limit, start). - Find(&res) - if err != nil { - return 0, nil, err - } - for _, r := range res { - ids = append(ids, r.ID) - } - - total, err := db.GetEngine(ctx).Distinct("id").Table("issue").Where(cond).Count() - if err != nil { - return 0, nil, err - } - - return total, ids, nil -} - -func init() { - indexer_issues.RegisterReFilterFunc(filterIssuesOfSearchResult) - indexer_issues.RegisterDBSearch(searchIssues) -} - -func searchIssues(ctx context.Context, options *indexer_issues.SearchOptions) ([]int64, int64, error) { - repoCond := builder.In("repo_id", options.RepoIDs) - subQuery := builder.Select("id").From("issue").Where(repoCond) - cond := builder.And( - repoCond, - builder.Or( - db.BuildCaseInsensitiveLike("name", options.Keyword), - db.BuildCaseInsensitiveLike("content", options.Keyword), - builder.In("id", builder.Select("issue_id"). - From("comment"). - Where(builder.And( - builder.Eq{"type": CommentTypeComment}, - builder.In("issue_id", subQuery), - db.BuildCaseInsensitiveLike("content", options.Keyword), - )), - ), - ), - ) - - ids := make([]int64, 0, options.Limit) - res := make([]struct { - ID int64 - UpdatedUnix int64 - }, 0, options.Limit) - err := db.GetEngine(ctx).Distinct("id", "updated_unix").Table("issue").Where(cond). - OrderBy("`updated_unix` DESC").Limit(options.Limit, options.Skip). - Find(&res) - if err != nil { - return nil, 0, err - } - for _, r := range res { - ids = append(ids, r.ID) - } - - total, err := db.GetEngine(ctx).Distinct("id").Table("issue").Where(cond).Count() - if err != nil { - return nil, 0, err - } - - return ids, total, nil -} - -func filterIssuesOfSearchResult(ctx context.Context, issuesIDs []int64, options *indexer_issues.SearchOptions) ([]int64, error) { - convertID := func(id *int64) int64 { - if id == nil { - return db.NoConditionID - } - return *id - } - convertIDs := func(ids []int64, no bool) []int64 { - if no { - return []int64{db.NoConditionID} - } - return ids - } - convertBool := func(b *bool) util.OptionalBool { - if b == nil { - return util.OptionalBoolNone - } - return util.OptionalBoolOf(*b) - } - convertLabelIDs := func(includes, excludes []int64, no bool) []int64 { - if no { - return []int64{0} // It's zero, not db.NoConditionID, - } - ret := make([]int64, 0, len(includes)+len(excludes)) - ret = append(ret, includes...) - for _, id := range excludes { - ret = append(ret, -id) - } - return ret - } - convertInt64 := func(i *int64) int64 { - if i == nil { - return 0 - } - return *i - } - sortType := "" - switch options.SortBy { - case indexer_issues.SearchOptionsSortByCreatedAsc: - sortType = "oldest" - case indexer_issues.SearchOptionsSortByUpdatedAsc: - sortType = "leastupdate" - case indexer_issues.SearchOptionsSortByCommentsAsc: - sortType = "leastcomment" - case indexer_issues.SearchOptionsSortByDueAsc: - sortType = "farduedate" - case indexer_issues.SearchOptionsSortByCreatedDesc: - sortType = "" // default - case indexer_issues.SearchOptionsSortByUpdatedDesc: - sortType = "recentupdate" - case indexer_issues.SearchOptionsSortByCommentsDesc: - sortType = "mostcomment" - case indexer_issues.SearchOptionsSortByDueDesc: - sortType = "nearduedate" - } - - opts := &IssuesOptions{ - ListOptions: db.ListOptions{ - ListAll: true, - }, - RepoIDs: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - RepoCond: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - AssigneeID: convertID(options.AssigneeID), - PosterID: convertID(options.PosterID), - MentionedID: convertID(options.MentionID), - ReviewRequestedID: convertID(options.ReviewRequestedID), - ReviewedID: convertID(options.ReviewedID), - SubscriberID: convertID(options.SubscriberID), - MilestoneIDs: convertIDs(options.MilestoneIDs, options.NoMilestone), - ProjectID: convertID(options.ProjectID), - ProjectBoardID: convertID(options.ProjectBoardID), - IsClosed: convertBool(options.IsClosed), - IsPull: convertBool(options.IsPull), - LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.NoLabel), - IncludedLabelNames: nil, // use LabelIDs instead - ExcludedLabelNames: nil, // use LabelIDs instead - IncludeMilestones: nil, // use MilestoneIDs instead - SortType: sortType, - IssueIDs: issuesIDs, - UpdatedAfterUnix: convertInt64(options.UpdatedAfterUnix), - UpdatedBeforeUnix: convertInt64(options.UpdatedBeforeUnix), - PriorityRepoID: 0, // don't use priority repo since it isn't supported by search to sort by priorityrepo - IsArchived: 0, // it's unnecessary since issuesIDs are already filtered by repoIDs - Org: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - Team: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - User: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - } - // FIXME: use a new function which returns ids only, to avoid unnecessary issues loading - issues, err := Issues(ctx, opts) - if err != nil { - return nil, err - } - issueIDs := make([]int64, 0, len(issues)) - for _, issue := range issues { - issueIDs = append(issueIDs, issue.ID) - } - return issueIDs, nil -} diff --git a/models/issues/issue_test.go b/models/issues/issue_test.go index 8acfe33eb55b8..019f5dac62fa6 100644 --- a/models/issues/issue_test.go +++ b/models/issues/issue_test.go @@ -472,7 +472,19 @@ func TestCorrectIssueStats(t *testing.T) { wg.Wait() // Now we will get all issueID's that match the "Bugs are nasty" query. - total, ids, err := issues_model.SearchIssueIDsByKeyword(context.TODO(), "Bugs are nasty", []int64{1}, issueAmount, 0) + issues, err := issues_model.Issues(context.TODO(), &issues_model.IssuesOptions{ + ListOptions: db.ListOptions{ + PageSize: issueAmount, + }, + RepoIDs: []int64{1}, + }) + total := int64(len(issues)) + var ids []int64 + for _, issue := range issues { + if issue.Content == "Bugs are nasty" { + ids = append(ids, issue.ID) + } + } // Just to be sure. assert.NoError(t, err) diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 5d0bd8cc29d25..0790d2a07066f 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -5,11 +5,14 @@ package db import ( "context" - "fmt" + "code.gitea.io/gitea/models/db" + issue_model "code.gitea.io/gitea/models/issues" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_db "code.gitea.io/gitea/modules/indexer/internal/db" "code.gitea.io/gitea/modules/indexer/issues/internal" + + "xorm.io/builder" ) var _ internal.Indexer = &Indexer{} @@ -35,12 +38,64 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error { return nil } -var SearchFunc func(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) - // Search searches for issues func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { - if SearchFunc != nil { - return SearchFunc(ctx, options) + // FIXME: I tried to avoid importing models here, but it seems to be impossible. + // We can provide a function to register the search function, so models/issues can register it. + // So models/issues will import modules/indexer/issues, it's OK because it's by design. + // But modules/indexer/issues has already imported models/issues to do UpdateRepoIndexer and UpdateIssueIndexer. + // And to avoid circular import, we have to move the functions to another package. + // I believe it should be services/indexer, sounds great! + // But the two functions are used in modules/notification/indexer, that means we will import services/indexer in modules/notification/indexer. + // So that's the root problem, the notification is defined in modules, but it's using lots of things should be in services. + + repoCond := builder.In("repo_id", options.RepoIDs) + subQuery := builder.Select("id").From("issue").Where(repoCond) + cond := builder.And( + repoCond, + builder.Or( + db.BuildCaseInsensitiveLike("name", options.Keyword), + db.BuildCaseInsensitiveLike("content", options.Keyword), + builder.In("id", builder.Select("issue_id"). + From("comment"). + Where(builder.And( + builder.Eq{"type": issue_model.CommentTypeComment}, + builder.In("issue_id", subQuery), + db.BuildCaseInsensitiveLike("content", options.Keyword), + )), + ), + ), + ) + + ids := make([]int64, 0, options.Limit) + res := make([]struct { + ID int64 + UpdatedUnix int64 + }, 0, options.Limit) + err := db.GetEngine(ctx).Distinct("id", "updated_unix").Table("issue").Where(cond). + OrderBy("`updated_unix` DESC").Limit(options.Limit, options.Skip). + Find(&res) + if err != nil { + return nil, err + } + for _, r := range res { + ids = append(ids, r.ID) + } + + total, err := db.GetEngine(ctx).Distinct("id").Table("issue").Where(cond).Count() + if err != nil { + return nil, err + } + + hits := make([]internal.Match, 0, len(ids)) + for _, id := range ids { + hits = append(hits, internal.Match{ + ID: id, + }) } - return nil, fmt.Errorf("SearchFunc is not registered") + return &internal.SearchResult{ + Total: total, + Hits: hits, + Imprecise: true, + }, nil } diff --git a/modules/indexer/issues/dbfilter.go b/modules/indexer/issues/dbfilter.go index 05dc05d10e3a4..dc4442b8659fd 100644 --- a/modules/indexer/issues/dbfilter.go +++ b/modules/indexer/issues/dbfilter.go @@ -5,50 +5,110 @@ package issues import ( "context" - "fmt" - "code.gitea.io/gitea/modules/indexer/issues/db" - "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/models/db" + issue_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/util" ) // reFilter filters the given issuesIDs by the database. // It is used to filter out issues coming from some indexers that are not supported fining filtering. -// Once all indexers support filtering, this function can be removed. +// Once all indexers support filtering, this function and this file can be removed. func reFilter(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) { - if reFilterFunc != nil { - return reFilterFunc(ctx, issuesIDs, options) + convertID := func(id *int64) int64 { + if id == nil { + return db.NoConditionID + } + return *id } - return nil, fmt.Errorf("reFilterFunc is not registered") -} - -var reFilterFunc func(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) - -// Why not just put the function body here to avoid RegisterReFilterFunc and RegisterDBSearch? -// Because modules can't depend on models by design. -// Although some packages have broken this rule, it's still a good practice to follow it. - -// RegisterReFilterFunc registers a function to filter issues by database. -// It's for issue_model to register its own filter function. -func RegisterReFilterFunc(f func(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error)) { - reFilterFunc = f -} - -func RegisterDBSearch(f func(ctx context.Context, options *SearchOptions) ([]int64, int64, error)) { - db.SearchFunc = func(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { - ids, total, err := f(ctx, (*SearchOptions)(options)) - if err != nil { - return nil, err + convertIDs := func(ids []int64, no bool) []int64 { + if no { + return []int64{db.NoConditionID} + } + return ids + } + convertBool := func(b *bool) util.OptionalBool { + if b == nil { + return util.OptionalBoolNone } - hits := make([]internal.Match, 0, len(ids)) - for _, id := range ids { - hits = append(hits, internal.Match{ - ID: id, - }) + return util.OptionalBoolOf(*b) + } + convertLabelIDs := func(includes, excludes []int64, no bool) []int64 { + if no { + return []int64{0} // It's zero, not db.NoConditionID, + } + ret := make([]int64, 0, len(includes)+len(excludes)) + ret = append(ret, includes...) + for _, id := range excludes { + ret = append(ret, -id) } - return &internal.SearchResult{ - Hits: hits, - Total: total, - Imprecise: false, - }, nil + return ret + } + convertInt64 := func(i *int64) int64 { + if i == nil { + return 0 + } + return *i + } + sortType := "" + switch options.SortBy { + case SearchOptionsSortByCreatedAsc: + sortType = "oldest" + case SearchOptionsSortByUpdatedAsc: + sortType = "leastupdate" + case SearchOptionsSortByCommentsAsc: + sortType = "leastcomment" + case SearchOptionsSortByDueAsc: + sortType = "farduedate" + case SearchOptionsSortByCreatedDesc: + sortType = "" // default + case SearchOptionsSortByUpdatedDesc: + sortType = "recentupdate" + case SearchOptionsSortByCommentsDesc: + sortType = "mostcomment" + case SearchOptionsSortByDueDesc: + sortType = "nearduedate" + } + + opts := &issue_model.IssuesOptions{ + ListOptions: db.ListOptions{ + ListAll: true, + }, + RepoIDs: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + RepoCond: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + AssigneeID: convertID(options.AssigneeID), + PosterID: convertID(options.PosterID), + MentionedID: convertID(options.MentionID), + ReviewRequestedID: convertID(options.ReviewRequestedID), + ReviewedID: convertID(options.ReviewedID), + SubscriberID: convertID(options.SubscriberID), + MilestoneIDs: convertIDs(options.MilestoneIDs, options.NoMilestone), + ProjectID: convertID(options.ProjectID), + ProjectBoardID: convertID(options.ProjectBoardID), + IsClosed: convertBool(options.IsClosed), + IsPull: convertBool(options.IsPull), + LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.NoLabel), + IncludedLabelNames: nil, // use LabelIDs instead + ExcludedLabelNames: nil, // use LabelIDs instead + IncludeMilestones: nil, // use MilestoneIDs instead + SortType: sortType, + IssueIDs: issuesIDs, + UpdatedAfterUnix: convertInt64(options.UpdatedAfterUnix), + UpdatedBeforeUnix: convertInt64(options.UpdatedBeforeUnix), + PriorityRepoID: 0, // don't use priority repo since it isn't supported by search to sort by priorityrepo + IsArchived: 0, // it's unnecessary since issuesIDs are already filtered by repoIDs + Org: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + Team: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + User: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs + } + // TODO: use a new function which returns ids only, to avoid unnecessary issues loading + issues, err := issue_model.Issues(ctx, opts) + if err != nil { + return nil, err + } + issueIDs := make([]int64, 0, len(issues)) + for _, issue := range issues { + issueIDs = append(issueIDs, issue.ID) } + return issueIDs, nil } From b640caabe3ead965701532d7c090f444f6386559 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 14 Jul 2023 18:02:39 +0800 Subject: [PATCH 18/99] feat: db indexer --- models/issues/issue_search.go | 41 ++++++++--- models/issues/issue_test.go | 8 +-- modules/indexer/issues/db/db.go | 40 ++++------- modules/indexer/issues/db/options.go | 99 +++++++++++++++++++++++++ modules/indexer/issues/dbfilter.go | 103 ++------------------------- modules/indexer/issues/indexer.go | 11 --- routers/api/v1/repo/issue.go | 8 +-- routers/web/repo/issue.go | 10 +-- routers/web/user/home.go | 6 +- routers/web/user/notification.go | 2 +- 10 files changed, 169 insertions(+), 159 deletions(-) create mode 100644 modules/indexer/issues/db/options.go diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index fd4077ce3e03d..d022b9af9debc 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -21,7 +21,7 @@ import ( // IssuesOptions represents options of an issue. type IssuesOptions struct { //nolint - db.ListOptions + db.Paginator RepoIDs []int64 // overwrites RepoCond if the length is not 0 RepoCond builder.Cond AssigneeID int64 @@ -99,14 +99,9 @@ func applySorts(sess *xorm.Session, sortType string, priorityRepoID int64) { } func applyLimit(sess *xorm.Session, opts *IssuesOptions) *xorm.Session { - if opts.Page >= 0 && opts.PageSize > 0 { - var start int - if opts.Page == 0 { - start = 0 - } else { - start = (opts.Page - 1) * opts.PageSize - } - sess.Limit(opts.PageSize, start) + if opts.Paginator != nil { + skip, take := opts.GetSkipTake() + sess.Limit(take, skip) } return sess } @@ -435,7 +430,7 @@ func Issues(ctx context.Context, opts *IssuesOptions) ([]*Issue, error) { applyConditions(sess, opts) applySorts(sess, opts.SortType, opts.PriorityRepoID) - issues := make(IssueList, 0, opts.ListOptions.PageSize) + issues := IssueList{} if err := sess.Find(&issues); err != nil { return nil, fmt.Errorf("unable to query Issues: %w", err) } @@ -446,3 +441,29 @@ func Issues(ctx context.Context, opts *IssuesOptions) ([]*Issue, error) { return issues, nil } + +// IssueIDs returns a list of issue ids by given conditions. +func IssueIDs(ctx context.Context, opts *IssuesOptions, otherConds ...builder.Cond) ([]int64, int64, error) { + sess := db.GetEngine(ctx). + Join("INNER", "repository", "`issue`.repo_id = `repository`.id") + applyConditions(sess, opts) + for _, cond := range otherConds { + sess.And(cond) + } + + total, err := sess.Count(&Issue{}) + if err != nil { + return nil, 0, err + } + + applyLimit(sess, opts) + applySorts(sess, opts.SortType, opts.PriorityRepoID) + + var res []int64 + if err := sess.Select("`issue`.id").Table("issue"). + Find(&res); err != nil { + return nil, 0, err + } + + return res, total, nil +} diff --git a/models/issues/issue_test.go b/models/issues/issue_test.go index 019f5dac62fa6..e36030c1f3920 100644 --- a/models/issues/issue_test.go +++ b/models/issues/issue_test.go @@ -165,7 +165,7 @@ func TestIssues(t *testing.T) { issues_model.IssuesOptions{ RepoCond: builder.In("repo_id", 1, 3), SortType: "oldest", - ListOptions: db.ListOptions{ + Paginator: &db.ListOptions{ Page: 1, PageSize: 4, }, @@ -175,7 +175,7 @@ func TestIssues(t *testing.T) { { issues_model.IssuesOptions{ LabelIDs: []int64{1}, - ListOptions: db.ListOptions{ + Paginator: &db.ListOptions{ Page: 1, PageSize: 4, }, @@ -185,7 +185,7 @@ func TestIssues(t *testing.T) { { issues_model.IssuesOptions{ LabelIDs: []int64{1, 2}, - ListOptions: db.ListOptions{ + Paginator: &db.ListOptions{ Page: 1, PageSize: 4, }, @@ -473,7 +473,7 @@ func TestCorrectIssueStats(t *testing.T) { // Now we will get all issueID's that match the "Bugs are nasty" query. issues, err := issues_model.Issues(context.TODO(), &issues_model.IssuesOptions{ - ListOptions: db.ListOptions{ + Paginator: &db.ListOptions{ PageSize: issueAmount, }, RepoIDs: []int64{1}, diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 0790d2a07066f..cb63e287ed668 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -47,42 +47,32 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( // And to avoid circular import, we have to move the functions to another package. // I believe it should be services/indexer, sounds great! // But the two functions are used in modules/notification/indexer, that means we will import services/indexer in modules/notification/indexer. - // So that's the root problem, the notification is defined in modules, but it's using lots of things should be in services. + // So that's the root problem: + // The notification is defined in modules, but it's using lots of things should be in services. repoCond := builder.In("repo_id", options.RepoIDs) subQuery := builder.Select("id").From("issue").Where(repoCond) cond := builder.And( repoCond, builder.Or( - db.BuildCaseInsensitiveLike("name", options.Keyword), - db.BuildCaseInsensitiveLike("content", options.Keyword), - builder.In("id", builder.Select("issue_id"). - From("comment"). - Where(builder.And( - builder.Eq{"type": issue_model.CommentTypeComment}, - builder.In("issue_id", subQuery), - db.BuildCaseInsensitiveLike("content", options.Keyword), - )), + builder.If(options.Keyword != "", + db.BuildCaseInsensitiveLike("name", options.Keyword), + db.BuildCaseInsensitiveLike("content", options.Keyword), + builder.In("id", builder.Select("issue_id"). + From("comment"). + Where(builder.And( + builder.Eq{"type": issue_model.CommentTypeComment}, + builder.In("issue_id", subQuery), + db.BuildCaseInsensitiveLike("content", options.Keyword), + )), + ), ), ), ) - ids := make([]int64, 0, options.Limit) - res := make([]struct { - ID int64 - UpdatedUnix int64 - }, 0, options.Limit) - err := db.GetEngine(ctx).Distinct("id", "updated_unix").Table("issue").Where(cond). - OrderBy("`updated_unix` DESC").Limit(options.Limit, options.Skip). - Find(&res) - if err != nil { - return nil, err - } - for _, r := range res { - ids = append(ids, r.ID) - } + opt := ToDBOptions(options) - total, err := db.GetEngine(ctx).Distinct("id").Table("issue").Where(cond).Count() + ids, total, err := issue_model.IssueIDs(ctx, opt, cond) if err != nil { return nil, err } diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go new file mode 100644 index 0000000000000..846536de23acd --- /dev/null +++ b/modules/indexer/issues/db/options.go @@ -0,0 +1,99 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import ( + "code.gitea.io/gitea/models/db" + issue_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/util" +) + +func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { + convertID := func(id *int64) int64 { + if id == nil { + return db.NoConditionID + } + return *id + } + convertIDs := func(ids []int64, no bool) []int64 { + if no { + return []int64{db.NoConditionID} + } + return ids + } + convertBool := func(b *bool) util.OptionalBool { + if b == nil { + return util.OptionalBoolNone + } + return util.OptionalBoolOf(*b) + } + convertLabelIDs := func(includes, excludes []int64, no bool) []int64 { + if no { + return []int64{0} // Be careful, it's zero, not db.NoConditionID, + } + ret := make([]int64, 0, len(includes)+len(excludes)) + ret = append(ret, includes...) + for _, id := range excludes { + ret = append(ret, -id) + } + return ret + } + convertInt64 := func(i *int64) int64 { + if i == nil { + return 0 + } + return *i + } + sortType := "" + switch options.SortBy { + case internal.SearchOptionsSortByCreatedAsc: + sortType = "oldest" + case internal.SearchOptionsSortByUpdatedAsc: + sortType = "leastupdate" + case internal.SearchOptionsSortByCommentsAsc: + sortType = "leastcomment" + case internal.SearchOptionsSortByDueAsc: + sortType = "farduedate" + case internal.SearchOptionsSortByCreatedDesc: + sortType = "" // default + case internal.SearchOptionsSortByUpdatedDesc: + sortType = "recentupdate" + case internal.SearchOptionsSortByCommentsDesc: + sortType = "mostcomment" + case internal.SearchOptionsSortByDueDesc: + sortType = "nearduedate" + } + + opts := &issue_model.IssuesOptions{ + Paginator: db.NewAbsoluteListOptions(options.Skip, options.Limit), + RepoIDs: options.RepoIDs, + RepoCond: nil, + AssigneeID: convertID(options.AssigneeID), + PosterID: convertID(options.PosterID), + MentionedID: convertID(options.MentionID), + ReviewRequestedID: convertID(options.ReviewRequestedID), + ReviewedID: convertID(options.ReviewedID), + SubscriberID: convertID(options.SubscriberID), + MilestoneIDs: convertIDs(options.MilestoneIDs, options.NoMilestone), + ProjectID: convertID(options.ProjectID), + ProjectBoardID: convertID(options.ProjectBoardID), + IsClosed: convertBool(options.IsClosed), + IsPull: convertBool(options.IsPull), + LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.NoLabel), + IncludedLabelNames: nil, + ExcludedLabelNames: nil, + IncludeMilestones: nil, + SortType: sortType, + IssueIDs: nil, + UpdatedAfterUnix: convertInt64(options.UpdatedAfterUnix), + UpdatedBeforeUnix: convertInt64(options.UpdatedBeforeUnix), + PriorityRepoID: 0, + IsArchived: 0, + Org: nil, + Team: nil, + User: nil, + } + return opts +} diff --git a/modules/indexer/issues/dbfilter.go b/modules/indexer/issues/dbfilter.go index dc4442b8659fd..3633bf40342e8 100644 --- a/modules/indexer/issues/dbfilter.go +++ b/modules/indexer/issues/dbfilter.go @@ -6,109 +6,18 @@ package issues import ( "context" - "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/indexer/issues/db" + "code.gitea.io/gitea/modules/indexer/issues/internal" ) // reFilter filters the given issuesIDs by the database. // It is used to filter out issues coming from some indexers that are not supported fining filtering. // Once all indexers support filtering, this function and this file can be removed. func reFilter(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) { - convertID := func(id *int64) int64 { - if id == nil { - return db.NoConditionID - } - return *id - } - convertIDs := func(ids []int64, no bool) []int64 { - if no { - return []int64{db.NoConditionID} - } - return ids - } - convertBool := func(b *bool) util.OptionalBool { - if b == nil { - return util.OptionalBoolNone - } - return util.OptionalBoolOf(*b) - } - convertLabelIDs := func(includes, excludes []int64, no bool) []int64 { - if no { - return []int64{0} // It's zero, not db.NoConditionID, - } - ret := make([]int64, 0, len(includes)+len(excludes)) - ret = append(ret, includes...) - for _, id := range excludes { - ret = append(ret, -id) - } - return ret - } - convertInt64 := func(i *int64) int64 { - if i == nil { - return 0 - } - return *i - } - sortType := "" - switch options.SortBy { - case SearchOptionsSortByCreatedAsc: - sortType = "oldest" - case SearchOptionsSortByUpdatedAsc: - sortType = "leastupdate" - case SearchOptionsSortByCommentsAsc: - sortType = "leastcomment" - case SearchOptionsSortByDueAsc: - sortType = "farduedate" - case SearchOptionsSortByCreatedDesc: - sortType = "" // default - case SearchOptionsSortByUpdatedDesc: - sortType = "recentupdate" - case SearchOptionsSortByCommentsDesc: - sortType = "mostcomment" - case SearchOptionsSortByDueDesc: - sortType = "nearduedate" - } + opts := db.ToDBOptions((*internal.SearchOptions)(options)) + opts.IssueIDs = issuesIDs - opts := &issue_model.IssuesOptions{ - ListOptions: db.ListOptions{ - ListAll: true, - }, - RepoIDs: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - RepoCond: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - AssigneeID: convertID(options.AssigneeID), - PosterID: convertID(options.PosterID), - MentionedID: convertID(options.MentionID), - ReviewRequestedID: convertID(options.ReviewRequestedID), - ReviewedID: convertID(options.ReviewedID), - SubscriberID: convertID(options.SubscriberID), - MilestoneIDs: convertIDs(options.MilestoneIDs, options.NoMilestone), - ProjectID: convertID(options.ProjectID), - ProjectBoardID: convertID(options.ProjectBoardID), - IsClosed: convertBool(options.IsClosed), - IsPull: convertBool(options.IsPull), - LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.NoLabel), - IncludedLabelNames: nil, // use LabelIDs instead - ExcludedLabelNames: nil, // use LabelIDs instead - IncludeMilestones: nil, // use MilestoneIDs instead - SortType: sortType, - IssueIDs: issuesIDs, - UpdatedAfterUnix: convertInt64(options.UpdatedAfterUnix), - UpdatedBeforeUnix: convertInt64(options.UpdatedBeforeUnix), - PriorityRepoID: 0, // don't use priority repo since it isn't supported by search to sort by priorityrepo - IsArchived: 0, // it's unnecessary since issuesIDs are already filtered by repoIDs - Org: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - Team: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - User: nil, // it's unnecessary since issuesIDs are already filtered by repoIDs - } - // TODO: use a new function which returns ids only, to avoid unnecessary issues loading - issues, err := issue_model.Issues(ctx, opts) - if err != nil { - return nil, err - } - issueIDs := make([]int64, 0, len(issues)) - for _, issue := range issues { - issueIDs = append(issueIDs, issue.ID) - } - return issueIDs, nil + ids, _, err := issue_model.IssueIDs(ctx, opts) + return ids, err } diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 297b693a00dee..fe94237ccb850 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -319,17 +319,6 @@ func IsAvailable(ctx context.Context) bool { // SearchOptions indicates the options for searching issues type SearchOptions internal.SearchOptions -const ( - SearchOptionsSortByCreatedAsc = internal.SearchOptionsSortByCreatedAsc - SearchOptionsSortByUpdatedAsc = internal.SearchOptionsSortByUpdatedAsc - SearchOptionsSortByCommentsAsc = internal.SearchOptionsSortByCommentsAsc - SearchOptionsSortByDueAsc = internal.SearchOptionsSortByDueAsc - SearchOptionsSortByCreatedDesc = internal.SearchOptionsSortByCreatedDesc - SearchOptionsSortByUpdatedDesc = internal.SearchOptionsSortByUpdatedDesc - SearchOptionsSortByCommentsDesc = internal.SearchOptionsSortByCommentsDesc - SearchOptionsSortByDueDesc = internal.SearchOptionsSortByDueDesc -) - // SearchIssues search issues by options. // It returns issue ids and a bool value indicates if the result is imprecise. func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, error) { diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index e76775ae82226..98dadda938c0b 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -236,7 +236,7 @@ func SearchIssues(ctx *context.APIContext) { // This would otherwise return all issues if no issues were found by the search. if len(keyword) == 0 || len(issueIDs) > 0 || len(includedLabelNames) > 0 || len(includedMilestones) > 0 { issuesOpt := &issues_model.IssuesOptions{ - ListOptions: db.ListOptions{ + Paginator: &db.ListOptions{ Page: ctx.FormInt("page"), PageSize: limit, }, @@ -279,7 +279,7 @@ func SearchIssues(ctx *context.APIContext) { return } - issuesOpt.ListOptions = db.ListOptions{ + issuesOpt.Paginator = &db.ListOptions{ Page: -1, } if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil { @@ -469,7 +469,7 @@ func ListIssues(ctx *context.APIContext) { // This would otherwise return all issues if no issues were found by the search. if len(keyword) == 0 || len(issueIDs) > 0 || len(labelIDs) > 0 { issuesOpt := &issues_model.IssuesOptions{ - ListOptions: listOptions, + Paginator: &listOptions, RepoIDs: []int64{ctx.Repo.Repository.ID}, IsClosed: isClosed, IssueIDs: issueIDs, @@ -488,7 +488,7 @@ func ListIssues(ctx *context.APIContext) { return } - issuesOpt.ListOptions = db.ListOptions{ + issuesOpt.Paginator = &db.ListOptions{ Page: -1, } if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil { diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index bd8959846c0ed..3e1e0662378bd 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -254,7 +254,7 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti issues = []*issues_model.Issue{} } else { issues, err = issues_model.Issues(ctx, &issues_model.IssuesOptions{ - ListOptions: db.ListOptions{ + Paginator: &db.ListOptions{ Page: pager.Paginater.Current(), PageSize: setting.UI.IssuePagingNum, }, @@ -2509,7 +2509,7 @@ func SearchIssues(ctx *context.Context) { // This would otherwise return all issues if no issues were found by the search. if len(keyword) == 0 || len(issueIDs) > 0 || len(includedLabelNames) > 0 || len(includedMilestones) > 0 { issuesOpt := &issues_model.IssuesOptions{ - ListOptions: db.ListOptions{ + Paginator: &db.ListOptions{ Page: ctx.FormInt("page"), PageSize: limit, }, @@ -2553,7 +2553,7 @@ func SearchIssues(ctx *context.Context) { return } - issuesOpt.ListOptions = db.ListOptions{ + issuesOpt.Paginator = &db.ListOptions{ Page: -1, } if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil { @@ -2694,7 +2694,7 @@ func ListIssues(ctx *context.Context) { // This would otherwise return all issues if no issues were found by the search. if len(keyword) == 0 || len(issueIDs) > 0 || len(labelIDs) > 0 { issuesOpt := &issues_model.IssuesOptions{ - ListOptions: listOptions, + Paginator: &listOptions, RepoIDs: []int64{ctx.Repo.Repository.ID}, IsClosed: isClosed, IssueIDs: issueIDs, @@ -2714,7 +2714,7 @@ func ListIssues(ctx *context.Context) { return } - issuesOpt.ListOptions = db.ListOptions{ + issuesOpt.Paginator = &db.ListOptions{ Page: -1, } if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil { diff --git a/routers/web/user/home.go b/routers/web/user/home.go index 5f1e0eb4277b8..9cbc8e9025a0c 100644 --- a/routers/web/user/home.go +++ b/routers/web/user/home.go @@ -503,8 +503,10 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { if page <= 1 { page = 1 } - opts.Page = page - opts.PageSize = setting.UI.IssuePagingNum + opts.Paginator = &db.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + } // Get IDs for labels (a filter option for issues/pulls). // Required for IssuesOptions. diff --git a/routers/web/user/notification.go b/routers/web/user/notification.go index cae12f4126775..60ae628445a93 100644 --- a/routers/web/user/notification.go +++ b/routers/web/user/notification.go @@ -263,7 +263,7 @@ func NotificationSubscriptions(ctx *context.Context) { return } issues, err := issues_model.Issues(ctx, &issues_model.IssuesOptions{ - ListOptions: db.ListOptions{ + Paginator: &db.ListOptions{ PageSize: setting.UI.IssuePagingNum, Page: page, }, From da43c801619d733acd25c7152eaa4258d1c471db Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 14 Jul 2023 18:42:35 +0800 Subject: [PATCH 19/99] feat: db search --- models/issues/issue_list.go | 18 +++++ modules/indexer/issues/db/options.go | 13 +--- modules/indexer/issues/indexer.go | 34 +++++---- modules/indexer/issues/internal/model.go | 17 ++--- routers/api/v1/repo/issue.go | 88 ++++++++++++------------ 5 files changed, 96 insertions(+), 74 deletions(-) diff --git a/models/issues/issue_list.go b/models/issues/issue_list.go index 9cc41ec6ab37e..6e3a66d8cda34 100644 --- a/models/issues/issue_list.go +++ b/models/issues/issue_list.go @@ -6,6 +6,7 @@ package issues import ( "context" "fmt" + "sort" "code.gitea.io/gitea/models/db" project_model "code.gitea.io/gitea/models/project" @@ -605,3 +606,20 @@ func (issues IssueList) GetApprovalCounts(ctx context.Context) (map[int64][]*Rev return approvalCountMap, nil } + +func FindIssuesByIDs(ctx context.Context, ids []int64) (IssueList, error) { + sess := db.GetEngine(ctx).In("id", ids) + issues := make(IssueList, 0, len(ids)) + if err := sess.Find(&issues); err != nil { + return nil, err + } + + order := map[int64]int{} + for i, id := range ids { + order[id] = i + } + sort.Slice(issues, func(i, j int) bool { + return order[issues[i].ID] < order[issues[j].ID] + }) + return issues, nil +} diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index 846536de23acd..0a3157d35cfbb 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -7,7 +7,6 @@ import ( "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/modules/indexer/issues/internal" - "code.gitea.io/gitea/modules/util" ) func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { @@ -23,12 +22,6 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { } return ids } - convertBool := func(b *bool) util.OptionalBool { - if b == nil { - return util.OptionalBoolNone - } - return util.OptionalBoolOf(*b) - } convertLabelIDs := func(includes, excludes []int64, no bool) []int64 { if no { return []int64{0} // Be careful, it's zero, not db.NoConditionID, @@ -67,7 +60,7 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { } opts := &issue_model.IssuesOptions{ - Paginator: db.NewAbsoluteListOptions(options.Skip, options.Limit), + Paginator: options.Paginator, RepoIDs: options.RepoIDs, RepoCond: nil, AssigneeID: convertID(options.AssigneeID), @@ -79,8 +72,8 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { MilestoneIDs: convertIDs(options.MilestoneIDs, options.NoMilestone), ProjectID: convertID(options.ProjectID), ProjectBoardID: convertID(options.ProjectBoardID), - IsClosed: convertBool(options.IsClosed), - IsPull: convertBool(options.IsPull), + IsClosed: options.IsClosed, + IsPull: options.IsPull, LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.NoLabel), IncludedLabelNames: nil, ExcludedLabelNames: nil, diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index fe94237ccb850..54b5a7dbd99ab 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -297,10 +297,9 @@ func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) var issueIDs []int64 indexer := *globalIndexer.Load() res, err := indexer.Search(ctx, &internal.SearchOptions{ - Keyword: keyword, - RepoIDs: repoIDs, - Limit: 50, - Skip: 0, + Keyword: keyword, + RepoIDs: repoIDs, + Paginator: db_model.NewAbsoluteListOptions(0, 50), }) if err != nil { return nil, err @@ -319,19 +318,28 @@ func IsAvailable(ctx context.Context) bool { // SearchOptions indicates the options for searching issues type SearchOptions internal.SearchOptions +const ( + SearchOptionsSortByCreatedDesc internal.SearchOptionsSortBy = "-created" + SearchOptionsSortByUpdatedDesc internal.SearchOptionsSortBy = "-updated" + SearchOptionsSortByCommentsDesc internal.SearchOptionsSortBy = "-comments" + SearchOptionsSortByDueDesc internal.SearchOptionsSortBy = "-due" + SearchOptionsSortByCreatedAsc internal.SearchOptionsSortBy = "created" + SearchOptionsSortByUpdatedAsc internal.SearchOptionsSortBy = "updated" + SearchOptionsSortByCommentsAsc internal.SearchOptionsSortBy = "comments" + SearchOptionsSortByDueAsc internal.SearchOptionsSortBy = "due" +) + // SearchIssues search issues by options. // It returns issue ids and a bool value indicates if the result is imprecise. -func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, error) { - if opts.Limit <= 0 { - // It's meaningless to search with limit <= 0, probably the caller missed to set it. - // If the caller really wants to search all issues, set limit to a large number. - opts.Limit = 50 +func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, error) { + if opts.Paginator == nil { + opts.Paginator = db_model.NewAbsoluteListOptions(0, 50) } indexer := *globalIndexer.Load() result, err := indexer.Search(ctx, (*internal.SearchOptions)(opts)) if err != nil { - return nil, err + return nil, 0, err } ret := make([]int64, 0, len(result.Hits)) @@ -342,10 +350,10 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, error) { if result.Imprecise { ret, err := reFilter(ctx, ret, opts) if err != nil { - return nil, err + return nil, 0, err } - return ret, nil + return ret, 0, nil } - return ret, nil + return ret, result.Total, nil } diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 9e57fd7dd272c..b3d10929b832a 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -4,7 +4,9 @@ package internal import ( + "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" ) // IndexerData data stored in the issue indexer @@ -67,8 +69,8 @@ type SearchOptions struct { RepoIDs []int64 // repository IDs which the issues belong to - IsPull *bool // if the issues is a pull request - IsClosed *bool // if the issues is closed + IsPull util.OptionalBool // if the issues is a pull request + IsClosed util.OptionalBool // if the issues is closed IncludedLabelIDs []int64 // labels the issues have ExcludedLabelIDs []int64 // labels the issues don't have @@ -95,8 +97,7 @@ type SearchOptions struct { UpdatedAfterUnix *int64 UpdatedBeforeUnix *int64 - Skip int // skip the first N results - Limit int // limit the number of results + db.Paginator SortBy SearchOptionsSortBy // sort by field } @@ -104,12 +105,12 @@ type SearchOptions struct { type SearchOptionsSortBy string const ( - SearchOptionsSortByCreatedAsc SearchOptionsSortBy = "created" - SearchOptionsSortByUpdatedAsc SearchOptionsSortBy = "updated" - SearchOptionsSortByCommentsAsc SearchOptionsSortBy = "comments" - SearchOptionsSortByDueAsc SearchOptionsSortBy = "due" SearchOptionsSortByCreatedDesc SearchOptionsSortBy = "-created" SearchOptionsSortByUpdatedDesc SearchOptionsSortBy = "-updated" SearchOptionsSortByCommentsDesc SearchOptionsSortBy = "-comments" SearchOptionsSortByDueDesc SearchOptionsSortBy = "-due" + SearchOptionsSortByCreatedAsc SearchOptionsSortBy = "created" + SearchOptionsSortByUpdatedAsc SearchOptionsSortBy = "updated" + SearchOptionsSortByCommentsAsc SearchOptionsSortBy = "comments" + SearchOptionsSortByDueAsc SearchOptionsSortBy = "due" ) diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 98dadda938c0b..507130c890cc3 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -195,7 +195,7 @@ func SearchIssues(ctx *context.APIContext) { } var issueIDs []int64 if len(keyword) > 0 && len(repoIDs) > 0 { - if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil { + if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil { ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err) return } @@ -384,23 +384,12 @@ func ListIssues(ctx *context.APIContext) { isClosed = util.OptionalBoolFalse } - var issues []*issues_model.Issue - var filteredCount int64 - keyword := ctx.FormTrim("q") if strings.IndexByte(keyword, 0) >= 0 { keyword = "" } - var issueIDs []int64 - var labelIDs []int64 - if len(keyword) > 0 { - issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state")) - if err != nil { - ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err) - return - } - } + var labelIDs []int64 if splitted := strings.Split(ctx.FormString("labels"), ","); len(splitted) > 0 { labelIDs, err = issues_model.GetLabelIDsInRepoByNames(ctx.Repo.Repository.ID, splitted) if err != nil { @@ -465,40 +454,53 @@ func ListIssues(ctx *context.APIContext) { return } - // Only fetch the issues if we either don't have a keyword or the search returned issues - // This would otherwise return all issues if no issues were found by the search. - if len(keyword) == 0 || len(issueIDs) > 0 || len(labelIDs) > 0 { - issuesOpt := &issues_model.IssuesOptions{ - Paginator: &listOptions, - RepoIDs: []int64{ctx.Repo.Repository.ID}, - IsClosed: isClosed, - IssueIDs: issueIDs, - LabelIDs: labelIDs, - MilestoneIDs: mileIDs, - IsPull: isPull, - UpdatedBeforeUnix: before, - UpdatedAfterUnix: since, - PosterID: createdByID, - AssigneeID: assignedByID, - MentionedID: mentionedByID, - } - - if issues, err = issues_model.Issues(ctx, issuesOpt); err != nil { - ctx.Error(http.StatusInternalServerError, "Issues", err) - return + searchOpt := &issue_indexer.SearchOptions{ + Keyword: keyword, + RepoIDs: []int64{ctx.Repo.Repository.ID}, + IsPull: isPull, + IsClosed: isClosed, + IncludedLabelIDs: nil, + ExcludedLabelIDs: nil, + NoLabel: false, + MilestoneIDs: nil, + NoMilestone: false, + PosterID: &createdByID, + AssigneeID: &assignedByID, + MentionID: &mentionedByID, + UpdatedAfterUnix: &since, + UpdatedBeforeUnix: &before, + SortBy: issue_indexer.SearchOptionsSortByCreatedDesc, + } + if len(labelIDs) == 0 && labelIDs[0] == 0 { + searchOpt.NoLabel = true + } else { + for _, labelID := range labelIDs { + if labelID > 0 { + searchOpt.IncludedLabelIDs = append(searchOpt.IncludedLabelIDs, labelID) + } else { + searchOpt.ExcludedLabelIDs = append(searchOpt.ExcludedLabelIDs, -labelID) + } } + } + if len(mileIDs) == 0 && mileIDs[0] == db.NoConditionID { + searchOpt.NoMilestone = true + } else { + searchOpt.MilestoneIDs = mileIDs + } - issuesOpt.Paginator = &db.ListOptions{ - Page: -1, - } - if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil { - ctx.Error(http.StatusInternalServerError, "CountIssues", err) - return - } + ids, total, err := issue_indexer.SearchIssues(ctx, searchOpt) + if err != nil { + ctx.Error(http.StatusInternalServerError, "SearchIssues", err) + return + } + issues, err := issues_model.FindIssuesByIDs(ctx, ids) + if err != nil { + ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err) + return } - ctx.SetLinkHeader(int(filteredCount), listOptions.PageSize) - ctx.SetTotalCountHeader(filteredCount) + ctx.SetLinkHeader(int(total), listOptions.PageSize) + ctx.SetTotalCountHeader(total) ctx.JSON(http.StatusOK, convert.ToAPIIssueList(ctx, issues)) } From 0dcbb40ac40b82dd4bcd20f424e418f3c3a1baa8 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 17 Jul 2023 15:10:11 +0800 Subject: [PATCH 20/99] feat: ParsePaginator --- modules/indexer/internal/paginator.go | 25 +++++++++++++++++++ modules/indexer/issues/bleve/bleve.go | 3 ++- .../issues/elasticsearch/elasticsearch.go | 5 ++-- .../indexer/issues/meilisearch/meilisearch.go | 5 ++-- 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 modules/indexer/internal/paginator.go diff --git a/modules/indexer/internal/paginator.go b/modules/indexer/internal/paginator.go new file mode 100644 index 0000000000000..004e3ad2e9704 --- /dev/null +++ b/modules/indexer/internal/paginator.go @@ -0,0 +1,25 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package internal + +import ( + "math" + + "code.gitea.io/gitea/models/db" +) + +// ParsePaginator parses a db.Paginator into a skip and limit +func ParsePaginator(paginator db.Paginator) (int, int) { + if paginator == nil { + // Use default values + return 0, 50 + } + + if paginator.IsListAll() { + // Use a very large number to list all + return 0, math.MaxInt + } + + return paginator.GetSkipTake() +} diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 61c8878d561f1..4cde9c9c3dad9 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -168,7 +168,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer), inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer), )) - search := bleve.NewSearchRequestOptions(indexerQuery, options.Limit, options.Skip, false) + skip, limit := indexer_internal.ParsePaginator(options.Paginator) + search := bleve.NewSearchRequestOptions(indexerQuery, limit, skip, false) search.SortBy([]string{"-_score"}) result, err := b.inner.Indexer.SearchInContext(ctx, search) diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index afcc6bcf6da77..0becff6e81acd 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -152,17 +152,18 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) query = query.Must(repoQuery) } + skip, limit := indexer_internal.ParsePaginator(options.Paginator) searchResult, err := b.inner.Client.Search(). Index(b.inner.VersionedIndexName()). Query(query). Sort("_score", false). - From(options.Skip).Size(options.Limit). + From(skip).Size(limit). Do(ctx) if err != nil { return nil, err } - hits := make([]internal.Match, 0, options.Limit) + hits := make([]internal.Match, 0, limit) for _, hit := range searchResult.Hits.Hits { id, _ := strconv.ParseInt(hit.Id, 10, 64) hits = append(hits, internal.Match{ diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 651f4fe24be82..5990e0de53f4a 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -76,6 +76,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10)) } filter := strings.Join(repoFilters, " OR ") + skip, limit := indexer_internal.ParsePaginator(options.Paginator) // TBC: /* @@ -90,8 +91,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(options.Keyword, &meilisearch.SearchRequest{ Filter: filter, - Limit: int64(options.Limit), - Offset: int64(options.Skip), + Limit: int64(limit), + Offset: int64(skip), }) if err != nil { return nil, err From 35eec573d8ade967da288597bc69c8754ce368fd Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 17 Jul 2023 15:42:37 +0800 Subject: [PATCH 21/99] fix: public repo --- modules/indexer/issues/internal/model.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index b3d10929b832a..8b5d2f6642564 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -11,8 +11,9 @@ import ( // IndexerData data stored in the issue indexer type IndexerData struct { - ID int64 `json:"id"` - RepoID int64 `json:"repo_id"` + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + IsPublic bool `json:"is_public"` // If the repo is public, so if the visibility of the repo has changed, we should reindex the issues. // Fields used for keyword searching Title string `json:"title"` @@ -67,7 +68,8 @@ type SearchResult struct { type SearchOptions struct { Keyword string // keyword to search - RepoIDs []int64 // repository IDs which the issues belong to + RepoIDs []int64 // repository IDs which the issues belong to + AllPublic bool // if include all public repositories IsPull util.OptionalBool // if the issues is a pull request IsClosed util.OptionalBool // if the issues is closed @@ -113,4 +115,7 @@ const ( SearchOptionsSortByUpdatedAsc SearchOptionsSortBy = "updated" SearchOptionsSortByCommentsAsc SearchOptionsSortBy = "comments" SearchOptionsSortByDueAsc SearchOptionsSortBy = "due" + // Unsupported sort types which are supported by issues.IssuesOptions.SortType: + // - "priorityrepo" + // - "project-column-sorting" ) From a6788361ddb6ad97f8708ac6206e7dd1bdb516c9 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 17 Jul 2023 17:52:45 +0800 Subject: [PATCH 22/99] feat: use SearchIssues --- models/db/common.go | 17 +++ models/issues/label.go | 12 ++ models/issues/milestone.go | 12 ++ routers/api/v1/repo/issue.go | 241 ++++++++++++++++++----------------- routers/web/repo/issue.go | 8 +- 5 files changed, 171 insertions(+), 119 deletions(-) diff --git a/models/db/common.go b/models/db/common.go index af6130c9f255b..2a5043a8e7849 100644 --- a/models/db/common.go +++ b/models/db/common.go @@ -20,3 +20,20 @@ func BuildCaseInsensitiveLike(key, value string) builder.Cond { } return builder.Like{"UPPER(" + key + ")", strings.ToUpper(value)} } + +// BuildCaseInsensitiveIn returns a condition to check if the given value is in the given values case-insensitively. +// Handles especially SQLite correctly as UPPER there only transforms ASCII letters. +func BuildCaseInsensitiveIn(key string, values []string) builder.Cond { + uppers := make([]string, 0, len(values)) + if setting.Database.Type.IsSQLite3() { + for _, value := range values { + uppers = append(uppers, util.ToUpperASCII(value)) + } + } else { + for _, value := range values { + uppers = append(uppers, strings.ToUpper(value)) + } + } + + return builder.In("UPPER("+key+")", uppers) +} diff --git a/models/issues/label.go b/models/issues/label.go index 8f2cf05a28d65..7fda0b5804e9c 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -476,6 +476,18 @@ func GetLabelsByOrgID(ctx context.Context, orgID int64, sortType string, listOpt return labels, sess.Find(&labels) } +// GetLabelIDsByNames returns a list of labelIDs by names. +// It doesn't filter them by repo or org, so it could return labels belonging to different repos/orgs. +// It's used for filtering issues via indexer, otherwise it would be useless. +// Since it could return labels with the same name, so the length of returned ids could be more than the length of names. +func GetLabelIDsByNames(ctx context.Context, labelNames []string) ([]int64, error) { + labelIDs := make([]int64, 0, len(labelNames)) + return labelIDs, db.GetEngine(ctx).Table("label"). + In("name", labelNames). + Cols("id"). + Find(&labelIDs) +} + // CountLabelsByOrgID count all labels that belong to given organization by ID. func CountLabelsByOrgID(orgID int64) (int64, error) { return db.GetEngine(db.DefaultContext).Where("org_id = ?", orgID).Count(&Label{}) diff --git a/models/issues/milestone.go b/models/issues/milestone.go index ffe5c8eb509ba..1418e0869d376 100644 --- a/models/issues/milestone.go +++ b/models/issues/milestone.go @@ -396,6 +396,18 @@ func GetMilestones(opts GetMilestonesOption) (MilestoneList, int64, error) { return miles, total, err } +// GetMilestoneIDsByNames returns a list of milestone ids by given names. +// It doesn't filter them by repo, so it could return milestones belonging to different repos. +// It's used for filtering issues via indexer, otherwise it would be useless. +// Since it could return milestones with the same name, so the length of returned ids could be more than the length of names. +func GetMilestoneIDsByNames(ctx context.Context, names []string) ([]int64, error) { + var ids []int64 + return ids, db.GetEngine(ctx).Table("milestone"). + Where(db.BuildCaseInsensitiveIn("name", names)). + Cols("id"). + Find(&ids) +} + // SearchMilestones search milestones func SearchMilestones(repoCond builder.Cond, page int, isClosed bool, sortType, keyword string) (MilestoneList, error) { miles := make([]*Milestone, 0, setting.UI.IssuePagingNum) diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 507130c890cc3..40eb4b31e9292 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -132,74 +132,73 @@ func SearchIssues(ctx *context.APIContext) { isClosed = util.OptionalBoolFalse } - // find repos user can access (for issue search) - opts := &repo_model.SearchRepoOptions{ - Private: false, - AllPublic: true, - TopicOnly: false, - Collaborate: util.OptionalBoolNone, - // This needs to be a column that is not nil in fixtures or - // MySQL will return different results when sorting by null in some cases - OrderBy: db.SearchOrderByAlphabetically, - Actor: ctx.Doer, - } - if ctx.IsSigned { - opts.Private = true - opts.AllLimited = true - } - if ctx.FormString("owner") != "" { - owner, err := user_model.GetUserByName(ctx, ctx.FormString("owner")) - if err != nil { - if user_model.IsErrUserNotExist(err) { - ctx.Error(http.StatusBadRequest, "Owner not found", err) - } else { - ctx.Error(http.StatusInternalServerError, "GetUserByName", err) + var ( + repoIDs []int64 + allPublic bool + ) + { + // find repos user can access (for issue search) + opts := &repo_model.SearchRepoOptions{ + Private: false, + AllPublic: true, + TopicOnly: false, + Collaborate: util.OptionalBoolNone, + // This needs to be a column that is not nil in fixtures or + // MySQL will return different results when sorting by null in some cases + OrderBy: db.SearchOrderByAlphabetically, + Actor: ctx.Doer, + } + if ctx.IsSigned { + opts.Private = true + opts.AllLimited = true + } + if ctx.FormString("owner") != "" { + owner, err := user_model.GetUserByName(ctx, ctx.FormString("owner")) + if err != nil { + if user_model.IsErrUserNotExist(err) { + ctx.Error(http.StatusBadRequest, "Owner not found", err) + } else { + ctx.Error(http.StatusInternalServerError, "GetUserByName", err) + } + return } - return + opts.OwnerID = owner.ID + opts.AllLimited = false + opts.AllPublic = false + opts.Collaborate = util.OptionalBoolFalse + } + if ctx.FormString("team") != "" { + if ctx.FormString("owner") == "" { + ctx.Error(http.StatusBadRequest, "", "Owner organisation is required for filtering on team") + return + } + team, err := organization.GetTeam(ctx, opts.OwnerID, ctx.FormString("team")) + if err != nil { + if organization.IsErrTeamNotExist(err) { + ctx.Error(http.StatusBadRequest, "Team not found", err) + } else { + ctx.Error(http.StatusInternalServerError, "GetUserByName", err) + } + return + } + opts.TeamID = team.ID } - opts.OwnerID = owner.ID - opts.AllLimited = false - opts.AllPublic = false - opts.Collaborate = util.OptionalBoolFalse - } - if ctx.FormString("team") != "" { - if ctx.FormString("owner") == "" { - ctx.Error(http.StatusBadRequest, "", "Owner organisation is required for filtering on team") - return + + if opts.AllPublic { + allPublic = true + opts.AllPublic = false // set it false to avoid returning too many repos, we could filter by indexer } - team, err := organization.GetTeam(ctx, opts.OwnerID, ctx.FormString("team")) + repoIDs, _, err = repo_model.SearchRepositoryIDs(opts) if err != nil { - if organization.IsErrTeamNotExist(err) { - ctx.Error(http.StatusBadRequest, "Team not found", err) - } else { - ctx.Error(http.StatusInternalServerError, "GetUserByName", err) - } + ctx.Error(http.StatusInternalServerError, "SearchRepositoryIDs", err) return } - opts.TeamID = team.ID - } - - repoCond := repo_model.SearchRepositoryCondition(opts) - repoIDs, _, err := repo_model.SearchRepositoryIDs(opts) - if err != nil { - ctx.Error(http.StatusInternalServerError, "SearchRepositoryIDs", err) - return } - var issues []*issues_model.Issue - var filteredCount int64 - keyword := ctx.FormTrim("q") if strings.IndexByte(keyword, 0) >= 0 { keyword = "" } - var issueIDs []int64 - if len(keyword) > 0 && len(repoIDs) > 0 { - if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil { - ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err) - return - } - } var isPull util.OptionalBool switch ctx.FormString("type") { @@ -211,16 +210,33 @@ func SearchIssues(ctx *context.APIContext) { isPull = util.OptionalBoolNone } - labels := ctx.FormTrim("labels") - var includedLabelNames []string - if len(labels) > 0 { - includedLabelNames = strings.Split(labels, ",") + var includedLabels []int64 + { + + labels := ctx.FormTrim("labels") + var includedLabelNames []string + if len(labels) > 0 { + includedLabelNames = strings.Split(labels, ",") + } + includedLabels, err = issues_model.GetLabelIDsByNames(ctx, includedLabelNames) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetLabelIDsByNames", err) + return + } } - milestones := ctx.FormTrim("milestones") - var includedMilestones []string - if len(milestones) > 0 { - includedMilestones = strings.Split(milestones, ",") + var includedMilestones []int64 + { + milestones := ctx.FormTrim("milestones") + var includedMilestoneNames []string + if len(milestones) > 0 { + includedMilestoneNames = strings.Split(milestones, ",") + } + includedMilestones, err = issues_model.GetMilestoneIDsByNames(ctx, includedMilestoneNames) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetMilestoneIDsByNames", err) + return + } } // this api is also used in UI, @@ -232,64 +248,57 @@ func SearchIssues(ctx *context.APIContext) { limit = setting.API.MaxResponseItems } - // Only fetch the issues if we either don't have a keyword or the search returned issues - // This would otherwise return all issues if no issues were found by the search. - if len(keyword) == 0 || len(issueIDs) > 0 || len(includedLabelNames) > 0 || len(includedMilestones) > 0 { - issuesOpt := &issues_model.IssuesOptions{ - Paginator: &db.ListOptions{ - Page: ctx.FormInt("page"), - PageSize: limit, - }, - RepoCond: repoCond, - IsClosed: isClosed, - IssueIDs: issueIDs, - IncludedLabelNames: includedLabelNames, - IncludeMilestones: includedMilestones, - SortType: "priorityrepo", - PriorityRepoID: ctx.FormInt64("priority_repo_id"), - IsPull: isPull, - UpdatedBeforeUnix: before, - UpdatedAfterUnix: since, - } + searchOpt := &issue_indexer.SearchOptions{ + Keyword: keyword, + RepoIDs: repoIDs, + AllPublic: allPublic, + IsPull: isPull, + IsClosed: isClosed, + IncludedLabelIDs: includedLabels, + MilestoneIDs: includedMilestones, + UpdatedAfterUnix: &since, + UpdatedBeforeUnix: &before, + SortBy: issue_indexer.SearchOptionsSortByCreatedDesc, + } - ctxUserID := int64(0) - if ctx.IsSigned { - ctxUserID = ctx.Doer.ID - } + ctxUserID := int64(0) + if ctx.IsSigned { + ctxUserID = ctx.Doer.ID + } - // Filter for: Created by User, Assigned to User, Mentioning User, Review of User Requested - if ctx.FormBool("created") { - issuesOpt.PosterID = ctxUserID - } - if ctx.FormBool("assigned") { - issuesOpt.AssigneeID = ctxUserID - } - if ctx.FormBool("mentioned") { - issuesOpt.MentionedID = ctxUserID - } - if ctx.FormBool("review_requested") { - issuesOpt.ReviewRequestedID = ctxUserID - } - if ctx.FormBool("reviewed") { - issuesOpt.ReviewedID = ctxUserID - } + if ctx.FormBool("created") { + searchOpt.PosterID = &ctxUserID + } + if ctx.FormBool("assigned") { + searchOpt.AssigneeID = &ctxUserID + } + if ctx.FormBool("mentioned") { + searchOpt.MentionID = &ctxUserID + } + if ctx.FormBool("review_requested") { + searchOpt.ReviewRequestedID = &ctxUserID + } + if ctx.FormBool("reviewed") { + searchOpt.ReviewedID = &ctxUserID + } - if issues, err = issues_model.Issues(ctx, issuesOpt); err != nil { - ctx.Error(http.StatusInternalServerError, "Issues", err) - return - } + // FIXME: It's unsupported to sort by priority repo when searching by indexer, + // it's indeed an regression, but I think it is worth to support filtering by indexer first. + _ = ctx.FormInt64("priority_repo_id") - issuesOpt.Paginator = &db.ListOptions{ - Page: -1, - } - if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil { - ctx.Error(http.StatusInternalServerError, "CountIssues", err) - return - } + ids, total, err := issue_indexer.SearchIssues(ctx, searchOpt) + if err != nil { + ctx.Error(http.StatusInternalServerError, "SearchIssues", err) + return + } + issues, err := issues_model.FindIssuesByIDs(ctx, ids) + if err != nil { + ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err) + return } - ctx.SetLinkHeader(int(filteredCount), limit) - ctx.SetTotalCountHeader(filteredCount) + ctx.SetLinkHeader(int(total), limit) + ctx.SetTotalCountHeader(total) ctx.JSON(http.StatusOK, convert.ToAPIIssueList(ctx, issues)) } diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 3e1e0662378bd..e2317f1945839 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -189,7 +189,7 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti var issueIDs []int64 if len(keyword) > 0 { - issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword, ctx.FormString("state")) + issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword) if err != nil { if issue_indexer.IsAvailable(ctx) { ctx.ServerError("issueIndexer.Search", err) @@ -2466,7 +2466,8 @@ func SearchIssues(ctx *context.Context) { } var issueIDs []int64 if len(keyword) > 0 && len(repoIDs) > 0 { - if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil { + // TBC: use issue_indexer.SearchIssues instead + if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil { ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err.Error()) return } @@ -2614,7 +2615,8 @@ func ListIssues(ctx *context.Context) { var issueIDs []int64 var labelIDs []int64 if len(keyword) > 0 { - issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state")) + // TBC: use issue_indexer.SearchIssues instead + issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return From 840d7e314ce5f917a13dd4f9501ee802ad9e2e5f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 17 Jul 2023 18:44:15 +0800 Subject: [PATCH 23/99] fix: index issue --- modules/indexer/issues/db/options.go | 20 +++--- modules/indexer/issues/indexer.go | 83 +++++++++++------------- modules/indexer/issues/internal/model.go | 23 +++---- modules/indexer/issues/util.go | 39 +++++++++-- 4 files changed, 93 insertions(+), 72 deletions(-) diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index 0a3157d35cfbb..cebeacd87234a 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -16,17 +16,17 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { } return *id } - convertIDs := func(ids []int64, no bool) []int64 { - if no { + convertIDs := func(ids []int64) []int64 { + if len(ids) == 0 { return []int64{db.NoConditionID} } return ids } - convertLabelIDs := func(includes, excludes []int64, no bool) []int64 { - if no { - return []int64{0} // Be careful, it's zero, not db.NoConditionID, + convertLabelIDs := func(includes, excludes []int64, includeNo bool) []int64 { + ret := make([]int64, 0, len(includes)+len(excludes)+1) + if includeNo { + ret = append(ret, 0) } - ret := make([]int64, 0, len(includes)+len(excludes)) ret = append(ret, includes...) for _, id := range excludes { ret = append(ret, -id) @@ -47,7 +47,7 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { sortType = "leastupdate" case internal.SearchOptionsSortByCommentsAsc: sortType = "leastcomment" - case internal.SearchOptionsSortByDueAsc: + case internal.SearchOptionsSortByDeadlineAsc: sortType = "farduedate" case internal.SearchOptionsSortByCreatedDesc: sortType = "" // default @@ -55,7 +55,7 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { sortType = "recentupdate" case internal.SearchOptionsSortByCommentsDesc: sortType = "mostcomment" - case internal.SearchOptionsSortByDueDesc: + case internal.SearchOptionsSortByDeadlineDesc: sortType = "nearduedate" } @@ -69,12 +69,12 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { ReviewRequestedID: convertID(options.ReviewRequestedID), ReviewedID: convertID(options.ReviewedID), SubscriberID: convertID(options.SubscriberID), - MilestoneIDs: convertIDs(options.MilestoneIDs, options.NoMilestone), + MilestoneIDs: convertIDs(options.MilestoneIDs), ProjectID: convertID(options.ProjectID), ProjectBoardID: convertID(options.ProjectBoardID), IsClosed: options.IsClosed, IsPull: options.IsPull, - LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.NoLabel), + LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.ExcludedNoLabel), IncludedLabelNames: nil, ExcludedLabelNames: nil, IncludeMilestones: nil, diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 54b5a7dbd99ab..9cb13bcd0c7c1 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -65,50 +65,7 @@ func InitIssueIndexer(syncReindex bool) { indexerInitWaitChannel := make(chan time.Duration, 1) // Create the Queue - switch setting.Indexer.IssueType { - case "bleve", "elasticsearch", "meilisearch": - handler := func(items ...*IndexerMetadata) (unhandled []*IndexerMetadata) { - indexer := *globalIndexer.Load() - for _, item := range items { - log.Trace("IndexerMetadata Process: %d %v %t", item.ID, item.IDs, item.IsDelete) - if item.IsDelete { - if err := indexer.Delete(ctx, item.IDs...); err != nil { - log.Error("Issue indexer handler: failed to from index: %v Error: %v", item.IDs, err) - unhandled = append(unhandled, item) - } - continue - } - data, existed, err := getIssueIndexerData(ctx, item.ID) - if err != nil { - log.Error("Issue indexer handler: failed to get issue data of %d: %v", item.ID, err) - unhandled = append(unhandled, item) - continue - } - if !existed { - if err := indexer.Delete(ctx, item.ID); err != nil { - log.Error("Issue indexer handler: failed to delete issue %d from index: %v", item.ID, err) - unhandled = append(unhandled, item) - } - continue - } - if err := indexer.Index(ctx, data); err != nil { - log.Error("Issue indexer handler: failed to index issue %d: %v", item.ID, err) - unhandled = append(unhandled, item) - continue - } - } - - return unhandled - } - - issueIndexerQueue = queue.CreateUniqueQueue(ctx, "issue_indexer", handler) - - if issueIndexerQueue == nil { - log.Fatal("Unable to create issue indexer queue") - } - default: - issueIndexerQueue = queue.CreateUniqueQueue[*IndexerMetadata](ctx, "issue_indexer", nil) - } + issueIndexerQueue = queue.CreateUniqueQueue(ctx, "issue_indexer", getIssueIndexerQueueHandler(ctx)) graceful.GetManager().RunAtTerminate(finished) @@ -204,6 +161,44 @@ func InitIssueIndexer(syncReindex bool) { } } +func getIssueIndexerQueueHandler(ctx context.Context) func(items ...*IndexerMetadata) []*IndexerMetadata { + return func(items ...*IndexerMetadata) []*IndexerMetadata { + var unhandled []*IndexerMetadata + + indexer := *globalIndexer.Load() + for _, item := range items { + log.Trace("IndexerMetadata Process: %d %v %t", item.ID, item.IDs, item.IsDelete) + if item.IsDelete { + if err := indexer.Delete(ctx, item.IDs...); err != nil { + log.Error("Issue indexer handler: failed to from index: %v Error: %v", item.IDs, err) + unhandled = append(unhandled, item) + } + continue + } + data, existed, err := getIssueIndexerData(ctx, item.ID) + if err != nil { + log.Error("Issue indexer handler: failed to get issue data of %d: %v", item.ID, err) + unhandled = append(unhandled, item) + continue + } + if !existed { + if err := indexer.Delete(ctx, item.ID); err != nil { + log.Error("Issue indexer handler: failed to delete issue %d from index: %v", item.ID, err) + unhandled = append(unhandled, item) + } + continue + } + if err := indexer.Index(ctx, data); err != nil { + log.Error("Issue indexer handler: failed to index issue %d: %v", item.ID, err) + unhandled = append(unhandled, item) + continue + } + } + + return unhandled + } +} + // populateIssueIndexer populate the issue indexer with issue data func populateIssueIndexer(ctx context.Context) { ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Service: PopulateIssueIndexer", process.SystemProcessType, true) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 8b5d2f6642564..ff94c135a535e 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -22,14 +22,12 @@ type IndexerData struct { // Fields used for filtering IsPull bool `json:"is_pull"` - IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. - Labels []int64 `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. - NoLabels bool `json:"no_labels"` // True if Labels is empty - MilestoneIDs []int64 `json:"milestone_ids"` // So if the milestones of an issue have changed, we should reindex the issue. - NoMilestone bool `json:"no_milestone"` // True if Milestones is empty - ProjectIDs []int64 `json:"project_ids"` // So if the projects of an issue have changed, we should reindex the issue. - ProjectBoardIDs []int64 `json:"project_board_ids"` // So if the projects of an issue have changed, we should reindex the issue. - NoProject bool `json:"no_project"` // True if ProjectIDs is empty + IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. + Labels []int64 `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. + NoLabel bool `json:"no_label"` // True if Labels is empty + MilestoneID int64 `json:"milestone_id"` // So if the milestones of an issue have changed, we should reindex the issue. + ProjectID int64 `json:"project_id"` // So if the projects of an issue have changed, we should reindex the issue. + ProjectBoardID int64 `json:"project_board_id"` // So if the projects of an issue have changed, we should reindex the issue. PosterID int64 `json:"poster_id"` AssigneeID int64 `json:"assignee_id"` // So if the assignee of an issue has changed, we should reindex the issue. MentionIDs []int64 `json:"mention_ids"` @@ -40,7 +38,7 @@ type IndexerData struct { // Fields used for sorting CreatedUnix timeutil.TimeStamp `json:"created_unix"` - DueUnix timeutil.TimeStamp `json:"due_unix"` + DeadlineUnix timeutil.TimeStamp `json:"deadline_unix"` CommentCount int64 `json:"comment_count"` } @@ -76,10 +74,9 @@ type SearchOptions struct { IncludedLabelIDs []int64 // labels the issues have ExcludedLabelIDs []int64 // labels the issues don't have - NoLabel bool // if the issues have no label, if true, IncludedLabelIDs and ExcludedLabelIDs will be ignored + ExcludedNoLabel bool // if include issues without labels MilestoneIDs []int64 // milestones the issues have - NoMilestone bool // if the issues have no milestones, if true, MilestoneIDs will be ignored ProjectID *int64 // project the issues belong to ProjectBoardID *int64 // project board the issues belong to @@ -110,11 +107,11 @@ const ( SearchOptionsSortByCreatedDesc SearchOptionsSortBy = "-created" SearchOptionsSortByUpdatedDesc SearchOptionsSortBy = "-updated" SearchOptionsSortByCommentsDesc SearchOptionsSortBy = "-comments" - SearchOptionsSortByDueDesc SearchOptionsSortBy = "-due" + SearchOptionsSortByDeadlineDesc SearchOptionsSortBy = "-deadline" SearchOptionsSortByCreatedAsc SearchOptionsSortBy = "created" SearchOptionsSortByUpdatedAsc SearchOptionsSortBy = "updated" SearchOptionsSortByCommentsAsc SearchOptionsSortBy = "comments" - SearchOptionsSortByDueAsc SearchOptionsSortBy = "due" + SearchOptionsSortByDeadlineAsc SearchOptionsSortBy = "deadline" // Unsupported sort types which are supported by issues.IssuesOptions.SortType: // - "priorityrepo" // - "project-column-sorting" diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index 6284d9407030c..bff0e01470f16 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -34,11 +34,40 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD } } + if err := issue.LoadAttributes(ctx); err != nil { + return nil, false, err + } + + labels := make([]int64, 0, len(issue.Labels)) + for _, label := range issue.Labels { + labels = append(labels, label.ID) + } + + // TBC: MentionIDs ReviewedIDs ReviewRequestedIDs SubscriberIDs + return &internal.IndexerData{ - ID: issue.ID, - RepoID: issue.RepoID, - Title: issue.Title, - Content: issue.Content, - Comments: comments, + ID: issue.ID, + RepoID: issue.RepoID, + IsPublic: !issue.Repo.IsPrivate, + Title: issue.Title, + Content: issue.Content, + Comments: comments, + IsPull: issue.IsPull, + IsClosed: issue.IsClosed, + Labels: labels, + NoLabel: len(labels) == 0, + MilestoneID: issue.MilestoneID, + ProjectID: issue.Project.ID, + ProjectBoardID: issue.ProjectBoardID(), + PosterID: issue.PosterID, + AssigneeID: issue.AssigneeID, + MentionIDs: nil, + ReviewedIDs: nil, + ReviewRequestedIDs: nil, + SubscriberIDs: nil, + UpdatedUnix: issue.UpdatedUnix, + CreatedUnix: issue.CreatedUnix, + DeadlineUnix: issue.DeadlineUnix, + CommentCount: int64(len(issue.Comments)), }, true, nil } From 2feead27ddcadbe389d554fd18650730374e6544 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 18 Jul 2023 15:16:40 +0800 Subject: [PATCH 24/99] feat: getIssueIndexerData --- models/issues/issue_user.go | 10 +++++++ modules/indexer/issues/util.go | 50 ++++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/models/issues/issue_user.go b/models/issues/issue_user.go index 4a537752a2ca4..47da5c62c4d4a 100644 --- a/models/issues/issue_user.go +++ b/models/issues/issue_user.go @@ -84,3 +84,13 @@ func UpdateIssueUsersByMentions(ctx context.Context, issueID int64, uids []int64 } return nil } + +// GetIssueMentionIDs returns all mentioned user IDs of an issue. +func GetIssueMentionIDs(ctx context.Context, issueID int64) ([]int64, error) { + var ids []int64 + return ids, db.GetEngine(ctx).Table(IssueUser{}). + Where("issue_id=?", issueID). + And("is_mentioned=?", true). + Select("uid"). + Find(&ids) +} diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index bff0e01470f16..b744502443032 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -6,10 +6,13 @@ package issues import ( "context" + "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/indexer/issues/internal" ) +// getIssueIndexerData returns the indexer data of an issue and a bool value indicating whether the issue exists. func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerData, bool, error) { issue, err := issue_model.GetIssueByID(ctx, issueID) if err != nil { @@ -43,7 +46,44 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD labels = append(labels, label.ID) } - // TBC: MentionIDs ReviewedIDs ReviewRequestedIDs SubscriberIDs + mentionIDs, err := issue_model.GetIssueMentionIDs(ctx, issueID) + if err != nil { + return nil, false, err + } + + var ( + reviewedIDs []int64 + reviewRequestedIDs []int64 + ) + { + reviews, err := issue_model.FindReviews(ctx, issue_model.FindReviewOptions{ + ListOptions: db.ListOptions{ + ListAll: true, + }, + IssueID: issueID, + OfficialOnly: false, + }) + if err != nil { + return nil, false, err + } + + reviewedIDsSet := make(container.Set[int64], len(reviews)) + reviewRequestedIDsSet := make(container.Set[int64], len(reviews)) + for _, review := range reviews { + if review.Type == issue_model.ReviewTypeRequest { + reviewRequestedIDsSet.Add(review.ReviewerID) + } else { + reviewedIDsSet.Add(review.ReviewerID) + } + } + reviewedIDs = reviewedIDsSet.Values() + reviewRequestedIDs = reviewRequestedIDsSet.Values() + } + + subscriberIDs, err := issue_model.GetIssueWatchersIDs(ctx, issue.ID, true) + if err != nil { + return nil, false, err + } return &internal.IndexerData{ ID: issue.ID, @@ -61,10 +101,10 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD ProjectBoardID: issue.ProjectBoardID(), PosterID: issue.PosterID, AssigneeID: issue.AssigneeID, - MentionIDs: nil, - ReviewedIDs: nil, - ReviewRequestedIDs: nil, - SubscriberIDs: nil, + MentionIDs: mentionIDs, + ReviewedIDs: reviewedIDs, + ReviewRequestedIDs: reviewRequestedIDs, + SubscriberIDs: subscriberIDs, UpdatedUnix: issue.UpdatedUnix, CreatedUnix: issue.CreatedUnix, DeadlineUnix: issue.DeadlineUnix, From 25eda692298ccbd63f5673231e9a4531caa6a376 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 18 Jul 2023 15:26:48 +0800 Subject: [PATCH 25/99] feat: update option --- modules/indexer/issues/db/options.go | 10 +++++----- modules/indexer/issues/internal/model.go | 2 +- routers/api/v1/repo/issue.go | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index cebeacd87234a..bd7a05b4f0d36 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -22,11 +22,11 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { } return ids } - convertLabelIDs := func(includes, excludes []int64, includeNo bool) []int64 { - ret := make([]int64, 0, len(includes)+len(excludes)+1) - if includeNo { - ret = append(ret, 0) + convertLabelIDs := func(includes, excludes []int64, noLabelOnly bool) []int64 { + if noLabelOnly { + return []int64{0} } + ret := make([]int64, 0, len(includes)+len(excludes)) ret = append(ret, includes...) for _, id := range excludes { ret = append(ret, -id) @@ -74,7 +74,7 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { ProjectBoardID: convertID(options.ProjectBoardID), IsClosed: options.IsClosed, IsPull: options.IsPull, - LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.ExcludedNoLabel), + LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.NoLabelOnly), IncludedLabelNames: nil, ExcludedLabelNames: nil, IncludeMilestones: nil, diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index ff94c135a535e..d98bae88936ed 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -74,7 +74,7 @@ type SearchOptions struct { IncludedLabelIDs []int64 // labels the issues have ExcludedLabelIDs []int64 // labels the issues don't have - ExcludedNoLabel bool // if include issues without labels + NoLabelOnly bool // if the issues have no label, if true, IncludedLabelIDs and ExcludedLabelIDs will be ignored MilestoneIDs []int64 // milestones the issues have diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 40eb4b31e9292..1b112d08f1b52 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -470,9 +470,8 @@ func ListIssues(ctx *context.APIContext) { IsClosed: isClosed, IncludedLabelIDs: nil, ExcludedLabelIDs: nil, - NoLabel: false, + NoLabelOnly: false, MilestoneIDs: nil, - NoMilestone: false, PosterID: &createdByID, AssigneeID: &assignedByID, MentionID: &mentionedByID, @@ -481,7 +480,7 @@ func ListIssues(ctx *context.APIContext) { SortBy: issue_indexer.SearchOptionsSortByCreatedDesc, } if len(labelIDs) == 0 && labelIDs[0] == 0 { - searchOpt.NoLabel = true + searchOpt.NoLabelOnly = true } else { for _, labelID := range labelIDs { if labelID > 0 { @@ -491,8 +490,9 @@ func ListIssues(ctx *context.APIContext) { } } } - if len(mileIDs) == 0 && mileIDs[0] == db.NoConditionID { - searchOpt.NoMilestone = true + + if len(mileIDs) == 1 && mileIDs[0] == db.NoConditionID { + searchOpt.MilestoneIDs = []int64{0} } else { searchOpt.MilestoneIDs = mileIDs } From cff4ad6708209b7100ab8c07594f823e3cb54f87 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 18 Jul 2023 15:48:05 +0800 Subject: [PATCH 26/99] feat: update issue indexer --- modules/indexer/issues/indexer.go | 38 ++++++++++----------- modules/indexer/issues/internal/model.go | 6 ++-- modules/notification/indexer/indexer.go | 42 +++++++----------------- modules/repository/create.go | 4 +++ 4 files changed, 36 insertions(+), 54 deletions(-) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 9cb13bcd0c7c1..6f37395a1033b 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -232,15 +232,15 @@ func populateIssueIndexer(ctx context.Context) { return default: } - UpdateRepoIndexer(ctx, repo) + UpdateRepoIndexer(ctx, repo.ID) } } } // UpdateRepoIndexer add/update all issues of the repositories -func UpdateRepoIndexer(ctx context.Context, repo *repo_model.Repository) { +func UpdateRepoIndexer(ctx context.Context, repoID int64) { is, err := issues_model.Issues(ctx, &issues_model.IssuesOptions{ - RepoIDs: []int64{repo.ID}, + RepoIDs: []int64{repoID}, IsClosed: util.OptionalBoolNone, IsPull: util.OptionalBoolNone, }) @@ -248,26 +248,22 @@ func UpdateRepoIndexer(ctx context.Context, repo *repo_model.Repository) { log.Error("Issues: %v", err) return } - if err = issues_model.IssueList(is).LoadDiscussComments(ctx); err != nil { - log.Error("LoadDiscussComments: %v", err) - return - } for _, issue := range is { - UpdateIssueIndexer(issue) + UpdateIssueIndexer(issue.ID) } } // UpdateIssueIndexer add/update an issue to the issue indexer -func UpdateIssueIndexer(issue *issues_model.Issue) { - if err := issueIndexerQueue.Push(&IndexerMetadata{ID: issue.ID}); err != nil { - log.Error("Unable to push to issue indexer: %v: Error: %v", issue.ID, err) +func UpdateIssueIndexer(issueID int64) { + if err := issueIndexerQueue.Push(&IndexerMetadata{ID: issueID}); err != nil { + log.Error("Unable to push to issue indexer: %v: Error: %v", issueID, err) } } // DeleteRepoIssueIndexer deletes repo's all issues indexes -func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) { +func DeleteRepoIssueIndexer(ctx context.Context, repoID int64) { var ids []int64 - ids, err := issues_model.GetIssueIDsByRepoID(ctx, repo.ID) + ids, err := issues_model.GetIssueIDsByRepoID(ctx, repoID) if err != nil { log.Error("GetIssueIDsByRepoID failed: %v", err) return @@ -314,14 +310,14 @@ func IsAvailable(ctx context.Context) bool { type SearchOptions internal.SearchOptions const ( - SearchOptionsSortByCreatedDesc internal.SearchOptionsSortBy = "-created" - SearchOptionsSortByUpdatedDesc internal.SearchOptionsSortBy = "-updated" - SearchOptionsSortByCommentsDesc internal.SearchOptionsSortBy = "-comments" - SearchOptionsSortByDueDesc internal.SearchOptionsSortBy = "-due" - SearchOptionsSortByCreatedAsc internal.SearchOptionsSortBy = "created" - SearchOptionsSortByUpdatedAsc internal.SearchOptionsSortBy = "updated" - SearchOptionsSortByCommentsAsc internal.SearchOptionsSortBy = "comments" - SearchOptionsSortByDueAsc internal.SearchOptionsSortBy = "due" + SearchOptionsSortByCreatedDesc = internal.SearchOptionsSortByCreatedDesc + SearchOptionsSortByUpdatedDesc = internal.SearchOptionsSortByUpdatedDesc + SearchOptionsSortByCommentsDesc = internal.SearchOptionsSortByCommentsDesc + SearchOptionsSortByDeadlineDesc = internal.SearchOptionsSortByDeadlineDesc + SearchOptionsSortByCreatedAsc = internal.SearchOptionsSortByCreatedAsc + SearchOptionsSortByUpdatedAsc = internal.SearchOptionsSortByUpdatedAsc + SearchOptionsSortByCommentsAsc = internal.SearchOptionsSortByCommentsAsc + SearchOptionsSortByDeadlineAsc = internal.SearchOptionsSortByDeadlineAsc ) // SearchIssues search issues by options. diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index d98bae88936ed..5942971d04ca3 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -25,9 +25,9 @@ type IndexerData struct { IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. Labels []int64 `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. NoLabel bool `json:"no_label"` // True if Labels is empty - MilestoneID int64 `json:"milestone_id"` // So if the milestones of an issue have changed, we should reindex the issue. - ProjectID int64 `json:"project_id"` // So if the projects of an issue have changed, we should reindex the issue. - ProjectBoardID int64 `json:"project_board_id"` // So if the projects of an issue have changed, we should reindex the issue. + MilestoneID int64 `json:"milestone_id"` // So if the milestone of an issue has changed, we should reindex the issue. + ProjectID int64 `json:"project_id"` // So if the project of an issue have changed, we should reindex the issue. + ProjectBoardID int64 `json:"project_board_id"` // So if the project board of an issue have changed, we should reindex the issue. PosterID int64 `json:"poster_id"` AssigneeID int64 `json:"assignee_id"` // So if the assignee of an issue has changed, we should reindex the issue. MentionIDs []int64 `json:"mention_ids"` diff --git a/modules/notification/indexer/indexer.go b/modules/notification/indexer/indexer.go index bb652e39426be..96da23e58e3e5 100644 --- a/modules/notification/indexer/indexer.go +++ b/modules/notification/indexer/indexer.go @@ -46,40 +46,22 @@ func (r *indexerNotifier) NotifyCreateIssueComment(ctx context.Context, doer *us issue.Comments = append(issue.Comments, comment) } - issue_indexer.UpdateIssueIndexer(issue) + issue_indexer.UpdateIssueIndexer(issue.ID) } } func (r *indexerNotifier) NotifyNewIssue(ctx context.Context, issue *issues_model.Issue, mentions []*user_model.User) { - issue_indexer.UpdateIssueIndexer(issue) + issue_indexer.UpdateIssueIndexer(issue.ID) } func (r *indexerNotifier) NotifyNewPullRequest(ctx context.Context, pr *issues_model.PullRequest, mentions []*user_model.User) { - issue_indexer.UpdateIssueIndexer(pr.Issue) + issue_indexer.UpdateIssueIndexer(pr.Issue.ID) } func (r *indexerNotifier) NotifyUpdateComment(ctx context.Context, doer *user_model.User, c *issues_model.Comment, oldContent string) { - if c.Type == issues_model.CommentTypeComment { - var found bool - if c.Issue.Comments != nil { - for i := 0; i < len(c.Issue.Comments); i++ { - if c.Issue.Comments[i].ID == c.ID { - c.Issue.Comments[i] = c - found = true - break - } - } - } - - if !found { - if err := c.Issue.LoadDiscussComments(ctx); err != nil { - log.Error("LoadDiscussComments failed: %v", err) - return - } - } - - issue_indexer.UpdateIssueIndexer(c.Issue) - } + // Whatever the comment type is, just update the issue indexer. + // So that the issue indexer will be updated when Status/Assignee/Label and so on changed. + issue_indexer.UpdateIssueIndexer(c.Issue.ID) } func (r *indexerNotifier) NotifyDeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) { @@ -107,19 +89,19 @@ func (r *indexerNotifier) NotifyDeleteComment(ctx context.Context, doer *user_mo } } // reload comments to delete the old comment - issue_indexer.UpdateIssueIndexer(comment.Issue) + issue_indexer.UpdateIssueIndexer(comment.Issue.ID) } } func (r *indexerNotifier) NotifyDeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository) { - issue_indexer.DeleteRepoIssueIndexer(ctx, repo) + issue_indexer.DeleteRepoIssueIndexer(ctx, repo.ID) if setting.Indexer.RepoIndexerEnabled { code_indexer.UpdateRepoIndexer(repo) } } func (r *indexerNotifier) NotifyMigrateRepository(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository) { - issue_indexer.UpdateRepoIndexer(ctx, repo) + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) if setting.Indexer.RepoIndexerEnabled && !repo.IsEmpty { code_indexer.UpdateRepoIndexer(repo) } @@ -155,13 +137,13 @@ func (r *indexerNotifier) NotifySyncPushCommits(ctx context.Context, pusher *use } func (r *indexerNotifier) NotifyIssueChangeContent(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldContent string) { - issue_indexer.UpdateIssueIndexer(issue) + issue_indexer.UpdateIssueIndexer(issue.ID) } func (r *indexerNotifier) NotifyIssueChangeTitle(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldTitle string) { - issue_indexer.UpdateIssueIndexer(issue) + issue_indexer.UpdateIssueIndexer(issue.ID) } func (r *indexerNotifier) NotifyIssueChangeRef(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldRef string) { - issue_indexer.UpdateIssueIndexer(issue) + issue_indexer.UpdateIssueIndexer(issue.ID) } diff --git a/modules/repository/create.go b/modules/repository/create.go index e8a1b8ba2bf81..7635e63385642 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -23,6 +23,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/git" + issue_indexer "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -418,6 +419,9 @@ func UpdateRepository(ctx context.Context, repo *repo_model.Repository, visibili return fmt.Errorf("updateRepository[%d]: %w", forkRepos[i].ID, err) } } + + // Update the issue indexer + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) } return nil From 0a6c2d946a41b22687c0529193b980e482d7a425 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 18 Jul 2023 15:52:23 +0800 Subject: [PATCH 27/99] fix: docMapping --- modules/indexer/issues/bleve/bleve.go | 33 ++++++++++++++------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 4cde9c9c3dad9..929339ead9105 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -67,29 +67,30 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) { numberFieldMapping.Store = false numberFieldMapping.IncludeInAll = false + docMapping.AddFieldMappingsAt("is_public", boolFieldMapping) + docMapping.AddFieldMappingsAt("title", textFieldMapping) docMapping.AddFieldMappingsAt("content", textFieldMapping) docMapping.AddFieldMappingsAt("comments", textFieldMapping) - // TBC: IndexerData has been changed, but the mapping has not been updated docMapping.AddFieldMappingsAt("is_pull", boolFieldMapping) docMapping.AddFieldMappingsAt("is_closed", boolFieldMapping) docMapping.AddFieldMappingsAt("labels", numberFieldMapping) - docMapping.AddFieldMappingsAt("no_labels", boolFieldMapping) - docMapping.AddFieldMappingsAt("milestones", numberFieldMapping) - docMapping.AddFieldMappingsAt("no_milestones", boolFieldMapping) - docMapping.AddFieldMappingsAt("projects", numberFieldMapping) - docMapping.AddFieldMappingsAt("no_projects", boolFieldMapping) - docMapping.AddFieldMappingsAt("author", numberFieldMapping) - docMapping.AddFieldMappingsAt("assignee", numberFieldMapping) - docMapping.AddFieldMappingsAt("mentions", numberFieldMapping) - docMapping.AddFieldMappingsAt("reviewers", numberFieldMapping) - docMapping.AddFieldMappingsAt("requested_reviewers", numberFieldMapping) - - docMapping.AddFieldMappingsAt("created_at", numberFieldMapping) - docMapping.AddFieldMappingsAt("updated_at", numberFieldMapping) - docMapping.AddFieldMappingsAt("closed_at", numberFieldMapping) - docMapping.AddFieldMappingsAt("due_date", numberFieldMapping) + docMapping.AddFieldMappingsAt("no_label", boolFieldMapping) + docMapping.AddFieldMappingsAt("milestone_id", numberFieldMapping) + docMapping.AddFieldMappingsAt("project_id", numberFieldMapping) + docMapping.AddFieldMappingsAt("project_board_id", numberFieldMapping) + docMapping.AddFieldMappingsAt("poster_id", numberFieldMapping) + docMapping.AddFieldMappingsAt("assignee_id", numberFieldMapping) + docMapping.AddFieldMappingsAt("mention_ids", numberFieldMapping) + docMapping.AddFieldMappingsAt("reviewed_ids", numberFieldMapping) + docMapping.AddFieldMappingsAt("review_requested_ids", numberFieldMapping) + docMapping.AddFieldMappingsAt("subscriber_ids", numberFieldMapping) + docMapping.AddFieldMappingsAt("updated_unix", numberFieldMapping) + + docMapping.AddFieldMappingsAt("created_unix", numberFieldMapping) + docMapping.AddFieldMappingsAt("deadline_unix", numberFieldMapping) + docMapping.AddFieldMappingsAt("comment_count", numberFieldMapping) if err := addUnicodeNormalizeTokenFilter(mapping); err != nil { return nil, err From fa35921ba2f582284200ae95b4b5236da4d8edd8 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 18 Jul 2023 16:29:32 +0800 Subject: [PATCH 28/99] feat: bleve searching --- modules/indexer/internal/bleve/query.go | 27 ++++++ modules/indexer/issues/bleve/bleve.go | 113 ++++++++++++++++++++--- modules/indexer/issues/db/options.go | 16 ++-- modules/indexer/issues/indexer.go | 16 ++-- modules/indexer/issues/internal/model.go | 27 +++--- modules/indexer/issues/util.go | 2 +- routers/api/v1/repo/issue.go | 4 +- 7 files changed, 157 insertions(+), 48 deletions(-) diff --git a/modules/indexer/internal/bleve/query.go b/modules/indexer/internal/bleve/query.go index d2f6d13049bbd..c7d66538c1263 100644 --- a/modules/indexer/internal/bleve/query.go +++ b/modules/indexer/internal/bleve/query.go @@ -24,3 +24,30 @@ func MatchPhraseQuery(matchPhrase, field, analyzer string) *query.MatchPhraseQue q.Analyzer = analyzer return q } + +// BoolFieldQuery generates a bool field query for the given value and field +func BoolFieldQuery(value bool, field string) *query.BoolFieldQuery { + q := bleve.NewBoolFieldQuery(value) + q.SetField(field) + return q +} + +func NumericRangeInclusiveQuery(min, max *int64, field string) *query.NumericRangeQuery { + var minF, maxF *float64 + var minI, maxI *bool + if min != nil { + minF = new(float64) + *minF = float64(*min) + minI = new(bool) + *minI = true + } + if max != nil { + maxF = new(float64) + *maxF = float64(*max) + maxI = new(bool) + *maxI = true + } + q := bleve.NewNumericRangeInclusiveQuery(minF, maxF, minI, maxI) + q.SetField(field) + return q +} diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 929339ead9105..52b031c033665 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -75,7 +75,7 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) { docMapping.AddFieldMappingsAt("is_pull", boolFieldMapping) docMapping.AddFieldMappingsAt("is_closed", boolFieldMapping) - docMapping.AddFieldMappingsAt("labels", numberFieldMapping) + docMapping.AddFieldMappingsAt("label_ids", numberFieldMapping) docMapping.AddFieldMappingsAt("no_label", boolFieldMapping) docMapping.AddFieldMappingsAt("milestone_id", numberFieldMapping) docMapping.AddFieldMappingsAt("project_id", numberFieldMapping) @@ -153,25 +153,108 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { - var repoQueriesP []*query.NumericRangeQuery - for _, repoID := range options.RepoIDs { - repoQueriesP = append(repoQueriesP, inner_bleve.NumericEqualityQuery(repoID, "repo_id")) - } - repoQueries := make([]query.Query, len(repoQueriesP)) - for i, v := range repoQueriesP { - repoQueries[i] = query.Query(v) - } + var queries []query.Query - indexerQuery := bleve.NewConjunctionQuery( - bleve.NewDisjunctionQuery(repoQueries...), - bleve.NewDisjunctionQuery( + if options.Keyword != "" { + keywordQueries := []query.Query{ inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer), inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer), inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer), - )) + } + queries = append(queries, bleve.NewDisjunctionQuery(keywordQueries...)) + } + + if len(options.RepoIDs) > 0 || options.AllPublic { + var repoQueries []query.Query + for _, repoID := range options.RepoIDs { + repoQueries = append(repoQueries, inner_bleve.NumericEqualityQuery(repoID, "repo_id")) + } + if options.AllPublic { + repoQueries = append(repoQueries, inner_bleve.BoolFieldQuery(true, "is_public")) + } + queries = append(queries, bleve.NewDisjunctionQuery(repoQueries...)) + } + + if !options.IsPull.IsNone() { + queries = append(queries, inner_bleve.BoolFieldQuery(options.IsPull.IsTrue(), "is_pull")) + } + if !options.IsClosed.IsNone() { + queries = append(queries, inner_bleve.BoolFieldQuery(options.IsClosed.IsTrue(), "is_closed")) + } + + if options.NoLabelOnly { + queries = append(queries, inner_bleve.BoolFieldQuery(true, "no_label")) + } else { + if len(options.IncludedLabelIDs) > 0 { + var includeQueries []query.Query + for _, labelID := range options.IncludedLabelIDs { + includeQueries = append(includeQueries, inner_bleve.NumericEqualityQuery(labelID, "label_ids")) + } + queries = append(queries, bleve.NewDisjunctionQuery(includeQueries...)) + } + if len(options.ExcludedLabelIDs) > 0 { + var excludeQueries []query.Query + for _, labelID := range options.ExcludedLabelIDs { + q := bleve.NewBooleanQuery() + q.AddMustNot(inner_bleve.NumericEqualityQuery(labelID, "label_ids")) + excludeQueries = append(excludeQueries, q) + } + queries = append(queries, bleve.NewConjunctionQuery(excludeQueries...)) // Be careful, it's conjunction here, not disjunction. + } + } + + if len(options.MilestoneIDs) > 0 { + var milestoneQueries []query.Query + for _, milestoneID := range options.MilestoneIDs { + milestoneQueries = append(milestoneQueries, inner_bleve.NumericEqualityQuery(milestoneID, "milestone_id")) + } + queries = append(queries, bleve.NewDisjunctionQuery(milestoneQueries...)) + } + + if options.ProjectID != nil { + queries = append(queries, inner_bleve.NumericEqualityQuery(*options.ProjectID, "project_id")) + } + if options.ProjectBoardID != nil { + queries = append(queries, inner_bleve.NumericEqualityQuery(*options.ProjectBoardID, "project_board_id")) + } + + if options.PosterID != nil { + queries = append(queries, inner_bleve.NumericEqualityQuery(*options.PosterID, "poster_id")) + } + + if options.AssigneeID != nil { + queries = append(queries, inner_bleve.NumericEqualityQuery(*options.AssigneeID, "assignee_id")) + } + + if options.MentionID != nil { + queries = append(queries, inner_bleve.NumericEqualityQuery(*options.MentionID, "mention_ids")) + } + + if options.ReviewedID != nil { + queries = append(queries, inner_bleve.NumericEqualityQuery(*options.ReviewedID, "reviewed_ids")) + } + if options.ReviewRequestedID != nil { + queries = append(queries, inner_bleve.NumericEqualityQuery(*options.ReviewRequestedID, "review_requested_ids")) + } + + if options.SubscriberID != nil { + queries = append(queries, inner_bleve.NumericEqualityQuery(*options.SubscriberID, "subscriber_ids")) + } + + if options.UpdatedAfterUnix != nil || options.UpdatedBeforeUnix != nil { + queries = append(queries, inner_bleve.NumericRangeInclusiveQuery(options.UpdatedAfterUnix, options.UpdatedBeforeUnix, "updated_unix")) + } + + indexerQuery := bleve.NewConjunctionQuery(queries...) + skip, limit := indexer_internal.ParsePaginator(options.Paginator) search := bleve.NewSearchRequestOptions(indexerQuery, limit, skip, false) - search.SortBy([]string{"-_score"}) + + if options.SortBy == "" { + options.SortBy = internal.SortByCreatedAsc + } + + search.SortBy([]string{string(options.SortBy), "-_id"}) result, err := b.inner.Indexer.SearchInContext(ctx, search) if err != nil { @@ -181,7 +264,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( ret := &internal.SearchResult{ Total: int64(result.Total), Hits: make([]internal.Match, 0, len(result.Hits)), - Imprecise: true, + Imprecise: false, } for _, hit := range result.Hits { id, err := indexer_internal.ParseBase36(hit.ID) diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index bd7a05b4f0d36..0ae40d7d670da 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -41,21 +41,21 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { } sortType := "" switch options.SortBy { - case internal.SearchOptionsSortByCreatedAsc: + case internal.SortByCreatedAsc: sortType = "oldest" - case internal.SearchOptionsSortByUpdatedAsc: + case internal.SortByUpdatedAsc: sortType = "leastupdate" - case internal.SearchOptionsSortByCommentsAsc: + case internal.SortByCommentsAsc: sortType = "leastcomment" - case internal.SearchOptionsSortByDeadlineAsc: + case internal.SortByDeadlineAsc: sortType = "farduedate" - case internal.SearchOptionsSortByCreatedDesc: + case internal.SortByCreatedDesc: sortType = "" // default - case internal.SearchOptionsSortByUpdatedDesc: + case internal.SortByUpdatedDesc: sortType = "recentupdate" - case internal.SearchOptionsSortByCommentsDesc: + case internal.SortByCommentsDesc: sortType = "mostcomment" - case internal.SearchOptionsSortByDeadlineDesc: + case internal.SortByDeadlineDesc: sortType = "nearduedate" } diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 6f37395a1033b..67d866dd2456a 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -310,14 +310,14 @@ func IsAvailable(ctx context.Context) bool { type SearchOptions internal.SearchOptions const ( - SearchOptionsSortByCreatedDesc = internal.SearchOptionsSortByCreatedDesc - SearchOptionsSortByUpdatedDesc = internal.SearchOptionsSortByUpdatedDesc - SearchOptionsSortByCommentsDesc = internal.SearchOptionsSortByCommentsDesc - SearchOptionsSortByDeadlineDesc = internal.SearchOptionsSortByDeadlineDesc - SearchOptionsSortByCreatedAsc = internal.SearchOptionsSortByCreatedAsc - SearchOptionsSortByUpdatedAsc = internal.SearchOptionsSortByUpdatedAsc - SearchOptionsSortByCommentsAsc = internal.SearchOptionsSortByCommentsAsc - SearchOptionsSortByDeadlineAsc = internal.SearchOptionsSortByDeadlineAsc + SortByCreatedDesc = internal.SortByCreatedDesc + SortByUpdatedDesc = internal.SortByUpdatedDesc + SortByCommentsDesc = internal.SortByCommentsDesc + SortByDeadlineDesc = internal.SortByDeadlineDesc + SortByCreatedAsc = internal.SortByCreatedAsc + SortByUpdatedAsc = internal.SortByUpdatedAsc + SortByCommentsAsc = internal.SortByCommentsAsc + SortByDeadlineAsc = internal.SortByDeadlineAsc ) // SearchIssues search issues by options. diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 5942971d04ca3..8a8fb1a694723 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -23,8 +23,8 @@ type IndexerData struct { // Fields used for filtering IsPull bool `json:"is_pull"` IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. - Labels []int64 `json:"labels"` // So if the labels of an issue have changed, we should reindex the issue. - NoLabel bool `json:"no_label"` // True if Labels is empty + LabelIDs []int64 `json:"label_ids"` // So if the labels of an issue have changed, we should reindex the issue. + NoLabel bool `json:"no_label"` // True if LabelIDs is empty MilestoneID int64 `json:"milestone_id"` // So if the milestone of an issue has changed, we should reindex the issue. ProjectID int64 `json:"project_id"` // So if the project of an issue have changed, we should reindex the issue. ProjectBoardID int64 `json:"project_board_id"` // So if the project board of an issue have changed, we should reindex the issue. @@ -87,8 +87,7 @@ type SearchOptions struct { MentionID *int64 // mentioned user of the issues - ReviewedID *int64 // reviewer of the issues - + ReviewedID *int64 // reviewer of the issues ReviewRequestedID *int64 // requested reviewer of the issues SubscriberID *int64 // subscriber of the issues @@ -98,20 +97,20 @@ type SearchOptions struct { db.Paginator - SortBy SearchOptionsSortBy // sort by field + SortBy SortBy // sort by field } -type SearchOptionsSortBy string +type SortBy string const ( - SearchOptionsSortByCreatedDesc SearchOptionsSortBy = "-created" - SearchOptionsSortByUpdatedDesc SearchOptionsSortBy = "-updated" - SearchOptionsSortByCommentsDesc SearchOptionsSortBy = "-comments" - SearchOptionsSortByDeadlineDesc SearchOptionsSortBy = "-deadline" - SearchOptionsSortByCreatedAsc SearchOptionsSortBy = "created" - SearchOptionsSortByUpdatedAsc SearchOptionsSortBy = "updated" - SearchOptionsSortByCommentsAsc SearchOptionsSortBy = "comments" - SearchOptionsSortByDeadlineAsc SearchOptionsSortBy = "deadline" + SortByCreatedDesc SortBy = "-created_unix" + SortByUpdatedDesc SortBy = "-updated_unix" + SortByCommentsDesc SortBy = "-comment_count" + SortByDeadlineDesc SortBy = "-deadline_unix" + SortByCreatedAsc SortBy = "created_unix" + SortByUpdatedAsc SortBy = "updated_unix" + SortByCommentsAsc SortBy = "comment_count" + SortByDeadlineAsc SortBy = "deadline_unix" // Unsupported sort types which are supported by issues.IssuesOptions.SortType: // - "priorityrepo" // - "project-column-sorting" diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index b744502443032..7992bc407e1a1 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -94,7 +94,7 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD Comments: comments, IsPull: issue.IsPull, IsClosed: issue.IsClosed, - Labels: labels, + LabelIDs: labels, NoLabel: len(labels) == 0, MilestoneID: issue.MilestoneID, ProjectID: issue.Project.ID, diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 1b112d08f1b52..7a60444794527 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -258,7 +258,7 @@ func SearchIssues(ctx *context.APIContext) { MilestoneIDs: includedMilestones, UpdatedAfterUnix: &since, UpdatedBeforeUnix: &before, - SortBy: issue_indexer.SearchOptionsSortByCreatedDesc, + SortBy: issue_indexer.SortByCreatedDesc, } ctxUserID := int64(0) @@ -477,7 +477,7 @@ func ListIssues(ctx *context.APIContext) { MentionID: &mentionedByID, UpdatedAfterUnix: &since, UpdatedBeforeUnix: &before, - SortBy: issue_indexer.SearchOptionsSortByCreatedDesc, + SortBy: issue_indexer.SortByCreatedDesc, } if len(labelIDs) == 0 && labelIDs[0] == 0 { searchOpt.NoLabelOnly = true From c97c8c26714838828c5c8b7595abc475666b6f7e Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 18 Jul 2023 16:29:51 +0800 Subject: [PATCH 29/99] chore: update bleve version --- modules/indexer/issues/bleve/bleve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 52b031c033665..398238e94b3ae 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -23,7 +23,7 @@ import ( const ( issueIndexerAnalyzer = "issueIndexer" issueIndexerDocType = "issueIndexerDocType" - issueIndexerLatestVersion = 3 + issueIndexerLatestVersion = 4 ) const unicodeNormalizeName = "unicodeNormalize" From f5e0f6ccf547be90cd8626f9d5ac19bc3b23e820 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 18 Jul 2023 18:34:36 +0800 Subject: [PATCH 30/99] fix: some fix --- modules/indexer/issues/indexer.go | 4 ++++ routers/api/v1/repo/issue.go | 25 +++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 67d866dd2456a..b45fec4c2ea0d 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -5,6 +5,7 @@ package issues import ( "context" + "errors" "os" "runtime/pprof" "sync/atomic" @@ -257,6 +258,9 @@ func UpdateRepoIndexer(ctx context.Context, repoID int64) { func UpdateIssueIndexer(issueID int64) { if err := issueIndexerQueue.Push(&IndexerMetadata{ID: issueID}); err != nil { log.Error("Unable to push to issue indexer: %v: Error: %v", issueID, err) + if errors.Is(err, context.DeadlineExceeded) { + log.Error("It seems that issue indexer is slow and the queue is full. Please check the issue indexer or increase the queue size.") + } } } diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 7a60444794527..6b402b446bb26 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -249,16 +249,21 @@ func SearchIssues(ctx *context.APIContext) { } searchOpt := &issue_indexer.SearchOptions{ - Keyword: keyword, - RepoIDs: repoIDs, - AllPublic: allPublic, - IsPull: isPull, - IsClosed: isClosed, - IncludedLabelIDs: includedLabels, - MilestoneIDs: includedMilestones, - UpdatedAfterUnix: &since, - UpdatedBeforeUnix: &before, - SortBy: issue_indexer.SortByCreatedDesc, + Keyword: keyword, + RepoIDs: repoIDs, + AllPublic: allPublic, + IsPull: isPull, + IsClosed: isClosed, + IncludedLabelIDs: includedLabels, + MilestoneIDs: includedMilestones, + SortBy: issue_indexer.SortByCreatedDesc, + } + + if since != 0 { + searchOpt.UpdatedAfterUnix = &since + } + if before != 0 { + searchOpt.UpdatedBeforeUnix = &before } ctxUserID := int64(0) From 0f1d89816286c57d8e83597c00d04d4eb83a75bc Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 14:12:31 +0800 Subject: [PATCH 31/99] fix: reuse SearchIssues in SearchIssuesByKeyword --- modules/indexer/issues/indexer.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index b45fec4c2ea0d..836688def3b1f 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -289,20 +289,12 @@ func DeleteRepoIssueIndexer(ctx context.Context, repoID int64) { // SearchIssuesByKeyword search issue ids by keywords and repo id // WARNNING: You have to ensure user have permission to visit repoIDs' issues func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) { - var issueIDs []int64 - indexer := *globalIndexer.Load() - res, err := indexer.Search(ctx, &internal.SearchOptions{ + ids, _, err := SearchIssues(ctx, &SearchOptions{ Keyword: keyword, RepoIDs: repoIDs, Paginator: db_model.NewAbsoluteListOptions(0, 50), }) - if err != nil { - return nil, err - } - for _, r := range res.Hits { - issueIDs = append(issueIDs, r.ID) - } - return issueIDs, nil + return ids, err } // IsAvailable checks if issue indexer is available From fad43d41a2d7d85a6e77c8fe31b9820037b15a95 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 14:25:02 +0800 Subject: [PATCH 32/99] feat: web SearchIssues --- routers/api/v1/repo/issue.go | 34 +++-- routers/web/repo/issue.go | 232 +++++++++++++++++++---------------- 2 files changed, 139 insertions(+), 127 deletions(-) diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 6b402b446bb26..2055f52ccb6ac 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -266,25 +266,23 @@ func SearchIssues(ctx *context.APIContext) { searchOpt.UpdatedBeforeUnix = &before } - ctxUserID := int64(0) if ctx.IsSigned { - ctxUserID = ctx.Doer.ID - } - - if ctx.FormBool("created") { - searchOpt.PosterID = &ctxUserID - } - if ctx.FormBool("assigned") { - searchOpt.AssigneeID = &ctxUserID - } - if ctx.FormBool("mentioned") { - searchOpt.MentionID = &ctxUserID - } - if ctx.FormBool("review_requested") { - searchOpt.ReviewRequestedID = &ctxUserID - } - if ctx.FormBool("reviewed") { - searchOpt.ReviewedID = &ctxUserID + ctxUserID := ctx.Doer.ID + if ctx.FormBool("created") { + searchOpt.PosterID = &ctxUserID + } + if ctx.FormBool("assigned") { + searchOpt.AssigneeID = &ctxUserID + } + if ctx.FormBool("mentioned") { + searchOpt.MentionID = &ctxUserID + } + if ctx.FormBool("review_requested") { + searchOpt.ReviewRequestedID = &ctxUserID + } + if ctx.FormBool("reviewed") { + searchOpt.ReviewedID = &ctxUserID + } } // FIXME: It's unsupported to sort by priority repo when searching by indexer, diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index e2317f1945839..e90c6667d0958 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -2403,75 +2403,73 @@ func SearchIssues(ctx *context.Context) { isClosed = util.OptionalBoolFalse } - // find repos user can access (for issue search) - opts := &repo_model.SearchRepoOptions{ - Private: false, - AllPublic: true, - TopicOnly: false, - Collaborate: util.OptionalBoolNone, - // This needs to be a column that is not nil in fixtures or - // MySQL will return different results when sorting by null in some cases - OrderBy: db.SearchOrderByAlphabetically, - Actor: ctx.Doer, - } - if ctx.IsSigned { - opts.Private = true - opts.AllLimited = true - } - if ctx.FormString("owner") != "" { - owner, err := user_model.GetUserByName(ctx, ctx.FormString("owner")) - if err != nil { - if user_model.IsErrUserNotExist(err) { - ctx.Error(http.StatusBadRequest, "Owner not found", err.Error()) - } else { - ctx.Error(http.StatusInternalServerError, "GetUserByName", err.Error()) + var ( + repoIDs []int64 + allPublic bool + ) + { + // find repos user can access (for issue search) + opts := &repo_model.SearchRepoOptions{ + Private: false, + AllPublic: true, + TopicOnly: false, + Collaborate: util.OptionalBoolNone, + // This needs to be a column that is not nil in fixtures or + // MySQL will return different results when sorting by null in some cases + OrderBy: db.SearchOrderByAlphabetically, + Actor: ctx.Doer, + } + if ctx.IsSigned { + opts.Private = true + opts.AllLimited = true + } + if ctx.FormString("owner") != "" { + owner, err := user_model.GetUserByName(ctx, ctx.FormString("owner")) + if err != nil { + if user_model.IsErrUserNotExist(err) { + ctx.Error(http.StatusBadRequest, "Owner not found", err.Error()) + } else { + ctx.Error(http.StatusInternalServerError, "GetUserByName", err.Error()) + } + return } - return + opts.OwnerID = owner.ID + opts.AllLimited = false + opts.AllPublic = false + opts.Collaborate = util.OptionalBoolFalse } - opts.OwnerID = owner.ID - opts.AllLimited = false - opts.AllPublic = false - opts.Collaborate = util.OptionalBoolFalse - } - if ctx.FormString("team") != "" { - if ctx.FormString("owner") == "" { - ctx.Error(http.StatusBadRequest, "", "Owner organisation is required for filtering on team") - return + if ctx.FormString("team") != "" { + if ctx.FormString("owner") == "" { + ctx.Error(http.StatusBadRequest, "", "Owner organisation is required for filtering on team") + return + } + team, err := organization.GetTeam(ctx, opts.OwnerID, ctx.FormString("team")) + if err != nil { + if organization.IsErrTeamNotExist(err) { + ctx.Error(http.StatusBadRequest, "Team not found", err.Error()) + } else { + ctx.Error(http.StatusInternalServerError, "GetUserByName", err.Error()) + } + return + } + opts.TeamID = team.ID } - team, err := organization.GetTeam(ctx, opts.OwnerID, ctx.FormString("team")) + + if opts.AllPublic { + allPublic = true + opts.AllPublic = false // set it false to avoid returning too many repos, we could filter by indexer + } + repoIDs, _, err = repo_model.SearchRepositoryIDs(opts) if err != nil { - if organization.IsErrTeamNotExist(err) { - ctx.Error(http.StatusBadRequest, "Team not found", err.Error()) - } else { - ctx.Error(http.StatusInternalServerError, "GetUserByName", err.Error()) - } + ctx.Error(http.StatusInternalServerError, "SearchRepositoryIDs", err.Error()) return } - opts.TeamID = team.ID - } - - repoCond := repo_model.SearchRepositoryCondition(opts) - repoIDs, _, err := repo_model.SearchRepositoryIDs(opts) - if err != nil { - ctx.Error(http.StatusInternalServerError, "SearchRepositoryIDs", err.Error()) - return } - var issues []*issues_model.Issue - var filteredCount int64 - keyword := ctx.FormTrim("q") if strings.IndexByte(keyword, 0) >= 0 { keyword = "" } - var issueIDs []int64 - if len(keyword) > 0 && len(repoIDs) > 0 { - // TBC: use issue_indexer.SearchIssues instead - if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil { - ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err.Error()) - return - } - } var isPull util.OptionalBool switch ctx.FormString("type") { @@ -2483,19 +2481,39 @@ func SearchIssues(ctx *context.Context) { isPull = util.OptionalBoolNone } - labels := ctx.FormTrim("labels") - var includedLabelNames []string - if len(labels) > 0 { - includedLabelNames = strings.Split(labels, ",") + var includedLabels []int64 + { + + labels := ctx.FormTrim("labels") + var includedLabelNames []string + if len(labels) > 0 { + includedLabelNames = strings.Split(labels, ",") + } + includedLabels, err = issues_model.GetLabelIDsByNames(ctx, includedLabelNames) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetLabelIDsByNames", err.Error()) + return + } } - milestones := ctx.FormTrim("milestones") - var includedMilestones []string - if len(milestones) > 0 { - includedMilestones = strings.Split(milestones, ",") + var includedMilestones []int64 + { + milestones := ctx.FormTrim("milestones") + var includedMilestoneNames []string + if len(milestones) > 0 { + includedMilestoneNames = strings.Split(milestones, ",") + } + includedMilestones, err = issues_model.GetMilestoneIDsByNames(ctx, includedMilestoneNames) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetMilestoneIDsByNames", err.Error()) + return + } } - projectID := ctx.FormInt64("project") + var projectID *int64 + if v := ctx.FormInt64("project"); v > 0 { + projectID = &v + } // this api is also used in UI, // so the default limit is set to fit UI needs @@ -2506,64 +2524,60 @@ func SearchIssues(ctx *context.Context) { limit = setting.API.MaxResponseItems } - // Only fetch the issues if we either don't have a keyword or the search returned issues - // This would otherwise return all issues if no issues were found by the search. - if len(keyword) == 0 || len(issueIDs) > 0 || len(includedLabelNames) > 0 || len(includedMilestones) > 0 { - issuesOpt := &issues_model.IssuesOptions{ - Paginator: &db.ListOptions{ - Page: ctx.FormInt("page"), - PageSize: limit, - }, - RepoCond: repoCond, - IsClosed: isClosed, - IssueIDs: issueIDs, - IncludedLabelNames: includedLabelNames, - IncludeMilestones: includedMilestones, - ProjectID: projectID, - SortType: "priorityrepo", - PriorityRepoID: ctx.FormInt64("priority_repo_id"), - IsPull: isPull, - UpdatedBeforeUnix: before, - UpdatedAfterUnix: since, - } - - ctxUserID := int64(0) - if ctx.IsSigned { - ctxUserID = ctx.Doer.ID - } + searchOpt := &issue_indexer.SearchOptions{ + Keyword: keyword, + RepoIDs: repoIDs, + AllPublic: allPublic, + IsPull: isPull, + IsClosed: isClosed, + IncludedLabelIDs: includedLabels, + MilestoneIDs: includedMilestones, + ProjectID: projectID, + SortBy: issue_indexer.SortByCreatedDesc, + } - // Filter for: Created by User, Assigned to User, Mentioning User, Review of User Requested + if since != 0 { + searchOpt.UpdatedAfterUnix = &since + } + if before != 0 { + searchOpt.UpdatedBeforeUnix = &before + } + + if ctx.IsSigned { + ctxUserID := ctx.Doer.ID if ctx.FormBool("created") { - issuesOpt.PosterID = ctxUserID + searchOpt.PosterID = &ctxUserID } if ctx.FormBool("assigned") { - issuesOpt.AssigneeID = ctxUserID + searchOpt.AssigneeID = &ctxUserID } if ctx.FormBool("mentioned") { - issuesOpt.MentionedID = ctxUserID + searchOpt.MentionID = &ctxUserID } if ctx.FormBool("review_requested") { - issuesOpt.ReviewRequestedID = ctxUserID + searchOpt.ReviewRequestedID = &ctxUserID } if ctx.FormBool("reviewed") { - issuesOpt.ReviewedID = ctxUserID + searchOpt.ReviewedID = &ctxUserID } + } - if issues, err = issues_model.Issues(ctx, issuesOpt); err != nil { - ctx.Error(http.StatusInternalServerError, "Issues", err.Error()) - return - } + // FIXME: It's unsupported to sort by priority repo when searching by indexer, + // it's indeed an regression, but I think it is worth to support filtering by indexer first. + _ = ctx.FormInt64("priority_repo_id") - issuesOpt.Paginator = &db.ListOptions{ - Page: -1, - } - if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil { - ctx.Error(http.StatusInternalServerError, "CountIssues", err.Error()) - return - } + ids, total, err := issue_indexer.SearchIssues(ctx, searchOpt) + if err != nil { + ctx.Error(http.StatusInternalServerError, "SearchIssues", err.Error()) + return + } + issues, err := issues_model.FindIssuesByIDs(ctx, ids) + if err != nil { + ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err.Error()) + return } - ctx.SetTotalCountHeader(filteredCount) + ctx.SetTotalCountHeader(total) ctx.JSON(http.StatusOK, convert.ToIssueList(ctx, issues)) } From 4c9af8b72c629180ce916190e4b39784a772adbd Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 14:27:35 +0800 Subject: [PATCH 33/99] fix: api ListIssues --- routers/api/v1/repo/issue.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 2055f52ccb6ac..99a1449003278 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -467,20 +467,20 @@ func ListIssues(ctx *context.APIContext) { } searchOpt := &issue_indexer.SearchOptions{ - Keyword: keyword, - RepoIDs: []int64{ctx.Repo.Repository.ID}, - IsPull: isPull, - IsClosed: isClosed, - IncludedLabelIDs: nil, - ExcludedLabelIDs: nil, - NoLabelOnly: false, - MilestoneIDs: nil, - PosterID: &createdByID, - AssigneeID: &assignedByID, - MentionID: &mentionedByID, - UpdatedAfterUnix: &since, - UpdatedBeforeUnix: &before, - SortBy: issue_indexer.SortByCreatedDesc, + Keyword: keyword, + RepoIDs: []int64{ctx.Repo.Repository.ID}, + IsPull: isPull, + IsClosed: isClosed, + PosterID: &createdByID, + AssigneeID: &assignedByID, + MentionID: &mentionedByID, + SortBy: issue_indexer.SortByCreatedDesc, + } + if since != 0 { + searchOpt.UpdatedAfterUnix = &since + } + if before != 0 { + searchOpt.UpdatedBeforeUnix = &before } if len(labelIDs) == 0 && labelIDs[0] == 0 { searchOpt.NoLabelOnly = true From 6953a68c461ba92a10f347679b509af71047cc41 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 14:37:37 +0800 Subject: [PATCH 34/99] feat: web ListIssues --- routers/api/v1/repo/issue.go | 23 +++++--- routers/web/repo/issue.go | 104 +++++++++++++++++++---------------- 2 files changed, 72 insertions(+), 55 deletions(-) diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 99a1449003278..769f1707a3f82 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -467,14 +467,11 @@ func ListIssues(ctx *context.APIContext) { } searchOpt := &issue_indexer.SearchOptions{ - Keyword: keyword, - RepoIDs: []int64{ctx.Repo.Repository.ID}, - IsPull: isPull, - IsClosed: isClosed, - PosterID: &createdByID, - AssigneeID: &assignedByID, - MentionID: &mentionedByID, - SortBy: issue_indexer.SortByCreatedDesc, + Keyword: keyword, + RepoIDs: []int64{ctx.Repo.Repository.ID}, + IsPull: isPull, + IsClosed: isClosed, + SortBy: issue_indexer.SortByCreatedDesc, } if since != 0 { searchOpt.UpdatedAfterUnix = &since @@ -500,6 +497,16 @@ func ListIssues(ctx *context.APIContext) { searchOpt.MilestoneIDs = mileIDs } + if createdByID > 0 { + searchOpt.PosterID = &createdByID + } + if assignedByID > 0 { + searchOpt.AssigneeID = &assignedByID + } + if mentionedByID > 0 { + searchOpt.MentionID = &mentionedByID + } + ids, total, err := issue_indexer.SearchIssues(ctx, searchOpt) if err != nil { ctx.Error(http.StatusInternalServerError, "SearchIssues", err) diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index e90c6667d0958..76611f2fb0687 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -2619,24 +2619,12 @@ func ListIssues(ctx *context.Context) { isClosed = util.OptionalBoolFalse } - var issues []*issues_model.Issue - var filteredCount int64 - keyword := ctx.FormTrim("q") if strings.IndexByte(keyword, 0) >= 0 { keyword = "" } - var issueIDs []int64 - var labelIDs []int64 - if len(keyword) > 0 { - // TBC: use issue_indexer.SearchIssues instead - issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword) - if err != nil { - ctx.Error(http.StatusInternalServerError, err.Error()) - return - } - } + var labelIDs []int64 if splitted := strings.Split(ctx.FormString("labels"), ","); len(splitted) > 0 { labelIDs, err = issues_model.GetLabelIDsInRepoByNames(ctx.Repo.Repository.ID, splitted) if err != nil { @@ -2675,11 +2663,9 @@ func ListIssues(ctx *context.Context) { } } - projectID := ctx.FormInt64("project") - - listOptions := db.ListOptions{ - Page: ctx.FormInt("page"), - PageSize: convert.ToCorrectPageSize(ctx.FormInt("limit")), + var projectID *int64 + if v := ctx.FormInt64("project"); v > 0 { + projectID = &v } var isPull util.OptionalBool @@ -2706,40 +2692,64 @@ func ListIssues(ctx *context.Context) { return } - // Only fetch the issues if we either don't have a keyword or the search returned issues - // This would otherwise return all issues if no issues were found by the search. - if len(keyword) == 0 || len(issueIDs) > 0 || len(labelIDs) > 0 { - issuesOpt := &issues_model.IssuesOptions{ - Paginator: &listOptions, - RepoIDs: []int64{ctx.Repo.Repository.ID}, - IsClosed: isClosed, - IssueIDs: issueIDs, - LabelIDs: labelIDs, - MilestoneIDs: mileIDs, - ProjectID: projectID, - IsPull: isPull, - UpdatedBeforeUnix: before, - UpdatedAfterUnix: since, - PosterID: createdByID, - AssigneeID: assignedByID, - MentionedID: mentionedByID, + searchOpt := &issue_indexer.SearchOptions{ + Paginator: &db.ListOptions{ + Page: ctx.FormInt("page"), + PageSize: convert.ToCorrectPageSize(ctx.FormInt("limit")), + }, + Keyword: keyword, + RepoIDs: []int64{ctx.Repo.Repository.ID}, + IsPull: isPull, + IsClosed: isClosed, + ProjectBoardID: projectID, + SortBy: issue_indexer.SortByCreatedDesc, + } + if since != 0 { + searchOpt.UpdatedAfterUnix = &since + } + if before != 0 { + searchOpt.UpdatedBeforeUnix = &before + } + if len(labelIDs) == 0 && labelIDs[0] == 0 { + searchOpt.NoLabelOnly = true + } else { + for _, labelID := range labelIDs { + if labelID > 0 { + searchOpt.IncludedLabelIDs = append(searchOpt.IncludedLabelIDs, labelID) + } else { + searchOpt.ExcludedLabelIDs = append(searchOpt.ExcludedLabelIDs, -labelID) + } } + } - if issues, err = issues_model.Issues(ctx, issuesOpt); err != nil { - ctx.Error(http.StatusInternalServerError, err.Error()) - return - } + if len(mileIDs) == 1 && mileIDs[0] == db.NoConditionID { + searchOpt.MilestoneIDs = []int64{0} + } else { + searchOpt.MilestoneIDs = mileIDs + } - issuesOpt.Paginator = &db.ListOptions{ - Page: -1, - } - if filteredCount, err = issues_model.CountIssues(ctx, issuesOpt); err != nil { - ctx.Error(http.StatusInternalServerError, err.Error()) - return - } + if createdByID > 0 { + searchOpt.PosterID = &createdByID + } + if assignedByID > 0 { + searchOpt.AssigneeID = &assignedByID + } + if mentionedByID > 0 { + searchOpt.MentionID = &mentionedByID + } + + ids, total, err := issue_indexer.SearchIssues(ctx, searchOpt) + if err != nil { + ctx.Error(http.StatusInternalServerError, "SearchIssues", err.Error()) + return + } + issues, err := issues_model.FindIssuesByIDs(ctx, ids) + if err != nil { + ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err.Error()) + return } - ctx.SetTotalCountHeader(filteredCount) + ctx.SetTotalCountHeader(total) ctx.JSON(http.StatusOK, convert.ToIssueList(ctx, issues)) } From 3af64048316ad62bd85c16e6131aafce434a5ad3 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 14:48:54 +0800 Subject: [PATCH 35/99] fix: Paginator --- routers/api/v1/repo/issue.go | 15 ++++++++++----- routers/web/repo/issue.go | 4 ++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 769f1707a3f82..e2d8f84fdcd15 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -249,6 +249,10 @@ func SearchIssues(ctx *context.APIContext) { } searchOpt := &issue_indexer.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: limit, + Page: ctx.FormInt("page"), + }, Keyword: keyword, RepoIDs: repoIDs, AllPublic: allPublic, @@ -467,11 +471,12 @@ func ListIssues(ctx *context.APIContext) { } searchOpt := &issue_indexer.SearchOptions{ - Keyword: keyword, - RepoIDs: []int64{ctx.Repo.Repository.ID}, - IsPull: isPull, - IsClosed: isClosed, - SortBy: issue_indexer.SortByCreatedDesc, + Paginator: &listOptions, + Keyword: keyword, + RepoIDs: []int64{ctx.Repo.Repository.ID}, + IsPull: isPull, + IsClosed: isClosed, + SortBy: issue_indexer.SortByCreatedDesc, } if since != 0 { searchOpt.UpdatedAfterUnix = &since diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 76611f2fb0687..b21ef7b306b06 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -2525,6 +2525,10 @@ func SearchIssues(ctx *context.Context) { } searchOpt := &issue_indexer.SearchOptions{ + Paginator: &db.ListOptions{ + Page: ctx.FormInt("page"), + PageSize: limit, + }, Keyword: keyword, RepoIDs: repoIDs, AllPublic: allPublic, From 1df1ef4b4f6d9f5af48502fccf4587aedcacd427 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 15:48:14 +0800 Subject: [PATCH 36/99] feat: buildIssueOverview --- routers/web/user/home.go | 168 +++++++++++++++++++++++++++++---------- 1 file changed, 127 insertions(+), 41 deletions(-) diff --git a/routers/web/user/home.go b/routers/web/user/home.go index 9cbc8e9025a0c..42ab125ec9562 100644 --- a/routers/web/user/home.go +++ b/routers/web/user/home.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/context" issue_indexer "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/json" @@ -466,21 +467,14 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { keyword := strings.Trim(ctx.FormString("q"), " ") ctx.Data["Keyword"] = keyword - // Execute keyword search for issues. - // USING NON-FINAL STATE OF opts FOR A QUERY. - issueIDsFromSearch, err := issueIDsFromSearch(ctx, ctxUser, keyword, opts) - if err != nil { - ctx.ServerError("issueIDsFromSearch", err) + accessibleRepos := container.Set[int64]{} + if ids, err := issues_model.GetRepoIDsForIssuesOptions(opts, ctxUser); err != nil { + ctx.ServerError("GetRepoIDsForIssuesOptions", err) return - } - - // Ensure no issues are returned if a keyword was provided that didn't match any issues. - var forceEmpty bool - - if len(issueIDsFromSearch) > 0 { - opts.IssueIDs = issueIDsFromSearch - } else if len(keyword) > 0 { - forceEmpty = true + } else { + for _, id := range ids { + accessibleRepos.Add(id) + } } // Educated guess: Do or don't show closed issues. @@ -489,13 +483,10 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { // Filter repos and count issues in them. Count will be used later. // USING NON-FINAL STATE OF opts FOR A QUERY. - var issueCountByRepo map[int64]int64 - if !forceEmpty { - issueCountByRepo, err = issues_model.CountIssuesByRepo(ctx, opts) - if err != nil { - ctx.ServerError("CountIssuesByRepo", err) - return - } + issueCountByRepo, err := issues_model.CountIssuesByRepo(ctx, opts) + if err != nil { + ctx.ServerError("CountIssuesByRepo", err) + return } // Make sure page number is at least 1. Will be posted to ctx.Data. @@ -523,7 +514,16 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { // Parse ctx.FormString("repos") and remember matched repo IDs for later. // Gets set when clicking filters on the issues overview page. - opts.RepoIDs = getRepoIDs(ctx.FormString("repos")) + repoIDs := getRepoIDs(ctx.FormString("repos")) + if len(repoIDs) == 0 { + repoIDs = accessibleRepos.Values() + } else { + // Remove repo IDs that are not accessible to the user. + repoIDs = util.SliceRemoveAllFunc(repoIDs, func(v int64) bool { + return !accessibleRepos.Contains(v) + }) + } + opts.RepoIDs = repoIDs // ------------------------------ // Get issues as defined by opts. @@ -532,14 +532,17 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { // Slice of Issues that will be displayed on the overview page // USING FINAL STATE OF opts FOR A QUERY. var issues []*issues_model.Issue - if !forceEmpty { - issues, err = issues_model.Issues(ctx, opts) + { + issueIDs, err := issueIDsFromSearch(ctx, keyword, opts) if err != nil { - ctx.ServerError("Issues", err) + ctx.ServerError("issueIDsFromSearch", err) + return + } + issues, err = issues_model.GetIssuesByIDs(ctx, issueIDs) + if err != nil { + ctx.ServerError("GetIssuesByIDs", err) return } - } else { - issues = []*issues_model.Issue{} } // ---------------------------------- @@ -578,12 +581,12 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { // Fill stats to post to ctx.Data. // ------------------------------- var issueStats *issues_model.IssueStats - if !forceEmpty { + { statsOpts := issues_model.IssuesOptions{ User: ctx.Doer, IsPull: util.OptionalBoolOf(isPullList), IsClosed: util.OptionalBoolOf(isShowClosed), - IssueIDs: issueIDsFromSearch, + IssueIDs: nil, IsArchived: util.OptionalBoolFalse, LabelIDs: opts.LabelIDs, Org: org, @@ -591,13 +594,25 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { RepoCond: opts.RepoCond, } + if keyword != "" { + old := opts.Paginator + opts.Paginator = &db.ListOptions{ + ListAll: true, + } + allIssueIDs, err := issueIDsFromSearch(ctx, keyword, opts) + if err != nil { + ctx.ServerError("issueIDsFromSearch", err) + return + } + statsOpts.IssueIDs = allIssueIDs + opts.Paginator = old + } + issueStats, err = issues_model.GetUserIssueStats(filterMode, statsOpts) if err != nil { ctx.ServerError("GetUserIssueStats Shown", err) return } - } else { - issueStats = &issues_model.IssueStats{} } // Will be posted to ctx.Data. @@ -718,21 +733,92 @@ func getRepoIDs(reposQuery string) []int64 { return repoIDs } -func issueIDsFromSearch(ctx *context.Context, ctxUser *user_model.User, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) { - if len(keyword) == 0 { - return []int64{}, nil +func issueIDsFromSearch(ctx *context.Context, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) { + ids, _, err := issue_indexer.SearchIssues(ctx, convertOptionsToSearchOptions(keyword, opts)) + if err != nil { + return nil, fmt.Errorf("SearchIssues: %w", err) } + return ids, nil +} - searchRepoIDs, err := issues_model.GetRepoIDsForIssuesOptions(opts, ctxUser) - if err != nil { - return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %w", err) +func convertOptionsToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *issue_indexer.SearchOptions { + searchOpt := &issue_indexer.SearchOptions{ + Keyword: keyword, + RepoIDs: opts.RepoIDs, + AllPublic: false, + IsPull: opts.IsPull, + IsClosed: opts.IsClosed, } - issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword, ctx.FormString("state")) - if err != nil { - return nil, fmt.Errorf("SearchIssuesByKeyword: %w", err) + + if len(opts.LabelIDs) == 1 && opts.LabelIDs[0] == 0 { + searchOpt.NoLabelOnly = true + } else { + for _, labelID := range opts.LabelIDs { + if labelID > 0 { + searchOpt.IncludedLabelIDs = append(searchOpt.IncludedLabelIDs, labelID) + } else { + searchOpt.ExcludedLabelIDs = append(searchOpt.ExcludedLabelIDs, -labelID) + } + } + } + + if len(opts.MilestoneIDs) == 1 && opts.MilestoneIDs[0] == db.NoConditionID { + searchOpt.MilestoneIDs = []int64{0} + } else { + searchOpt.MilestoneIDs = opts.MilestoneIDs + } + + if opts.AssigneeID > 0 { + searchOpt.AssigneeID = &opts.AssigneeID + } + if opts.PosterID > 0 { + searchOpt.PosterID = &opts.PosterID + } + if opts.MentionedID > 0 { + searchOpt.MentionID = &opts.MentionedID + } + if opts.ReviewedID > 0 { + searchOpt.ReviewedID = &opts.ReviewedID + } + if opts.ReviewRequestedID > 0 { + searchOpt.ReviewRequestedID = &opts.ReviewRequestedID + } + if opts.SubscriberID > 0 { + searchOpt.SubscriberID = &opts.SubscriberID + } + + if opts.UpdatedAfterUnix > 0 { + searchOpt.UpdatedAfterUnix = &opts.UpdatedAfterUnix + } + if opts.UpdatedBeforeUnix > 0 { + searchOpt.UpdatedBeforeUnix = &opts.UpdatedBeforeUnix + } + + searchOpt.Paginator = opts.Paginator + + switch opts.SortType { + case "oldest": + searchOpt.SortBy = issue_indexer.SortByCreatedAsc + case "recentupdate": + searchOpt.SortBy = issue_indexer.SortByUpdatedDesc + case "leastupdate": + searchOpt.SortBy = issue_indexer.SortByUpdatedAsc + case "mostcomment": + searchOpt.SortBy = issue_indexer.SortByCommentsDesc + case "leastcomment": + searchOpt.SortBy = issue_indexer.SortByCommentsAsc + case "nearduedate": + searchOpt.SortBy = issue_indexer.SortByDeadlineAsc + case "farduedate": + searchOpt.SortBy = issue_indexer.SortByDeadlineDesc + case "priority", "priorityrepo", "project-column-sorting": + // Unsupported sort type for search + searchOpt.SortBy = issue_indexer.SortByUpdatedDesc + default: + searchOpt.SortBy = issue_indexer.SortByUpdatedDesc } - return issueIDsFromSearch, nil + return searchOpt } func loadRepoByIDs(ctxUser *user_model.User, issueCountByRepo map[int64]int64, unitType unit.Type) (map[int64]*repo_model.Repository, error) { From 2b8aa894fb9b322cfc6452464d35284aaa018357 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 15:57:31 +0800 Subject: [PATCH 37/99] fix: allIssueIDs --- routers/web/user/home.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/routers/web/user/home.go b/routers/web/user/home.go index 42ab125ec9562..c345209dec7a4 100644 --- a/routers/web/user/home.go +++ b/routers/web/user/home.go @@ -595,17 +595,13 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { } if keyword != "" { - old := opts.Paginator - opts.Paginator = &db.ListOptions{ - ListAll: true, - } - allIssueIDs, err := issueIDsFromSearch(ctx, keyword, opts) + statsOpts.RepoIDs = opts.RepoIDs + allIssueIDs, err := issueIDsFromSearch(ctx, keyword, &statsOpts) if err != nil { ctx.ServerError("issueIDsFromSearch", err) return } statsOpts.IssueIDs = allIssueIDs - opts.Paginator = old } issueStats, err = issues_model.GetUserIssueStats(filterMode, statsOpts) From 4ca06603c88cccfa132c548fb73d4d52a4ea5fb5 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 16:32:29 +0800 Subject: [PATCH 38/99] feat: issues --- models/issues/issue.go | 26 ++++++++- models/issues/issue_list.go | 18 ------ models/issues/issue_test.go | 3 +- modules/indexer/issues/dboptions.go | 89 +++++++++++++++++++++++++++++ routers/api/v1/repo/issue.go | 4 +- routers/web/repo/issue.go | 62 +++++++++++--------- routers/web/user/home.go | 84 +-------------------------- 7 files changed, 153 insertions(+), 133 deletions(-) create mode 100644 modules/indexer/issues/dboptions.go diff --git a/models/issues/issue.go b/models/issues/issue.go index c1a802c792a0d..b56fd32077c9b 100644 --- a/models/issues/issue.go +++ b/models/issues/issue.go @@ -13,6 +13,7 @@ import ( project_model "code.gitea.io/gitea/models/project" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -550,9 +551,30 @@ func GetIssueWithAttrsByID(id int64) (*Issue, error) { } // GetIssuesByIDs return issues with the given IDs. -func GetIssuesByIDs(ctx context.Context, issueIDs []int64) (IssueList, error) { +// If keepOrder is true, the order of the returned issues will be the same as the given IDs. +func GetIssuesByIDs(ctx context.Context, issueIDs []int64, keepOrder ...bool) (IssueList, error) { issues := make([]*Issue, 0, len(issueIDs)) - return issues, db.GetEngine(ctx).In("id", issueIDs).Find(&issues) + + if err := db.GetEngine(ctx).In("id", issueIDs).Find(&issues); err != nil { + return nil, err + } + + if len(keepOrder) > 0 && keepOrder[0] { + m := make(map[int64]*Issue, len(issues)) + appended := container.Set[int64]{} + for _, issue := range issues { + m[issue.ID] = issue + } + issues = issues[:0] + for _, id := range issueIDs { + if issue, ok := m[id]; ok && !appended.Contains(id) { // make sure the id is existed and not appended + appended.Add(id) + issues = append(issues, issue) + } + } + } + + return issues, nil } // GetIssueIDsByRepoID returns all issue ids by repo id diff --git a/models/issues/issue_list.go b/models/issues/issue_list.go index 6e3a66d8cda34..9cc41ec6ab37e 100644 --- a/models/issues/issue_list.go +++ b/models/issues/issue_list.go @@ -6,7 +6,6 @@ package issues import ( "context" "fmt" - "sort" "code.gitea.io/gitea/models/db" project_model "code.gitea.io/gitea/models/project" @@ -606,20 +605,3 @@ func (issues IssueList) GetApprovalCounts(ctx context.Context) (map[int64][]*Rev return approvalCountMap, nil } - -func FindIssuesByIDs(ctx context.Context, ids []int64) (IssueList, error) { - sess := db.GetEngine(ctx).In("id", ids) - issues := make(IssueList, 0, len(ids)) - if err := sess.Find(&issues); err != nil { - return nil, err - } - - order := map[int64]int{} - for i, id := range ids { - order[id] = i - } - sort.Slice(issues, func(i, j int) bool { - return order[issues[i].ID] < order[issues[j].ID] - }) - return issues, nil -} diff --git a/models/issues/issue_test.go b/models/issues/issue_test.go index e36030c1f3920..786a2d84247d2 100644 --- a/models/issues/issue_test.go +++ b/models/issues/issue_test.go @@ -73,7 +73,7 @@ func TestIssueAPIURL(t *testing.T) { func TestGetIssuesByIDs(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) testSuccess := func(expectedIssueIDs, nonExistentIssueIDs []int64) { - issues, err := issues_model.GetIssuesByIDs(db.DefaultContext, append(expectedIssueIDs, nonExistentIssueIDs...)) + issues, err := issues_model.GetIssuesByIDs(db.DefaultContext, append(expectedIssueIDs, nonExistentIssueIDs...), true) assert.NoError(t, err) actualIssueIDs := make([]int64, len(issues)) for i, issue := range issues { @@ -83,6 +83,7 @@ func TestGetIssuesByIDs(t *testing.T) { } testSuccess([]int64{1, 2, 3}, []int64{}) testSuccess([]int64{1, 2, 3}, []int64{unittest.NonexistentID}) + testSuccess([]int64{3, 2, 1}, []int64{}) } func TestGetParticipantIDsByIssue(t *testing.T) { diff --git a/modules/indexer/issues/dboptions.go b/modules/indexer/issues/dboptions.go new file mode 100644 index 0000000000000..9f90647779436 --- /dev/null +++ b/modules/indexer/issues/dboptions.go @@ -0,0 +1,89 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package issues + +import ( + "code.gitea.io/gitea/models/db" + issues_model "code.gitea.io/gitea/models/issues" +) + +func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOptions { + searchOpt := &SearchOptions{ + Keyword: keyword, + RepoIDs: opts.RepoIDs, + AllPublic: false, + IsPull: opts.IsPull, + IsClosed: opts.IsClosed, + } + + if len(opts.LabelIDs) == 1 && opts.LabelIDs[0] == 0 { + searchOpt.NoLabelOnly = true + } else { + for _, labelID := range opts.LabelIDs { + if labelID > 0 { + searchOpt.IncludedLabelIDs = append(searchOpt.IncludedLabelIDs, labelID) + } else { + searchOpt.ExcludedLabelIDs = append(searchOpt.ExcludedLabelIDs, -labelID) + } + } + } + + if len(opts.MilestoneIDs) == 1 && opts.MilestoneIDs[0] == db.NoConditionID { + searchOpt.MilestoneIDs = []int64{0} + } else { + searchOpt.MilestoneIDs = opts.MilestoneIDs + } + + if opts.AssigneeID > 0 { + searchOpt.AssigneeID = &opts.AssigneeID + } + if opts.PosterID > 0 { + searchOpt.PosterID = &opts.PosterID + } + if opts.MentionedID > 0 { + searchOpt.MentionID = &opts.MentionedID + } + if opts.ReviewedID > 0 { + searchOpt.ReviewedID = &opts.ReviewedID + } + if opts.ReviewRequestedID > 0 { + searchOpt.ReviewRequestedID = &opts.ReviewRequestedID + } + if opts.SubscriberID > 0 { + searchOpt.SubscriberID = &opts.SubscriberID + } + + if opts.UpdatedAfterUnix > 0 { + searchOpt.UpdatedAfterUnix = &opts.UpdatedAfterUnix + } + if opts.UpdatedBeforeUnix > 0 { + searchOpt.UpdatedBeforeUnix = &opts.UpdatedBeforeUnix + } + + searchOpt.Paginator = opts.Paginator + + switch opts.SortType { + case "oldest": + searchOpt.SortBy = SortByCreatedAsc + case "recentupdate": + searchOpt.SortBy = SortByUpdatedDesc + case "leastupdate": + searchOpt.SortBy = SortByUpdatedAsc + case "mostcomment": + searchOpt.SortBy = SortByCommentsDesc + case "leastcomment": + searchOpt.SortBy = SortByCommentsAsc + case "nearduedate": + searchOpt.SortBy = SortByDeadlineAsc + case "farduedate": + searchOpt.SortBy = SortByDeadlineDesc + case "priority", "priorityrepo", "project-column-sorting": + // Unsupported sort type for search + searchOpt.SortBy = SortByUpdatedDesc + default: + searchOpt.SortBy = SortByUpdatedDesc + } + + return searchOpt +} diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index e2d8f84fdcd15..74e228abb6496 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -298,7 +298,7 @@ func SearchIssues(ctx *context.APIContext) { ctx.Error(http.StatusInternalServerError, "SearchIssues", err) return } - issues, err := issues_model.FindIssuesByIDs(ctx, ids) + issues, err := issues_model.GetIssuesByIDs(ctx, ids, true) if err != nil { ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err) return @@ -517,7 +517,7 @@ func ListIssues(ctx *context.APIContext) { ctx.Error(http.StatusInternalServerError, "SearchIssues", err) return } - issues, err := issues_model.FindIssuesByIDs(ctx, ids) + issues, err := issues_model.GetIssuesByIDs(ctx, ids, true) if err != nil { ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err) return diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index b21ef7b306b06..aac3a4afe2c2c 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -150,7 +150,6 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti mentionedID int64 reviewRequestedID int64 reviewedID int64 - forceEmpty bool ) if ctx.IsSigned { @@ -187,19 +186,9 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti keyword = "" } - var issueIDs []int64 - if len(keyword) > 0 { - issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword) - if err != nil { - if issue_indexer.IsAvailable(ctx) { - ctx.ServerError("issueIndexer.Search", err) - return - } - ctx.Data["IssueIndexerUnavailable"] = true - } - if len(issueIDs) == 0 { - forceEmpty = true - } + if issue_indexer.IsAvailable(ctx) { + ctx.ServerError("issueIndexer.Search", err) + return } var mileIDs []int64 @@ -208,10 +197,8 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti } var issueStats *issues_model.IssueStats - if forceEmpty { - issueStats = &issues_model.IssueStats{} - } else { - issueStats, err = issues_model.GetIssueStats(&issues_model.IssuesOptions{ + { + statsOpts := &issues_model.IssuesOptions{ RepoIDs: []int64{repo.ID}, LabelIDs: labelIDs, MilestoneIDs: mileIDs, @@ -222,8 +209,17 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti ReviewRequestedID: reviewRequestedID, ReviewedID: reviewedID, IsPull: isPullOption, - IssueIDs: issueIDs, - }) + IssueIDs: nil, + } + if keyword != "" { + allIssueIDs, err := issueIDsFromSearch(ctx, keyword, statsOpts) + if err != nil { + ctx.ServerError("issueIDsFromSearch", err) + return + } + statsOpts.IssueIDs = allIssueIDs + } + issueStats, err = issues_model.GetIssueStats(statsOpts) if err != nil { ctx.ServerError("GetIssueStats", err) return @@ -250,10 +246,8 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti pager := context.NewPagination(total, setting.UI.IssuePagingNum, page, 5) var issues []*issues_model.Issue - if forceEmpty { - issues = []*issues_model.Issue{} - } else { - issues, err = issues_model.Issues(ctx, &issues_model.IssuesOptions{ + { + ids, err := issueIDsFromSearch(ctx, keyword, &issues_model.IssuesOptions{ Paginator: &db.ListOptions{ Page: pager.Paginater.Current(), PageSize: setting.UI.IssuePagingNum, @@ -270,10 +264,14 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti IsPull: isPullOption, LabelIDs: labelIDs, SortType: sortType, - IssueIDs: issueIDs, }) if err != nil { - ctx.ServerError("Issues", err) + ctx.ServerError("issueIDsFromSearch", err) + return + } + issues, err = issues_model.GetIssuesByIDs(ctx, ids, true) + if err != nil { + ctx.ServerError("GetIssuesByIDs", err) return } } @@ -425,6 +423,14 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti ctx.Data["Page"] = pager } +func issueIDsFromSearch(ctx *context.Context, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) { + ids, _, err := issue_indexer.SearchIssues(ctx, issue_indexer.ToSearchOptions(keyword, opts)) + if err != nil { + return nil, fmt.Errorf("SearchIssues: %w", err) + } + return ids, nil +} + // Issues render issues page func Issues(ctx *context.Context) { isPullList := ctx.Params(":type") == "pulls" @@ -2575,7 +2581,7 @@ func SearchIssues(ctx *context.Context) { ctx.Error(http.StatusInternalServerError, "SearchIssues", err.Error()) return } - issues, err := issues_model.FindIssuesByIDs(ctx, ids) + issues, err := issues_model.GetIssuesByIDs(ctx, ids, true) if err != nil { ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err.Error()) return @@ -2747,7 +2753,7 @@ func ListIssues(ctx *context.Context) { ctx.Error(http.StatusInternalServerError, "SearchIssues", err.Error()) return } - issues, err := issues_model.FindIssuesByIDs(ctx, ids) + issues, err := issues_model.GetIssuesByIDs(ctx, ids, true) if err != nil { ctx.Error(http.StatusInternalServerError, "FindIssuesByIDs", err.Error()) return diff --git a/routers/web/user/home.go b/routers/web/user/home.go index c345209dec7a4..7a4c9229ec98d 100644 --- a/routers/web/user/home.go +++ b/routers/web/user/home.go @@ -538,7 +538,7 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { ctx.ServerError("issueIDsFromSearch", err) return } - issues, err = issues_model.GetIssuesByIDs(ctx, issueIDs) + issues, err = issues_model.GetIssuesByIDs(ctx, issueIDs, true) if err != nil { ctx.ServerError("GetIssuesByIDs", err) return @@ -730,93 +730,13 @@ func getRepoIDs(reposQuery string) []int64 { } func issueIDsFromSearch(ctx *context.Context, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) { - ids, _, err := issue_indexer.SearchIssues(ctx, convertOptionsToSearchOptions(keyword, opts)) + ids, _, err := issue_indexer.SearchIssues(ctx, issue_indexer.ToSearchOptions(keyword, opts)) if err != nil { return nil, fmt.Errorf("SearchIssues: %w", err) } return ids, nil } -func convertOptionsToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *issue_indexer.SearchOptions { - searchOpt := &issue_indexer.SearchOptions{ - Keyword: keyword, - RepoIDs: opts.RepoIDs, - AllPublic: false, - IsPull: opts.IsPull, - IsClosed: opts.IsClosed, - } - - if len(opts.LabelIDs) == 1 && opts.LabelIDs[0] == 0 { - searchOpt.NoLabelOnly = true - } else { - for _, labelID := range opts.LabelIDs { - if labelID > 0 { - searchOpt.IncludedLabelIDs = append(searchOpt.IncludedLabelIDs, labelID) - } else { - searchOpt.ExcludedLabelIDs = append(searchOpt.ExcludedLabelIDs, -labelID) - } - } - } - - if len(opts.MilestoneIDs) == 1 && opts.MilestoneIDs[0] == db.NoConditionID { - searchOpt.MilestoneIDs = []int64{0} - } else { - searchOpt.MilestoneIDs = opts.MilestoneIDs - } - - if opts.AssigneeID > 0 { - searchOpt.AssigneeID = &opts.AssigneeID - } - if opts.PosterID > 0 { - searchOpt.PosterID = &opts.PosterID - } - if opts.MentionedID > 0 { - searchOpt.MentionID = &opts.MentionedID - } - if opts.ReviewedID > 0 { - searchOpt.ReviewedID = &opts.ReviewedID - } - if opts.ReviewRequestedID > 0 { - searchOpt.ReviewRequestedID = &opts.ReviewRequestedID - } - if opts.SubscriberID > 0 { - searchOpt.SubscriberID = &opts.SubscriberID - } - - if opts.UpdatedAfterUnix > 0 { - searchOpt.UpdatedAfterUnix = &opts.UpdatedAfterUnix - } - if opts.UpdatedBeforeUnix > 0 { - searchOpt.UpdatedBeforeUnix = &opts.UpdatedBeforeUnix - } - - searchOpt.Paginator = opts.Paginator - - switch opts.SortType { - case "oldest": - searchOpt.SortBy = issue_indexer.SortByCreatedAsc - case "recentupdate": - searchOpt.SortBy = issue_indexer.SortByUpdatedDesc - case "leastupdate": - searchOpt.SortBy = issue_indexer.SortByUpdatedAsc - case "mostcomment": - searchOpt.SortBy = issue_indexer.SortByCommentsDesc - case "leastcomment": - searchOpt.SortBy = issue_indexer.SortByCommentsAsc - case "nearduedate": - searchOpt.SortBy = issue_indexer.SortByDeadlineAsc - case "farduedate": - searchOpt.SortBy = issue_indexer.SortByDeadlineDesc - case "priority", "priorityrepo", "project-column-sorting": - // Unsupported sort type for search - searchOpt.SortBy = issue_indexer.SortByUpdatedDesc - default: - searchOpt.SortBy = issue_indexer.SortByUpdatedDesc - } - - return searchOpt -} - func loadRepoByIDs(ctxUser *user_model.User, issueCountByRepo map[int64]int64, unitType unit.Type) (map[int64]*repo_model.Repository, error) { totalRes := make(map[int64]*repo_model.Repository, len(issueCountByRepo)) repoIDs := make([]int64, 0, 500) From 719173b73754a9c87da55464df69d06237f0ce06 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 17:02:01 +0800 Subject: [PATCH 39/99] fix: meilisearch --- .../indexer/issues/meilisearch/meilisearch.go | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 5990e0de53f4a..cd81469462aa4 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -5,6 +5,7 @@ package meilisearch import ( "context" + "fmt" "strconv" "strings" @@ -16,7 +17,7 @@ import ( ) const ( - issueIndexerLatestVersion = 1 + issueIndexerLatestVersion = 2 ) var _ internal.Indexer = &Indexer{} @@ -78,16 +79,16 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( filter := strings.Join(repoFilters, " OR ") skip, limit := indexer_internal.ParsePaginator(options.Paginator) - // TBC: - /* - if state == "open" || state == "closed" { - if filter != "" { - filter = "(" + filter + ") AND state = " + state - } else { - filter = "state = " + state - } + if !options.IsClosed.IsNone() { + condition := fmt.Sprintf("is_closed = %t", options.IsClosed.IsTrue()) + if filter != "" { + filter = "(" + filter + ") AND " + condition + } else { + filter = "state = " + condition } - */ + } + + // TODO: support more conditions searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(options.Keyword, &meilisearch.SearchRequest{ Filter: filter, From 587c666d7d228675fd27de72d7a69c19a6ead02f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 17:05:56 +0800 Subject: [PATCH 40/99] fix: applyLimit --- models/issues/issue_search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index d022b9af9debc..d96ff8747258c 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -99,7 +99,7 @@ func applySorts(sess *xorm.Session, sortType string, priorityRepoID int64) { } func applyLimit(sess *xorm.Session, opts *IssuesOptions) *xorm.Session { - if opts.Paginator != nil { + if opts.Paginator != nil && !opts.Paginator.IsListAll() { skip, take := opts.GetSkipTake() sess.Limit(take, skip) } From 9bab9316c4c6b02c0f184f137b4512f9a9c7ae6a Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 17:14:05 +0800 Subject: [PATCH 41/99] fix: empty means SortByCreatedDesc --- modules/indexer/issues/dboptions.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/indexer/issues/dboptions.go b/modules/indexer/issues/dboptions.go index 9f90647779436..d04c7cd80404d 100644 --- a/modules/indexer/issues/dboptions.go +++ b/modules/indexer/issues/dboptions.go @@ -64,6 +64,8 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp searchOpt.Paginator = opts.Paginator switch opts.SortType { + case "": + searchOpt.SortBy = SortByCreatedDesc case "oldest": searchOpt.SortBy = SortByCreatedAsc case "recentupdate": From 86d8ee0316b1c9ce622a00083dc598578adbfe73 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 17:16:35 +0800 Subject: [PATCH 42/99] chore: remove SearchIssuesByKeyword --- modules/indexer/issues/indexer.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 836688def3b1f..eb9e216af9c73 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -285,18 +285,6 @@ func DeleteRepoIssueIndexer(ctx context.Context, repoID int64) { } } -// Deprecated: use SearchIssues instead -// SearchIssuesByKeyword search issue ids by keywords and repo id -// WARNNING: You have to ensure user have permission to visit repoIDs' issues -func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) { - ids, _, err := SearchIssues(ctx, &SearchOptions{ - Keyword: keyword, - RepoIDs: repoIDs, - Paginator: db_model.NewAbsoluteListOptions(0, 50), - }) - return ids, err -} - // IsAvailable checks if issue indexer is available func IsAvailable(ctx context.Context) bool { return (*globalIndexer.Load()).Ping(ctx) == nil From 4aa7fc77fb14093349cebb14c07eb133d8c9e9c7 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 17:19:28 +0800 Subject: [PATCH 43/99] chore: remove help comments --- modules/indexer/issues/internal/model.go | 26 ++++++++++-------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 8a8fb1a694723..20d6397469f7c 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -13,7 +13,7 @@ import ( type IndexerData struct { ID int64 `json:"id"` RepoID int64 `json:"repo_id"` - IsPublic bool `json:"is_public"` // If the repo is public, so if the visibility of the repo has changed, we should reindex the issues. + IsPublic bool `json:"is_public"` // If the repo is public // Fields used for keyword searching Title string `json:"title"` @@ -22,18 +22,18 @@ type IndexerData struct { // Fields used for filtering IsPull bool `json:"is_pull"` - IsClosed bool `json:"is_closed"` // So if the status of an issue has changed, we should reindex the issue. - LabelIDs []int64 `json:"label_ids"` // So if the labels of an issue have changed, we should reindex the issue. - NoLabel bool `json:"no_label"` // True if LabelIDs is empty - MilestoneID int64 `json:"milestone_id"` // So if the milestone of an issue has changed, we should reindex the issue. - ProjectID int64 `json:"project_id"` // So if the project of an issue have changed, we should reindex the issue. - ProjectBoardID int64 `json:"project_board_id"` // So if the project board of an issue have changed, we should reindex the issue. + IsClosed bool `json:"is_closed"` + LabelIDs []int64 `json:"label_ids"` + NoLabel bool `json:"no_label"` // True if LabelIDs is empty + MilestoneID int64 `json:"milestone_id"` + ProjectID int64 `json:"project_id"` + ProjectBoardID int64 `json:"project_board_id"` PosterID int64 `json:"poster_id"` - AssigneeID int64 `json:"assignee_id"` // So if the assignee of an issue has changed, we should reindex the issue. + AssigneeID int64 `json:"assignee_id"` MentionIDs []int64 `json:"mention_ids"` - ReviewedIDs []int64 `json:"reviewed_ids"` // So if the reviewers of an issue have changed, we should reindex the issue. - ReviewRequestedIDs []int64 `json:"review_requested_ids"` // So if the requested reviewers of an issue have changed, we should reindex the issue. - SubscriberIDs []int64 `json:"subscriber_ids"` // So if the subscribers of an issue have changed, we should reindex the issue. + ReviewedIDs []int64 `json:"reviewed_ids"` + ReviewRequestedIDs []int64 `json:"review_requested_ids"` + SubscriberIDs []int64 `json:"subscriber_ids"` UpdatedUnix timeutil.TimeStamp `json:"updated_unix"` // Fields used for sorting @@ -59,10 +59,6 @@ type SearchResult struct { } // SearchOptions represents search options -// So the search engine should support: -// - Filter by boolean/int value -// - Filter by "array contains any of specified elements" -// - Filter by "array doesn't contain any of specified elements" type SearchOptions struct { Keyword string // keyword to search From e7e4edb4bb05320264e0b0a4bbd2998af2d0661f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 17:27:05 +0800 Subject: [PATCH 44/99] fix: lint code --- routers/web/user/home.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/routers/web/user/home.go b/routers/web/user/home.go index 7a4c9229ec98d..1cdf5a8f0424b 100644 --- a/routers/web/user/home.go +++ b/routers/web/user/home.go @@ -468,10 +468,12 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { ctx.Data["Keyword"] = keyword accessibleRepos := container.Set[int64]{} - if ids, err := issues_model.GetRepoIDsForIssuesOptions(opts, ctxUser); err != nil { - ctx.ServerError("GetRepoIDsForIssuesOptions", err) - return - } else { + { + ids, err := issues_model.GetRepoIDsForIssuesOptions(opts, ctxUser) + if err != nil { + ctx.ServerError("GetRepoIDsForIssuesOptions", err) + return + } for _, id := range ids { accessibleRepos.Add(id) } From 6473b315e40b691ffa5619b84a79c88efd50528a Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 17:28:39 +0800 Subject: [PATCH 45/99] test: fix case --- tests/integration/issue_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/issue_test.go b/tests/integration/issue_test.go index ab2986906bfd4..f883ae2b2bfd0 100644 --- a/tests/integration/issue_test.go +++ b/tests/integration/issue_test.go @@ -99,7 +99,7 @@ func TestViewIssuesKeyword(t *testing.T) { RepoID: repo.ID, Index: 1, }) - issues.UpdateIssueIndexer(issue) + issues.UpdateIssueIndexer(issue.ID) time.Sleep(time.Second * 1) const keyword = "first" req := NewRequestf(t, "GET", "%s/issues?q=%s", repo.Link(), keyword) From 2759d46911852792ee37fe6420986e0fe4509361 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 17:33:27 +0800 Subject: [PATCH 46/99] test: fix case --- modules/indexer/issues/indexer_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index 762d6e2fbf746..d6812f714e086 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -51,7 +51,7 @@ func TestBleveSearchIssues(t *testing.T) { time.Sleep(5 * time.Second) t.Run("issue2", func(t *testing.T) { - ids, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "issue2", RepoIDs: []int64{1}, }) @@ -60,7 +60,7 @@ func TestBleveSearchIssues(t *testing.T) { }) t.Run("first", func(t *testing.T) { - ids, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "first", RepoIDs: []int64{1}, }) @@ -69,7 +69,7 @@ func TestBleveSearchIssues(t *testing.T) { }) t.Run("for", func(t *testing.T) { - ids, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "for", RepoIDs: []int64{1}, }) @@ -78,7 +78,7 @@ func TestBleveSearchIssues(t *testing.T) { }) t.Run("good", func(t *testing.T) { - ids, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "good", RepoIDs: []int64{1}, }) @@ -94,7 +94,7 @@ func TestDBSearchIssues(t *testing.T) { InitIssueIndexer(true) t.Run("issue2", func(t *testing.T) { - ids, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "issue2", RepoIDs: []int64{1}, }) @@ -103,7 +103,7 @@ func TestDBSearchIssues(t *testing.T) { }) t.Run("first", func(t *testing.T) { - ids, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "first", RepoIDs: []int64{1}, }) @@ -112,7 +112,7 @@ func TestDBSearchIssues(t *testing.T) { }) t.Run("for", func(t *testing.T) { - ids, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "for", RepoIDs: []int64{1}, }) @@ -121,7 +121,7 @@ func TestDBSearchIssues(t *testing.T) { }) t.Run("good", func(t *testing.T) { - ids, err := SearchIssues(context.TODO(), &SearchOptions{ + ids, _, err := SearchIssues(context.TODO(), &SearchOptions{ Keyword: "good", RepoIDs: []int64{1}, }) From 3c8cb1a1fb319b394811bfb91ddf488989c47c21 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 18:53:58 +0800 Subject: [PATCH 47/99] tests: new test framework for indexer --- modules/indexer/issues/bleve/bleve.go | 5 +- modules/indexer/issues/bleve/bleve_test.go | 77 +------ modules/indexer/issues/internal/tests/doc.go | 8 + .../indexer/issues/internal/tests/tests.go | 203 ++++++++++++++++++ 4 files changed, 218 insertions(+), 75 deletions(-) create mode 100644 modules/indexer/issues/internal/tests/doc.go create mode 100644 modules/indexer/issues/internal/tests/tests.go diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 398238e94b3ae..d7998af362a0b 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -245,7 +245,10 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( queries = append(queries, inner_bleve.NumericRangeInclusiveQuery(options.UpdatedAfterUnix, options.UpdatedBeforeUnix, "updated_unix")) } - indexerQuery := bleve.NewConjunctionQuery(queries...) + var indexerQuery query.Query = bleve.NewConjunctionQuery(queries...) + if len(queries) == 0 { + indexerQuery = bleve.NewMatchAllQuery() + } skip, limit := indexer_internal.ParsePaginator(options.Paginator) search := bleve.NewSearchRequestOptions(indexerQuery, limit, skip, false) diff --git a/modules/indexer/issues/bleve/bleve_test.go b/modules/indexer/issues/bleve/bleve_test.go index 0eb136d22b276..0a3e1e1abad54 100644 --- a/modules/indexer/issues/bleve/bleve_test.go +++ b/modules/indexer/issues/bleve/bleve_test.go @@ -4,86 +4,15 @@ package bleve import ( - "context" "testing" - "code.gitea.io/gitea/modules/indexer/issues/internal" - - "github.com/stretchr/testify/assert" + "code.gitea.io/gitea/modules/indexer/issues/internal/tests" ) -func TestBleveIndexAndSearch(t *testing.T) { +func TestIndexer(t *testing.T) { dir := t.TempDir() indexer := NewIndexer(dir) defer indexer.Close() - if _, err := indexer.Init(context.Background()); err != nil { - assert.Fail(t, "Unable to initialize bleve indexer: %v", err) - return - } - - err := indexer.Index(context.Background(), []*internal.IndexerData{ - { - ID: 1, - RepoID: 2, - Title: "Issue search should support Chinese", - Content: "As title", - Comments: []string{ - "test1", - "test2", - }, - }, - { - ID: 2, - RepoID: 2, - Title: "CJK support could be optional", - Content: "Chinese Korean and Japanese should be supported but I would like it's not enabled by default", - Comments: []string{ - "LGTM", - "Good idea", - }, - }, - }) - assert.NoError(t, err) - - keywords := []struct { - Keyword string - IDs []int64 - }{ - { - Keyword: "search", - IDs: []int64{1}, - }, - { - Keyword: "test1", - IDs: []int64{1}, - }, - { - Keyword: "test2", - IDs: []int64{1}, - }, - { - Keyword: "support", - IDs: []int64{1, 2}, - }, - { - Keyword: "chinese", - IDs: []int64{1, 2}, - }, - { - Keyword: "help", - IDs: []int64{}, - }, - } - - for _, kw := range keywords { - res, err := indexer.Search(context.TODO(), kw.Keyword, []int64{2}, 10, 0, "") - assert.NoError(t, err) - - ids := make([]int64, 0, len(res.Hits)) - for _, hit := range res.Hits { - ids = append(ids, hit.ID) - } - assert.ElementsMatch(t, kw.IDs, ids) - } + tests.TestIndexer(t, indexer) } diff --git a/modules/indexer/issues/internal/tests/doc.go b/modules/indexer/issues/internal/tests/doc.go new file mode 100644 index 0000000000000..df24bde7c9c01 --- /dev/null +++ b/modules/indexer/issues/internal/tests/doc.go @@ -0,0 +1,8 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tests + +// This package contains tests for the indexer module. +// All the code in this package is only used for testing. +// Do not put any production code in this package to avoid it being included in the final binary. diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go new file mode 100644 index 0000000000000..d350b3395c461 --- /dev/null +++ b/modules/indexer/issues/internal/tests/tests.go @@ -0,0 +1,203 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tests + +import ( + "context" + "fmt" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIndexer(t *testing.T, indexer internal.Indexer) { + t.Run("Init", func(t *testing.T) { + if _, err := indexer.Init(nil); err != nil { + t.Fatalf("Init failed: %v", err) + } + }) + + t.Run("Ping", func(t *testing.T) { + if err := indexer.Ping(nil); err != nil { + t.Fatalf("Ping failed: %v", err) + } + }) + + var ( + ids []int64 + data = map[int64]*internal.IndexerData{} + ) + + t.Run("Index", func(t *testing.T) { + d := generateIndexerData() + for _, v := range d { + ids = append(ids, v.ID) + data[v.ID] = v + } + if err := indexer.Index(context.Background(), d...); err != nil { + t.Fatalf("Index failed: %v", err) + } + }) + + defer t.Run("Delete", func(t *testing.T) { + if err := indexer.Delete(context.Background(), ids...); err != nil { + t.Fatalf("Delete failed: %v", err) + } + }) + + t.Run("Search", func(t *testing.T) { + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + if c.Before != nil { + c.Before(t, data, indexer) + } + result, err := indexer.Search(context.Background(), c.SearchOptions) + require.NoError(t, err) + + if c.Expected != nil { + c.Expected(t, data, result) + } else { + ids := make([]int64, 0, len(result.Hits)) + for _, hit := range result.Hits { + ids = append(ids, hit.ID) + } + assert.Equal(t, c.ExpectedIDs, ids) + assert.Equal(t, c.ExpectedTotal, result.Total) + } + if result.Imprecise { + // If an engine does not support complex queries, do not use TestIndexer to test it + t.Errorf("Expected imprecise to be false, got true") + } + + if c.After != nil { + c.After(t, data, indexer) + } + }) + } + }) +} + +var cases = []*testIndexerCase{ + { + Name: "empty", + SearchOptions: &internal.SearchOptions{}, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 50, len(result.Hits)) // the default limit is 50 + assert.Equal(t, len(data), int(result.Total)) + }, + }, + { + Name: "all", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + ListAll: true, + }, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, len(data), len(result.Hits)) // the default limit is 50 + assert.Equal(t, len(data), int(result.Total)) + }, + }, + { + Name: "keyword", + Before: func(t *testing.T, _ map[int64]*internal.IndexerData, indexer internal.Indexer) { + newData := []*internal.IndexerData{ + {ID: 1000, Title: "hi hello world"}, + {ID: 1001, Content: "hi hello world"}, + {ID: 1002, Comments: []string{"hi", "hello world"}}, + } + assert.NoError(t, indexer.Index(context.Background(), newData...)) + }, + After: func(t *testing.T, data map[int64]*internal.IndexerData, indexer internal.Indexer) { + assert.NoError(t, indexer.Delete(context.Background(), 1000, 1001, 1002)) + }, + SearchOptions: &internal.SearchOptions{ + Keyword: "hello", + }, + ExpectedIDs: []int64{1002, 1001, 1000}, + ExpectedTotal: 3, + }, + // TODO: add more cases +} + +type testIndexerCase struct { + Name string + Before func(t *testing.T, data map[int64]*internal.IndexerData, indexer internal.Indexer) + After func(t *testing.T, data map[int64]*internal.IndexerData, indexer internal.Indexer) + + SearchOptions *internal.SearchOptions + + Expected func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) // if nil, use ExpectedIDs, ExpectedTotal and ExpectedImprecise + ExpectedIDs []int64 + ExpectedTotal int64 +} + +func generateIndexerData() []*internal.IndexerData { + var id int64 + var data []*internal.IndexerData + for repoID := int64(1); repoID <= 10; repoID++ { + for issueIndex := int64(1); issueIndex <= 20; issueIndex++ { + id++ + + comments := make([]string, id%4) + for i := range comments { + comments[i] = fmt.Sprintf("comment%d", i) + } + + labelIDs := make([]int64, id%5) + for i := range labelIDs { + labelIDs[i] = int64(i) + } + mentionIDs := make([]int64, id%6) + for i := range mentionIDs { + mentionIDs[i] = int64(i) + } + reviewedIDs := make([]int64, id%7) + for i := range reviewedIDs { + reviewedIDs[i] = int64(i) + } + reviewRequestedIDs := make([]int64, id%8) + for i := range reviewRequestedIDs { + reviewRequestedIDs[i] = int64(i) + } + subscriberIDs := make([]int64, id%9) + for i := range subscriberIDs { + subscriberIDs[i] = int64(i) + } + + data = append(data, &internal.IndexerData{ + ID: id, + RepoID: repoID, + IsPublic: repoID%2 == 0, + Title: fmt.Sprintf("issue%d of repo%d", issueIndex, repoID), + Content: fmt.Sprintf("content%d", issueIndex), + Comments: comments, + IsPull: issueIndex%2 == 0, + IsClosed: issueIndex%3 == 0, + LabelIDs: labelIDs, + NoLabel: len(labelIDs) == 0, + MilestoneID: issueIndex % 4, + ProjectID: issueIndex % 5, + ProjectBoardID: issueIndex % 6, + PosterID: id % 10, + AssigneeID: issueIndex % 10, + MentionIDs: mentionIDs, + ReviewedIDs: reviewedIDs, + ReviewRequestedIDs: reviewRequestedIDs, + SubscriberIDs: subscriberIDs, + UpdatedUnix: timeutil.TimeStamp(id + issueIndex), + CreatedUnix: timeutil.TimeStamp(id), + DeadlineUnix: timeutil.TimeStamp(id + issueIndex + repoID), + CommentCount: int64(len(comments)), + }) + } + } + + return data +} From dc8b61aee4573cadf2b79ce936976fe42e6d02b9 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 18:55:42 +0800 Subject: [PATCH 48/99] test: add comments --- modules/indexer/issues/internal/tests/doc.go | 8 -------- modules/indexer/issues/internal/tests/tests.go | 4 ++++ 2 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 modules/indexer/issues/internal/tests/doc.go diff --git a/modules/indexer/issues/internal/tests/doc.go b/modules/indexer/issues/internal/tests/doc.go deleted file mode 100644 index df24bde7c9c01..0000000000000 --- a/modules/indexer/issues/internal/tests/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package tests - -// This package contains tests for the indexer module. -// All the code in this package is only used for testing. -// Do not put any production code in this package to avoid it being included in the final binary. diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index d350b3395c461..e0beff88bfd7e 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -1,6 +1,10 @@ // Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT +// This package contains tests for the indexer module. +// All the code in this package is only used for testing. +// Do not put any production code in this package to avoid it being included in the final binary. + package tests import ( From 3d00c2d1cbf7be49385091f6e5688837e30e3f2e Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Jul 2023 19:01:05 +0800 Subject: [PATCH 49/99] test: ExtraData --- .../indexer/issues/internal/tests/tests.go | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index e0beff88bfd7e..f180425ac208c 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -39,7 +39,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { ) t.Run("Index", func(t *testing.T) { - d := generateIndexerData() + d := generateDefaultIndexerData() for _, v := range d { ids = append(ids, v.ID) data[v.ID] = v @@ -58,9 +58,19 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { t.Run("Search", func(t *testing.T) { for _, c := range cases { t.Run(c.Name, func(t *testing.T) { - if c.Before != nil { - c.Before(t, data, indexer) + if len(c.ExtraData) > 0 { + require.NoError(t, indexer.Index(context.Background(), c.ExtraData...)) + for _, v := range c.ExtraData { + data[v.ID] = v + } + defer func() { + for _, v := range c.ExtraData { + require.NoError(t, indexer.Delete(context.Background(), v.ID)) + delete(data, v.ID) + } + }() } + result, err := indexer.Search(context.Background(), c.SearchOptions) require.NoError(t, err) @@ -78,10 +88,6 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { // If an engine does not support complex queries, do not use TestIndexer to test it t.Errorf("Expected imprecise to be false, got true") } - - if c.After != nil { - c.After(t, data, indexer) - } }) } }) @@ -110,16 +116,10 @@ var cases = []*testIndexerCase{ }, { Name: "keyword", - Before: func(t *testing.T, _ map[int64]*internal.IndexerData, indexer internal.Indexer) { - newData := []*internal.IndexerData{ - {ID: 1000, Title: "hi hello world"}, - {ID: 1001, Content: "hi hello world"}, - {ID: 1002, Comments: []string{"hi", "hello world"}}, - } - assert.NoError(t, indexer.Index(context.Background(), newData...)) - }, - After: func(t *testing.T, data map[int64]*internal.IndexerData, indexer internal.Indexer) { - assert.NoError(t, indexer.Delete(context.Background(), 1000, 1001, 1002)) + ExtraData: []*internal.IndexerData{ + {ID: 1000, Title: "hi hello world"}, + {ID: 1001, Content: "hi hello world"}, + {ID: 1002, Comments: []string{"hi", "hello world"}}, }, SearchOptions: &internal.SearchOptions{ Keyword: "hello", @@ -131,9 +131,8 @@ var cases = []*testIndexerCase{ } type testIndexerCase struct { - Name string - Before func(t *testing.T, data map[int64]*internal.IndexerData, indexer internal.Indexer) - After func(t *testing.T, data map[int64]*internal.IndexerData, indexer internal.Indexer) + Name string + ExtraData []*internal.IndexerData SearchOptions *internal.SearchOptions @@ -142,7 +141,7 @@ type testIndexerCase struct { ExpectedTotal int64 } -func generateIndexerData() []*internal.IndexerData { +func generateDefaultIndexerData() []*internal.IndexerData { var id int64 var data []*internal.IndexerData for repoID := int64(1); repoID <= 10; repoID++ { From 515677136b9db48142b53fd779adb9dee46b7edd Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 12:15:40 +0800 Subject: [PATCH 50/99] test: tidy --- .../indexer/issues/internal/tests/tests.go | 90 ++++++++----------- 1 file changed, 38 insertions(+), 52 deletions(-) diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index f180425ac208c..700798f99d771 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -21,76 +21,62 @@ import ( ) func TestIndexer(t *testing.T, indexer internal.Indexer) { - t.Run("Init", func(t *testing.T) { - if _, err := indexer.Init(nil); err != nil { - t.Fatalf("Init failed: %v", err) - } - }) + _, err := indexer.Init(context.Background()) + require.NoError(t, err) - t.Run("Ping", func(t *testing.T) { - if err := indexer.Ping(nil); err != nil { - t.Fatalf("Ping failed: %v", err) - } - }) + require.NoError(t, indexer.Ping(context.Background())) var ( ids []int64 data = map[int64]*internal.IndexerData{} ) - - t.Run("Index", func(t *testing.T) { + { d := generateDefaultIndexerData() for _, v := range d { ids = append(ids, v.ID) data[v.ID] = v } - if err := indexer.Index(context.Background(), d...); err != nil { - t.Fatalf("Index failed: %v", err) - } - }) + require.NoError(t, indexer.Index(context.Background(), d...)) + } - defer t.Run("Delete", func(t *testing.T) { - if err := indexer.Delete(context.Background(), ids...); err != nil { - t.Fatalf("Delete failed: %v", err) - } - }) + defer func() { + require.NoError(t, indexer.Delete(context.Background(), ids...)) + }() - t.Run("Search", func(t *testing.T) { - for _, c := range cases { - t.Run(c.Name, func(t *testing.T) { - if len(c.ExtraData) > 0 { - require.NoError(t, indexer.Index(context.Background(), c.ExtraData...)) + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + if len(c.ExtraData) > 0 { + require.NoError(t, indexer.Index(context.Background(), c.ExtraData...)) + for _, v := range c.ExtraData { + data[v.ID] = v + } + defer func() { for _, v := range c.ExtraData { - data[v.ID] = v + require.NoError(t, indexer.Delete(context.Background(), v.ID)) + delete(data, v.ID) } - defer func() { - for _, v := range c.ExtraData { - require.NoError(t, indexer.Delete(context.Background(), v.ID)) - delete(data, v.ID) - } - }() - } + }() + } - result, err := indexer.Search(context.Background(), c.SearchOptions) - require.NoError(t, err) + result, err := indexer.Search(context.Background(), c.SearchOptions) + require.NoError(t, err) - if c.Expected != nil { - c.Expected(t, data, result) - } else { - ids := make([]int64, 0, len(result.Hits)) - for _, hit := range result.Hits { - ids = append(ids, hit.ID) - } - assert.Equal(t, c.ExpectedIDs, ids) - assert.Equal(t, c.ExpectedTotal, result.Total) + if c.Expected != nil { + c.Expected(t, data, result) + } else { + ids := make([]int64, 0, len(result.Hits)) + for _, hit := range result.Hits { + ids = append(ids, hit.ID) } - if result.Imprecise { - // If an engine does not support complex queries, do not use TestIndexer to test it - t.Errorf("Expected imprecise to be false, got true") - } - }) - } - }) + assert.Equal(t, c.ExpectedIDs, ids) + assert.Equal(t, c.ExpectedTotal, result.Total) + } + if result.Imprecise { + // If an engine does not support complex queries, do not use TestIndexer to test it + t.Errorf("Expected imprecise to be false, got true") + } + }) + } } var cases = []*testIndexerCase{ From 38c80248b009c2252a1956e0bd76c5058545cf79 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 12:21:04 +0800 Subject: [PATCH 51/99] test: empty --- modules/indexer/issues/internal/tests/tests.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 700798f99d771..be5b0ec0e0d81 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -81,13 +81,21 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { var cases = []*testIndexerCase{ { - Name: "empty", + Name: "default", SearchOptions: &internal.SearchOptions{}, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Equal(t, 50, len(result.Hits)) // the default limit is 50 assert.Equal(t, len(data), int(result.Total)) }, }, + { + Name: "empty", + SearchOptions: &internal.SearchOptions{ + Keyword: "f1dfac73-fda6-4a6b-b8a4-2408fcb8ef69", + }, + ExpectedIDs: []int64{}, + ExpectedTotal: 0, + }, { Name: "all", SearchOptions: &internal.SearchOptions{ From 0c8269eb1147a191affa22ba7b5d12f9b420dee7 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 12:25:27 +0800 Subject: [PATCH 52/99] fix: panic to read nil project --- modules/indexer/issues/util.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index 7992bc407e1a1..7a8b64dcfbbfc 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -85,6 +85,11 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD return nil, false, err } + var projectID int64 + if issue.Project != nil { + projectID = issue.Project.ID + } + return &internal.IndexerData{ ID: issue.ID, RepoID: issue.RepoID, @@ -97,7 +102,7 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD LabelIDs: labels, NoLabel: len(labels) == 0, MilestoneID: issue.MilestoneID, - ProjectID: issue.Project.ID, + ProjectID: projectID, ProjectBoardID: issue.ProjectBoardID(), PosterID: issue.PosterID, AssigneeID: issue.AssigneeID, From c9d1399654fb25cedfd1f105bb0cc30d05514ac9 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 12:37:53 +0800 Subject: [PATCH 53/99] fix: no label check --- routers/api/v1/repo/issue.go | 2 +- routers/web/repo/issue.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 74e228abb6496..62004dc9d368f 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -484,7 +484,7 @@ func ListIssues(ctx *context.APIContext) { if before != 0 { searchOpt.UpdatedBeforeUnix = &before } - if len(labelIDs) == 0 && labelIDs[0] == 0 { + if len(labelIDs) == 1 && labelIDs[0] == 0 { searchOpt.NoLabelOnly = true } else { for _, labelID := range labelIDs { diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index aac3a4afe2c2c..24b1d20002db8 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -2720,7 +2720,7 @@ func ListIssues(ctx *context.Context) { if before != 0 { searchOpt.UpdatedBeforeUnix = &before } - if len(labelIDs) == 0 && labelIDs[0] == 0 { + if len(labelIDs) == 1 && labelIDs[0] == 0 { searchOpt.NoLabelOnly = true } else { for _, labelID := range labelIDs { From 6adf9d6188f32b644bea26ff40d4534ef69e4f03 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 15:37:03 +0800 Subject: [PATCH 54/99] fix: meilisearch condition --- modules/indexer/issues/meilisearch/meilisearch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index cd81469462aa4..61bcbcdc1e707 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -84,7 +84,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( if filter != "" { filter = "(" + filter + ") AND " + condition } else { - filter = "state = " + condition + filter = condition } } From d18a68ccb9c07f4b6aa7211612fa649e5f665463 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 16:03:18 +0800 Subject: [PATCH 55/99] fix: db search --- models/issues/issue_search.go | 9 ++------- modules/indexer/issues/db/db.go | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index d96ff8747258c..0e9c483202dd1 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -451,17 +451,12 @@ func IssueIDs(ctx context.Context, opts *IssuesOptions, otherConds ...builder.Co sess.And(cond) } - total, err := sess.Count(&Issue{}) - if err != nil { - return nil, 0, err - } - applyLimit(sess, opts) applySorts(sess, opts.SortType, opts.PriorityRepoID) var res []int64 - if err := sess.Select("`issue`.id").Table("issue"). - Find(&res); err != nil { + total, err := sess.Select("`issue`.id").Table(&Issue{}).FindAndCount(&res) + if err != nil { return nil, 0, err } diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index cb63e287ed668..b088be2c2fb00 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -56,8 +56,8 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( repoCond, builder.Or( builder.If(options.Keyword != "", - db.BuildCaseInsensitiveLike("name", options.Keyword), - db.BuildCaseInsensitiveLike("content", options.Keyword), + db.BuildCaseInsensitiveLike("issue.name", options.Keyword), + db.BuildCaseInsensitiveLike("issue.content", options.Keyword), builder.In("id", builder.Select("issue_id"). From("comment"). Where(builder.And( From e919d73995243427cf1265b548364fa335c4f546 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 16:09:41 +0800 Subject: [PATCH 56/99] fix: db test --- modules/indexer/issues/db/options.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index 0ae40d7d670da..a1b9ec7afb462 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -12,19 +12,22 @@ import ( func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { convertID := func(id *int64) int64 { if id == nil { + return 0 + } + if *id == 0 { return db.NoConditionID } return *id } convertIDs := func(ids []int64) []int64 { - if len(ids) == 0 { + if len(ids) == 1 && ids[0] == 0 { return []int64{db.NoConditionID} } return ids } convertLabelIDs := func(includes, excludes []int64, noLabelOnly bool) []int64 { if noLabelOnly { - return []int64{0} + return []int64{0} // Be careful, it's zero, not db.NoConditionID } ret := make([]int64, 0, len(includes)+len(excludes)) ret = append(ret, includes...) @@ -50,13 +53,15 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { case internal.SortByDeadlineAsc: sortType = "farduedate" case internal.SortByCreatedDesc: - sortType = "" // default + sortType = "newest" case internal.SortByUpdatedDesc: sortType = "recentupdate" case internal.SortByCommentsDesc: sortType = "mostcomment" case internal.SortByDeadlineDesc: sortType = "nearduedate" + default: + sortType = "newest" } opts := &issue_model.IssuesOptions{ From 6075007fbe5a954fc7bdb25b8fd1823892644f99 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 17:11:57 +0800 Subject: [PATCH 57/99] fix: db searching --- modules/indexer/issues/db/db.go | 41 +++++++++++++++++-------------- modules/indexer/issues/indexer.go | 3 ++- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index b088be2c2fb00..44f664c2492a3 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -50,25 +50,28 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( // So that's the root problem: // The notification is defined in modules, but it's using lots of things should be in services. - repoCond := builder.In("repo_id", options.RepoIDs) - subQuery := builder.Select("id").From("issue").Where(repoCond) - cond := builder.And( - repoCond, - builder.Or( - builder.If(options.Keyword != "", - db.BuildCaseInsensitiveLike("issue.name", options.Keyword), - db.BuildCaseInsensitiveLike("issue.content", options.Keyword), - builder.In("id", builder.Select("issue_id"). - From("comment"). - Where(builder.And( - builder.Eq{"type": issue_model.CommentTypeComment}, - builder.In("issue_id", subQuery), - db.BuildCaseInsensitiveLike("content", options.Keyword), - )), - ), + cond := builder.NewCond() + + if options.Keyword != "" { + repoCond := builder.In("repo_id", options.RepoIDs) + if len(options.RepoIDs) == 1 { + repoCond = builder.Eq{"repo_id": options.RepoIDs[0]} + } + subQuery := builder.Select("id").From("issue").Where(repoCond) + + cond = builder.Or( + db.BuildCaseInsensitiveLike("issue.name", options.Keyword), + db.BuildCaseInsensitiveLike("issue.content", options.Keyword), + builder.In("issue.id", builder.Select("issue_id"). + From("comment"). + Where(builder.And( + builder.Eq{"type": issue_model.CommentTypeComment}, + builder.In("issue_id", subQuery), + db.BuildCaseInsensitiveLike("content", options.Keyword), + )), ), - ), - ) + ) + } opt := ToDBOptions(options) @@ -86,6 +89,6 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( return &internal.SearchResult{ Total: total, Hits: hits, - Imprecise: true, + Imprecise: false, }, nil } diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index eb9e216af9c73..273e707dba583 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -322,7 +322,8 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, err ret = append(ret, hit.ID) } - if result.Imprecise { + if len(result.Hits) > 0 && result.Imprecise { + // The result is imprecise, we need to filter the result again. ret, err := reFilter(ctx, ret, opts) if err != nil { return nil, 0, err From 535a6916863bdb323d46f89c6855bb8efbcbbd6f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 17:40:48 +0800 Subject: [PATCH 58/99] fix: paginator --- models/issues/issue_search.go | 24 +++++++++++++++++++++--- modules/indexer/internal/paginator.go | 21 ++++++++++++++++----- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index 0e9c483202dd1..7652bd36f174d 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -99,10 +99,28 @@ func applySorts(sess *xorm.Session, sortType string, priorityRepoID int64) { } func applyLimit(sess *xorm.Session, opts *IssuesOptions) *xorm.Session { - if opts.Paginator != nil && !opts.Paginator.IsListAll() { - skip, take := opts.GetSkipTake() - sess.Limit(take, skip) + if opts.Paginator.IsListAll() { + return sess } + + // Warning: Do not use GetSkipTake() for *db.ListOptions + // Its implementation could reset the page size with setting.API.MaxResponseItems + if listOptions, ok := opts.Paginator.(*db.ListOptions); ok { + if listOptions.Page >= 0 && listOptions.PageSize > 0 { + var start int + if listOptions.Page == 0 { + start = 0 + } else { + start = (listOptions.Page - 1) * listOptions.PageSize + } + sess.Limit(listOptions.PageSize, start) + } + return sess + } + + start, limit := opts.Paginator.GetSkipTake() + sess.Limit(limit, start) + return sess } diff --git a/modules/indexer/internal/paginator.go b/modules/indexer/internal/paginator.go index 004e3ad2e9704..3f6a1ec512fc7 100644 --- a/modules/indexer/internal/paginator.go +++ b/modules/indexer/internal/paginator.go @@ -11,13 +11,24 @@ import ( // ParsePaginator parses a db.Paginator into a skip and limit func ParsePaginator(paginator db.Paginator) (int, int) { - if paginator == nil { - // Use default values - return 0, 50 + if paginator == nil || paginator.IsListAll() { + // Use a very large number to list all + return 0, math.MaxInt } - if paginator.IsListAll() { - // Use a very large number to list all + // Warning: Do not use GetSkipTake() for *db.ListOptions + // Its implementation could reset the page size with setting.API.MaxResponseItems + if listOptions, ok := paginator.(*db.ListOptions); ok { + if listOptions.Page >= 0 && listOptions.PageSize > 0 { + var start int + if listOptions.Page == 0 { + start = 0 + } else { + start = (listOptions.Page - 1) * listOptions.PageSize + } + return start, listOptions.PageSize + } + // Use a very large number to indicate no limit return 0, math.MaxInt } From 4af28d60787bb0d40d6ba85b963bbda334daa9ea Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 17:49:42 +0800 Subject: [PATCH 59/99] fix: do default Paginator --- modules/indexer/issues/indexer.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 273e707dba583..aa2e7b4bfe8e0 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -307,10 +307,6 @@ const ( // SearchIssues search issues by options. // It returns issue ids and a bool value indicates if the result is imprecise. func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, error) { - if opts.Paginator == nil { - opts.Paginator = db_model.NewAbsoluteListOptions(0, 50) - } - indexer := *globalIndexer.Load() result, err := indexer.Search(ctx, (*internal.SearchOptions)(opts)) if err != nil { From 934c526e6a75b428f04abf45c9d85674920e81cc Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 18:03:02 +0800 Subject: [PATCH 60/99] feat: use db if keyword is empty --- modules/indexer/issues/indexer.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index aa2e7b4bfe8e0..91619d3656c89 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -308,6 +308,17 @@ const ( // It returns issue ids and a bool value indicates if the result is imprecise. func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, error) { indexer := *globalIndexer.Load() + + if opts.Keyword == "" { + // This is a conservative shortcut. + // If the keyword is empty, db has better (at least not worse) performance to filter issues. + // When the keyword is empty, it tends to listing rather than searching issues. + // So if the user creates an issue and list issues immediately, the issue may not be listed because the indexer needs time to index the issue. + // Even worse, the external indexer like elastic search may not be available for a while, + // and the user may not be able to list issues completely until it is available again. + indexer = db.NewIndexer() + } + result, err := indexer.Search(ctx, (*internal.SearchOptions)(opts)) if err != nil { return nil, 0, err From b7668aaa9466908e8ea41948d03b8680624cc8da Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 18:16:35 +0800 Subject: [PATCH 61/99] fix: applyLimit --- models/issues/issue_search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index 7652bd36f174d..a3787f4337681 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -99,7 +99,7 @@ func applySorts(sess *xorm.Session, sortType string, priorityRepoID int64) { } func applyLimit(sess *xorm.Session, opts *IssuesOptions) *xorm.Session { - if opts.Paginator.IsListAll() { + if opts.Paginator == nil || opts.Paginator.IsListAll() { return sess } From f6fec6b72fb41d3fd8256d2ac679307f9c4468e7 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 18:21:18 +0800 Subject: [PATCH 62/99] test: cases for TestIndexer --- modules/indexer/issues/internal/tests/tests.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index be5b0ec0e0d81..8121fe0968b73 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -84,7 +84,7 @@ var cases = []*testIndexerCase{ Name: "default", SearchOptions: &internal.SearchOptions{}, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, 50, len(result.Hits)) // the default limit is 50 + assert.Equal(t, len(data), len(result.Hits)) assert.Equal(t, len(data), int(result.Total)) }, }, @@ -97,14 +97,14 @@ var cases = []*testIndexerCase{ ExpectedTotal: 0, }, { - Name: "all", + Name: "with limit", SearchOptions: &internal.SearchOptions{ Paginator: &db.ListOptions{ - ListAll: true, + PageSize: 5, }, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) // the default limit is 50 + assert.Equal(t, 5, len(result.Hits)) assert.Equal(t, len(data), int(result.Total)) }, }, From 6a4999ca1135f5c74942c42958c6706609a30b84 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 20:56:15 +0800 Subject: [PATCH 63/99] fix: LoadAttributes context --- models/issues/issue.go | 2 +- models/issues/issue_list.go | 4 ++-- models/issues/issue_list_test.go | 2 +- models/issues/issue_search.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/issues/issue.go b/models/issues/issue.go index b56fd32077c9b..cc690c82c7005 100644 --- a/models/issues/issue.go +++ b/models/issues/issue.go @@ -868,7 +868,7 @@ func GetPinnedIssues(ctx context.Context, repoID int64, isPull bool) ([]*Issue, return nil, err } - err = IssueList(issues).LoadAttributes() + err = IssueList(issues).LoadAttributes(ctx) if err != nil { return nil, err } diff --git a/models/issues/issue_list.go b/models/issues/issue_list.go index 9cc41ec6ab37e..d95dcce7ccdb2 100644 --- a/models/issues/issue_list.go +++ b/models/issues/issue_list.go @@ -564,8 +564,8 @@ func (issues IssueList) loadAttributes(ctx context.Context) error { // LoadAttributes loads attributes of the issues, except for attachments and // comments -func (issues IssueList) LoadAttributes() error { - return issues.loadAttributes(db.DefaultContext) +func (issues IssueList) LoadAttributes(ctx context.Context) error { + return issues.loadAttributes(ctx) } // LoadComments loads comments diff --git a/models/issues/issue_list_test.go b/models/issues/issue_list_test.go index 696c3b765d3d2..9069e1012da53 100644 --- a/models/issues/issue_list_test.go +++ b/models/issues/issue_list_test.go @@ -39,7 +39,7 @@ func TestIssueList_LoadAttributes(t *testing.T) { unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 4}), } - assert.NoError(t, issueList.LoadAttributes()) + assert.NoError(t, issueList.LoadAttributes(db.DefaultContext)) for _, issue := range issueList { assert.EqualValues(t, issue.RepoID, issue.Repo.ID) for _, label := range issue.Labels { diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index a3787f4337681..f9c1dbb38471d 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -453,7 +453,7 @@ func Issues(ctx context.Context, opts *IssuesOptions) ([]*Issue, error) { return nil, fmt.Errorf("unable to query Issues: %w", err) } - if err := issues.LoadAttributes(); err != nil { + if err := issues.LoadAttributes(ctx); err != nil { return nil, fmt.Errorf("unable to LoadAttributes for Issues: %w", err) } From f05b5b5e005287f8341bb0521084ae5e4588096c Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 21 Jul 2023 21:25:00 +0800 Subject: [PATCH 64/99] chore: lint code --- modules/indexer/issues/db/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index a1b9ec7afb462..6e3dae3aaf331 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -42,7 +42,7 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { } return *i } - sortType := "" + var sortType string switch options.SortBy { case internal.SortByCreatedAsc: sortType = "oldest" From 05fb5f499c26206481408d1e2126272371739644 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 14:04:33 +0800 Subject: [PATCH 65/99] fix: filter by lables --- modules/indexer/issues/bleve/bleve.go | 4 ++-- modules/indexer/issues/internal/tests/tests.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index d7998af362a0b..91f9754aefcaa 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -190,7 +190,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( for _, labelID := range options.IncludedLabelIDs { includeQueries = append(includeQueries, inner_bleve.NumericEqualityQuery(labelID, "label_ids")) } - queries = append(queries, bleve.NewDisjunctionQuery(includeQueries...)) + queries = append(queries, bleve.NewConjunctionQuery(includeQueries...)) } if len(options.ExcludedLabelIDs) > 0 { var excludeQueries []query.Query @@ -199,7 +199,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( q.AddMustNot(inner_bleve.NumericEqualityQuery(labelID, "label_ids")) excludeQueries = append(excludeQueries, q) } - queries = append(queries, bleve.NewConjunctionQuery(excludeQueries...)) // Be careful, it's conjunction here, not disjunction. + queries = append(queries, bleve.NewConjunctionQuery(excludeQueries...)) } } diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 8121fe0968b73..9a7c9eb4a4336 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -121,6 +121,23 @@ var cases = []*testIndexerCase{ ExpectedIDs: []int64{1002, 1001, 1000}, ExpectedTotal: 3, }, + { + Name: "labels", + ExtraData: []*internal.IndexerData{ + {ID: 1000, Title: "hello a", LabelIDs: []int64{2000, 2001, 2002}}, + {ID: 1001, Title: "hello b", LabelIDs: []int64{2000, 2001}}, + {ID: 1002, Title: "hello c", LabelIDs: []int64{2000, 2001, 2003}}, + {ID: 1003, Title: "hello d", LabelIDs: []int64{2000}}, + {ID: 1004, Title: "hello e", LabelIDs: []int64{}}, + }, + SearchOptions: &internal.SearchOptions{ + Keyword: "hello", + IncludedLabelIDs: []int64{2000, 2001}, + ExcludedLabelIDs: []int64{2003}, + }, + ExpectedIDs: []int64{1001, 1000}, + ExpectedTotal: 2, + }, // TODO: add more cases } From 376a381466257c54dd6f2094593ccbb0321b3e4e Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 14:43:33 +0800 Subject: [PATCH 66/99] chore: add TBC --- models/issues/label.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/issues/label.go b/models/issues/label.go index 7fda0b5804e9c..127e2e7d50a48 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -480,6 +480,7 @@ func GetLabelsByOrgID(ctx context.Context, orgID int64, sortType string, listOpt // It doesn't filter them by repo or org, so it could return labels belonging to different repos/orgs. // It's used for filtering issues via indexer, otherwise it would be useless. // Since it could return labels with the same name, so the length of returned ids could be more than the length of names. +// TBC: incorrect, it should return labels belonging to the same repo/org. func GetLabelIDsByNames(ctx context.Context, labelNames []string) ([]int64, error) { labelIDs := make([]int64, 0, len(labelNames)) return labelIDs, db.GetEngine(ctx).Table("label"). From 8a64027ccf0515388336f57ed2f8b27a90fa0a1e Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 15:54:58 +0800 Subject: [PATCH 67/99] feat: IncludedAnyLabelIDs --- modules/indexer/issues/bleve/bleve.go | 6 ++ modules/indexer/issues/db/db.go | 5 +- modules/indexer/issues/db/options.go | 59 ++++++++++++------- modules/indexer/issues/dbfilter.go | 6 +- modules/indexer/issues/dboptions.go | 2 + modules/indexer/issues/internal/model.go | 7 ++- .../indexer/issues/internal/tests/tests.go | 17 ++++++ 7 files changed, 76 insertions(+), 26 deletions(-) diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 91f9754aefcaa..2422066fd7035 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -191,6 +191,12 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( includeQueries = append(includeQueries, inner_bleve.NumericEqualityQuery(labelID, "label_ids")) } queries = append(queries, bleve.NewConjunctionQuery(includeQueries...)) + } else if len(options.IncludedAnyLabelIDs) > 0 { + var includeQueries []query.Query + for _, labelID := range options.IncludedAnyLabelIDs { + includeQueries = append(includeQueries, inner_bleve.NumericEqualityQuery(labelID, "label_ids")) + } + queries = append(queries, bleve.NewDisjunctionQuery(includeQueries...)) } if len(options.ExcludedLabelIDs) > 0 { var excludeQueries []query.Query diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 44f664c2492a3..4544a0577a1d6 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -73,7 +73,10 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( ) } - opt := ToDBOptions(options) + opt, err := ToDBOptions(ctx, options) + if err != nil { + return nil, err + } ids, total, err := issue_model.IssueIDs(ctx, opt, cond) if err != nil { diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index 6e3dae3aaf331..fb3da668d82b7 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -4,12 +4,16 @@ package db import ( + "context" + "fmt" + "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/indexer/issues/internal" ) -func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { +func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_model.IssuesOptions, error) { convertID := func(id *int64) int64 { if id == nil { return 0 @@ -19,23 +23,6 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { } return *id } - convertIDs := func(ids []int64) []int64 { - if len(ids) == 1 && ids[0] == 0 { - return []int64{db.NoConditionID} - } - return ids - } - convertLabelIDs := func(includes, excludes []int64, noLabelOnly bool) []int64 { - if noLabelOnly { - return []int64{0} // Be careful, it's zero, not db.NoConditionID - } - ret := make([]int64, 0, len(includes)+len(excludes)) - ret = append(ret, includes...) - for _, id := range excludes { - ret = append(ret, -id) - } - return ret - } convertInt64 := func(i *int64) int64 { if i == nil { return 0 @@ -74,12 +61,10 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { ReviewRequestedID: convertID(options.ReviewRequestedID), ReviewedID: convertID(options.ReviewedID), SubscriberID: convertID(options.SubscriberID), - MilestoneIDs: convertIDs(options.MilestoneIDs), ProjectID: convertID(options.ProjectID), ProjectBoardID: convertID(options.ProjectBoardID), IsClosed: options.IsClosed, IsPull: options.IsPull, - LabelIDs: convertLabelIDs(options.IncludedLabelIDs, options.ExcludedLabelIDs, options.NoLabelOnly), IncludedLabelNames: nil, ExcludedLabelNames: nil, IncludeMilestones: nil, @@ -93,5 +78,37 @@ func ToDBOptions(options *internal.SearchOptions) *issue_model.IssuesOptions { Team: nil, User: nil, } - return opts + + if len(options.MilestoneIDs) == 1 && options.MilestoneIDs[0] == 0 { + opts.MilestoneIDs = []int64{db.NoConditionID} + } else { + opts.MilestoneIDs = options.MilestoneIDs + } + + if options.NoLabelOnly { + opts.LabelIDs = []int64{0} // Be careful, it's zero, not db.NoConditionID + } else { + opts.LabelIDs = make([]int64, 0, len(options.IncludedLabelIDs)+len(options.ExcludedLabelIDs)) + opts.LabelIDs = append(opts.LabelIDs, options.IncludedLabelIDs...) + for _, id := range options.ExcludedLabelIDs { + opts.LabelIDs = append(opts.LabelIDs, -id) + } + + if len(options.IncludedLabelIDs) == 0 && len(options.IncludedAnyLabelIDs) > 0 { + _ = ctx // issue_model.GetLabelsByIDs should be called with ctx, this line can be removed when it's done. + labels, err := issue_model.GetLabelsByIDs(options.IncludedAnyLabelIDs) + if err != nil { + return nil, fmt.Errorf("GetLabelsByIDs: %v", err) + } + set := container.Set[string]{} + for _, label := range labels { + if !set.Contains(label.Name) { + set.Add(label.Name) + opts.IncludedLabelNames = append(opts.IncludedLabelNames, label.Name) + } + } + } + } + + return opts, nil } diff --git a/modules/indexer/issues/dbfilter.go b/modules/indexer/issues/dbfilter.go index 3633bf40342e8..ee44df6484880 100644 --- a/modules/indexer/issues/dbfilter.go +++ b/modules/indexer/issues/dbfilter.go @@ -15,7 +15,11 @@ import ( // It is used to filter out issues coming from some indexers that are not supported fining filtering. // Once all indexers support filtering, this function and this file can be removed. func reFilter(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) { - opts := db.ToDBOptions((*internal.SearchOptions)(options)) + opts, err := db.ToDBOptions(ctx, (*internal.SearchOptions)(options)) + if err != nil { + return nil, err + } + opts.IssueIDs = issuesIDs ids, _, err := issue_model.IssueIDs(ctx, opts) diff --git a/modules/indexer/issues/dboptions.go b/modules/indexer/issues/dboptions.go index d04c7cd80404d..6a41afadd7077 100644 --- a/modules/indexer/issues/dboptions.go +++ b/modules/indexer/issues/dboptions.go @@ -27,6 +27,8 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp searchOpt.ExcludedLabelIDs = append(searchOpt.ExcludedLabelIDs, -labelID) } } + // opts.IncludedLabelNames and opts.ExcludedLabelNames are not supported here. + // It's not a TO DO, it's just unnecessary. } if len(opts.MilestoneIDs) == 1 && opts.MilestoneIDs[0] == db.NoConditionID { diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 20d6397469f7c..e816996d93fa4 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -68,9 +68,10 @@ type SearchOptions struct { IsPull util.OptionalBool // if the issues is a pull request IsClosed util.OptionalBool // if the issues is closed - IncludedLabelIDs []int64 // labels the issues have - ExcludedLabelIDs []int64 // labels the issues don't have - NoLabelOnly bool // if the issues have no label, if true, IncludedLabelIDs and ExcludedLabelIDs will be ignored + IncludedLabelIDs []int64 // labels the issues have + ExcludedLabelIDs []int64 // labels the issues don't have + IncludedAnyLabelIDs []int64 // labels the issues have at least one. It will be ignored if IncludedLabelIDs is not empty. It's an uncommon filter, but it has been supported accidentally by issues.IssuesOptions.IncludedLabelNames. + NoLabelOnly bool // if the issues have no label, if true, IncludedLabelIDs and ExcludedLabelIDs, IncludedAnyLabelIDs will be ignored MilestoneIDs []int64 // milestones the issues have diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 9a7c9eb4a4336..644e6fba07567 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -138,6 +138,23 @@ var cases = []*testIndexerCase{ ExpectedIDs: []int64{1001, 1000}, ExpectedTotal: 2, }, + { + Name: "include any labels", + ExtraData: []*internal.IndexerData{ + {ID: 1000, Title: "hello a", LabelIDs: []int64{2000, 2001, 2002}}, + {ID: 1001, Title: "hello b", LabelIDs: []int64{2001}}, + {ID: 1002, Title: "hello c", LabelIDs: []int64{2000, 2001, 2003}}, + {ID: 1003, Title: "hello d", LabelIDs: []int64{2002}}, + {ID: 1004, Title: "hello e", LabelIDs: []int64{}}, + }, + SearchOptions: &internal.SearchOptions{ + Keyword: "hello", + IncludedAnyLabelIDs: []int64{2001, 2002}, + ExcludedLabelIDs: []int64{2003}, + }, + ExpectedIDs: []int64{1003, 1001, 1000}, + ExpectedTotal: 3, + }, // TODO: add more cases } From 639a393e72a8d57f9f6e0fc1c1d6ae319f9bef00 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 16:24:31 +0800 Subject: [PATCH 68/99] fix: use includedAnyLabels --- models/issues/label.go | 1 - routers/api/v1/repo/issue.go | 20 ++++++++++---------- routers/web/repo/issue.go | 22 +++++++++++----------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/models/issues/label.go b/models/issues/label.go index 127e2e7d50a48..7fda0b5804e9c 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -480,7 +480,6 @@ func GetLabelsByOrgID(ctx context.Context, orgID int64, sortType string, listOpt // It doesn't filter them by repo or org, so it could return labels belonging to different repos/orgs. // It's used for filtering issues via indexer, otherwise it would be useless. // Since it could return labels with the same name, so the length of returned ids could be more than the length of names. -// TBC: incorrect, it should return labels belonging to the same repo/org. func GetLabelIDsByNames(ctx context.Context, labelNames []string) ([]int64, error) { labelIDs := make([]int64, 0, len(labelNames)) return labelIDs, db.GetEngine(ctx).Table("label"). diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index c95c4dc81f0d6..861e63a9b82f8 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -210,7 +210,7 @@ func SearchIssues(ctx *context.APIContext) { isPull = util.OptionalBoolNone } - var includedLabels []int64 + var includedAnyLabels []int64 { labels := ctx.FormTrim("labels") @@ -218,7 +218,7 @@ func SearchIssues(ctx *context.APIContext) { if len(labels) > 0 { includedLabelNames = strings.Split(labels, ",") } - includedLabels, err = issues_model.GetLabelIDsByNames(ctx, includedLabelNames) + includedAnyLabels, err = issues_model.GetLabelIDsByNames(ctx, includedLabelNames) if err != nil { ctx.Error(http.StatusInternalServerError, "GetLabelIDsByNames", err) return @@ -253,14 +253,14 @@ func SearchIssues(ctx *context.APIContext) { PageSize: limit, Page: ctx.FormInt("page"), }, - Keyword: keyword, - RepoIDs: repoIDs, - AllPublic: allPublic, - IsPull: isPull, - IsClosed: isClosed, - IncludedLabelIDs: includedLabels, - MilestoneIDs: includedMilestones, - SortBy: issue_indexer.SortByCreatedDesc, + Keyword: keyword, + RepoIDs: repoIDs, + AllPublic: allPublic, + IsPull: isPull, + IsClosed: isClosed, + IncludedAnyLabelIDs: includedAnyLabels, + MilestoneIDs: includedMilestones, + SortBy: issue_indexer.SortByCreatedDesc, } if since != 0 { diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index d8aa5f68d71be..251771dc90ab8 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -2499,7 +2499,7 @@ func SearchIssues(ctx *context.Context) { isPull = util.OptionalBoolNone } - var includedLabels []int64 + var includedAnyLabels []int64 { labels := ctx.FormTrim("labels") @@ -2507,7 +2507,7 @@ func SearchIssues(ctx *context.Context) { if len(labels) > 0 { includedLabelNames = strings.Split(labels, ",") } - includedLabels, err = issues_model.GetLabelIDsByNames(ctx, includedLabelNames) + includedAnyLabels, err = issues_model.GetLabelIDsByNames(ctx, includedLabelNames) if err != nil { ctx.Error(http.StatusInternalServerError, "GetLabelIDsByNames", err.Error()) return @@ -2547,15 +2547,15 @@ func SearchIssues(ctx *context.Context) { Page: ctx.FormInt("page"), PageSize: limit, }, - Keyword: keyword, - RepoIDs: repoIDs, - AllPublic: allPublic, - IsPull: isPull, - IsClosed: isClosed, - IncludedLabelIDs: includedLabels, - MilestoneIDs: includedMilestones, - ProjectID: projectID, - SortBy: issue_indexer.SortByCreatedDesc, + Keyword: keyword, + RepoIDs: repoIDs, + AllPublic: allPublic, + IsPull: isPull, + IsClosed: isClosed, + IncludedAnyLabelIDs: includedAnyLabels, + MilestoneIDs: includedMilestones, + ProjectID: projectID, + SortBy: issue_indexer.SortByCreatedDesc, } if since != 0 { From 96180022448834c3f1a54351c550adbfa17d0f95 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 16:48:24 +0800 Subject: [PATCH 69/99] fix: load label names --- models/issues/label.go | 4 ++-- modules/indexer/issues/db/options.go | 2 +- routers/api/v1/repo/issue_label.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/models/issues/label.go b/models/issues/label.go index 7fda0b5804e9c..57a2e67f8cd51 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -272,12 +272,12 @@ func GetLabelByID(ctx context.Context, labelID int64) (*Label, error) { } // GetLabelsByIDs returns a list of labels by IDs -func GetLabelsByIDs(labelIDs []int64) ([]*Label, error) { +func GetLabelsByIDs(labelIDs []int64, cols ...string) ([]*Label, error) { labels := make([]*Label, 0, len(labelIDs)) return labels, db.GetEngine(db.DefaultContext).Table("label"). In("id", labelIDs). Asc("name"). - Cols("id", "repo_id", "org_id"). + Cols(cols...). Find(&labels) } diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index fb3da668d82b7..ebd672a6954a4 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -96,7 +96,7 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m if len(options.IncludedLabelIDs) == 0 && len(options.IncludedAnyLabelIDs) > 0 { _ = ctx // issue_model.GetLabelsByIDs should be called with ctx, this line can be removed when it's done. - labels, err := issue_model.GetLabelsByIDs(options.IncludedAnyLabelIDs) + labels, err := issue_model.GetLabelsByIDs(options.IncludedAnyLabelIDs, "name") if err != nil { return nil, fmt.Errorf("GetLabelsByIDs: %v", err) } diff --git a/routers/api/v1/repo/issue_label.go b/routers/api/v1/repo/issue_label.go index fc83b6f14c94e..a2814a03db81e 100644 --- a/routers/api/v1/repo/issue_label.go +++ b/routers/api/v1/repo/issue_label.go @@ -309,7 +309,7 @@ func prepareForReplaceOrAdd(ctx *context.APIContext, form api.IssueLabelsOption) return nil, nil, err } - labels, err := issues_model.GetLabelsByIDs(form.Labels) + labels, err := issues_model.GetLabelsByIDs(form.Labels, "id", "repo_id", "org_id") if err != nil { ctx.Error(http.StatusInternalServerError, "GetLabelsByIDs", err) return nil, nil, err From e840b6c2b5bfafb3ce8610a229fc968e45e99942 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 17:22:35 +0800 Subject: [PATCH 70/99] test: more cases --- modules/indexer/issues/bleve/bleve_test.go | 2 +- .../indexer/issues/internal/tests/tests.go | 112 ++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/modules/indexer/issues/bleve/bleve_test.go b/modules/indexer/issues/bleve/bleve_test.go index 0a3e1e1abad54..908514a01a2d6 100644 --- a/modules/indexer/issues/bleve/bleve_test.go +++ b/modules/indexer/issues/bleve/bleve_test.go @@ -9,7 +9,7 @@ import ( "code.gitea.io/gitea/modules/indexer/issues/internal/tests" ) -func TestIndexer(t *testing.T) { +func TestBleveIndexer(t *testing.T) { dir := t.TempDir() indexer := NewIndexer(dir) defer indexer.Close() diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 644e6fba07567..f1f6968dbf28c 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/indexer/issues/internal" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -121,6 +122,107 @@ var cases = []*testIndexerCase{ ExpectedIDs: []int64{1002, 1001, 1000}, ExpectedTotal: 3, }, + { + Name: "repo ids", + ExtraData: []*internal.IndexerData{ + {ID: 1001, Title: "hello world", RepoID: 1, IsPublic: false}, + {ID: 1002, Title: "hello world", RepoID: 1, IsPublic: false}, + {ID: 1003, Title: "hello world", RepoID: 2, IsPublic: true}, + {ID: 1004, Title: "hello world", RepoID: 2, IsPublic: true}, + {ID: 1005, Title: "hello world", RepoID: 3, IsPublic: true}, + {ID: 1006, Title: "hello world", RepoID: 4, IsPublic: false}, + {ID: 1007, Title: "hello world", RepoID: 5, IsPublic: false}, + }, + SearchOptions: &internal.SearchOptions{ + Keyword: "hello", + RepoIDs: []int64{1, 4}, + }, + ExpectedIDs: []int64{1006, 1002, 1001}, + ExpectedTotal: 3, + }, + { + Name: "repo ids and public", + ExtraData: []*internal.IndexerData{ + {ID: 1001, Title: "hello world", RepoID: 1, IsPublic: false}, + {ID: 1002, Title: "hello world", RepoID: 1, IsPublic: false}, + {ID: 1003, Title: "hello world", RepoID: 2, IsPublic: true}, + {ID: 1004, Title: "hello world", RepoID: 2, IsPublic: true}, + {ID: 1005, Title: "hello world", RepoID: 3, IsPublic: true}, + {ID: 1006, Title: "hello world", RepoID: 4, IsPublic: false}, + {ID: 1007, Title: "hello world", RepoID: 5, IsPublic: false}, + }, + SearchOptions: &internal.SearchOptions{ + Keyword: "hello", + RepoIDs: []int64{1, 4}, + AllPublic: true, + }, + ExpectedIDs: []int64{1006, 1005, 1004, 1003, 1002, 1001}, + ExpectedTotal: 6, + }, + { + Name: "issue only", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + IsPull: util.OptionalBoolFalse, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.False(t, data[v.ID].IsPull) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { return !v.IsPull }), result.Total) + }, + }, + { + Name: "pull only", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + IsPull: util.OptionalBoolTrue, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.True(t, data[v.ID].IsPull) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { return v.IsPull }), result.Total) + }, + }, + { + Name: "opened only", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + IsClosed: util.OptionalBoolFalse, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.False(t, data[v.ID].IsClosed) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { return !v.IsClosed }), result.Total) + }, + }, + { + Name: "closed only", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + IsClosed: util.OptionalBoolTrue, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.True(t, data[v.ID].IsClosed) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { return v.IsClosed }), result.Total) + }, + }, { Name: "labels", ExtraData: []*internal.IndexerData{ @@ -232,3 +334,13 @@ func generateDefaultIndexerData() []*internal.IndexerData { return data } + +func countIndexerData(data map[int64]*internal.IndexerData, f func(v *internal.IndexerData) bool) int64 { + var count int64 + for _, v := range data { + if f(v) { + count++ + } + } + return count +} From 28f1be60e71e4589ecbab25788fbe96fe3a93298 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 17:45:43 +0800 Subject: [PATCH 71/99] fix: check IsAvailable --- routers/web/repo/issue.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 251771dc90ab8..e1fa3c7e4346d 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -186,11 +186,6 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti keyword = "" } - if issue_indexer.IsAvailable(ctx) { - ctx.ServerError("issueIndexer.Search", err) - return - } - var mileIDs []int64 if milestoneID > 0 || milestoneID == db.NoConditionID { // -1 to get those issues which have no any milestone assigned mileIDs = []int64{milestoneID} @@ -214,7 +209,11 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti if keyword != "" { allIssueIDs, err := issueIDsFromSearch(ctx, keyword, statsOpts) if err != nil { - ctx.ServerError("issueIDsFromSearch", err) + if issue_indexer.IsAvailable(ctx) { + ctx.ServerError("issueIDsFromSearch", err) + return + } + ctx.Data["IssueIndexerUnavailable"] = true return } statsOpts.IssueIDs = allIssueIDs @@ -266,7 +265,11 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti SortType: sortType, }) if err != nil { - ctx.ServerError("issueIDsFromSearch", err) + if issue_indexer.IsAvailable(ctx) { + ctx.ServerError("issueIDsFromSearch", err) + return + } + ctx.Data["IssueIndexerUnavailable"] = true return } issues, err = issues_model.GetIssuesByIDs(ctx, ids, true) From e6383f31fc653281136e073e4040a484b1f7b326 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 18:17:29 +0800 Subject: [PATCH 72/99] test: more cases --- .../indexer/issues/internal/tests/tests.go | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index f1f6968dbf28c..b0f77f30cba8b 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -257,6 +257,147 @@ var cases = []*testIndexerCase{ ExpectedIDs: []int64{1003, 1001, 1000}, ExpectedTotal: 3, }, + { + Name: "milestone", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + MilestoneIDs: []int64{1, 2, 6}, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Contains(t, []int64{1, 2, 6}, data[v.ID].MilestoneID) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.MilestoneID == 1 || v.MilestoneID == 2 || v.MilestoneID == 6 + }), result.Total) + }, + }, + { + Name: "no milestone", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + MilestoneIDs: []int64{0}, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Equal(t, int64(0), data[v.ID].MilestoneID) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.MilestoneID == 0 + }), result.Total) + }, + }, + { + Name: "project", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + ProjectID: func() *int64 { + id := int64(1) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Equal(t, int64(1), data[v.ID].ProjectID) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.ProjectID == 1 + }), result.Total) + }, + }, + { + Name: "no project", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + ProjectID: func() *int64 { + id := int64(0) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Equal(t, int64(0), data[v.ID].ProjectID) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.ProjectID == 0 + }), result.Total) + }, + }, + { + Name: "project board", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + ProjectBoardID: func() *int64 { + id := int64(1) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Equal(t, int64(1), data[v.ID].ProjectBoardID) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.ProjectBoardID == 1 + }), result.Total) + }, + }, + { + Name: "no project board", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + ProjectBoardID: func() *int64 { + id := int64(0) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Equal(t, int64(0), data[v.ID].ProjectBoardID) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.ProjectBoardID == 0 + }), result.Total) + }, + }, + { + Name: "poster", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + PosterID: func() *int64 { + id := int64(1) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Equal(t, int64(1), data[v.ID].PosterID) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.PosterID == 0 + }), result.Total) + }, + }, // TODO: add more cases } From 032363f8de79d5bbc01cb65219f7399ab72e57fb Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 18:42:02 +0800 Subject: [PATCH 73/99] docs: add comments --- modules/repository/create.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/repository/create.go b/modules/repository/create.go index 7635e63385642..10a1e872df989 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -420,7 +420,8 @@ func UpdateRepository(ctx context.Context, repo *repo_model.Repository, visibili } } - // Update the issue indexer + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. issue_indexer.UpdateRepoIndexer(ctx, repo.ID) } From 30a33756eec8fe4ce5d85acf651cebc1de4e724b Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 24 Jul 2023 18:42:44 +0800 Subject: [PATCH 74/99] debug: pq "gtestschema.label_issue" does not exist --- models/issues/issue_list.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/issues/issue_list.go b/models/issues/issue_list.go index a932ac2554369..e6f1127958b99 100644 --- a/models/issues/issue_list.go +++ b/models/issues/issue_list.go @@ -157,7 +157,9 @@ func (issues IssueList) loadLabels(ctx context.Context) error { if left < limit { limit = left } - rows, err := db.GetEngine(ctx).Table("label"). + sess := db.GetEngine(ctx).Table("label") // debug: + sess.MustLogSQL(true) + rows, err := sess. Join("LEFT", "issue_label", "issue_label.label_id = label.id"). In("issue_label.issue_id", issueIDs[:limit]). Asc("label.name"). From ddd5f2b9d711fef30d1fb49a926316d3838c1942 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 25 Jul 2023 18:08:54 +0800 Subject: [PATCH 75/99] feat: improve enqueue --- modules/indexer/issues/indexer.go | 58 ++++++++++--------------------- modules/indexer/issues/util.go | 48 +++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 40 deletions(-) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 91619d3656c89..87497dbe8fb1e 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -5,14 +5,12 @@ package issues import ( "context" - "errors" "os" "runtime/pprof" "sync/atomic" "time" db_model "code.gitea.io/gitea/models/db" - issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/indexer/issues/bleve" @@ -227,61 +225,41 @@ func populateIssueIndexer(ctx context.Context) { } for _, repo := range repos { - select { - case <-ctx.Done(): - log.Info("Issue Indexer population shutdown before completion") - return - default: + for { + select { + case <-ctx.Done(): + log.Info("Issue Indexer population shutdown before completion") + return + default: + } + if err := updateRepoIndexer(ctx, repo.ID); err != nil { + log.Warn("Retry to populate issue indexer for repo %d: %v", repo.ID, err) + continue + } + break } - UpdateRepoIndexer(ctx, repo.ID) } } } // UpdateRepoIndexer add/update all issues of the repositories func UpdateRepoIndexer(ctx context.Context, repoID int64) { - is, err := issues_model.Issues(ctx, &issues_model.IssuesOptions{ - RepoIDs: []int64{repoID}, - IsClosed: util.OptionalBoolNone, - IsPull: util.OptionalBoolNone, - }) - if err != nil { - log.Error("Issues: %v", err) - return - } - for _, issue := range is { - UpdateIssueIndexer(issue.ID) + if err := updateRepoIndexer(ctx, repoID); err != nil { + log.Error("Unable to push repo %d to issue indexer: %v", repoID, err) } } // UpdateIssueIndexer add/update an issue to the issue indexer func UpdateIssueIndexer(issueID int64) { - if err := issueIndexerQueue.Push(&IndexerMetadata{ID: issueID}); err != nil { - log.Error("Unable to push to issue indexer: %v: Error: %v", issueID, err) - if errors.Is(err, context.DeadlineExceeded) { - log.Error("It seems that issue indexer is slow and the queue is full. Please check the issue indexer or increase the queue size.") - } + if err := updateIssueIndexer(issueID); err != nil { + log.Error("Unable to push issue %d to issue indexer: %v", issueID, err) } } // DeleteRepoIssueIndexer deletes repo's all issues indexes func DeleteRepoIssueIndexer(ctx context.Context, repoID int64) { - var ids []int64 - ids, err := issues_model.GetIssueIDsByRepoID(ctx, repoID) - if err != nil { - log.Error("GetIssueIDsByRepoID failed: %v", err) - return - } - - if len(ids) == 0 { - return - } - indexerData := &IndexerMetadata{ - IDs: ids, - IsDelete: true, - } - if err := issueIndexerQueue.Push(indexerData); err != nil { - log.Error("Unable to push to issue indexer: %v: Error: %v", indexerData, err) + if err := deleteRepoIssueIndexer(ctx, repoID); err != nil { + log.Error("Unable to push deleted repo %d to issue indexer: %v", repoID, err) } } diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index 7a8b64dcfbbfc..b3eb9ddd7d896 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -5,11 +5,15 @@ package issues import ( "context" + "errors" + "fmt" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/queue" ) // getIssueIndexerData returns the indexer data of an issue and a bool value indicating whether the issue exists. @@ -116,3 +120,47 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD CommentCount: int64(len(issue.Comments)), }, true, nil } + +func updateRepoIndexer(ctx context.Context, repoID int64) error { + ids, err := issue_model.GetIssueIDsByRepoID(ctx, repoID) + if err != nil { + return fmt.Errorf("issue_model.GetIssueIDsByRepoID: %w", err) + } + for _, id := range ids { + if err := updateIssueIndexer(id); err != nil { + return err + } + } + return nil +} + +func updateIssueIndexer(issueID int64) error { + return pushIssueIndexerQueue(&IndexerMetadata{ID: issueID}) +} + +func deleteRepoIssueIndexer(ctx context.Context, repoID int64) error { + var ids []int64 + ids, err := issue_model.GetIssueIDsByRepoID(ctx, repoID) + if err != nil { + return fmt.Errorf("issue_model.GetIssueIDsByRepoID: %w", err) + } + + if len(ids) == 0 { + return nil + } + return pushIssueIndexerQueue(&IndexerMetadata{ + IDs: ids, + IsDelete: true, + }) +} + +func pushIssueIndexerQueue(data *IndexerMetadata) error { + err := issueIndexerQueue.Push(data) + if errors.Is(err, queue.ErrAlreadyInQueue) { + return nil + } + if errors.Is(err, context.DeadlineExceeded) { + log.Warn("It seems that issue indexer is slow and the queue is full. Please check the issue indexer or increase the queue size.") + } + return err +} From 37cf4c1998418a6680d4abfa3d0661fae0f2a5a6 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 25 Jul 2023 18:53:07 +0800 Subject: [PATCH 76/99] chore: use hotfix xorm --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9038bdd689f25..ba673bbdf1a61 100644 --- a/go.mod +++ b/go.mod @@ -120,7 +120,7 @@ require ( mvdan.cc/xurls/v2 v2.5.0 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 xorm.io/builder v0.3.12 - xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75 + xorm.io/xorm v1.3.3-0.20230725104838-539cbdc983c0 // FIXME: this is a hotfix, it should be removed before this PR is merged ) require ( diff --git a/go.sum b/go.sum index 976c0ead3835f..87d92dbd6805b 100644 --- a/go.sum +++ b/go.sum @@ -1893,5 +1893,5 @@ strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1: xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= xorm.io/builder v0.3.12 h1:ASZYX7fQmy+o8UJdhlLHSW57JDOkM8DNhcAF5d0LiJM= xorm.io/builder v0.3.12/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= -xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75 h1:ReBAlO50dCIXCWF8Gbi0ZRa62AGAwCJNCPaUNUa7JSg= -xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw= +xorm.io/xorm v1.3.3-0.20230725104838-539cbdc983c0 h1:1xCS/IoFEZBmfqkxsEIujJG4mOodidTD4DXDBG55Mmc= +xorm.io/xorm v1.3.3-0.20230725104838-539cbdc983c0/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw= From ef27f6a78e9774ddfee0a972d5b3657e40220c36 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 25 Jul 2023 19:17:17 +0800 Subject: [PATCH 77/99] fix: es page size --- modules/indexer/internal/paginator.go | 15 ++++++++++----- .../indexer/issues/elasticsearch/elasticsearch.go | 7 ++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/modules/indexer/internal/paginator.go b/modules/indexer/internal/paginator.go index 3f6a1ec512fc7..de0a33c06ff80 100644 --- a/modules/indexer/internal/paginator.go +++ b/modules/indexer/internal/paginator.go @@ -10,10 +10,16 @@ import ( ) // ParsePaginator parses a db.Paginator into a skip and limit -func ParsePaginator(paginator db.Paginator) (int, int) { +func ParsePaginator(paginator db.Paginator, max ...int) (int, int) { + // Use a very large number to indicate no limit + unlimited := math.MaxInt32 + if len(max) > 0 { + // Some indexer engines have a limit on the page size, respect that + unlimited = max[0] + } + if paginator == nil || paginator.IsListAll() { - // Use a very large number to list all - return 0, math.MaxInt + return 0, unlimited } // Warning: Do not use GetSkipTake() for *db.ListOptions @@ -28,8 +34,7 @@ func ParsePaginator(paginator db.Paginator) (int, int) { } return start, listOptions.PageSize } - // Use a very large number to indicate no limit - return 0, math.MaxInt + return 0, unlimited } return paginator.GetSkipTake() diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 0becff6e81acd..f8d1e38d57584 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -152,7 +152,12 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) query = query.Must(repoQuery) } - skip, limit := indexer_internal.ParsePaginator(options.Paginator) + + // See https://stackoverflow.com/questions/35206409/elasticsearch-2-1-result-window-is-too-large-index-max-result-window/35221900 + // TODO: make it configurable since it's configurable in elasticsearch + const maxPageSize = 10000 + + skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxPageSize) searchResult, err := b.inner.Client.Search(). Index(b.inner.VersionedIndexName()). Query(query). From 61b2317e2ce136a432f1f92396ef19572f5a0407 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 26 Jul 2023 10:04:10 +0800 Subject: [PATCH 78/99] Revert "chore: use hotfix xorm" This reverts commit 37cf4c1998418a6680d4abfa3d0661fae0f2a5a6. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ba673bbdf1a61..9038bdd689f25 100644 --- a/go.mod +++ b/go.mod @@ -120,7 +120,7 @@ require ( mvdan.cc/xurls/v2 v2.5.0 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 xorm.io/builder v0.3.12 - xorm.io/xorm v1.3.3-0.20230725104838-539cbdc983c0 // FIXME: this is a hotfix, it should be removed before this PR is merged + xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75 ) require ( diff --git a/go.sum b/go.sum index 87d92dbd6805b..976c0ead3835f 100644 --- a/go.sum +++ b/go.sum @@ -1893,5 +1893,5 @@ strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1: xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= xorm.io/builder v0.3.12 h1:ASZYX7fQmy+o8UJdhlLHSW57JDOkM8DNhcAF5d0LiJM= xorm.io/builder v0.3.12/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= -xorm.io/xorm v1.3.3-0.20230725104838-539cbdc983c0 h1:1xCS/IoFEZBmfqkxsEIujJG4mOodidTD4DXDBG55Mmc= -xorm.io/xorm v1.3.3-0.20230725104838-539cbdc983c0/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw= +xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75 h1:ReBAlO50dCIXCWF8Gbi0ZRa62AGAwCJNCPaUNUa7JSg= +xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw= From 5797332aeeb0c1ea34e8ee8145cb8402e13210bf Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 26 Jul 2023 10:27:04 +0800 Subject: [PATCH 79/99] fix: fix unit test --- modules/indexer/issues/util.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index b3eb9ddd7d896..2dec3b71db0b4 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -155,6 +155,13 @@ func deleteRepoIssueIndexer(ctx context.Context, repoID int64) error { } func pushIssueIndexerQueue(data *IndexerMetadata) error { + if issueIndexerQueue == nil { + // Some unit tests will trigger indexing, but the queue is not initialized. + // It's OK to ignore it, but log a warning message in case it's not a unit test. + log.Warn("Trying to push %+v to issue indexer queue, but the queue is not initialized, it's OK if it's a unit test", data) + return nil + } + err := issueIndexerQueue.Push(data) if errors.Is(err, queue.ErrAlreadyInQueue) { return nil From d4d540d6029d6918dfb9839cb586c524f2767013 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 26 Jul 2023 11:00:48 +0800 Subject: [PATCH 80/99] test: more cases --- .../indexer/issues/internal/tests/tests.go | 333 +++++++++++++++++- 1 file changed, 314 insertions(+), 19 deletions(-) diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index b0f77f30cba8b..b22432ac43770 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -1,7 +1,7 @@ // Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -// This package contains tests for the indexer module. +// This package contains tests for issues indexer modules. // All the code in this package is only used for testing. // Do not put any production code in this package to avoid it being included in the final binary. @@ -110,7 +110,7 @@ var cases = []*testIndexerCase{ }, }, { - Name: "keyword", + Name: "Keyword", ExtraData: []*internal.IndexerData{ {ID: 1000, Title: "hi hello world"}, {ID: 1001, Content: "hi hello world"}, @@ -123,7 +123,7 @@ var cases = []*testIndexerCase{ ExpectedTotal: 3, }, { - Name: "repo ids", + Name: "RepoIDs", ExtraData: []*internal.IndexerData{ {ID: 1001, Title: "hello world", RepoID: 1, IsPublic: false}, {ID: 1002, Title: "hello world", RepoID: 1, IsPublic: false}, @@ -141,7 +141,7 @@ var cases = []*testIndexerCase{ ExpectedTotal: 3, }, { - Name: "repo ids and public", + Name: "RepoIDs and AllPublic", ExtraData: []*internal.IndexerData{ {ID: 1001, Title: "hello world", RepoID: 1, IsPublic: false}, {ID: 1002, Title: "hello world", RepoID: 1, IsPublic: false}, @@ -258,7 +258,7 @@ var cases = []*testIndexerCase{ ExpectedTotal: 3, }, { - Name: "milestone", + Name: "MilestoneIDs", SearchOptions: &internal.SearchOptions{ Paginator: &db.ListOptions{ PageSize: 5, @@ -276,7 +276,7 @@ var cases = []*testIndexerCase{ }, }, { - Name: "no milestone", + Name: "no MilestoneIDs", SearchOptions: &internal.SearchOptions{ Paginator: &db.ListOptions{ PageSize: 5, @@ -294,7 +294,7 @@ var cases = []*testIndexerCase{ }, }, { - Name: "project", + Name: "ProjectID", SearchOptions: &internal.SearchOptions{ Paginator: &db.ListOptions{ PageSize: 5, @@ -315,7 +315,7 @@ var cases = []*testIndexerCase{ }, }, { - Name: "no project", + Name: "no ProjectID", SearchOptions: &internal.SearchOptions{ Paginator: &db.ListOptions{ PageSize: 5, @@ -336,7 +336,7 @@ var cases = []*testIndexerCase{ }, }, { - Name: "project board", + Name: "ProjectBoardID", SearchOptions: &internal.SearchOptions{ Paginator: &db.ListOptions{ PageSize: 5, @@ -357,7 +357,7 @@ var cases = []*testIndexerCase{ }, }, { - Name: "no project board", + Name: "no ProjectBoardID", SearchOptions: &internal.SearchOptions{ Paginator: &db.ListOptions{ PageSize: 5, @@ -378,7 +378,7 @@ var cases = []*testIndexerCase{ }, }, { - Name: "poster", + Name: "PosterID", SearchOptions: &internal.SearchOptions{ Paginator: &db.ListOptions{ PageSize: 5, @@ -394,11 +394,306 @@ var cases = []*testIndexerCase{ assert.Equal(t, int64(1), data[v.ID].PosterID) } assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { - return v.PosterID == 0 + return v.PosterID == 1 }), result.Total) }, }, - // TODO: add more cases + { + Name: "AssigneeID", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + AssigneeID: func() *int64 { + id := int64(1) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Equal(t, int64(1), data[v.ID].AssigneeID) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.AssigneeID == 1 + }), result.Total) + }, + }, + { + Name: "no AssigneeID", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + AssigneeID: func() *int64 { + id := int64(0) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Equal(t, int64(0), data[v.ID].AssigneeID) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.AssigneeID == 0 + }), result.Total) + }, + }, + { + Name: "MentionID", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + MentionID: func() *int64 { + id := int64(1) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Contains(t, data[v.ID].MentionIDs, int64(1)) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return util.SliceContains(v.MentionIDs, 1) + }), result.Total) + }, + }, + { + Name: "ReviewedID", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + ReviewedID: func() *int64 { + id := int64(1) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Contains(t, data[v.ID].ReviewedIDs, int64(1)) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return util.SliceContains(v.ReviewedIDs, 1) + }), result.Total) + }, + }, + { + Name: "ReviewRequestedID", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + ReviewRequestedID: func() *int64 { + id := int64(1) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Contains(t, data[v.ID].ReviewRequestedIDs, int64(1)) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return util.SliceContains(v.ReviewRequestedIDs, 1) + }), result.Total) + }, + }, + { + Name: "SubscriberID", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + SubscriberID: func() *int64 { + id := int64(1) + return &id + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.Contains(t, data[v.ID].SubscriberIDs, int64(1)) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return util.SliceContains(v.SubscriberIDs, 1) + }), result.Total) + }, + }, + { + Name: "updated", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 5, + }, + UpdatedAfterUnix: func() *int64 { + var t int64 = 20 + return &t + }(), + UpdatedBeforeUnix: func() *int64 { + var t int64 = 30 + return &t + }(), + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, 5, len(result.Hits)) + for _, v := range result.Hits { + assert.GreaterOrEqual(t, data[v.ID].UpdatedUnix, int64(20)) + assert.LessOrEqual(t, data[v.ID].UpdatedUnix, int64(30)) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return data[v.ID].UpdatedUnix >= 20 && data[v.ID].UpdatedUnix <= 30 + }), result.Total) + }, + }, + { + Name: "SortByCreatedDesc", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + ListAll: true, + }, + SortBy: internal.SortByCreatedDesc, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, len(data), len(result.Hits)) + assert.Equal(t, len(data), int(result.Total)) + for i, v := range result.Hits { + if i < len(result.Hits)-1 { + assert.GreaterOrEqual(t, data[v.ID].CreatedUnix, data[result.Hits[i+1].ID].CreatedUnix) + } + } + }, + }, + { + Name: "SortByUpdatedDesc", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + ListAll: true, + }, + SortBy: internal.SortByUpdatedDesc, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, len(data), len(result.Hits)) + assert.Equal(t, len(data), int(result.Total)) + for i, v := range result.Hits { + if i < len(result.Hits)-1 { + assert.GreaterOrEqual(t, data[v.ID].UpdatedUnix, data[result.Hits[i+1].ID].UpdatedUnix) + } + } + }, + }, + { + Name: "SortByCommentsDesc", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + ListAll: true, + }, + SortBy: internal.SortByCommentsDesc, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, len(data), len(result.Hits)) + assert.Equal(t, len(data), int(result.Total)) + for i, v := range result.Hits { + if i < len(result.Hits)-1 { + assert.GreaterOrEqual(t, data[v.ID].CommentCount, data[result.Hits[i+1].ID].CommentCount) + } + } + }, + }, + { + Name: "SortByDeadlineDesc", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + ListAll: true, + }, + SortBy: internal.SortByCommentsDesc, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, len(data), len(result.Hits)) + assert.Equal(t, len(data), int(result.Total)) + for i, v := range result.Hits { + if i < len(result.Hits)-1 { + assert.GreaterOrEqual(t, data[v.ID].DeadlineUnix, data[result.Hits[i+1].ID].DeadlineUnix) + } + } + }, + }, + { + Name: "SortByCreatedAsc", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + ListAll: true, + }, + SortBy: internal.SortByCreatedDesc, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, len(data), len(result.Hits)) + assert.Equal(t, len(data), int(result.Total)) + for i, v := range result.Hits { + if i < len(result.Hits)-1 { + assert.LessOrEqual(t, data[v.ID].CreatedUnix, data[result.Hits[i+1].ID].CreatedUnix) + } + } + }, + }, + { + Name: "SortByUpdatedAsc", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + ListAll: true, + }, + SortBy: internal.SortByUpdatedDesc, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, len(data), len(result.Hits)) + assert.Equal(t, len(data), int(result.Total)) + for i, v := range result.Hits { + if i < len(result.Hits)-1 { + assert.LessOrEqual(t, data[v.ID].UpdatedUnix, data[result.Hits[i+1].ID].UpdatedUnix) + } + } + }, + }, + { + Name: "SortByCommentsAsc", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + ListAll: true, + }, + SortBy: internal.SortByCommentsDesc, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, len(data), len(result.Hits)) + assert.Equal(t, len(data), int(result.Total)) + for i, v := range result.Hits { + if i < len(result.Hits)-1 { + assert.LessOrEqual(t, data[v.ID].CommentCount, data[result.Hits[i+1].ID].CommentCount) + } + } + }, + }, + { + Name: "SortByDeadlineAsc", + SearchOptions: &internal.SearchOptions{ + Paginator: &db.ListOptions{ + ListAll: true, + }, + SortBy: internal.SortByCommentsDesc, + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Equal(t, len(data), len(result.Hits)) + assert.Equal(t, len(data), int(result.Total)) + for i, v := range result.Hits { + if i < len(result.Hits)-1 { + assert.LessOrEqual(t, data[v.ID].DeadlineUnix, data[result.Hits[i+1].ID].DeadlineUnix) + } + } + }, + }, } type testIndexerCase struct { @@ -426,23 +721,23 @@ func generateDefaultIndexerData() []*internal.IndexerData { labelIDs := make([]int64, id%5) for i := range labelIDs { - labelIDs[i] = int64(i) + labelIDs[i] = int64(i) + 1 // LabelID should not be 0 } mentionIDs := make([]int64, id%6) for i := range mentionIDs { - mentionIDs[i] = int64(i) + mentionIDs[i] = int64(i) + 1 // MentionID should not be 0 } reviewedIDs := make([]int64, id%7) for i := range reviewedIDs { - reviewedIDs[i] = int64(i) + reviewedIDs[i] = int64(i) + 1 // ReviewID should not be 0 } reviewRequestedIDs := make([]int64, id%8) for i := range reviewRequestedIDs { - reviewRequestedIDs[i] = int64(i) + reviewRequestedIDs[i] = int64(i) + 1 // ReviewRequestedID should not be 0 } subscriberIDs := make([]int64, id%9) for i := range subscriberIDs { - subscriberIDs[i] = int64(i) + subscriberIDs[i] = int64(i) + 1 // SubscriberID should not be 0 } data = append(data, &internal.IndexerData{ @@ -459,7 +754,7 @@ func generateDefaultIndexerData() []*internal.IndexerData { MilestoneID: issueIndex % 4, ProjectID: issueIndex % 5, ProjectBoardID: issueIndex % 6, - PosterID: id % 10, + PosterID: id%10 + 1, // PosterID should not be 0 AssigneeID: issueIndex % 10, MentionIDs: mentionIDs, ReviewedIDs: reviewedIDs, From f1b75ae72de809b1daf0863894b887d1acbb1819 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 26 Jul 2023 11:06:08 +0800 Subject: [PATCH 81/99] test: fix cases --- modules/indexer/issues/internal/tests/tests.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index b22432ac43770..89c3a01474ff4 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -610,7 +610,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ ListAll: true, }, - SortBy: internal.SortByCommentsDesc, + SortBy: internal.SortByDeadlineDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Equal(t, len(data), len(result.Hits)) @@ -628,7 +628,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ ListAll: true, }, - SortBy: internal.SortByCreatedDesc, + SortBy: internal.SortByCreatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Equal(t, len(data), len(result.Hits)) @@ -646,7 +646,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ ListAll: true, }, - SortBy: internal.SortByUpdatedDesc, + SortBy: internal.SortByUpdatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Equal(t, len(data), len(result.Hits)) @@ -664,7 +664,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ ListAll: true, }, - SortBy: internal.SortByCommentsDesc, + SortBy: internal.SortByCommentsAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Equal(t, len(data), len(result.Hits)) @@ -682,7 +682,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ ListAll: true, }, - SortBy: internal.SortByCommentsDesc, + SortBy: internal.SortByDeadlineAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Equal(t, len(data), len(result.Hits)) From 0549ba3ec2e7ac68cf40ca98ca9ae491ca98c510 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 26 Jul 2023 17:46:35 +0800 Subject: [PATCH 82/99] feat: support elasticsearch --- .../issues/elasticsearch/elasticsearch.go | 194 +++++++++++++----- 1 file changed, 145 insertions(+), 49 deletions(-) diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index f8d1e38d57584..0b137ea422460 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "strconv" + "strings" "code.gitea.io/gitea/modules/graceful" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" @@ -17,7 +18,7 @@ import ( ) const ( - issueIndexerLatestVersion = 0 + issueIndexerLatestVersion = 1 ) var _ internal.Indexer = &Indexer{} @@ -39,32 +40,40 @@ func NewIndexer(url, indexerName string) *Indexer { } const ( - defaultMapping = `{ - "mappings": { - "properties": { - "id": { - "type": "integer", - "index": true - }, - "repo_id": { - "type": "integer", - "index": true - }, - "title": { - "type": "text", - "index": true - }, - "content": { - "type": "text", - "index": true - }, - "comments": { - "type" : "text", - "index": true - } - } + defaultMapping = ` +{ + "mappings": { + "properties": { + "id": { "type": "integer", "index": true }, + "repo_id": { "type": "integer", "index": true }, + "is_public": { "type": "boolean", "index": true }, + + "title": { "type": "text", "index": true }, + "content": { "type": "text", "index": true }, + "comments": { "type" : "text", "index": true }, + + "is_pull": { "type": "boolean", "index": true }, + "is_closed": { "type": "boolean", "index": true }, + "label_ids": { "type": "integer", "index": true }, + "no_label": { "type": "boolean", "index": true }, + "milestone_id": { "type": "integer", "index": true }, + "project_id": { "type": "integer", "index": true }, + "project_board_id": { "type": "integer", "index": true }, + "poster_id": { "type": "integer", "index": true }, + "assignee_id": { "type": "integer", "index": true }, + "mention_ids": { "type": "integer", "index": true }, + "reviewed_ids": { "type": "integer", "index": true }, + "review_requested_ids": { "type": "integer", "index": true }, + "subscriber_ids": { "type": "integer", "index": true }, + "updated_unix": { "type": "integer", "index": true }, + + "created_unix": { "type": "integer", "index": true }, + "deadline_unix": { "type": "integer", "index": true }, + "comment_count": { "type": "integer", "index": true } } - }` + } +} +` ) // Index will save the index data @@ -76,13 +85,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er _, err := b.inner.Client.Index(). Index(b.inner.VersionedIndexName()). Id(fmt.Sprintf("%d", issue.ID)). - BodyJson(map[string]any{ - "id": issue.ID, - "repo_id": issue.RepoID, - "title": issue.Title, - "content": issue.Content, - "comments": issue.Comments, - }). + BodyJson(issue). Do(ctx) return err } @@ -93,13 +96,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er elastic.NewBulkIndexRequest(). Index(b.inner.VersionedIndexName()). Id(fmt.Sprintf("%d", issue.ID)). - Doc(map[string]any{ - "id": issue.ID, - "repo_id": issue.RepoID, - "title": issue.Title, - "content": issue.Content, - "comments": issue.Comments, - }), + Doc(issue), ) } @@ -141,16 +138,98 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { - kwQuery := elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments") query := elastic.NewBoolQuery() - query = query.Must(kwQuery) + + if options.Keyword != "" { + query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments")) + } + if len(options.RepoIDs) > 0 { - repoStrs := make([]any, 0, len(options.RepoIDs)) - for _, repoID := range options.RepoIDs { - repoStrs = append(repoStrs, repoID) + q := elastic.NewBoolQuery() + q.Should(elastic.NewTermsQuery("repo_id", toAnySlice(options.RepoIDs)...)) + if options.AllPublic { + q.Should(elastic.NewTermQuery("is_public", true)) + } + query.Must(q) + } + + if !options.IsPull.IsNone() { + query.Must(elastic.NewTermQuery("is_pull", options.IsPull.IsTrue())) + } + if !options.IsClosed.IsNone() { + query.Must(elastic.NewTermQuery("is_closed", options.IsClosed.IsTrue())) + } + + if options.NoLabelOnly { + // queries = append(queries, inner_bleve.BoolFieldQuery(true, "no_label")) + query.Must(elastic.NewTermQuery("no_label", true)) + } else { + if len(options.IncludedLabelIDs) > 0 { + q := elastic.NewBoolQuery() + for _, labelID := range options.IncludedLabelIDs { + q.Must(elastic.NewTermQuery("label_ids", labelID)) + } + query.Must(q) + } else if len(options.IncludedAnyLabelIDs) > 0 { + query.Must(elastic.NewTermsQuery("label_ids", toAnySlice(options.IncludedAnyLabelIDs)...)) + } + if len(options.ExcludedLabelIDs) > 0 { + q := elastic.NewBoolQuery() + for _, labelID := range options.ExcludedLabelIDs { + q.MustNot(elastic.NewTermQuery("label_ids", labelID)) + } + query.Must(q) + } + } + + if len(options.MilestoneIDs) > 0 { + query.Must(elastic.NewTermsQuery("milestone_id", toAnySlice(options.MilestoneIDs)...)) + } + + if options.ProjectID != nil { + query.Must(elastic.NewTermQuery("project_id", *options.ProjectID)) + } + if options.ProjectBoardID != nil { + query.Must(elastic.NewTermQuery("project_board_id", *options.ProjectBoardID)) + } + + if options.PosterID != nil { + query.Must(elastic.NewTermQuery("poster_id", *options.PosterID)) + } + + if options.AssigneeID != nil { + query.Must(elastic.NewTermQuery("assignee_id", *options.AssigneeID)) + } + + if options.MentionID != nil { + query.Must(elastic.NewTermQuery("mention_ids", *options.MentionID)) + } + + if options.ReviewedID != nil { + query.Must(elastic.NewTermQuery("reviewed_ids", *options.ReviewedID)) + } + if options.ReviewRequestedID != nil { + query.Must(elastic.NewTermQuery("review_requested_ids", *options.ReviewRequestedID)) + } + + if options.SubscriberID != nil { + query.Must(elastic.NewTermQuery("subscriber_ids", *options.SubscriberID)) + } + + if options.UpdatedAfterUnix != nil || options.UpdatedBeforeUnix != nil { + q := elastic.NewRangeQuery("updated_unix") + if options.UpdatedAfterUnix != nil { + q.Gte(*options.UpdatedAfterUnix) + } + if options.UpdatedBeforeUnix != nil { + q.Lte(*options.UpdatedBeforeUnix) } - repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) - query = query.Must(repoQuery) + query.Must(q) + } + + sortBy := []elastic.Sorter{ + parseSortBy(options.SortBy), + elastic.NewFieldSort("id").Desc(), } // See https://stackoverflow.com/questions/35206409/elasticsearch-2-1-result-window-is-too-large-index-max-result-window/35221900 @@ -161,7 +240,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( searchResult, err := b.inner.Client.Search(). Index(b.inner.VersionedIndexName()). Query(query). - Sort("_score", false). + SortBy(sortBy...). From(skip).Size(limit). Do(ctx) if err != nil { @@ -179,6 +258,23 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( return &internal.SearchResult{ Total: searchResult.TotalHits(), Hits: hits, - Imprecise: true, + Imprecise: false, }, nil } + +func toAnySlice[T any](s []T) []any { + ret := make([]any, 0, len(s)) + for _, item := range s { + ret = append(ret, item) + } + return ret +} + +func parseSortBy(sortBy internal.SortBy) elastic.Sorter { + field := strings.TrimPrefix(string(sortBy), "-") + ret := elastic.NewFieldSort(field) + if strings.HasPrefix(string(sortBy), "-") { + ret.Desc() + } + return ret +} From 4789728d8d7f5b968c8f1d8308073d7fd856cd1e Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 26 Jul 2023 18:47:11 +0800 Subject: [PATCH 83/99] feat: support es --- modules/indexer/issues/bleve/bleve_test.go | 2 +- .../issues/elasticsearch/elasticsearch.go | 3 ++ .../elasticsearch/elasticsearch_test.go | 49 +++++++++++++++++++ .../indexer/issues/internal/tests/tests.go | 8 +-- 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 modules/indexer/issues/elasticsearch/elasticsearch_test.go diff --git a/modules/indexer/issues/bleve/bleve_test.go b/modules/indexer/issues/bleve/bleve_test.go index 908514a01a2d6..63bc86bd2d902 100644 --- a/modules/indexer/issues/bleve/bleve_test.go +++ b/modules/indexer/issues/bleve/bleve_test.go @@ -14,5 +14,5 @@ func TestBleveIndexer(t *testing.T) { indexer := NewIndexer(dir) defer indexer.Close() - tests.TestIndexer(t, indexer) + tests.TestIndexer(t, indexer, 0) } diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 0b137ea422460..dc60e6b829e75 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -227,6 +227,9 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query.Must(q) } + if options.SortBy == "" { + options.SortBy = internal.SortByCreatedAsc + } sortBy := []elastic.Sorter{ parseSortBy(options.SortBy), elastic.NewFieldSort("id").Desc(), diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_test.go b/modules/indexer/issues/elasticsearch/elasticsearch_test.go new file mode 100644 index 0000000000000..41990f0ced78b --- /dev/null +++ b/modules/indexer/issues/elasticsearch/elasticsearch_test.go @@ -0,0 +1,49 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package elasticsearch + +import ( + "fmt" + "net/http" + "os" + "testing" + "time" + + "code.gitea.io/gitea/modules/indexer/issues/internal/tests" +) + +func TestElasticsearchIndexer(t *testing.T) { + url := "http://elastic:changeme@elasticsearch:9200" + + if os.Getenv("CI") == "" { + // Make it possible to run tests against a local elasticsearch instance + url = os.Getenv("TEST_ELASTICSEARCH_URL") + if url == "" { + t.Skip("TEST_ELASTICSEARCH_URL not set and not running in CI") + return + } + } + + ok := false + for i := 0; i < 60; i++ { + resp, err := http.Get(url) + if err == nil && resp.StatusCode == http.StatusOK { + ok = true + break + } + t.Logf("Waiting for elasticsearch to be up: %v", err) + } + if !ok { + t.Fatalf("Failed to wait for elasticsearch to be up") + return + } + + indexer := NewIndexer(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) + defer indexer.Close() + + // When writing to elasticsearch, it can take a while for the data to be available for search + delayAfterWrite := time.Second + + tests.TestIndexer(t, indexer, delayAfterWrite) +} diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 89c3a01474ff4..dca28b7ec5277 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "testing" + "time" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/indexer/issues/internal" @@ -21,12 +22,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestIndexer(t *testing.T, indexer internal.Indexer) { +func TestIndexer(t *testing.T, indexer internal.Indexer, delayAfterWrite time.Duration) { _, err := indexer.Init(context.Background()) require.NoError(t, err) - require.NoError(t, indexer.Ping(context.Background())) - var ( ids []int64 data = map[int64]*internal.IndexerData{} @@ -38,6 +37,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { data[v.ID] = v } require.NoError(t, indexer.Index(context.Background(), d...)) + time.Sleep(delayAfterWrite) } defer func() { @@ -48,6 +48,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { t.Run(c.Name, func(t *testing.T) { if len(c.ExtraData) > 0 { require.NoError(t, indexer.Index(context.Background(), c.ExtraData...)) + time.Sleep(delayAfterWrite) for _, v := range c.ExtraData { data[v.ID] = v } @@ -56,6 +57,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { require.NoError(t, indexer.Delete(context.Background(), v.ID)) delete(data, v.ID) } + time.Sleep(delayAfterWrite) }() } From 91ba549c19d5e34f705e7409f034bc867e35af9f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 26 Jul 2023 18:58:17 +0800 Subject: [PATCH 84/99] feat: support meilisearch setting --- modules/indexer/internal/meilisearch/indexer.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/indexer/internal/meilisearch/indexer.go b/modules/indexer/internal/meilisearch/indexer.go index 06747ff7e07ae..b037249d43210 100644 --- a/modules/indexer/internal/meilisearch/indexer.go +++ b/modules/indexer/internal/meilisearch/indexer.go @@ -17,14 +17,16 @@ type Indexer struct { url, apiKey string indexName string version int + settings *meilisearch.Settings } -func NewIndexer(url, apiKey, indexName string, version int) *Indexer { +func NewIndexer(url, apiKey, indexName string, version int, settings *meilisearch.Settings) *Indexer { return &Indexer{ url: url, apiKey: apiKey, indexName: indexName, version: version, + settings: settings, } } @@ -57,7 +59,7 @@ func (i *Indexer) Init(_ context.Context) (bool, error) { i.checkOldIndexes() - _, err = i.Client.Index(i.VersionedIndexName()).UpdateFilterableAttributes(&[]string{"repo_id"}) + _, err = i.Client.Index(i.VersionedIndexName()).UpdateSettings(i.settings) return false, err } From 97e455f65b44b7091e000e4cc3ee96dd1499402b Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 26 Jul 2023 20:31:34 +0800 Subject: [PATCH 85/99] test: waitData --- modules/indexer/issues/bleve/bleve_test.go | 2 +- .../elasticsearch/elasticsearch_test.go | 5 +--- .../indexer/issues/internal/tests/tests.go | 30 ++++++++++++++++--- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/modules/indexer/issues/bleve/bleve_test.go b/modules/indexer/issues/bleve/bleve_test.go index 63bc86bd2d902..908514a01a2d6 100644 --- a/modules/indexer/issues/bleve/bleve_test.go +++ b/modules/indexer/issues/bleve/bleve_test.go @@ -14,5 +14,5 @@ func TestBleveIndexer(t *testing.T) { indexer := NewIndexer(dir) defer indexer.Close() - tests.TestIndexer(t, indexer, 0) + tests.TestIndexer(t, indexer) } diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_test.go b/modules/indexer/issues/elasticsearch/elasticsearch_test.go index 41990f0ced78b..93cba79287f1b 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_test.go @@ -42,8 +42,5 @@ func TestElasticsearchIndexer(t *testing.T) { indexer := NewIndexer(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) defer indexer.Close() - // When writing to elasticsearch, it can take a while for the data to be available for search - delayAfterWrite := time.Second - - tests.TestIndexer(t, indexer, delayAfterWrite) + tests.TestIndexer(t, indexer) } diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index dca28b7ec5277..ebde14af12b84 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestIndexer(t *testing.T, indexer internal.Indexer, delayAfterWrite time.Duration) { +func TestIndexer(t *testing.T, indexer internal.Indexer) { _, err := indexer.Init(context.Background()) require.NoError(t, err) @@ -37,7 +37,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer, delayAfterWrite time.Du data[v.ID] = v } require.NoError(t, indexer.Index(context.Background(), d...)) - time.Sleep(delayAfterWrite) + require.NoError(t, waitData(indexer, int64(len(data)))) } defer func() { @@ -48,16 +48,16 @@ func TestIndexer(t *testing.T, indexer internal.Indexer, delayAfterWrite time.Du t.Run(c.Name, func(t *testing.T) { if len(c.ExtraData) > 0 { require.NoError(t, indexer.Index(context.Background(), c.ExtraData...)) - time.Sleep(delayAfterWrite) for _, v := range c.ExtraData { data[v.ID] = v } + require.NoError(t, waitData(indexer, int64(len(data)))) defer func() { for _, v := range c.ExtraData { require.NoError(t, indexer.Delete(context.Background(), v.ID)) delete(data, v.ID) } - time.Sleep(delayAfterWrite) + require.NoError(t, waitData(indexer, int64(len(data)))) }() } @@ -782,3 +782,25 @@ func countIndexerData(data map[int64]*internal.IndexerData, f func(v *internal.I } return count } + +// waitData waits for the indexer to index all data. +// Some engines like Elasticsearch index data asynchronously, so we need to wait for a while. +func waitData(indexer internal.Indexer, total int64) error { + var actual int64 + for i := 0; i < 100; i++ { + result, err := indexer.Search(context.Background(), &internal.SearchOptions{ + Paginator: &db.ListOptions{ + PageSize: 0, + }, + }) + if err != nil { + return err + } + actual = result.Total + if actual == total { + return nil + } + time.Sleep(100 * time.Millisecond) + } + return fmt.Errorf("waitData: expected %d, actual %d", total, actual) +} From 95e1f49191a3085092cb1049fc3a95e9d23aeb2f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 26 Jul 2023 20:41:28 +0800 Subject: [PATCH 86/99] feat: settings for melisearch --- .../indexer/issues/meilisearch/meilisearch.go | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 61bcbcdc1e707..d6e25a9494708 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -18,6 +18,9 @@ import ( const ( issueIndexerLatestVersion = 2 + + // TODO: make this configurable if necessary + maxTotalHits = 10000 ) var _ internal.Indexer = &Indexer{} @@ -30,7 +33,45 @@ type Indexer struct { // NewIndexer creates a new meilisearch indexer func NewIndexer(url, apiKey, indexerName string) *Indexer { - inner := inner_meilisearch.NewIndexer(url, apiKey, indexerName, issueIndexerLatestVersion) + settings := &meilisearch.Settings{ + SearchableAttributes: []string{ + "title", + "content", + "comments", + }, + DisplayedAttributes: []string{ + "id", + }, + FilterableAttributes: []string{ + "repo_id", + "is_public", + "is_pull", + "is_closed", + "label_ids", + "no_label", + "milestone_id", + "project_id", + "project_board_id", + "poster_id", + "assignee_id", + "mention_ids", + "reviewed_ids", + "review_requested_ids", + "subscriber_ids", + "updated_unix", + }, + SortableAttributes: []string{ + "updated_unix", + "created_unix", + "deadline_unix", + "comment_count", + }, + Pagination: &meilisearch.Pagination{ + MaxTotalHits: maxTotalHits, + }, + } + + inner := inner_meilisearch.NewIndexer(url, apiKey, indexerName, issueIndexerLatestVersion, settings) indexer := &Indexer{ inner: inner, Indexer: inner, From 1e1328f0c0f9033a43f5b39ab5f56eb36c3c84cb Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 09:49:00 +0800 Subject: [PATCH 87/99] fix: accept es yellow --- modules/indexer/internal/elasticsearch/indexer.go | 3 ++- modules/indexer/issues/internal/tests/tests.go | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/indexer/internal/elasticsearch/indexer.go b/modules/indexer/internal/elasticsearch/indexer.go index 2c60efad564fc..395eea3bce652 100644 --- a/modules/indexer/internal/elasticsearch/indexer.go +++ b/modules/indexer/internal/elasticsearch/indexer.go @@ -76,7 +76,8 @@ func (i *Indexer) Ping(ctx context.Context) error { if err != nil { return err } - if resp.Status != "green" { + if resp.Status != "green" && resp.Status != "yellow" { + // It's healthy if the status is green, and it's available if the status is yellow, // see https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html return fmt.Errorf("status of elasticsearch cluster is %s", resp.Status) } diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index ebde14af12b84..0477a966257c5 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -26,6 +26,8 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { _, err := indexer.Init(context.Background()) require.NoError(t, err) + require.NoError(t, indexer.Ping(context.Background())) + var ( ids []int64 data = map[int64]*internal.IndexerData{} From bc7cb96e0ad0a87e553cefb330bc55df93f24ee7 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 10:49:19 +0800 Subject: [PATCH 88/99] feat: support meilisearch --- .../indexer/internal/meilisearch/filter.go | 119 ++++++++++++++++++ .../issues/elasticsearch/elasticsearch.go | 1 - .../indexer/issues/meilisearch/meilisearch.go | 112 ++++++++++++++--- .../issues/meilisearch/meilisearch_test.go | 48 +++++++ 4 files changed, 264 insertions(+), 16 deletions(-) create mode 100644 modules/indexer/internal/meilisearch/filter.go create mode 100644 modules/indexer/issues/meilisearch/meilisearch_test.go diff --git a/modules/indexer/internal/meilisearch/filter.go b/modules/indexer/internal/meilisearch/filter.go new file mode 100644 index 0000000000000..593177f163731 --- /dev/null +++ b/modules/indexer/internal/meilisearch/filter.go @@ -0,0 +1,119 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package meilisearch + +import ( + "fmt" + "strings" +) + +// Filter represents a filter for meilisearch queries. +// It's just a simple wrapper around a string. +// DO NOT assume that it is a complete implementation. +type Filter interface { + Statement() string +} + +type FilterAnd struct { + filters []Filter +} + +func (f *FilterAnd) Statement() string { + var statements []string + for _, filter := range f.filters { + if s := filter.Statement(); s != "" { + statements = append(statements, fmt.Sprintf("(%s)", s)) + } + } + return strings.Join(statements, " AND ") +} + +func (f *FilterAnd) And(filter Filter) *FilterAnd { + f.filters = append(f.filters, filter) + return f +} + +type FilterOr struct { + filters []Filter +} + +func (f *FilterOr) Statement() string { + var statements []string + for _, filter := range f.filters { + if s := filter.Statement(); s != "" { + statements = append(statements, fmt.Sprintf("(%s)", s)) + } + } + return strings.Join(statements, " OR ") +} + +func (f *FilterOr) Or(filter Filter) *FilterOr { + f.filters = append(f.filters, filter) + return f +} + +type FilterIn string + +// NewFilterIn creates a new FilterIn. +// It supports int64 only, to avoid extra works to handle strings with special characters. +func NewFilterIn[T int64](field string, values ...T) FilterIn { + if len(values) == 0 { + return "" + } + vs := make([]string, len(values)) + for i, v := range values { + vs[i] = fmt.Sprintf("%v", v) + } + return FilterIn(fmt.Sprintf("%s IN [%v]", field, strings.Join(vs, ", "))) +} + +func (f FilterIn) Statement() string { + return string(f) +} + +type FilterEq string + +// NewFilterEq creates a new FilterEq. +// It supports int64 and bool only, to avoid extra works to handle strings with special characters. +func NewFilterEq[T bool | int64](field string, value T) FilterEq { + return FilterEq(fmt.Sprintf("%s = %v", field, value)) +} + +func (f FilterEq) Statement() string { + return string(f) +} + +type FilterNot string + +func NewFilterNot(filter Filter) FilterNot { + return FilterNot(fmt.Sprintf("NOT (%s)", filter.Statement())) +} + +func (f FilterNot) Statement() string { + return string(f) +} + +type FilterGte string + +// NewFilterGte creates a new FilterGte. +// It supports int64 only, to avoid extra works to handle strings with special characters. +func NewFilterGte[T int64](field string, value T) FilterGte { + return FilterGte(fmt.Sprintf("%s >= %v", field, value)) +} + +func (f FilterGte) Statement() string { + return string(f) +} + +type FilterLte string + +// NewFilterLte creates a new FilterLte. +// It supports int64 only, to avoid extra works to handle strings with special characters. +func NewFilterLte[T int64](field string, value T) FilterLte { + return FilterLte(fmt.Sprintf("%s <= %v", field, value)) +} + +func (f FilterLte) Statement() string { + return string(f) +} diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index dc60e6b829e75..21526e5accde6 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -161,7 +161,6 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( } if options.NoLabelOnly { - // queries = append(queries, inner_bleve.BoolFieldQuery(true, "no_label")) query.Must(elastic.NewTermQuery("no_label", true)) } else { if len(options.IncludedLabelIDs) > 0 { diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index d6e25a9494708..a59b460676345 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -5,7 +5,6 @@ package meilisearch import ( "context" - "fmt" "strconv" "strings" @@ -65,6 +64,7 @@ func NewIndexer(url, apiKey, indexerName string) *Indexer { "created_unix", "deadline_unix", "comment_count", + "id", }, Pagination: &meilisearch.Pagination{ MaxTotalHits: maxTotalHits, @@ -113,28 +113,101 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { - repoFilters := make([]string, 0, len(options.RepoIDs)) - for _, repoID := range options.RepoIDs { - repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10)) + query := inner_meilisearch.FilterAnd{} + + if len(options.RepoIDs) > 0 { + q := &inner_meilisearch.FilterOr{} + q.Or(inner_meilisearch.NewFilterIn("repo_id", options.RepoIDs...)) + if options.AllPublic { + q.Or(inner_meilisearch.NewFilterEq("is_public", true)) + } + query.And(q) } - filter := strings.Join(repoFilters, " OR ") - skip, limit := indexer_internal.ParsePaginator(options.Paginator) + if !options.IsPull.IsNone() { + query.And(inner_meilisearch.NewFilterEq("is_pull", options.IsPull.IsTrue())) + } if !options.IsClosed.IsNone() { - condition := fmt.Sprintf("is_closed = %t", options.IsClosed.IsTrue()) - if filter != "" { - filter = "(" + filter + ") AND " + condition - } else { - filter = condition + query.And(inner_meilisearch.NewFilterEq("is_closed", options.IsClosed.IsTrue())) + } + + if options.NoLabelOnly { + query.And(inner_meilisearch.NewFilterEq("no_label", true)) + } else { + if len(options.IncludedLabelIDs) > 0 { + q := &inner_meilisearch.FilterAnd{} + for _, labelID := range options.IncludedLabelIDs { + q.And(inner_meilisearch.NewFilterEq("label_ids", labelID)) + } + query.And(q) + } else if len(options.IncludedAnyLabelIDs) > 0 { + query.And(inner_meilisearch.NewFilterIn("label_ids", options.IncludedAnyLabelIDs...)) + } + if len(options.ExcludedLabelIDs) > 0 { + q := &inner_meilisearch.FilterAnd{} + for _, labelID := range options.ExcludedLabelIDs { + q.And(inner_meilisearch.NewFilterNot(inner_meilisearch.NewFilterEq("label_ids", labelID))) + } + query.And(q) } } - // TODO: support more conditions + if len(options.MilestoneIDs) > 0 { + query.And(inner_meilisearch.NewFilterIn("milestone_id", options.MilestoneIDs...)) + } + + if options.ProjectID != nil { + query.And(inner_meilisearch.NewFilterEq("project_id", *options.ProjectID)) + } + if options.ProjectBoardID != nil { + query.And(inner_meilisearch.NewFilterEq("project_board_id", *options.ProjectBoardID)) + } + + if options.PosterID != nil { + query.And(inner_meilisearch.NewFilterEq("poster_id", *options.PosterID)) + } + + if options.AssigneeID != nil { + query.And(inner_meilisearch.NewFilterEq("assignee_id", *options.AssigneeID)) + } + + if options.MentionID != nil { + query.And(inner_meilisearch.NewFilterEq("mention_ids", *options.MentionID)) + } + + if options.ReviewedID != nil { + query.And(inner_meilisearch.NewFilterEq("reviewed_ids", *options.ReviewedID)) + } + if options.ReviewRequestedID != nil { + query.And(inner_meilisearch.NewFilterEq("review_requested_ids", *options.ReviewRequestedID)) + } + + if options.SubscriberID != nil { + query.And(inner_meilisearch.NewFilterEq("subscriber_ids", *options.SubscriberID)) + } + + if options.UpdatedAfterUnix != nil { + query.And(inner_meilisearch.NewFilterGte("updated_unix", *options.UpdatedAfterUnix)) + } + if options.UpdatedBeforeUnix != nil { + query.And(inner_meilisearch.NewFilterLte("updated_unix", *options.UpdatedBeforeUnix)) + } + + if options.SortBy == "" { + options.SortBy = internal.SortByCreatedAsc + } + sortBy := []string{ + parseSortBy(options.SortBy), + "id:desc", + } + + skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxTotalHits) searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(options.Keyword, &meilisearch.SearchRequest{ - Filter: filter, + Filter: query.Statement(), Limit: int64(limit), Offset: int64(skip), + Sort: sortBy, }) if err != nil { return nil, err @@ -146,9 +219,18 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( ID: int64(hit.(map[string]any)["id"].(float64)), }) } + return &internal.SearchResult{ - Total: searchRes.TotalHits, + Total: searchRes.EstimatedTotalHits, Hits: hits, - Imprecise: true, + Imprecise: false, }, nil } + +func parseSortBy(sortBy internal.SortBy) string { + field := strings.TrimPrefix(string(sortBy), "-") + if strings.HasPrefix(string(sortBy), "-") { + return field + ":desc" + } + return field + ":asc" +} diff --git a/modules/indexer/issues/meilisearch/meilisearch_test.go b/modules/indexer/issues/meilisearch/meilisearch_test.go new file mode 100644 index 0000000000000..f9fb36a473b5d --- /dev/null +++ b/modules/indexer/issues/meilisearch/meilisearch_test.go @@ -0,0 +1,48 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package meilisearch + +import ( + "fmt" + "net/http" + "os" + "testing" + "time" + + "code.gitea.io/gitea/modules/indexer/issues/internal/tests" +) + +func TestMeilisearchIndexer(t *testing.T) { + url := "" // TODO: set meilisearch in unit tests + key := "" + + if os.Getenv("CI") == "" { + // Make it possible to run tests against a local meilisearch instance + url = os.Getenv("TEST_MEILISEARCH_URL") + if url == "" { + t.Skip("TEST_MEILISEARCH_URL not set and not running in CI") + return + } + key = os.Getenv("TEST_MEILISEARCH_KEY") + } + + ok := false + for i := 0; i < 60; i++ { + resp, err := http.Get(url) + if err == nil && resp.StatusCode == http.StatusOK { + ok = true + break + } + t.Logf("Waiting for meilisearch to be up: %v", err) + } + if !ok { + t.Fatalf("Failed to wait for meilisearch to be up") + return + } + + indexer := NewIndexer(url, key, fmt.Sprintf("test_meilisearch_indexer_%d", time.Now().Unix())) + defer indexer.Close() + + tests.TestIndexer(t, indexer) +} From da3a67846089d476f116222caefd9fc792d7ffa0 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 11:19:33 +0800 Subject: [PATCH 89/99] fix: RankingRules --- modules/indexer/issues/meilisearch/meilisearch.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index a59b460676345..d258f80885018 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -33,6 +33,13 @@ type Indexer struct { // NewIndexer creates a new meilisearch indexer func NewIndexer(url, apiKey, indexerName string) *Indexer { settings := &meilisearch.Settings{ + // The default ranking rules of meilisearch are: ["words", "typo", "proximity", "attribute", "sort", "exactness"] + // So even if we specify the sort order, it could not be respected because the priority of "sort" is so low. + // So we need to specify the ranking rules to make sure the sort order is respected. + // See https://www.meilisearch.com/docs/learn/core_concepts/relevancy + RankingRules: []string{"sort", // make sure "sort" has the highest priority + "words", "typo", "proximity", "attribute", "exactness"}, + SearchableAttributes: []string{ "title", "content", From 71d5ad2cf6811516f9cfabb46492df90795daa54 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 11:24:07 +0800 Subject: [PATCH 90/99] chore: remove Imprecise --- modules/indexer/issues/bleve/bleve.go | 5 ++-- modules/indexer/issues/db/db.go | 5 ++-- modules/indexer/issues/dbfilter.go | 27 ------------------- .../issues/elasticsearch/elasticsearch.go | 5 ++-- modules/indexer/issues/indexer.go | 9 ------- modules/indexer/issues/internal/model.go | 4 --- .../indexer/issues/internal/tests/tests.go | 6 +---- .../indexer/issues/meilisearch/meilisearch.go | 5 ++-- 8 files changed, 9 insertions(+), 57 deletions(-) delete mode 100644 modules/indexer/issues/dbfilter.go diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 2422066fd7035..7c82cfbb792ef 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -271,9 +271,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( } ret := &internal.SearchResult{ - Total: int64(result.Total), - Hits: make([]internal.Match, 0, len(result.Hits)), - Imprecise: false, + Total: int64(result.Total), + Hits: make([]internal.Match, 0, len(result.Hits)), } for _, hit := range result.Hits { id, err := indexer_internal.ParseBase36(hit.ID) diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 4544a0577a1d6..1016523b7291e 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -90,8 +90,7 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( }) } return &internal.SearchResult{ - Total: total, - Hits: hits, - Imprecise: false, + Total: total, + Hits: hits, }, nil } diff --git a/modules/indexer/issues/dbfilter.go b/modules/indexer/issues/dbfilter.go deleted file mode 100644 index ee44df6484880..0000000000000 --- a/modules/indexer/issues/dbfilter.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package issues - -import ( - "context" - - issue_model "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/modules/indexer/issues/db" - "code.gitea.io/gitea/modules/indexer/issues/internal" -) - -// reFilter filters the given issuesIDs by the database. -// It is used to filter out issues coming from some indexers that are not supported fining filtering. -// Once all indexers support filtering, this function and this file can be removed. -func reFilter(ctx context.Context, issuesIDs []int64, options *SearchOptions) ([]int64, error) { - opts, err := db.ToDBOptions(ctx, (*internal.SearchOptions)(options)) - if err != nil { - return nil, err - } - - opts.IssueIDs = issuesIDs - - ids, _, err := issue_model.IssueIDs(ctx, opts) - return ids, err -} diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 21526e5accde6..d059f76b3288a 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -258,9 +258,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( } return &internal.SearchResult{ - Total: searchResult.TotalHits(), - Hits: hits, - Imprecise: false, + Total: searchResult.TotalHits(), + Hits: hits, }, nil } diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 87497dbe8fb1e..42279cbddb648 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -307,14 +307,5 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, err ret = append(ret, hit.ID) } - if len(result.Hits) > 0 && result.Imprecise { - // The result is imprecise, we need to filter the result again. - ret, err := reFilter(ctx, ret, opts) - if err != nil { - return nil, 0, err - } - return ret, 0, nil - } - return ret, result.Total, nil } diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index e816996d93fa4..a799263fe8d23 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -52,10 +52,6 @@ type Match struct { type SearchResult struct { Total int64 Hits []Match - - // Imprecise indicates that the result is not accurate, and it needs second filtering and sorting by database. - // It could be removed when all engines support filtering and sorting. - Imprecise bool } // SearchOptions represents search options diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 0477a966257c5..93d38a0b37996 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -76,10 +76,6 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { assert.Equal(t, c.ExpectedIDs, ids) assert.Equal(t, c.ExpectedTotal, result.Total) } - if result.Imprecise { - // If an engine does not support complex queries, do not use TestIndexer to test it - t.Errorf("Expected imprecise to be false, got true") - } }) } } @@ -706,7 +702,7 @@ type testIndexerCase struct { SearchOptions *internal.SearchOptions - Expected func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) // if nil, use ExpectedIDs, ExpectedTotal and ExpectedImprecise + Expected func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) // if nil, use ExpectedIDs, ExpectedTotal ExpectedIDs []int64 ExpectedTotal int64 } diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index d258f80885018..335395f2f6714 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -228,9 +228,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( } return &internal.SearchResult{ - Total: searchRes.EstimatedTotalHits, - Hits: hits, - Imprecise: false, + Total: searchRes.EstimatedTotalHits, + Hits: hits, }, nil } From c65718be5905337e4c6e643f2e34b9312d21edf2 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 11:31:15 +0800 Subject: [PATCH 91/99] test: env for meilisearch --- .github/workflows/pull-db-tests.yml | 6 ++++++ modules/indexer/issues/elasticsearch/elasticsearch_test.go | 1 + modules/indexer/issues/meilisearch/meilisearch_test.go | 5 +++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 12e5e64e80763..d310731bb927a 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -98,6 +98,12 @@ jobs: discovery.type: single-node ports: - "9200:9200" + meilisearch: + image: getmeili/meilisearch:v1.2.0 + env: + MEILI_ENV: development # disable auth + ports: + - "7700:7700" smtpimap: image: tabascoterrier/docker-imap-devel:latest ports: diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_test.go b/modules/indexer/issues/elasticsearch/elasticsearch_test.go index 93cba79287f1b..236cbd65db475 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_test.go @@ -14,6 +14,7 @@ import ( ) func TestElasticsearchIndexer(t *testing.T) { + // The elasticsearch instance started by pull-db-tests.yml > test-unit > services > elasticsearch url := "http://elastic:changeme@elasticsearch:9200" if os.Getenv("CI") == "" { diff --git a/modules/indexer/issues/meilisearch/meilisearch_test.go b/modules/indexer/issues/meilisearch/meilisearch_test.go index f9fb36a473b5d..024e5a36e9006 100644 --- a/modules/indexer/issues/meilisearch/meilisearch_test.go +++ b/modules/indexer/issues/meilisearch/meilisearch_test.go @@ -14,8 +14,9 @@ import ( ) func TestMeilisearchIndexer(t *testing.T) { - url := "" // TODO: set meilisearch in unit tests - key := "" + // The meilisearch instance started by pull-db-tests.yml > test-unit > services > meilisearch + url := "http://meilisearch:7700" + key := "" // auth has been disabled in test environment if os.Getenv("CI") == "" { // Make it possible to run tests against a local meilisearch instance From 439c7984fee5be1c7afda93f288730ed7d3e6fb3 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 11:33:09 +0800 Subject: [PATCH 92/99] chore: remove debug code --- models/issues/issue_list.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/models/issues/issue_list.go b/models/issues/issue_list.go index e6f1127958b99..a932ac2554369 100644 --- a/models/issues/issue_list.go +++ b/models/issues/issue_list.go @@ -157,9 +157,7 @@ func (issues IssueList) loadLabels(ctx context.Context) error { if left < limit { limit = left } - sess := db.GetEngine(ctx).Table("label") // debug: - sess.MustLogSQL(true) - rows, err := sess. + rows, err := db.GetEngine(ctx).Table("label"). Join("LEFT", "issue_label", "issue_label.label_id = label.id"). In("issue_label.issue_id", issueIDs[:limit]). Asc("label.name"). From 70e780e293db17e3e2a9410ef3fafe3151cd7e62 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 11:43:28 +0800 Subject: [PATCH 93/99] docs: add comments --- modules/indexer/issues/internal/model.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index a799263fe8d23..d6fe39df3f2ad 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -37,6 +37,8 @@ type IndexerData struct { UpdatedUnix timeutil.TimeStamp `json:"updated_unix"` // Fields used for sorting + // UpdatedUnix is both used for filtering and sorting. + // ID is used for sorting too, to make the sorting stable. CreatedUnix timeutil.TimeStamp `json:"created_unix"` DeadlineUnix timeutil.TimeStamp `json:"deadline_unix"` CommentCount int64 `json:"comment_count"` From f12aba529e936ba1873d5888e8355b99a3f006cf Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 12:07:33 +0800 Subject: [PATCH 94/99] docs: add comments --- modules/indexer/issues/internal/model.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index d6fe39df3f2ad..31acd16bd44e0 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -107,6 +107,15 @@ const ( SortByCommentsAsc SortBy = "comment_count" SortByDeadlineAsc SortBy = "deadline_unix" // Unsupported sort types which are supported by issues.IssuesOptions.SortType: - // - "priorityrepo" - // - "project-column-sorting" + // + // - "priorityrepo": + // It's impossible to support it in the indexer. + // It is based on the specified repository in the request, so we cannot add static field to the indexer. + // If we do something like that query the issues in the specified repository first then append other issues, + // it will break the pagination. + // + // - "project-column-sorting": + // Although it's possible to support it by adding project.ProjectIssue.Sorting to the indexer, + // but what if the issue belongs to multiple projects? + // Since it's unsupported to search issues with keyword in project page, we don't need to support it. ) From 923b15fe71f04738b4b70bd0d350800e8553e16b Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 12:13:05 +0800 Subject: [PATCH 95/99] chore: add meilisearch to hosts --- .github/workflows/pull-db-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index d310731bb927a..7cddaff63b5ab 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -134,7 +134,7 @@ jobs: go-version: ">=1.20" check-latest: true - name: Add hosts to /etc/hosts - run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts' + run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch meilisearch smtpimap" | sudo tee -a /etc/hosts' - run: make deps-backend - run: make backend env: From 8f265d485cf991544e557caaa94544ac7b089d7f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 27 Jul 2023 12:14:54 +0800 Subject: [PATCH 96/99] test: sleep when waiting --- modules/indexer/issues/elasticsearch/elasticsearch_test.go | 1 + modules/indexer/issues/meilisearch/meilisearch_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_test.go b/modules/indexer/issues/elasticsearch/elasticsearch_test.go index 236cbd65db475..ffd85b1aa1dbb 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_test.go @@ -34,6 +34,7 @@ func TestElasticsearchIndexer(t *testing.T) { break } t.Logf("Waiting for elasticsearch to be up: %v", err) + time.Sleep(time.Second) } if !ok { t.Fatalf("Failed to wait for elasticsearch to be up") diff --git a/modules/indexer/issues/meilisearch/meilisearch_test.go b/modules/indexer/issues/meilisearch/meilisearch_test.go index 024e5a36e9006..3d7237268e1bd 100644 --- a/modules/indexer/issues/meilisearch/meilisearch_test.go +++ b/modules/indexer/issues/meilisearch/meilisearch_test.go @@ -36,6 +36,7 @@ func TestMeilisearchIndexer(t *testing.T) { break } t.Logf("Waiting for meilisearch to be up: %v", err) + time.Sleep(time.Second) } if !ok { t.Fatalf("Failed to wait for meilisearch to be up") From 60638bb90677e3c30d8b8a587ead0d480bb8103a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Jul 2023 21:44:23 +0800 Subject: [PATCH 97/99] Fix bug --- routers/web/user/home.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/routers/web/user/home.go b/routers/web/user/home.go index 1cdf5a8f0424b..02b9f9bd6b82d 100644 --- a/routers/web/user/home.go +++ b/routers/web/user/home.go @@ -533,7 +533,7 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { // Slice of Issues that will be displayed on the overview page // USING FINAL STATE OF opts FOR A QUERY. - var issues []*issues_model.Issue + var issues issues_model.IssueList { issueIDs, err := issueIDsFromSearch(ctx, keyword, opts) if err != nil { @@ -644,9 +644,13 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { ctx.Data["IssueRefEndNames"], ctx.Data["IssueRefURLs"] = issue_service.GetRefEndNamesAndURLs(issues, ctx.FormString("RepoLink")) + if err := issues.LoadAttributes(ctx); err != nil { + ctx.ServerError("issues.LoadAttributes", err) + return + } ctx.Data["Issues"] = issues - approvalCounts, err := issues_model.IssueList(issues).GetApprovalCounts(ctx) + approvalCounts, err := issues.GetApprovalCounts(ctx) if err != nil { ctx.ServerError("ApprovalCounts", err) return From 60fcb9c7f0437b446216009e44d78d1cf58db954 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Jul 2023 21:56:26 +0800 Subject: [PATCH 98/99] Fix bug --- routers/web/repo/issue.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index f38920261dca8..38ef782c83458 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -248,7 +248,7 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti } pager := context.NewPagination(total, setting.UI.IssuePagingNum, page, 5) - var issues []*issues_model.Issue + var issues issues_model.IssueList { ids, err := issueIDsFromSearch(ctx, keyword, &issues_model.IssuesOptions{ Paginator: &db.ListOptions{ @@ -283,8 +283,7 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti } } - issueList := issues_model.IssueList(issues) - approvalCounts, err := issueList.GetApprovalCounts(ctx) + approvalCounts, err := issues.GetApprovalCounts(ctx) if err != nil { ctx.ServerError("ApprovalCounts", err) return @@ -307,6 +306,11 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti return } + if err := issues.LoadAttributes(ctx); err != nil { + ctx.ServerError("issues.LoadAttributes", err) + return + } + ctx.Data["Issues"] = issues ctx.Data["CommitLastStatus"] = lastStatus ctx.Data["CommitStatuses"] = commitStatuses From 7010d536963e4e8e58a0fe8db58c58772a4fc7bf Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 28 Jul 2023 10:32:39 +0800 Subject: [PATCH 99/99] fix: stats if zero found --- routers/web/repo/issue.go | 17 +++++++++++++---- routers/web/user/home.go | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 38ef782c83458..aef3902005c83 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -222,11 +222,20 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti } statsOpts.IssueIDs = allIssueIDs } - issueStats, err = issues_model.GetIssueStats(statsOpts) - if err != nil { - ctx.ServerError("GetIssueStats", err) - return + if keyword != "" && len(statsOpts.IssueIDs) == 0 { + // So it did search with the keyword, but no issue found. + // Just set issueStats to empty. + issueStats = &issues_model.IssueStats{} + } else { + // So it did search with the keyword, and found some issues. It needs to get issueStats of these issues. + // Or the keyword is empty, so it doesn't need issueIDs as filter, just get issueStats with statsOpts. + issueStats, err = issues_model.GetIssueStats(statsOpts) + if err != nil { + ctx.ServerError("GetIssueStats", err) + return + } } + } isShowClosed := ctx.FormString("state") == "closed" diff --git a/routers/web/user/home.go b/routers/web/user/home.go index 02b9f9bd6b82d..77974a84a242e 100644 --- a/routers/web/user/home.go +++ b/routers/web/user/home.go @@ -485,10 +485,22 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { // Filter repos and count issues in them. Count will be used later. // USING NON-FINAL STATE OF opts FOR A QUERY. - issueCountByRepo, err := issues_model.CountIssuesByRepo(ctx, opts) - if err != nil { - ctx.ServerError("CountIssuesByRepo", err) - return + var issueCountByRepo map[int64]int64 + { + issueIDs, err := issueIDsFromSearch(ctx, keyword, opts) + if err != nil { + ctx.ServerError("issueIDsFromSearch", err) + return + } + if len(issueIDs) > 0 { // else, no issues found, just leave issueCountByRepo empty + opts.IssueIDs = issueIDs + issueCountByRepo, err = issues_model.CountIssuesByRepo(ctx, opts) + if err != nil { + ctx.ServerError("CountIssuesByRepo", err) + return + } + opts.IssueIDs = nil // reset, the opts will be used later + } } // Make sure page number is at least 1. Will be posted to ctx.Data. @@ -506,6 +518,7 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { var labelIDs []int64 selectedLabels := ctx.FormString("labels") if len(selectedLabels) > 0 && selectedLabels != "0" { + var err error labelIDs, err = base.StringsToInt64s(strings.Split(selectedLabels, ",")) if err != nil { ctx.ServerError("StringsToInt64s", err) @@ -606,10 +619,18 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) { statsOpts.IssueIDs = allIssueIDs } - issueStats, err = issues_model.GetUserIssueStats(filterMode, statsOpts) - if err != nil { - ctx.ServerError("GetUserIssueStats Shown", err) - return + if keyword != "" && len(statsOpts.IssueIDs) == 0 { + // So it did search with the keyword, but no issue found. + // Just set issueStats to empty. + issueStats = &issues_model.IssueStats{} + } else { + // So it did search with the keyword, and found some issues. It needs to get issueStats of these issues. + // Or the keyword is empty, so it doesn't need issueIDs as filter, just get issueStats with statsOpts. + issueStats, err = issues_model.GetUserIssueStats(filterMode, statsOpts) + if err != nil { + ctx.ServerError("GetUserIssueStats", err) + return + } } }