Fix doctor deleting orphaned issues attachments (#34142) (#34571)

Backport #34142 by @lunny

Fix the bug when deleting orphaned issues attachments. The attachments
maybe stored on other storages service rather than disk.

Co-authored-by: Lunny Xiao <[email protected]>

Author: GiteaBot

models/issues/issue_update.go

@@ -12,9 +12,7 @@ import (
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/organization"
 	access_model "code.gitea.io/gitea/models/perm/access"
-	project_model "code.gitea.io/gitea/models/project"
 	repo_model "code.gitea.io/gitea/models/repo"
-	system_model "code.gitea.io/gitea/models/system"
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/git"
@@ -715,138 +713,13 @@ func UpdateReactionsMigrationsByType(ctx context.Context, gitServiceType api.Git
 	return err
 }
 
-// DeleteIssuesByRepoID deletes issues by repositories id
-func DeleteIssuesByRepoID(ctx context.Context, repoID int64) (attachmentPaths []string, err error) {
-	// MariaDB has a performance bug: https://jira.mariadb.org/browse/MDEV-16289
-	// so here it uses "DELETE ... WHERE IN" with pre-queried IDs.
-	sess := db.GetEngine(ctx)
-
-	for {
-		issueIDs := make([]int64, 0, db.DefaultMaxInSize)
-
-		err := sess.Table(&Issue{}).Where("repo_id = ?", repoID).OrderBy("id").Limit(db.DefaultMaxInSize).Cols("id").Find(&issueIDs)
-		if err != nil {
-			return nil, err
-		}
-
-		if len(issueIDs) == 0 {
-			break
-		}
-
-		// Delete content histories
-		_, err = sess.In("issue_id", issueIDs).Delete(&ContentHistory{})
-		if err != nil {
-			return nil, err
-		}
-
-		// Delete comments and attachments
-		_, err = sess.In("issue_id", issueIDs).Delete(&Comment{})
-		if err != nil {
-			return nil, err
-		}
-
-		// Dependencies for issues in this repository
-		_, err = sess.In("issue_id", issueIDs).Delete(&IssueDependency{})
-		if err != nil {
-			return nil, err
-		}
-
-		// Delete dependencies for issues in other repositories
-		_, err = sess.In("dependency_id", issueIDs).Delete(&IssueDependency{})
-		if err != nil {
-			return nil, err
-		}
-
-		_, err = sess.In("issue_id", issueIDs).Delete(&IssueUser{})
-		if err != nil {
-			return nil, err
-		}
-
-		_, err = sess.In("issue_id", issueIDs).Delete(&Reaction{})
-		if err != nil {
-			return nil, err
-		}
-
-		_, err = sess.In("issue_id", issueIDs).Delete(&IssueWatch{})
-		if err != nil {
-			return nil, err
-		}
-
-		_, err = sess.In("issue_id", issueIDs).Delete(&Stopwatch{})
-		if err != nil {
-			return nil, err
-		}
-
-		_, err = sess.In("issue_id", issueIDs).Delete(&TrackedTime{})
-		if err != nil {
-			return nil, err
-		}
-
-		_, err = sess.In("issue_id", issueIDs).Delete(&project_model.ProjectIssue{})
-		if err != nil {
-			return nil, err
-		}
-
-		_, err = sess.In("dependent_issue_id", issueIDs).Delete(&Comment{})
-		if err != nil {
-			return nil, err
-		}
-
-		var attachments []*repo_model.Attachment
-		err = sess.In("issue_id", issueIDs).Find(&attachments)
-		if err != nil {
-			return nil, err
-		}
-
-		for j := range attachments {
-			attachmentPaths = append(attachmentPaths, attachments[j].RelativePath())
-		}
-
-		_, err = sess.In("issue_id", issueIDs).Delete(&repo_model.Attachment{})
-		if err != nil {
-			return nil, err
-		}
-
-		_, err = sess.In("id", issueIDs).Delete(&Issue{})
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return attachmentPaths, err
-}
-
-// DeleteOrphanedIssues delete issues without a repo
-func DeleteOrphanedIssues(ctx context.Context) error {
-	var attachmentPaths []string
-	err := db.WithTx(ctx, func(ctx context.Context) error {
-		var ids []int64
-
-		if err := db.GetEngine(ctx).Table("issue").Distinct("issue.repo_id").
-			Join("LEFT", "repository", "issue.repo_id=repository.id").
-			Where(builder.IsNull{"repository.id"}).GroupBy("issue.repo_id").
-			Find(&ids); err != nil {
-			return err
-		}
-
-		for i := range ids {
-			paths, err := DeleteIssuesByRepoID(ctx, ids[i])
-			if err != nil {
-				return err
-			}
-			attachmentPaths = append(attachmentPaths, paths...)
-		}
-
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-
-	// Remove issue attachment files.
-	for i := range attachmentPaths {
-		// FIXME: it's not right, because the attachment might not be on local filesystem
-		system_model.RemoveAllWithNotice(ctx, "Delete issue attachment", attachmentPaths[i])
+func GetOrphanedIssueRepoIDs(ctx context.Context) ([]int64, error) {
+	var repoIDs []int64
+	if err := db.GetEngine(ctx).Table("issue").Distinct("issue.repo_id").
+		Join("LEFT", "repository", "issue.repo_id=repository.id").
+		Where(builder.IsNull{"repository.id"}).
+		Find(&repoIDs); err != nil {
+		return nil, err
 	}
-	return nil
+	return repoIDs, nil
 }

services/doctor/dbconsistency.go

@@ -15,6 +15,7 @@ import (
 	secret_model "code.gitea.io/gitea/models/secret"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	issue_service "code.gitea.io/gitea/services/issue"
 )
 
 type consistencyCheck struct {
@@ -93,7 +94,7 @@ func prepareDBConsistencyChecks() []consistencyCheck {
 			// find issues without existing repository
 			Name:    "Orphaned Issues without existing repository",
 			Counter: issues_model.CountOrphanedIssues,
-			Fixer:   asFixer(issues_model.DeleteOrphanedIssues),
+			Fixer:   asFixer(issue_service.DeleteOrphanedIssues),
 		},
 		// find releases without existing repository
 		genericOrphanCheck("Orphaned Releases without existing repository",

services/issue/issue.go

@@ -190,9 +190,13 @@ func DeleteIssue(ctx context.Context, doer *user_model.User, gitRepo *git.Reposi
 	}
 
 	// delete entries in database
-	if err := deleteIssue(ctx, issue); err != nil {
+	attachmentPaths, err := deleteIssue(ctx, issue)
+	if err != nil {
 		return err
 	}
+	for _, attachmentPath := range attachmentPaths {
+		system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", attachmentPath)
+	}
 
 	// delete pull request related git data
 	if issue.IsPull && gitRepo != nil {
@@ -256,45 +260,45 @@ func GetRefEndNamesAndURLs(issues []*issues_model.Issue, repoLink string) (map[i
 }
 
 // deleteIssue deletes the issue
-func deleteIssue(ctx context.Context, issue *issues_model.Issue) error {
+func deleteIssue(ctx context.Context, issue *issues_model.Issue) ([]string, error) {
 	ctx, committer, err := db.TxContext(ctx)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer committer.Close()
 
-	e := db.GetEngine(ctx)
-	if _, err := e.ID(issue.ID).NoAutoCondition().Delete(issue); err != nil {
-		return err
+	if _, err := db.GetEngine(ctx).ID(issue.ID).NoAutoCondition().Delete(issue); err != nil {
+		return nil, err
 	}
 
 	// update the total issue numbers
 	if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, false); err != nil {
-		return err
+		return nil, err
 	}
 	// if the issue is closed, update the closed issue numbers
 	if issue.IsClosed {
 		if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, true); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	if err := issues_model.UpdateMilestoneCounters(ctx, issue.MilestoneID); err != nil {
-		return fmt.Errorf("error updating counters for milestone id %d: %w",
+		return nil, fmt.Errorf("error updating counters for milestone id %d: %w",
 			issue.MilestoneID, err)
 	}
 
 	if err := activities_model.DeleteIssueActions(ctx, issue.RepoID, issue.ID, issue.Index); err != nil {
-		return err
+		return nil, err
 	}
 
 	// find attachments related to this issue and remove them
-	if err := issue.LoadAttributes(ctx); err != nil {
-		return err
+	if err := issue.LoadAttachments(ctx); err != nil {
+		return nil, err
 	}
 
+	var attachmentPaths []string
 	for i := range issue.Attachments {
-		system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", issue.Attachments[i].RelativePath())
+		attachmentPaths = append(attachmentPaths, issue.Attachments[i].RelativePath())
 	}
 
 	// delete all database data still assigned to this issue
@@ -318,8 +322,68 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue) error {
 		&issues_model.Comment{DependentIssueID: issue.ID},
 		&issues_model.IssuePin{IssueID: issue.ID},
 	); err != nil {
+		return nil, err
+	}
+
+	if err := committer.Commit(); err != nil {
+		return nil, err
+	}
+	return attachmentPaths, nil
+}
+
+// DeleteOrphanedIssues delete issues without a repo
+func DeleteOrphanedIssues(ctx context.Context) error {
+	var attachmentPaths []string
+	err := db.WithTx(ctx, func(ctx context.Context) error {
+		repoIDs, err := issues_model.GetOrphanedIssueRepoIDs(ctx)
+		if err != nil {
+			return err
+		}
+		for i := range repoIDs {
+			paths, err := DeleteIssuesByRepoID(ctx, repoIDs[i])
+			if err != nil {
+				return err
+			}
+			attachmentPaths = append(attachmentPaths, paths...)
+		}
+		return nil
+	})
+	if err != nil {
 		return err
 	}
 
-	return committer.Commit()
+	// Remove issue attachment files.
+	for i := range attachmentPaths {
+		system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", attachmentPaths[i])
+	}
+	return nil
+}
+
+// DeleteIssuesByRepoID deletes issues by repositories id
+func DeleteIssuesByRepoID(ctx context.Context, repoID int64) (attachmentPaths []string, err error) {
+	for {
+		issues := make([]*issues_model.Issue, 0, db.DefaultMaxInSize)
+		if err := db.GetEngine(ctx).
+			Where("repo_id = ?", repoID).
+			OrderBy("id").
+			Limit(db.DefaultMaxInSize).
+			Find(&issues); err != nil {
+			return nil, err
+		}
+
+		if len(issues) == 0 {
+			break
+		}
+
+		for _, issue := range issues {
+			issueAttachPaths, err := deleteIssue(ctx, issue)
+			if err != nil {
+				return nil, fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err)
+			}
+
+			attachmentPaths = append(attachmentPaths, issueAttachPaths...)
+		}
+	}
+
+	return attachmentPaths, err
 }

services/issue/issue_test.go

@@ -44,7 +44,7 @@ func TestIssue_DeleteIssue(t *testing.T) {
 		ID:     issueIDs[2],
 	}
 
-	err = deleteIssue(db.DefaultContext, issue)
+	_, err = deleteIssue(db.DefaultContext, issue)
 	assert.NoError(t, err)
 	issueIDs, err = issues_model.GetIssueIDsByRepoID(db.DefaultContext, 1)
 	assert.NoError(t, err)
@@ -55,7 +55,7 @@ func TestIssue_DeleteIssue(t *testing.T) {
 	assert.NoError(t, err)
 	issue, err = issues_model.GetIssueByID(db.DefaultContext, 4)
 	assert.NoError(t, err)
-	err = deleteIssue(db.DefaultContext, issue)
+	_, err = deleteIssue(db.DefaultContext, issue)
 	assert.NoError(t, err)
 	assert.Len(t, attachments, 2)
 	for i := range attachments {
@@ -78,7 +78,7 @@ func TestIssue_DeleteIssue(t *testing.T) {
 	assert.NoError(t, err)
 	assert.False(t, left)
 
-	err = deleteIssue(db.DefaultContext, issue2)
+	_, err = deleteIssue(db.DefaultContext, issue2)
 	assert.NoError(t, err)
 	left, err = issues_model.IssueNoDependenciesLeft(db.DefaultContext, issue1)
 	assert.NoError(t, err)

services/repository/delete.go

@@ -29,6 +29,7 @@ import (
 	"code.gitea.io/gitea/modules/storage"
 	actions_service "code.gitea.io/gitea/services/actions"
 	asymkey_service "code.gitea.io/gitea/services/asymkey"
+	issue_service "code.gitea.io/gitea/services/issue"
 
 	"xorm.io/builder"
 )
@@ -193,7 +194,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID
 
 	// Delete Issues and related objects
 	var attachmentPaths []string
-	if attachmentPaths, err = issues_model.DeleteIssuesByRepoID(ctx, repoID); err != nil {
+	if attachmentPaths, err = issue_service.DeleteIssuesByRepoID(ctx, repoID); err != nil {
 		return err
 	}