Add Docker /v2/_catalog endpoint (#20469) (#20556)

* Added properties for packages.
* Fixed authenticate header format.
* Added _catalog endpoint.
* Check owner visibility.
* Extracted condition.
* Added test for _catalog.

Co-authored-by: 6543 <[email protected]>
Co-authored-by: KN4CK3R <[email protected]>
Co-authored-by: Lunny Xiao <[email protected]>
Co-authored-by: Lauris BH <[email protected]>
Co-authored-by: wxiaoguang <[email protected]>

Author: 5 people

integrations/api_packages_container_test.go

@@ -26,6 +26,7 @@ import (
 
 func TestPackageContainer(t *testing.T) {
 	defer prepareTestEnv(t)()
+
 	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User)
 
 	has := func(l packages_model.PackagePropertyList, name string) bool {
@@ -36,6 +37,15 @@ func TestPackageContainer(t *testing.T) {
 		}
 		return false
 	}
+	getAllByName := func(l packages_model.PackagePropertyList, name string) []string {
+		values := make([]string, 0, len(l))
+		for _, pp := range l {
+			if pp.Name == name {
+				values = append(values, pp.Value)
+			}
+		}
+		return values
+	}
 
 	images := []string{"test", "te/st"}
 	tags := []string{"latest", "main"}
@@ -66,7 +76,7 @@ func TestPackageContainer(t *testing.T) {
 			Token string `json:"token"`
 		}
 
-		authenticate := []string{`Bearer realm="` + setting.AppURL + `v2/token"`}
+		authenticate := []string{`Bearer realm="` + setting.AppURL + `v2/token",service="container_registry",scope="*"`}
 
 		t.Run("Anonymous", func(t *testing.T) {
 			defer PrintCurrentTest(t)()
@@ -236,7 +246,8 @@ func TestPackageContainer(t *testing.T) {
 						assert.Nil(t, pd.SemVer)
 						assert.Equal(t, image, pd.Package.Name)
 						assert.Equal(t, tag, pd.Version.Version)
-						assert.True(t, has(pd.Properties, container_module.PropertyManifestTagged))
+						assert.ElementsMatch(t, []string{strings.ToLower(user.LowerName + "/" + image)}, getAllByName(pd.PackageProperties, container_module.PropertyRepository))
+						assert.True(t, has(pd.VersionProperties, container_module.PropertyManifestTagged))
 
 						assert.IsType(t, &container_module.Metadata{}, pd.Metadata)
 						metadata := pd.Metadata.(*container_module.Metadata)
@@ -330,7 +341,8 @@ func TestPackageContainer(t *testing.T) {
 				assert.Nil(t, pd.SemVer)
 				assert.Equal(t, image, pd.Package.Name)
 				assert.Equal(t, untaggedManifestDigest, pd.Version.Version)
-				assert.False(t, has(pd.Properties, container_module.PropertyManifestTagged))
+				assert.ElementsMatch(t, []string{strings.ToLower(user.LowerName + "/" + image)}, getAllByName(pd.PackageProperties, container_module.PropertyRepository))
+				assert.False(t, has(pd.VersionProperties, container_module.PropertyManifestTagged))
 
 				assert.IsType(t, &container_module.Metadata{}, pd.Metadata)
 
@@ -362,18 +374,10 @@ func TestPackageContainer(t *testing.T) {
 				assert.Nil(t, pd.SemVer)
 				assert.Equal(t, image, pd.Package.Name)
 				assert.Equal(t, multiTag, pd.Version.Version)
-				assert.True(t, has(pd.Properties, container_module.PropertyManifestTagged))
+				assert.ElementsMatch(t, []string{strings.ToLower(user.LowerName + "/" + image)}, getAllByName(pd.PackageProperties, container_module.PropertyRepository))
+				assert.True(t, has(pd.VersionProperties, container_module.PropertyManifestTagged))
 
-				getAllByName := func(l packages_model.PackagePropertyList, name string) []string {
-					values := make([]string, 0, len(l))
-					for _, pp := range l {
-						if pp.Name == name {
-							values = append(values, pp.Value)
-						}
-					}
-					return values
-				}
-				assert.ElementsMatch(t, []string{manifestDigest, untaggedManifestDigest}, getAllByName(pd.Properties, container_module.PropertyManifestReference))
+				assert.ElementsMatch(t, []string{manifestDigest, untaggedManifestDigest}, getAllByName(pd.VersionProperties, container_module.PropertyManifestReference))
 
 				assert.IsType(t, &container_module.Metadata{}, pd.Metadata)
 				metadata := pd.Metadata.(*container_module.Metadata)
@@ -528,4 +532,56 @@ func TestPackageContainer(t *testing.T) {
 			})
 		})
 	}
+
+	t.Run("OwnerNameChange", func(t *testing.T) {
+		defer PrintCurrentTest(t)()
+
+		checkCatalog := func(owner string) func(t *testing.T) {
+			return func(t *testing.T) {
+				defer PrintCurrentTest(t)()
+
+				req := NewRequest(t, "GET", fmt.Sprintf("%sv2/_catalog", setting.AppURL))
+				addTokenAuthHeader(req, userToken)
+				resp := MakeRequest(t, req, http.StatusOK)
+
+				type RepositoryList struct {
+					Repositories []string `json:"repositories"`
+				}
+
+				repoList := &RepositoryList{}
+				DecodeJSON(t, resp, &repoList)
+
+				assert.Len(t, repoList.Repositories, len(images))
+				names := make([]string, 0, len(images))
+				for _, image := range images {
+					names = append(names, strings.ToLower(owner+"/"+image))
+				}
+				assert.ElementsMatch(t, names, repoList.Repositories)
+			}
+		}
+
+		t.Run(fmt.Sprintf("Catalog[%s]", user.LowerName), checkCatalog(user.LowerName))
+
+		session := loginUser(t, user.Name)
+
+		newOwnerName := "newUsername"
+
+		req := NewRequestWithValues(t, "POST", "/user/settings", map[string]string{
+			"_csrf":    GetCSRF(t, session, "/user/settings"),
+			"name":     newOwnerName,
+			"email":    "[email protected]",
+			"language": "en-US",
+		})
+		session.MakeRequest(t, req, http.StatusSeeOther)
+
+		t.Run(fmt.Sprintf("Catalog[%s]", newOwnerName), checkCatalog(newOwnerName))
+
+		req = NewRequestWithValues(t, "POST", "/user/settings", map[string]string{
+			"_csrf":    GetCSRF(t, session, "/user/settings"),
+			"name":     user.Name,
+			"email":    "[email protected]",
+			"language": "en-US",
+		})
+		session.MakeRequest(t, req, http.StatusSeeOther)
+	})
 }

integrations/api_packages_npm_test.go

@@ -85,9 +85,9 @@ func TestPackageNpm(t *testing.T) {
 		assert.IsType(t, &npm.Metadata{}, pd.Metadata)
 		assert.Equal(t, packageName, pd.Package.Name)
 		assert.Equal(t, packageVersion, pd.Version.Version)
-		assert.Len(t, pd.Properties, 1)
-		assert.Equal(t, npm.TagProperty, pd.Properties[0].Name)
-		assert.Equal(t, packageTag, pd.Properties[0].Value)
+		assert.Len(t, pd.VersionProperties, 1)
+		assert.Equal(t, npm.TagProperty, pd.VersionProperties[0].Name)
+		assert.Equal(t, packageTag, pd.VersionProperties[0].Value)
 
 		pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
 		assert.NoError(t, err)

models/migrations/migrations.go

@@ -396,6 +396,10 @@ var migrations = []Migration{
 	NewMigration("Alter hook_task table TEXT fields to LONGTEXT", alterHookTaskTextFieldsToLongText),
 	// v218 -> v219
 	NewMigration("Improve Action table indices v2", improveActionTableIndices),
+	// v219 -> v220
+	NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnCommitColForPushMirror),
+	// v220 -> v221
+	NewMigration("Add container repository property", addContainerRepositoryProperty),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v219.go

@@ -0,0 +1,31 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func addSyncOnCommitColForPushMirror(x *xorm.Engine) error {
+	type PushMirror struct {
+		ID         int64            `xorm:"pk autoincr"`
+		RepoID     int64            `xorm:"INDEX"`
+		Repo       *repo.Repository `xorm:"-"`
+		RemoteName string
+
+		SyncOnCommit   bool `xorm:"NOT NULL DEFAULT true"`
+		Interval       time.Duration
+		CreatedUnix    timeutil.TimeStamp `xorm:"created"`
+		LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
+		LastError      string             `xorm:"text"`
+	}
+
+	return x.Sync2(new(PushMirror))
+}

models/migrations/v220.go

@@ -0,0 +1,29 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	packages_model "code.gitea.io/gitea/models/packages"
+	container_module "code.gitea.io/gitea/modules/packages/container"
+
+	"xorm.io/xorm"
+	"xorm.io/xorm/schemas"
+)
+
+func addContainerRepositoryProperty(x *xorm.Engine) error {
+	switch x.Dialect().URI().DBType {
+	case schemas.SQLITE:
+		_, err := x.Exec("INSERT INTO package_property (ref_type, ref_id, name, value) SELECT ?, p.id, ?, u.lower_name || '/' || p.lower_name FROM package p JOIN `user` u ON p.owner_id = u.id WHERE p.type = ?", packages_model.PropertyTypePackage, container_module.PropertyRepository, packages_model.TypeContainer)
+		if err != nil {
+			return err
+		}
+	default:
+		_, err := x.Exec("INSERT INTO package_property (ref_type, ref_id, name, value) SELECT ?, p.id, ?, CONCAT(u.lower_name, '/', p.lower_name) FROM package p JOIN `user` u ON p.owner_id = u.id WHERE p.type = ?", packages_model.PropertyTypePackage, container_module.PropertyRepository, packages_model.TypeContainer)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

models/packages/container/search.go

@@ -12,6 +12,7 @@ import (
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/packages"
+	user_model "code.gitea.io/gitea/models/user"
 	container_module "code.gitea.io/gitea/modules/packages/container"
 
 	"xorm.io/builder"
@@ -210,6 +211,7 @@ func SearchImageTags(ctx context.Context, opts *ImageTagsSearchOptions) ([]*pack
 	return pvs, count, err
 }
 
+// SearchExpiredUploadedBlobs gets all uploaded blobs which are older than specified
 func SearchExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) ([]*packages.PackageFile, error) {
 	var cond builder.Cond = builder.Eq{
 		"package_version.is_internal":   true,
@@ -225,3 +227,37 @@ func SearchExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) ([
 		Where(cond).
 		Find(&pfs)
 }
+
+// GetRepositories gets a sorted list of all repositories
+func GetRepositories(ctx context.Context, actor *user_model.User, n int, last string) ([]string, error) {
+	var cond builder.Cond = builder.Eq{
+		"package.type":              packages.TypeContainer,
+		"package_property.ref_type": packages.PropertyTypePackage,
+		"package_property.name":     container_module.PropertyRepository,
+	}
+
+	cond = cond.And(builder.Exists(
+		builder.
+			Select("package_version.id").
+			Where(builder.Eq{"package_version.is_internal": false}.And(builder.Expr("package.id = package_version.package_id"))).
+			From("package_version"),
+	))
+
+	if last != "" {
+		cond = cond.And(builder.Gt{"package_property.value": strings.ToLower(last)})
+	}
+
+	cond = cond.And(user_model.BuildCanSeeUserCondition(actor))
+
+	sess := db.GetEngine(ctx).
+		Table("package").
+		Select("package_property.value").
+		Join("INNER", "user", "`user`.id = package.owner_id").
+		Join("INNER", "package_property", "package_property.ref_id = package.id").
+		Where(cond).
+		Asc("package_property.value").
+		Limit(n)
+
+	repositories := make([]string, 0, n)
+	return repositories, sess.Find(&repositories)
+}

models/packages/descriptor.go

@@ -40,15 +40,16 @@ func (l PackagePropertyList) GetByName(name string) string {
 
 // PackageDescriptor describes a package
 type PackageDescriptor struct {
-	Package    *Package
-	Owner      *user_model.User
-	Repository *repo_model.Repository
-	Version    *PackageVersion
-	SemVer     *version.Version
-	Creator    *user_model.User
-	Properties PackagePropertyList
-	Metadata   interface{}
-	Files      []*PackageFileDescriptor
+	Package           *Package
+	Owner             *user_model.User
+	Repository        *repo_model.Repository
+	Version           *PackageVersion
+	SemVer            *version.Version
+	Creator           *user_model.User
+	PackageProperties PackagePropertyList
+	VersionProperties PackagePropertyList
+	Metadata          interface{}
+	Files             []*PackageFileDescriptor
 }
 
 // PackageFileDescriptor describes a package file
@@ -102,6 +103,10 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
 			return nil, err
 		}
 	}
+	pps, err := GetProperties(ctx, PropertyTypePackage, p.ID)
+	if err != nil {
+		return nil, err
+	}
 	pvps, err := GetProperties(ctx, PropertyTypeVersion, pv.ID)
 	if err != nil {
 		return nil, err
@@ -152,15 +157,16 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
 	}
 
 	return &PackageDescriptor{
-		Package:    p,
-		Owner:      o,
-		Repository: repository,
-		Version:    pv,
-		SemVer:     semVer,
-		Creator:    creator,
-		Properties: PackagePropertyList(pvps),
-		Metadata:   metadata,
-		Files:      pfds,
+		Package:           p,
+		Owner:             o,
+		Repository:        repository,
+		Version:           pv,
+		SemVer:            semVer,
+		Creator:           creator,
+		PackageProperties: PackagePropertyList(pps),
+		VersionProperties: PackagePropertyList(pvps),
+		Metadata:          metadata,
+		Files:             pfds,
 	}, nil
 }
 

models/packages/package.go

@@ -131,6 +131,12 @@ func TryInsertPackage(ctx context.Context, p *Package) (*Package, error) {
 	return p, nil
 }
 
+// DeletePackageByID deletes a package by id
+func DeletePackageByID(ctx context.Context, packageID int64) error {
+	_, err := db.GetEngine(ctx).ID(packageID).Delete(&Package{})
+	return err
+}
+
 // SetRepositoryLink sets the linked repository
 func SetRepositoryLink(ctx context.Context, packageID, repoID int64) error {
 	_, err := db.GetEngine(ctx).ID(packageID).Cols("repo_id").Update(&Package{RepoID: repoID})
@@ -192,21 +198,20 @@ func GetPackagesByType(ctx context.Context, ownerID int64, packageType Type) ([]
 		Find(&ps)
 }
 
-// DeletePackagesIfUnreferenced deletes a package if there are no associated versions
-func DeletePackagesIfUnreferenced(ctx context.Context) error {
+// FindUnreferencedPackages gets all packages without associated versions
+func FindUnreferencedPackages(ctx context.Context) ([]*Package, error) {
 	in := builder.
 		Select("package.id").
 		From("package").
 		LeftJoin("package_version", "package_version.package_id = package.id").
 		Where(builder.Expr("package_version.id IS NULL"))
 
-	_, err := db.GetEngine(ctx).
+	ps := make([]*Package, 0, 10)
+	return ps, db.GetEngine(ctx).
 		// double select workaround for MySQL
 		// https://stackoverflow.com/questions/4471277/mysql-delete-from-with-subquery-as-condition
 		Where(builder.In("package.id", builder.Select("id").From(in, "temp"))).
-		Delete(&Package{})
-
-	return err
+		Find(&ps)
 }
 
 // HasOwnerPackages tests if a user/org has packages

models/packages/package_property.go

@@ -21,9 +21,11 @@ const (
 	PropertyTypeVersion PropertyType = iota // 0
 	// PropertyTypeFile means the reference is a package file
 	PropertyTypeFile // 1
+	// PropertyTypePackage means the reference is a package
+	PropertyTypePackage // 2
 )
 
-// PackageProperty represents a property of a package version or file
+// PackageProperty represents a property of a package, version or file
 type PackageProperty struct {
 	ID      int64        `xorm:"pk autoincr"`
 	RefType PropertyType `xorm:"INDEX NOT NULL"`
@@ -68,3 +70,9 @@ func DeletePropertyByID(ctx context.Context, propertyID int64) error {
 	_, err := db.GetEngine(ctx).ID(propertyID).Delete(&PackageProperty{})
 	return err
 }
+
+// DeletePropertyByName deletes properties by name
+func DeletePropertyByName(ctx context.Context, refType PropertyType, refID int64, name string) error {
+	_, err := db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Delete(&PackageProperty{})
+	return err
+}