From 51bc02ecb9bc53cfeee6e81b3aff4992561f6759 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 11 Jul 2021 19:02:13 +0000 Subject: [PATCH 001/130] Added package store settings. --- custom/conf/app.example.ini | 10 ++++++++++ modules/setting/repository.go | 5 +++++ modules/storage/storage.go | 15 ++++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 251ec7a80e1cd..6186f7c638332 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2085,6 +2085,16 @@ PATH = ;[lfs] ;STORAGE_TYPE = local +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; settings for packages, will override storage setting +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;[storage.packages] +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; storage type +;STORAGE_TYPE = local + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; customize storage diff --git a/modules/setting/repository.go b/modules/setting/repository.go index c2a6357d94622..63230468a6e5b 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -255,6 +255,10 @@ var ( RepoArchive = struct { Storage }{} + + Packages = struct { + Storage + }{} ) func newRepository() { @@ -334,4 +338,5 @@ func newRepository() { } RepoArchive.Storage = getStorage("repo-archive", "", nil) + Packages.Storage = getStorage("packages", "", nil) } diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 2fe14b41ef0ec..58b953745f075 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -117,6 +117,9 @@ var ( // RepoArchives represents repository archives storage RepoArchives ObjectStorage + + // Packages represents packages storage + Packages ObjectStorage ) // Init init the stoarge @@ -137,7 +140,11 @@ func Init() error { return err } - return initRepoArchives() + if err := initRepoArchives(); err != nil { + return err + } + + return initPackages() } // NewStorage takes a storage type and some config and returns an ObjectStorage or an error @@ -182,3 +189,9 @@ func initRepoArchives() (err error) { RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage) return } + +func initPackages() (err error) { + log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type) + Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage) + return +} From 48c2fafe62fc985bcab83d8767d99f80d56db521 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 11 Jul 2021 19:30:23 +0000 Subject: [PATCH 002/130] Added models. --- models/migrations/migrations.go | 2 + models/migrations/v188.go | 44 +++++++++ models/models.go | 2 + models/package.go | 158 ++++++++++++++++++++++++++++++++ 4 files changed, 206 insertions(+) create mode 100644 models/migrations/v188.go create mode 100644 models/package.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index de60d89bbe781..d9a51314ea616 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -325,6 +325,8 @@ var migrations = []Migration{ NewMigration("Create protected tag table", createProtectedTagTable), // v187 -> v188 NewMigration("Drop unneeded webhook related columns", dropWebhookColumns), + // v188 -> v189 + NewMigration("Add package tables", addPackageTables), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v188.go b/models/migrations/v188.go new file mode 100644 index 0000000000000..4f41b54f9f35b --- /dev/null +++ b/models/migrations/v188.go @@ -0,0 +1,44 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func addPackageTables(x *xorm.Engine) error { + type Package struct { + ID int64 `xorm:"pk autoincr"` + RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatorID int64 + Type int `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` + MetaData interface{} `xorm:"TEXT JSON"` + + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + } + + if err := x.Sync2(new(Package)); err != nil { + return err + } + + type PackageFile struct { + ID int64 `xorm:"pk autoincr"` + PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + Size int64 + Name string `xorm:"UNIQUE(s) NOT NULL"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + } + + return x.Sync2(new(PackageFile)) +} diff --git a/models/models.go b/models/models.go index 610933d3270bd..c2e0a3df3b7bc 100644 --- a/models/models.go +++ b/models/models.go @@ -138,6 +138,8 @@ func init() { new(PushMirror), new(RepoArchiver), new(ProtectedTag), + new(Package), + new(PackageFile), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/package.go b/models/package.go new file mode 100644 index 0000000000000..e6572682e3f29 --- /dev/null +++ b/models/package.go @@ -0,0 +1,158 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "errors" + "strings" + + "code.gitea.io/gitea/modules/timeutil" +) + +// PackageType +type PackageType int + +// Note: new type must append to the end of list to maintain compatibility. +const ( + PackageGeneric PackageType = iota +) + +var ( + // ErrDuplicatePackage indicates a duplicated package error + ErrDuplicatePackage = errors.New("Package does exist already") + // ErrPackageNotExist indicates a package not exist error + ErrPackageNotExist = errors.New("Package does not exist") +) + +// Package represents a package +type Package struct { + ID int64 `xorm:"pk autoincr"` + RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatorID int64 + Type PackageType `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` + MetaData interface{} `xorm:"TEXT JSON"` + + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` +} + +// PackageFile represents files associated with a package +type PackageFile struct { + ID int64 `xorm:"pk autoincr"` + PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + Size int64 + Name string `xorm:"UNIQUE(s) NOT NULL"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` +} + +// GetFiles loads all files associated with the package +func (p *Package) GetFiles() ([]*PackageFile, error) { + packageFiles := make([]*PackageFile, 0, 10) + return packageFiles, x.Where("package_id = ?", p.ID).Find(&packageFiles) +} + +// TryInsertPackage inserts a package +// If a package already exists ErrDuplicatePackage is returned +func TryInsertPackage(p *Package) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + has, err := sess.Get(p) + if err != nil { + return err + } + + if has { + return ErrDuplicatePackage + } + + if _, err = sess.Insert(p); err != nil { + return err + } + + return sess.Commit() +} + +// DeletePackageByID deletes a package and its files by ID +func DeletePackageByID(packageID int64) error { + if err := DeletePackageFilesByPackageID(packageID); err != nil { + return err + } + + _, err := x.ID(packageID).Delete(&Package{}) + return err +} + +// DeletePackagesByRepositoryID deletes all packages of a repository +func DeletePackagesByRepositoryID(repositoryID int64) error { + packages, err := GetPackagesByRepositoryID(repositoryID) + if err != nil { + return err + } + + for _, p := range packages { + if err := DeletePackageByID(p.ID); err != nil { + return err + } + } + + return nil +} + +// GetPackagesByRepositoryID returns all packages of a repository +func GetPackagesByRepositoryID(repositoryID int64) ([]*Package, error) { + packages := make([]*Package, 0, 10) + return packages, x.Where("repository_id = ?", repositoryID).Find(&packages) +} + +// GetPackagesByName gets all repository packages with the specific name +func GetPackagesByName(repositoryID int64, packageType PackageType, packageName string) ([]*Package, error) { + packages := make([]*Package, 0, 10) + return packages, x.Where("repository_id = ? AND type = ? AND lower_name = ?", repositoryID, packageType, strings.ToLower(packageName)).Find(&packages) +} + +// GetPackageByNameAndVersion gets a repository package by name and version +func GetPackageByNameAndVersion(repositoryID int64, packageType PackageType, packageName, packageVersion string) (*Package, error) { + p := &Package{ + RepositoryID: repositoryID, + Type: packageType, + LowerName: strings.ToLower(packageName), + Version: packageVersion, + } + has, err := x.Get(p) + if err != nil { + return nil, err + } else if !has { + return nil, ErrPackageNotExist + } + return p, nil +} + +// InsertPackageFile inserts a package file +func InsertPackageFile(pf *PackageFile) error { + _, err := x.Insert(pf) + return err +} + +// DeletePackageFileByID deletes a package file +func DeletePackageFileByID(fileID int64) error { + _, err := x.ID(fileID).Delete(&PackageFile{}) + return err +} + +// DeletePackageFilesByPackageID deletes all files associated with the package +func DeletePackageFilesByPackageID(packageID int64) error { + _, err := x.Where("package_id = ?", packageID).Delete(&PackageFile{}) + return err +} From 3e801f9ec5e86da36210d7932e6bc581ca1802f9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 11 Jul 2021 19:50:17 +0000 Subject: [PATCH 003/130] Added generic package registry. --- models/package.go | 2 +- modules/packages/content_store.go | 43 +++++ modules/util/filebuffer/file_backed_buffer.go | 88 ++++++++++ routers/api/v1/api.go | 10 ++ routers/api/v1/packages/generic/generic.go | 152 ++++++++++++++++++ services/packages/packages.go | 95 +++++++++++ 6 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 modules/packages/content_store.go create mode 100644 modules/util/filebuffer/file_backed_buffer.go create mode 100644 routers/api/v1/packages/generic/generic.go create mode 100644 services/packages/packages.go diff --git a/models/package.go b/models/package.go index e6572682e3f29..351a78fd25979 100644 --- a/models/package.go +++ b/models/package.go @@ -11,7 +11,7 @@ import ( "code.gitea.io/gitea/modules/timeutil" ) -// PackageType +// PackageType specifies the different package types type PackageType int // Note: new type must append to the end of list to maintain compatibility. diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go new file mode 100644 index 0000000000000..5c22c8eaf5119 --- /dev/null +++ b/modules/packages/content_store.go @@ -0,0 +1,43 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "fmt" + "io" + + "code.gitea.io/gitea/modules/storage" +) + +// ContentStore is a wrapper around ObjectStorage +type ContentStore struct { + store storage.ObjectStorage +} + +// NewContentStore creates the default package store +func NewContentStore() *ContentStore { + contentStore := &ContentStore{storage.Packages} + return contentStore +} + +// Get get the package file content +func (s *ContentStore) Get(packageFileID int64) (storage.Object, error) { + return s.store.Open(toRelativePath(packageFileID)) +} + +// Save stores the package file content +func (s *ContentStore) Save(packageFileID int64, r io.Reader, size int64) error { + _, err := s.store.Save(toRelativePath(packageFileID), r, size) + return err +} + +// Delete deletes the package file content +func (s *ContentStore) Delete(packageFileID int64) error { + return s.store.Delete(toRelativePath(packageFileID)) +} + +func toRelativePath(packageFileID int64) string { + return fmt.Sprintf("%d/package", packageFileID) +} diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go new file mode 100644 index 0000000000000..fca4aa3f0f8aa --- /dev/null +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -0,0 +1,88 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package filebuffer + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "math" + "os" +) + +var ( + // ErrInvalidMemorySize occurs if the memory size is not in a valid range + ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") +) + +// FileBackedBuffer implements io.ReadSeekCloser +type FileBackedBuffer struct { + size int64 + buffer *bytes.Reader + tmpFile *os.File +} + +// CreateFromReader creates a file backed buffer which uses a buffer with a fixed memory size. +// If more data is available a temporary file is used to store the data. +func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) { + if maxMemorySize < 0 || maxMemorySize == math.MaxInt32 { + return nil, ErrInvalidMemorySize + } + + var buf bytes.Buffer + n, err := io.CopyN(&buf, r, int64(maxMemorySize+1)) + if err == io.EOF { + return &FileBackedBuffer{ + size: n, + buffer: bytes.NewReader(buf.Bytes()), + }, nil + } + file, err := ioutil.TempFile("", "buffer-") + if err != nil { + return nil, err + } + + n, err = io.Copy(file, io.MultiReader(&buf, r)) + if err != nil { + return nil, err + } + + return &FileBackedBuffer{ + size: n, + tmpFile: file, + }, nil +} + +// Size returns the byte size of the buffered data +func (b *FileBackedBuffer) Size() int64 { + return b.size +} + +// Read implements io.Reader +func (b *FileBackedBuffer) Read(p []byte) (int, error) { + if b.tmpFile != nil { + return b.tmpFile.Read(p) + } + return b.buffer.Read(p) +} + +// Seek implements io.Seeker +func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { + if b.tmpFile != nil { + return b.tmpFile.Seek(offset, whence) + } + return b.buffer.Seek(offset, whence) +} + +// Close implements io.Closer +func (b *FileBackedBuffer) Close() error { + if b.tmpFile != nil { + err := b.tmpFile.Close() + os.Remove(b.tmpFile.Name()) + return err + } + return nil +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index b4f14bf2d1e03..e860443c05f3f 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -79,6 +79,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/misc" "code.gitea.io/gitea/routers/api/v1/notify" "code.gitea.io/gitea/routers/api/v1/org" + "code.gitea.io/gitea/routers/api/v1/packages/generic" "code.gitea.io/gitea/routers/api/v1/repo" "code.gitea.io/gitea/routers/api/v1/settings" _ "code.gitea.io/gitea/routers/api/v1/swagger" // for swagger generation @@ -970,6 +971,15 @@ func Routes() *web.Route { }, reqAnyRepoReader()) m.Get("/issue_templates", context.ReferencesGitRepo(false), repo.GetIssueTemplates) m.Get("/languages", reqRepoReader(models.UnitTypeCode), repo.GetLanguages) + m.Group("/packages", func() { + m.Group("/generic", func() { + m.Group("/{packagename}/{packageversion}/{filename}", func() { + m.Get("", generic.DownloadPackage) + m.Put("" /*reqRepoWriter(models.UnitTypePackage),*/, generic.UploadPackage) + m.Delete("" /*reqRepoWriter(models.UnitTypePackage),*/, generic.DeletePackage) + }) + }) + }, reqToken()) }, repoAssignment()) }) diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go new file mode 100644 index 0000000000000..c713fa7153064 --- /dev/null +++ b/routers/api/v1/packages/generic/generic.go @@ -0,0 +1,152 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package generic + +import ( + "errors" + "fmt" + "io" + "net/http" + "regexp" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/util/filebuffer" + + package_service "code.gitea.io/gitea/services/packages" +) + +var packageNameRegex = regexp.MustCompile(`\A[A-Za-z0-9\.\_\-\+]+\z`) +var packageVersionRegex = regexp.MustCompile(`\A(?:\.?[\w\+-]+\.?)+\z`) +var filenameRegex = packageNameRegex + +// DownloadPackage serves the specific generic package. +func DownloadPackage(ctx *context.APIContext) { + packageName, packageVersion, filename, err := sanitizeParameters(ctx) + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + + p, err := models.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, models.PackageGeneric, packageName, packageVersion) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.Error(http.StatusBadRequest, "", err) + return + } + log.Error("Error getting package: %v", err) + ctx.Error(http.StatusInternalServerError, "", "") + return + } + + pfs, err := p.GetFiles() + if err != nil { + log.Error("Error getting package files: %v", err) + ctx.Error(http.StatusInternalServerError, "", "") + return + } + + pf := pfs[0] + + if !strings.EqualFold(pf.LowerName, filename) { + ctx.Error(http.StatusBadRequest, "", models.ErrPackageNotExist) + return + } + + packageStore := packages.NewContentStore() + s, err := packageStore.Get(pf.ID) + if err != nil { + log.Error("Error reading package file: %v", err) + ctx.Error(http.StatusInternalServerError, "", "") + return + } + defer s.Close() + + ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, pf.Name)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + _, err = io.Copy(ctx.Resp, s) + if err != nil { + log.Error("Error in io.Copy: %v", err) + } +} + +// UploadPackage uploads the specific generic package. +// Duplicated packages get rejected. +func UploadPackage(ctx *context.APIContext) { + packageName, packageVersion, filename, err := sanitizeParameters(ctx) + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + + defer ctx.Req.Body.Close() + r, err := filebuffer.CreateFromReader(ctx.Req.Body, 32*1024*1024) + if err != nil { + log.Error("Error in CreateFromReader: %v", err) + ctx.Error(http.StatusInternalServerError, "", "") + return + } + defer r.Close() + + p, err := package_service.CreatePackage( + ctx.User, + ctx.Repo.Repository, + models.PackageGeneric, + packageName, + packageVersion, + nil, + false, + ) + if err != nil { + if err == models.ErrDuplicatePackage { + ctx.Error(http.StatusBadRequest, "", err) + return + } + log.Error("Error in CreatePackage: %v", err) + ctx.Error(http.StatusInternalServerError, "", "") + return + } + + _, err = package_service.AddFileToPackage(p, filename, r.Size(), r) + if err != nil { + log.Error("Error in AddFileToPackage: %v", err) + if err := models.DeletePackageByID(p.ID); err != nil { + log.Error("Error in DeletePackageByID: %v", err) + } + ctx.Error(http.StatusInternalServerError, "", "") + } +} + +// DeletePackage deletes the specific generic package. +func DeletePackage(ctx *context.APIContext) { + packageName, packageVersion, _, err := sanitizeParameters(ctx) + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + + err = package_service.DeletePackage(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.Error(http.StatusBadRequest, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", "") + } +} + +func sanitizeParameters(ctx *context.APIContext) (packageName, packageVersion, filename string, err error) { + packageName = ctx.Params("packagename") + packageVersion = ctx.Params("packageversion") + filename = ctx.Params("filename") + + if !packageNameRegex.MatchString(packageName) || !packageVersionRegex.MatchString(packageVersion) || !filenameRegex.MatchString(filename) { + err = errors.New("Invalid package name, package version or filename") + } + return +} diff --git a/services/packages/packages.go b/services/packages/packages.go new file mode 100644 index 0000000000000..f9b4f18bbbf33 --- /dev/null +++ b/services/packages/packages.go @@ -0,0 +1,95 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "io" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" +) + +// CreatePackage creates a new package +func CreatePackage(creator *models.User, repository *models.Repository, packageType models.PackageType, name, version string, metaData interface{}, allowDuplicate bool) (*models.Package, error) { + p := &models.Package{ + RepositoryID: repository.ID, + CreatorID: creator.ID, + Type: packageType, + Name: name, + LowerName: strings.ToLower(name), + Version: version, + MetaData: metaData, + } + if err := models.TryInsertPackage(p); err != nil { + if err == models.ErrDuplicatePackage { + if allowDuplicate { + return p, nil + } + return nil, err + } + log.Error("Error inserting package: %v", err) + return nil, err + } + return p, nil +} + +// AddFileToPackage adds a new file to package and stores its content +func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reader) (*models.PackageFile, error) { + pf := &models.PackageFile{ + PackageID: p.ID, + Size: size, + Name: filename, + LowerName: strings.ToLower(filename), + } + if err := models.InsertPackageFile(pf); err != nil { + log.Error("Error inserting package file: %v", err) + return nil, err + } + + packageStore := packages_module.NewContentStore() + if err := packageStore.Save(pf.ID, r, size); err != nil { + log.Error("Error saving package file: %v", err) + if err := models.DeletePackageFileByID(pf.ID); err != nil { + log.Error("Error deleting package file: %v", err) + } + return nil, err + } + return pf, nil +} + +// DeletePackage deletes a package and all associated files +func DeletePackage(repository *models.Repository, packageType models.PackageType, name, version string) error { + p, err := models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) + if err != nil { + if err == models.ErrPackageNotExist { + return err + } + log.Error("Error getting package: %v", err) + return err + } + + pfs, err := p.GetFiles() + if err != nil { + log.Error("Error getting package files: %v", err) + return err + } + + contentStore := packages_module.NewContentStore() + for _, pf := range pfs { + if err := contentStore.Delete(pf.ID); err != nil { + log.Error("Error deleting package file: %v", err) + return err + } + } + + if err := models.DeletePackageByID(p.ID); err != nil { + log.Error("Error deleting package: %v", err) + return err + } + + return nil +} From 3c86d2b7f8658dd384d1a2375455028b928b262e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 11 Jul 2021 20:24:22 +0000 Subject: [PATCH 004/130] Added tests. --- integrations/api_packages_generic_test.go | 79 ++++++++++++++++++++++ modules/context/context.go | 2 +- routers/api/v1/packages/generic/generic.go | 17 ++--- 3 files changed, 86 insertions(+), 12 deletions(-) create mode 100644 integrations/api_packages_generic_test.go diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go new file mode 100644 index 0000000000000..a73a14051005c --- /dev/null +++ b/integrations/api_packages_generic_test.go @@ -0,0 +1,79 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "bytes" + "fmt" + "net/http" + "testing" + + "code.gitea.io/gitea/models" + + "github.com/stretchr/testify/assert" +) + +func TestPackageGeneric(t *testing.T) { + defer prepareTestEnv(t)() + repository := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) + user := models.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + session := loginUser(t, user.Name) + token := getTokenForLoggedInUser(t, session) + + packageName := "te-st_pac.kage" + packageVersion := "v1.0.3" + filename := "fi-le_na.me" + content := []byte{1,2,3} + + url := fmt.Sprintf("/api/v1/repos/%s/%s/packages/generic/%s/%s/%s?token=%s", user.Name, repository.Name, packageName, packageVersion, filename, token) + + t.Run("Upload", func(t *testing.T) { + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) + MakeRequest(t, req, http.StatusCreated) + + ps, err := models.GetPackagesByRepositoryID(repository.ID) + assert.NoError(t, err) + assert.Len(t, ps, 1) + assert.Equal(t, packageName, ps[0].Name) + assert.Equal(t, packageVersion, ps[0].Version) + + pfs, err := ps[0].GetFiles() + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, filename, pfs[0].Name) + assert.Equal(t, int64(len(content)), pfs[0].Size) + }) + + t.Run("UploadExists", func(t *testing.T) { + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) + MakeRequest(t, req, http.StatusBadRequest) + }) + + t.Run("Download", func(t *testing.T) { + req := NewRequest(t, "GET", url) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, content, resp.Body.Bytes()) + }) + + t.Run("Delete", func(t *testing.T) { + req := NewRequest(t, "DELETE", url) + MakeRequest(t, req, http.StatusOK) + + ps, err := models.GetPackagesByRepositoryID(repository.ID) + assert.NoError(t, err) + assert.Empty(t, ps) + }) + + t.Run("DownloadNotExists", func(t *testing.T) { + req := NewRequest(t, "GET", url) + MakeRequest(t, req, http.StatusNotFound) + }) + + t.Run("DeleteNotExists", func(t *testing.T) { + req := NewRequest(t, "DELETE", url) + MakeRequest(t, req, http.StatusNotFound) + }) +} diff --git a/modules/context/context.go b/modules/context/context.go index 64f8b12084576..978459ed725c6 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -358,7 +358,7 @@ func (ctx *Context) PlainText(status int, bs []byte) { ctx.Resp.WriteHeader(status) ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") if _, err := ctx.Resp.Write(bs); err != nil { - ctx.ServerError("Render JSON failed", err) + ctx.ServerError("Write failed", err) } } diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index c713fa7153064..57023144cc7a2 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -6,8 +6,6 @@ package generic import ( "errors" - "fmt" - "io" "net/http" "regexp" "strings" @@ -36,7 +34,7 @@ func DownloadPackage(ctx *context.APIContext) { p, err := models.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, models.PackageGeneric, packageName, packageVersion) if err != nil { if err == models.ErrPackageNotExist { - ctx.Error(http.StatusBadRequest, "", err) + ctx.Error(http.StatusNotFound, "", err) return } log.Error("Error getting package: %v", err) @@ -54,7 +52,7 @@ func DownloadPackage(ctx *context.APIContext) { pf := pfs[0] if !strings.EqualFold(pf.LowerName, filename) { - ctx.Error(http.StatusBadRequest, "", models.ErrPackageNotExist) + ctx.Error(http.StatusNotFound, "", models.ErrPackageNotExist) return } @@ -67,12 +65,7 @@ func DownloadPackage(ctx *context.APIContext) { } defer s.Close() - ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, pf.Name)) - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - _, err = io.Copy(ctx.Resp, s) - if err != nil { - log.Error("Error in io.Copy: %v", err) - } + ctx.ServeStream(s, pf.Name) } // UploadPackage uploads the specific generic package. @@ -120,6 +113,8 @@ func UploadPackage(ctx *context.APIContext) { } ctx.Error(http.StatusInternalServerError, "", "") } + + ctx.PlainText(http.StatusCreated, nil) } // DeletePackage deletes the specific generic package. @@ -133,7 +128,7 @@ func DeletePackage(ctx *context.APIContext) { err = package_service.DeletePackage(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) if err != nil { if err == models.ErrPackageNotExist { - ctx.Error(http.StatusBadRequest, "", err) + ctx.Error(http.StatusNotFound, "", err) return } ctx.Error(http.StatusInternalServerError, "", "") From 8d922ff1777bcc69ecded48f7b8dccfc893e4a08 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 13 Jul 2021 16:29:37 +0000 Subject: [PATCH 005/130] Added NuGet package registry. --- integrations/api_packages_generic_test.go | 4 +- integrations/api_packages_nuget_test.go | 192 ++++++++++++++ models/migrations/v188.go | 16 +- models/package.go | 58 +++- modules/context/context.go | 20 ++ modules/packages/nuget/metadata.go | 151 +++++++++++ modules/packages/nuget/metadata_test.go | 112 ++++++++ modules/util/filebuffer/file_backed_buffer.go | 10 +- routers/api/v1/api.go | 24 +- routers/api/v1/packages/generic/generic.go | 77 +++--- routers/api/v1/packages/nuget/api.go | 251 ++++++++++++++++++ routers/api/v1/packages/nuget/links.go | 28 ++ routers/api/v1/packages/nuget/nuget.go | 248 +++++++++++++++++ routers/api/v1/packages/nuget/package.go | 65 +++++ services/packages/packages.go | 48 +++- 15 files changed, 1223 insertions(+), 81 deletions(-) create mode 100644 integrations/api_packages_nuget_test.go create mode 100644 modules/packages/nuget/metadata.go create mode 100644 modules/packages/nuget/metadata_test.go create mode 100644 routers/api/v1/packages/nuget/api.go create mode 100644 routers/api/v1/packages/nuget/links.go create mode 100644 routers/api/v1/packages/nuget/nuget.go create mode 100644 routers/api/v1/packages/nuget/package.go diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index a73a14051005c..89e099b3fd365 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -23,9 +23,9 @@ func TestPackageGeneric(t *testing.T) { token := getTokenForLoggedInUser(t, session) packageName := "te-st_pac.kage" - packageVersion := "v1.0.3" + packageVersion := "1.0.3" filename := "fi-le_na.me" - content := []byte{1,2,3} + content := []byte{1, 2, 3} url := fmt.Sprintf("/api/v1/repos/%s/%s/packages/generic/%s/%s/%s?token=%s", user.Name, repository.Name, packageName, packageVersion, filename, token) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go new file mode 100644 index 0000000000000..c5e08a96cc156 --- /dev/null +++ b/integrations/api_packages_nuget_test.go @@ -0,0 +1,192 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "archive/zip" + "bytes" + "fmt" + "net/http" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/v1/packages/nuget" + + "github.com/stretchr/testify/assert" +) + +func TestPackageNuGet(t *testing.T) { + defer prepareTestEnv(t)() + repository := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) + user := models.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + + packageName := "test.package" + packageVersion := "1.0.3" + packageAuthors := "KN4CK3R" + packageDescription := "Gitea Test Package" + + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create("package.nuspec") + w.Write([]byte(` + + + `+packageName+` + `+packageVersion+` + `+packageAuthors+` + `+packageDescription+` + + `)) + archive.Close() + content := buf.Bytes() + + url := fmt.Sprintf("/api/v1/repos/%s/%s/packages/nuget", user.Name, repository.Name) + + t.Run("ServiceIndex", func(t *testing.T) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/index.json", url)) + req = AddBasicAuthHeader(req, user.Name) + _ = MakeRequest(t, req, http.StatusOK) + + + }) + + t.Run("Upload", func(t *testing.T) { + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + ps, err := models.GetPackagesByRepositoryID(repository.ID) + assert.NoError(t, err) + assert.Len(t, ps, 1) + assert.Equal(t, packageName, ps[0].Name) + assert.Equal(t, packageVersion, ps[0].Version) + + pfs, err := ps[0].GetFiles() + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name) + assert.Equal(t, int64(len(content)), pfs[0].Size) + }) + + t.Run("UploadExists", func(t *testing.T) { + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) + + t.Run("Download", func(t *testing.T) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.nupkg", url, packageName, packageVersion, packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, content, resp.Body.Bytes()) + }) + + t.Run("SearchService", func(t *testing.T) { + cases := []struct{ + Query string + Skip int + Take int + ExpectedTotal int64 + ExpectedResults int + } { + {"", 0, 0, 1, 1}, + {"", 0, 10, 1, 1}, + {"gitea", 0, 10, 0, 0}, + {"test", 0, 10, 1, 1}, + {"test", 1, 10, 1, 0}, + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s/query?q=%s&skip=%d&take=%d", url, c.Query, c.Skip, c.Take)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result nuget.SearchResultResponse + DecodeJSON(t, resp, &result) + + assert.Equal(t, c.ExpectedTotal, result.TotalHits, "case %d: unexpected total hits", i) + assert.Len(t, result.Data, c.ExpectedResults, "case %d: unexpected result count", i) + } + }) + + t.Run("RegistrationService", func(t *testing.T) { + indexURL := fmt.Sprintf("%s%s/registration/%s/index.json", setting.AppURL, url[1:], packageName) + leafURL := fmt.Sprintf("%s%s/registration/%s/%s.json", setting.AppURL, url[1:], packageName, packageVersion) + contentURL := fmt.Sprintf("%s%s/package/%s/%s/%s.%s.nupkg", setting.AppURL, url[1:], packageName, packageVersion, packageName, packageVersion) + + t.Run("RegistrationIndex", func(t *testing.T) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/registration/%s/index.json", url, packageName)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result nuget.RegistrationIndexResponse + DecodeJSON(t, resp, &result) + + assert.Equal(t, indexURL, result.RegistrationIndexURL) + assert.Equal(t, 1, result.Count) + assert.Len(t, result.Pages, 1) + assert.Equal(t, indexURL, result.Pages[0].RegistrationPageURL) + assert.Equal(t, packageVersion, result.Pages[0].Lower) + assert.Equal(t, packageVersion, result.Pages[0].Upper) + assert.Equal(t, 1, result.Pages[0].Count) + assert.Len(t, result.Pages[0].Items, 1) + assert.Equal(t, packageName, result.Pages[0].Items[0].CatalogEntry.ID) + assert.Equal(t, packageVersion, result.Pages[0].Items[0].CatalogEntry.Version) + assert.Equal(t, packageAuthors, result.Pages[0].Items[0].CatalogEntry.Authors) + assert.Equal(t, packageDescription, result.Pages[0].Items[0].CatalogEntry.Description) + assert.Equal(t, leafURL, result.Pages[0].Items[0].CatalogEntry.CatalogLeafURL) + assert.Equal(t, contentURL, result.Pages[0].Items[0].CatalogEntry.PackageContentURL) + }) + + t.Run("RegistrationLeaf", func(t *testing.T) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/registration/%s/%s.json", url, packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result nuget.RegistrationLeafResponse + DecodeJSON(t, resp, &result) + + assert.Equal(t, leafURL, result.RegistrationLeafURL) + assert.Equal(t, contentURL, result.PackageContentURL) + assert.Equal(t, indexURL, result.RegistrationIndexURL) + }) + }) + + t.Run("PackageService", func(t *testing.T) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/index.json", url, packageName)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result nuget.PackageVersionsResponse + DecodeJSON(t, resp, &result) + + assert.Len(t, result.Versions, 1) + assert.Equal(t, packageVersion, result.Versions[0]) + }) + + t.Run("Delete", func(t *testing.T) { + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/package/%s/%s", url, packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + ps, err := models.GetPackagesByRepositoryID(repository.ID) + assert.NoError(t, err) + assert.Empty(t, ps) + }) + + t.Run("DownloadNotExists", func(t *testing.T) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.nupkg", url, packageName, packageVersion, packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusNotFound) + }) + + t.Run("DeleteNotExists", func(t *testing.T) { + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/package/%s/%s", url, packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusNotFound) + }) +} diff --git a/models/migrations/v188.go b/models/migrations/v188.go index 4f41b54f9f35b..5f8fb3aea52e5 100644 --- a/models/migrations/v188.go +++ b/models/migrations/v188.go @@ -12,14 +12,14 @@ import ( func addPackageTables(x *xorm.Engine) error { type Package struct { - ID int64 `xorm:"pk autoincr"` - RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - CreatorID int64 - Type int `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` - MetaData interface{} `xorm:"TEXT JSON"` + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatorID int64 + Type int `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` + MetadataRaw string `xorm:"TEXT"` CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"updated"` diff --git a/models/package.go b/models/package.go index 351a78fd25979..0ee54f91ef661 100644 --- a/models/package.go +++ b/models/package.go @@ -9,6 +9,9 @@ import ( "strings" "code.gitea.io/gitea/modules/timeutil" + + "github.com/hashicorp/go-version" + "xorm.io/builder" ) // PackageType specifies the different package types @@ -17,6 +20,7 @@ type PackageType int // Note: new type must append to the end of list to maintain compatibility. const ( PackageGeneric PackageType = iota + PackageNuGet // 1 ) var ( @@ -28,14 +32,15 @@ var ( // Package represents a package type Package struct { - ID int64 `xorm:"pk autoincr"` - RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - CreatorID int64 - Type PackageType `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` - MetaData interface{} `xorm:"TEXT JSON"` + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatorID int64 + Type PackageType `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` + SemVer *version.Version `xorm:"-"` + MetadataRaw string `xorm:"TEXT"` CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"updated"` @@ -113,22 +118,22 @@ func DeletePackagesByRepositoryID(repositoryID int64) error { // GetPackagesByRepositoryID returns all packages of a repository func GetPackagesByRepositoryID(repositoryID int64) ([]*Package, error) { packages := make([]*Package, 0, 10) - return packages, x.Where("repository_id = ?", repositoryID).Find(&packages) + return packages, x.Where("repo_id = ?", repositoryID).Find(&packages) } // GetPackagesByName gets all repository packages with the specific name func GetPackagesByName(repositoryID int64, packageType PackageType, packageName string) ([]*Package, error) { packages := make([]*Package, 0, 10) - return packages, x.Where("repository_id = ? AND type = ? AND lower_name = ?", repositoryID, packageType, strings.ToLower(packageName)).Find(&packages) + return packages, x.Where("repo_id = ? AND type = ? AND lower_name = ?", repositoryID, packageType, strings.ToLower(packageName)).Find(&packages) } // GetPackageByNameAndVersion gets a repository package by name and version func GetPackageByNameAndVersion(repositoryID int64, packageType PackageType, packageName, packageVersion string) (*Package, error) { p := &Package{ - RepositoryID: repositoryID, - Type: packageType, - LowerName: strings.ToLower(packageName), - Version: packageVersion, + RepoID: repositoryID, + Type: packageType, + LowerName: strings.ToLower(packageName), + Version: packageVersion, } has, err := x.Get(p) if err != nil { @@ -139,6 +144,31 @@ func GetPackageByNameAndVersion(repositoryID int64, packageType PackageType, pac return p, nil } +// SearchPackages searches for packages by name and can be used to navigate through the package list +func SearchPackages(repositoryID int64, packageType PackageType, query string, skip, take int) (int64, []*Package, error) { + cond := builder.NewCond() + cond = cond.And(builder.Eq{"repo_id": repositoryID}) + cond = cond.And(builder.Eq{"type": packageType}) + if query != "" { + cond = cond.And(builder.Like{"lower_name", strings.ToLower(query)}) + } + + if take <= 0 || take > 100 { + take = 100 + } + + sess := x.Where(cond) + if skip > 0 { + sess = sess.Limit(take, skip) + } else { + sess = sess.Limit(take) + } + + packages := make([]*Package, 0, take) + count, err := sess.FindAndCount(&packages) + return count, packages, err +} + // InsertPackageFile inserts a package file func InsertPackageFile(pf *PackageFile) error { _, err := x.Insert(pf) diff --git a/modules/context/context.go b/modules/context/context.go index 978459ed725c6..e575e90e10158 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -325,6 +325,26 @@ func (ctx *Context) QueryOptionalBool(key string, defaults ...util.OptionalBool) return (*Forms)(ctx.Req).MustOptionalBool(key, defaults...) } +// UploadStream returns the request body or the first form file +func (ctx *Context) UploadStream() (io.ReadCloser, error) { + contentType := strings.ToLower(ctx.Req.Header.Get("Content-Type")) + if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") || strings.HasPrefix(contentType, "multipart/form-data") { + if err := ctx.Req.ParseMultipartForm(32 * 1024 * 1024); err != nil { + return nil, err + } + if ctx.Req.MultipartForm.File == nil { + return nil, http.ErrMissingFile + } + for _, files := range ctx.Req.MultipartForm.File { + if len(files) > 0 { + return files[0].Open() + } + } + return nil, http.ErrMissingFile + } + return ctx.Req.Body, nil +} + // HandleText handles HTTP status code func (ctx *Context) HandleText(status int, title string) { if (status/100 == 4) || (status/100 == 5) { diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go new file mode 100644 index 0000000000000..6ca2de0d9cc95 --- /dev/null +++ b/modules/packages/nuget/metadata.go @@ -0,0 +1,151 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "archive/zip" + "encoding/xml" + "errors" + "io" + "path/filepath" + "regexp" + "strings" + + "github.com/hashicorp/go-version" +) + +var ( + // ErrMissingNuspecFile indicates a missing Nuspec file + ErrMissingNuspecFile = errors.New("Nuspec file is missing") + // ErrNuspecFileTooLarge indicates a Nuspec file which is too large + ErrNuspecFileTooLarge = errors.New("Nuspec file is too large") + // ErrNuspecInvalidID indicates an invalid id in the Nuspec file + ErrNuspecInvalidID = errors.New("Nuspec file contains an invalid id") + // ErrNuspecInvalidVersion indicates an invalid version in the Nuspec file + ErrNuspecInvalidVersion = errors.New("Nuspec file contains an invalid version") +) + +var idmatch = regexp.MustCompile(`\A\w+(?:[.-]\w+)*\z`) + +const maxNuspecFileSize = 3 * 1024 * 1024 + +// Metadata represents the metadata of a Nuget package +type Metadata struct { + ID string `json:"-"` + Version string `json:"-"` + Description string + Summary string + ReleaseNotes string + Authors string + RequireLicenseAcceptance bool + ProjectURL string `json:"project_url"` + RepositoryURL string `json:"repository_url"` + Dependencies map[string][]Dependency +} + +// Dependency represents a dependency of a Nuget package +type Dependency struct { + ID string + Version string +} + +type nuspecPackage struct { + Metadata struct { + ID string `xml:"id"` + Version string `xml:"version"` + Authors string `xml:"authors"` + RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + ProjectURL string `xml:"projectUrl"` + Description string `xml:"description"` + Summary string `xml:"summary"` + ReleaseNotes string `xml:"releaseNotes"` + Repository struct { + URL string `xml:"url,attr"` + } `xml:"repository"` + Dependencies struct { + Group []struct { + TargetFramework string `xml:"targetFramework,attr"` + Dependency []struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + Exclude string `xml:"exclude,attr"` + } `xml:"dependency"` + } `xml:"group"` + } `xml:"dependencies"` + } `xml:"metadata"` +} + +// ParsePackageMetaData parses the metadata of a Nuget package file +func ParsePackageMetaData(r io.ReaderAt, size int64) (*Metadata, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range archive.File { + if filepath.Dir(file.Name) != "." { + continue + } + if strings.HasSuffix(strings.ToLower(file.Name), ".nuspec") { + if file.UncompressedSize64 > maxNuspecFileSize { + return nil, ErrNuspecFileTooLarge + } + f, err := archive.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + return ParseNuspecMetaData(f) + } + } + return nil, ErrMissingNuspecFile +} + +// ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package +func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { + var p nuspecPackage + dec := xml.NewDecoder(r) + err := dec.Decode(&p) + if err != nil { + return nil, err + } + + if !idmatch.MatchString(p.Metadata.ID) { + return nil, ErrNuspecInvalidID + } + + v, err := version.NewSemver(p.Metadata.Version) + if err != nil { + return nil, ErrNuspecInvalidVersion + } + + m := &Metadata{ + ID: p.Metadata.ID, + Version: v.String(), + Description: p.Metadata.Description, + Summary: p.Metadata.Summary, + ReleaseNotes: p.Metadata.ReleaseNotes, + Authors: p.Metadata.Authors, + RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance, + ProjectURL: p.Metadata.ProjectURL, + RepositoryURL: p.Metadata.Repository.URL, + Dependencies: make(map[string][]Dependency), + } + for _, group := range p.Metadata.Dependencies.Group { + deps := make([]Dependency, 0, len(group.Dependency)) + for _, dep := range group.Dependency { + if dep.ID == "" || dep.Version == "" { + continue + } + deps = append(deps, Dependency{ + ID: dep.ID, + Version: dep.Version, + }) + } + m.Dependencies[group.TargetFramework] = deps + } + return m, nil +} diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go new file mode 100644 index 0000000000000..87c73c53ebfe2 --- /dev/null +++ b/modules/packages/nuget/metadata_test.go @@ -0,0 +1,112 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "archive/zip" + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + id = "System.Gitea" + semver = "1.0.1" + authors = "Gitea Authors" + projectURL = "https://gitea.com" + description = "Package Description" + summary = "Package Summary" + releaseNotes = "Package Release Notes" + repositoryURL = "https://gitea.com/gitea/gitea" + targetFramework = ".NETStandard2.1" + dependencyID = "System.Text.Json" + dependencyVersion = "5.0.0" +) + +const nuspecContent = ` + + + ` + id + ` + ` + semver + ` + ` + authors + ` + true + ` + projectURL + ` + ` + description + ` + ` + summary + ` + ` + releaseNotes + ` + + + + + + + +` + +func TestParsePackageMetaData(t *testing.T) { + createArchive := func(name, content string) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create(name) + w.Write([]byte(content)) + archive.Close() + return buf.Bytes() + } + + t.Run("MissingNuspecFile", func(t *testing.T) { + data := createArchive("dummy.txt", "") + + m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, m) + assert.ErrorIs(t, err, ErrMissingNuspecFile) + }) + + t.Run("MissingNuspecFileInRoot", func(t *testing.T) { + data := createArchive("sub/package.nuspec", "") + + m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, m) + assert.ErrorIs(t, err, ErrMissingNuspecFile) + }) + + t.Run("InvalidNuspecFile", func(t *testing.T) { + data := createArchive("package.nuspec", "") + + m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, m) + assert.ErrorIs(t, err, ErrNuspecInvalidID) + }) + + t.Run("Valid", func(t *testing.T) { + data := createArchive("package.nuspec", nuspecContent) + + m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.NoError(t, err) + assert.NotNil(t, m) + }) +} + +func TestParseNuspecMetaData(t *testing.T) { + m, err := ParseNuspecMetaData(strings.NewReader(nuspecContent)) + assert.NoError(t, err) + assert.NotNil(t, m) + + assert.Equal(t, id, m.ID) + assert.Equal(t, semver, m.Version) + assert.Equal(t, authors, m.Authors) + assert.Equal(t, projectURL, m.ProjectURL) + assert.Equal(t, description, m.Description) + assert.Equal(t, summary, m.Summary) + assert.Equal(t, releaseNotes, m.ReleaseNotes) + assert.Equal(t, repositoryURL, m.RepositoryURL) + assert.Len(t, m.Dependencies, 1) + assert.Contains(t, m.Dependencies, targetFramework) + deps := m.Dependencies[targetFramework] + assert.Len(t, deps, 1) + assert.Equal(t, dependencyID, deps[0].ID) + assert.Equal(t, dependencyVersion, deps[0].Version) +} diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index fca4aa3f0f8aa..136f7d661e008 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -18,7 +18,7 @@ var ( ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") ) -// FileBackedBuffer implements io.ReadSeekCloser +// FileBackedBuffer implements io.ReadSeekCloser and io.ReaderAt type FileBackedBuffer struct { size int64 buffer *bytes.Reader @@ -69,6 +69,14 @@ func (b *FileBackedBuffer) Read(p []byte) (int, error) { return b.buffer.Read(p) } +// ReadAt implements io.ReaderAt +func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) { + if b.tmpFile != nil { + return b.tmpFile.ReadAt(p, off) + } + return b.buffer.ReadAt(p, off) +} + // Seek implements io.Seeker func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { if b.tmpFile != nil { diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index e860443c05f3f..ae64cb3b3b8ad 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -80,6 +80,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/notify" "code.gitea.io/gitea/routers/api/v1/org" "code.gitea.io/gitea/routers/api/v1/packages/generic" + "code.gitea.io/gitea/routers/api/v1/packages/nuget" "code.gitea.io/gitea/routers/api/v1/repo" "code.gitea.io/gitea/routers/api/v1/settings" _ "code.gitea.io/gitea/routers/api/v1/swagger" // for swagger generation @@ -217,6 +218,7 @@ func reqExploreSignIn() func(ctx *context.APIContext) { func reqBasicAuth() func(ctx *context.APIContext) { return func(ctx *context.APIContext) { if !ctx.Context.IsBasicAuth { + ctx.Resp.Header().Set("WWW-Authenticate", `Basic realm="API"`) ctx.Error(http.StatusUnauthorized, "reqBasicAuth", "basic auth required") return } @@ -974,12 +976,28 @@ func Routes() *web.Route { m.Group("/packages", func() { m.Group("/generic", func() { m.Group("/{packagename}/{packageversion}/{filename}", func() { - m.Get("", generic.DownloadPackage) + m.Get("", generic.DownloadPackageContent) m.Put("" /*reqRepoWriter(models.UnitTypePackage),*/, generic.UploadPackage) m.Delete("" /*reqRepoWriter(models.UnitTypePackage),*/, generic.DeletePackage) }) - }) - }, reqToken()) + }, reqToken()) + m.Group("/nuget", func() { + m.Put("/" /*reqRepoWriter(models.UnitTypePackage),*/, nuget.UploadPackage) + m.Get("/index.json", nuget.ServiceIndex) + m.Get("/query", nuget.SearchService) + m.Group("/registration/{id}", func() { + m.Get("/index.json", nuget.RegistrationIndex) + m.Get("/{version}", nuget.RegistrationLeaf) + }) + m.Group("/package/{id}", func() { + m.Get("/index.json", nuget.EnumeratePackageVersions) + m.Group("/{version}", func() { + m.Delete("/" /*reqRepoWriter(models.UnitTypePackage),*/, nuget.DeletePackage) + m.Get("/{filename}", nuget.DownloadPackageContent) + }) + }) + }, reqBasicAuth()) + }) }, repoAssignment()) }) diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 57023144cc7a2..5b65264ed1e3d 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -8,59 +8,35 @@ import ( "errors" "net/http" "regexp" - "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/util/filebuffer" package_service "code.gitea.io/gitea/services/packages" + + "github.com/hashicorp/go-version" ) var packageNameRegex = regexp.MustCompile(`\A[A-Za-z0-9\.\_\-\+]+\z`) -var packageVersionRegex = regexp.MustCompile(`\A(?:\.?[\w\+-]+\.?)+\z`) var filenameRegex = packageNameRegex -// DownloadPackage serves the specific generic package. -func DownloadPackage(ctx *context.APIContext) { +// DownloadPackageContent serves the specific generic package. +func DownloadPackageContent(ctx *context.APIContext) { packageName, packageVersion, filename, err := sanitizeParameters(ctx) if err != nil { ctx.Error(http.StatusBadRequest, "", err) return } - p, err := models.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, models.PackageGeneric, packageName, packageVersion) + s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion, filename) if err != nil { if err == models.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) return } - log.Error("Error getting package: %v", err) - ctx.Error(http.StatusInternalServerError, "", "") - return - } - - pfs, err := p.GetFiles() - if err != nil { - log.Error("Error getting package files: %v", err) - ctx.Error(http.StatusInternalServerError, "", "") - return - } - - pf := pfs[0] - - if !strings.EqualFold(pf.LowerName, filename) { - ctx.Error(http.StatusNotFound, "", models.ErrPackageNotExist) - return - } - - packageStore := packages.NewContentStore() - s, err := packageStore.Get(pf.ID) - if err != nil { - log.Error("Error reading package file: %v", err) - ctx.Error(http.StatusInternalServerError, "", "") + ctx.Error(http.StatusInternalServerError, "", err) return } defer s.Close() @@ -77,14 +53,20 @@ func UploadPackage(ctx *context.APIContext) { return } - defer ctx.Req.Body.Close() - r, err := filebuffer.CreateFromReader(ctx.Req.Body, 32*1024*1024) + upload, err := ctx.UploadStream() + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + defer upload.Close() + + buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) if err != nil { - log.Error("Error in CreateFromReader: %v", err) + log.Error("Error creating file buffer: %v", err) ctx.Error(http.StatusInternalServerError, "", "") return } - defer r.Close() + defer buf.Close() p, err := package_service.CreatePackage( ctx.User, @@ -100,16 +82,16 @@ func UploadPackage(ctx *context.APIContext) { ctx.Error(http.StatusBadRequest, "", err) return } - log.Error("Error in CreatePackage: %v", err) + log.Error("Error creating package: %v", err) ctx.Error(http.StatusInternalServerError, "", "") return } - _, err = package_service.AddFileToPackage(p, filename, r.Size(), r) + _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) if err != nil { - log.Error("Error in AddFileToPackage: %v", err) + log.Error("Error adding file to package: %v", err) if err := models.DeletePackageByID(p.ID); err != nil { - log.Error("Error in DeletePackageByID: %v", err) + log.Error("Error deleting package by id: %v", err) } ctx.Error(http.StatusInternalServerError, "", "") } @@ -135,13 +117,18 @@ func DeletePackage(ctx *context.APIContext) { } } -func sanitizeParameters(ctx *context.APIContext) (packageName, packageVersion, filename string, err error) { - packageName = ctx.Params("packagename") - packageVersion = ctx.Params("packageversion") - filename = ctx.Params("filename") +func sanitizeParameters(ctx *context.APIContext) (string, string, string, error) { + packageName := ctx.Params("packagename") + filename := ctx.Params("filename") + + if !packageNameRegex.MatchString(packageName) || !filenameRegex.MatchString(filename) { + return "", "", "", errors.New("Invalid package name or filename") + } - if !packageNameRegex.MatchString(packageName) || !packageVersionRegex.MatchString(packageVersion) || !filenameRegex.MatchString(filename) { - err = errors.New("Invalid package name, package version or filename") + v, err := version.NewSemver(ctx.Params("packageversion")) + if err != nil { + return "", "", "", err } - return + + return packageName, v.String(), filename, nil } diff --git a/routers/api/v1/packages/nuget/api.go b/routers/api/v1/packages/nuget/api.go new file mode 100644 index 0000000000000..f7586ab65b83c --- /dev/null +++ b/routers/api/v1/packages/nuget/api.go @@ -0,0 +1,251 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "bytes" + "fmt" + "time" + + "github.com/hashicorp/go-version" +) + +// RegistrationIndexResponse https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#response +type RegistrationIndexResponse struct { + RegistrationIndexURL string `json:"@id"` + Type []string `json:"@type"` + Count int `json:"count"` + Pages []*RegistrationIndexPage `json:"items"` +} + +// RegistrationIndexPage https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-page-object +type RegistrationIndexPage struct { + RegistrationPageURL string `json:"@id"` + Lower string `json:"lower"` + Upper string `json:"upper"` + Count int `json:"count"` + Items []*RegistrationIndexPageItem `json:"items"` +} + +// RegistrationIndexPageItem https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf-object-in-a-page +type RegistrationIndexPageItem struct { + RegistrationLeafURL string `json:"@id"` + PackageContentURL string `json:"packageContent"` + CatalogEntry *CatalogEntry `json:"catalogEntry"` +} + +// CatalogEntry https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#catalog-entry +type CatalogEntry struct { + CatalogLeafURL string `json:"@id"` + PackageContentURL string `json:"packageContent"` + ID string `json:"id"` + Version string `json:"version"` + Description string `json:"description"` + Summary string `json:"summary"` + ReleaseNotes string `json:"releaseNotes"` + Authors string `json:"authors"` + RequireLicenseAcceptance bool `json:"requireLicenseAcceptance"` + ProjectURL string `json:"projectURL"` + DependencyGroups []*PackageDependencyGroup `json:"dependencyGroups"` +} + +// PackageDependencyGroup https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#package-dependency-group +type PackageDependencyGroup struct { + TargetFramework string `json:"targetFramework"` + Dependencies []*PackageDependency `json:"dependencies"` +} + +// PackageDependency https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#package-dependency +type PackageDependency struct { + ID string `json:"id"` + Range string `json:"range"` +} + +func createRegistrationIndexResponse(l *linkBuilder, packages []*Package) *RegistrationIndexResponse { + sortedPackages := sortPackagesByVersionASC(packages) + + items := make([]*RegistrationIndexPageItem, 0, len(packages)) + for _, p := range sortedPackages { + items = append(items, createRegistrationIndexPageItem(l, p)) + } + + return &RegistrationIndexResponse{ + RegistrationIndexURL: l.GetRegistrationIndexURL(packages[0].Name), + Type: []string{"catalog:CatalogRoot", "PackageRegistration", "catalog:Permalink"}, + Count: 1, + Pages: []*RegistrationIndexPage{ + { + RegistrationPageURL: l.GetRegistrationIndexURL(packages[0].Name), + Count: len(packages), + Lower: normalizeVersion(packages[0].SemVer), + Upper: normalizeVersion(packages[len(packages)-1].SemVer), + Items: items, + }, + }, + } +} + +func createRegistrationIndexPageItem(l *linkBuilder, p *Package) *RegistrationIndexPageItem { + return &RegistrationIndexPageItem{ + RegistrationLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), + PackageContentURL: l.GetPackageDownloadURL(p.Name, p.Version), + CatalogEntry: &CatalogEntry{ + CatalogLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), + PackageContentURL: l.GetPackageDownloadURL(p.Name, p.Version), + ID: p.Name, + Version: p.Version, + Description: p.Metadata.Description, + Summary: p.Metadata.Summary, + ReleaseNotes: p.Metadata.ReleaseNotes, + Authors: p.Metadata.Authors, + RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance, + ProjectURL: p.Metadata.ProjectURL, + DependencyGroups: createDependencyGroups(p), + }, + } +} + +func createDependencyGroups(p *Package) []*PackageDependencyGroup { + dependencyGroups := make([]*PackageDependencyGroup, 0, len(p.Metadata.Dependencies)) + for k, v := range p.Metadata.Dependencies { + dependencies := make([]*PackageDependency, 0, len(v)) + for _, dep := range v { + dependencies = append(dependencies, &PackageDependency{ + ID: dep.ID, + Range: dep.Version, + }) + } + + dependencyGroups = append(dependencyGroups, &PackageDependencyGroup{ + TargetFramework: k, + Dependencies: dependencies, + }) + } + return dependencyGroups +} + +// RegistrationLeafResponse https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf +type RegistrationLeafResponse struct { + RegistrationLeafURL string `json:"@id"` + Type []string `json:"@type"` + Listed bool `json:"listed"` + PackageContentURL string `json:"packageContent"` + Published time.Time `json:"published"` + RegistrationIndexURL string `json:"registration"` +} + +func createRegistrationLeafResponse(l *linkBuilder, p *Package) *RegistrationLeafResponse { + return &RegistrationLeafResponse{ + Type: []string{"Package", "http://schema.nuget.org/catalog#Permalink"}, + Listed: true, + Published: time.Unix(int64(p.CreatedUnix), 0), + RegistrationLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), + PackageContentURL: l.GetPackageDownloadURL(p.Name, p.Version), + RegistrationIndexURL: l.GetRegistrationIndexURL(p.Name), + } +} + +// PackageVersionsResponse https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#response +type PackageVersionsResponse struct { + Versions []string `json:"versions"` +} + +func createPackageVersionsResponse(packages []*Package) *PackageVersionsResponse { + versions := make([]string, 0, len(packages)) + for _, p := range packages { + versions = append(versions, normalizeVersion(p.SemVer)) + } + + return &PackageVersionsResponse{ + Versions: versions, + } +} + +// SearchResultResponse https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#response +type SearchResultResponse struct { + TotalHits int64 `json:"totalHits"` + Data []*SearchResult `json:"data"` +} + +// SearchResult https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-result +type SearchResult struct { + ID string `json:"id"` + Version string `json:"version"` + Versions []*SearchResultVersion `json:"versions"` + Description string `json:"description"` + Summary string `json:"summary"` + Authors string `json:"authors"` + ProjectURL string `json:"projectURL"` + RegistrationIndexURL string `json:"registration"` +} + +// SearchResultVersion https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-result +type SearchResultVersion struct { + RegistrationLeafURL string `json:"@id"` + Version string `json:"version"` + Downloads int64 `json:"downloads"` +} + +func createSearchResultResponse(l *linkBuilder, totalHits int64, packages []*Package) *SearchResultResponse { + data := make([]*SearchResult, 0, len(packages)) + + if len(packages) > 0 { + groupID := packages[0].Name + group := make([]*Package, 0, 10) + + for i := 0; i < len(packages); i++ { + if groupID != packages[i].Name { + data = append(data, createSearchResult(l, group)) + groupID = packages[i].Name + group = group[:0] + } + group = append(group, packages[i]) + } + data = append(data, createSearchResult(l, group)) + } + + return &SearchResultResponse{ + TotalHits: totalHits, + Data: data, + } +} + +func createSearchResult(l *linkBuilder, packages []*Package) *SearchResult { + latest := packages[0] + versions := make([]*SearchResultVersion, 0, len(packages)) + for _, p := range packages { + if latest.SemVer.LessThan(p.SemVer) { + latest = p + } + + versions = append(versions, &SearchResultVersion{ + RegistrationLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), + Version: p.Version, + }) + } + + return &SearchResult{ + ID: latest.Name, + Version: latest.Version, + Versions: versions, + Description: latest.Metadata.Description, + Summary: latest.Metadata.Summary, + Authors: latest.Metadata.Authors, + ProjectURL: latest.Metadata.ProjectURL, + RegistrationIndexURL: l.GetRegistrationIndexURL(latest.Name), + } +} + +// normalizeVersion removes the metadata +func normalizeVersion(v *version.Version) string { + var buf bytes.Buffer + segments := v.Segments64() + fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) + pre := v.Prerelease() + if pre != "" { + fmt.Fprintf(&buf, "-%s", pre) + } + return buf.String() +} diff --git a/routers/api/v1/packages/nuget/links.go b/routers/api/v1/packages/nuget/links.go new file mode 100644 index 0000000000000..82fa8ec1f94df --- /dev/null +++ b/routers/api/v1/packages/nuget/links.go @@ -0,0 +1,28 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "fmt" +) + +type linkBuilder struct { + Base string +} + +// GetRegistrationIndexURL builds the registration index url +func (l *linkBuilder) GetRegistrationIndexURL(id string) string { + return fmt.Sprintf("%s/registration/%s/index.json", l.Base, id) +} + +// GetRegistrationIndexURL builds the registration leaf url +func (l *linkBuilder) GetRegistrationLeafURL(id, version string) string { + return fmt.Sprintf("%s/registration/%s/%s.json", l.Base, id, version) +} + +// GetRegistrationIndexURL builds the download url +func (l *linkBuilder) GetPackageDownloadURL(id, version string) string { + return fmt.Sprintf("%s/package/%s/%s/%s.%s.nupkg", l.Base, id, version, id, version) +} diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go new file mode 100644 index 0000000000000..6cda36e77c975 --- /dev/null +++ b/routers/api/v1/packages/nuget/nuget.go @@ -0,0 +1,248 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + //"errors" + "fmt" + "net/http" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + nuget_module "code.gitea.io/gitea/modules/packages/nuget" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util/filebuffer" + + package_service "code.gitea.io/gitea/services/packages" +) + +// ServiceIndex https://docs.microsoft.com/en-us/nuget/api/service-index +func ServiceIndex(ctx *context.APIContext) { + repoName := ctx.Repo.Repository.FullName() + + type resource struct { + ID string `json:"@id"` + Type string `json:"@type"` + } + + serviceIndex := struct { + Version string `json:"version"` + Resources []resource `json:"resources"` + }{ + Version: "3.0.0", + Resources: []resource{ + {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/query", Type: "SearchQueryService"}, + {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/query", Type: "SearchQueryService/3.0.0-beta"}, + {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/query", Type: "SearchQueryService/3.0.0-rc"}, + {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/registration", Type: "RegistrationsBaseUrl"}, + {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/registration", Type: "RegistrationsBaseUrl/3.0.0-beta"}, + {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/registration", Type: "RegistrationsBaseUrl/3.0.0-rc"}, + {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/package", Type: "PackageBaseAddress/3.0.0"}, + {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget", Type: "PackagePublish/2.0.0"}, + }, + } + + ctx.JSON(http.StatusOK, serviceIndex) +} + +// SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages +func SearchService(ctx *context.APIContext) { + query := ctx.QueryTrim("q") + skip := ctx.QueryInt("skip") + take := ctx.QueryInt("take") + + total, packages, err := models.SearchPackages(ctx.Repo.Repository.ID, models.PackageNuGet, query, skip, take) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + nugetPackages, err := intializePackages(packages) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + resp := createSearchResultResponse( + &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, + total, + nugetPackages, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// RegistrationIndex https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-index +func RegistrationIndex(ctx *context.APIContext) { + packageName := ctx.Params("id") + + packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageNuGet, packageName) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + if len(packages) == 0 { + ctx.Error(http.StatusNotFound, "", err) + return + } + + nugetPackages, err := intializePackages(packages) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + resp := createRegistrationIndexResponse( + &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, + nugetPackages, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// RegistrationLeaf https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf +func RegistrationLeaf(ctx *context.APIContext) { + packageName := ctx.Params("id") + packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") + + p, err := models.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, models.PackageNuGet, packageName, packageVersion) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.Error(http.StatusNotFound, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + nugetPackage, err := intializePackage(p) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + resp := createRegistrationLeafResponse( + &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, + nugetPackage, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// EnumeratePackageVersions https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#enumerate-package-versions +func EnumeratePackageVersions(ctx *context.APIContext) { + packageName := ctx.Params("id") + + packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageNuGet, packageName) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + if len(packages) == 0 { + ctx.Error(http.StatusNotFound, "", err) + return + } + + nugetPackages, err := intializePackages(packages) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + resp := createPackageVersionsResponse(nugetPackages) + + ctx.JSON(http.StatusOK, resp) +} + +// DownloadPackageContent https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg +func DownloadPackageContent(ctx *context.APIContext) { + packageName := ctx.Params("id") + packageVersion := ctx.Params("version") + filename := ctx.Params("filename") + + s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion, filename) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.Error(http.StatusNotFound, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackage creates a new package with the metadata contained in the uploaded nupgk file +// https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#push-a-package +func UploadPackage(ctx *context.APIContext) { + upload, err := ctx.UploadStream() + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + defer upload.Close() + + buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + defer buf.Close() + + meta, err := nuget_module.ParsePackageMetaData(buf, buf.Size()) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + p, err := package_service.CreatePackage( + ctx.User, + ctx.Repo.Repository, + models.PackageNuGet, + meta.ID, + meta.Version, + meta, + false, + ) + if err != nil { + if err == models.ErrDuplicatePackage { + ctx.Error(http.StatusBadRequest, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + filename := strings.ToLower(fmt.Sprintf("%s.%s.nupkg", meta.ID, meta.Version)) + _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) + if err != nil { + if err := models.DeletePackageByID(p.ID); err != nil { + log.Error("Error deleting package by id: %v", err) + } + ctx.Error(http.StatusInternalServerError, "", err) + } + + ctx.PlainText(http.StatusCreated, nil) +} + +// DeletePackage heard deletes the package +// https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#delete-a-package +func DeletePackage(ctx *context.APIContext) { + packageName := ctx.Params("id") + packageVersion := ctx.Params("version") + + err := package_service.DeletePackage(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.Error(http.StatusNotFound, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", "") + } +} diff --git a/routers/api/v1/packages/nuget/package.go b/routers/api/v1/packages/nuget/package.go new file mode 100644 index 0000000000000..8480e28e29bb2 --- /dev/null +++ b/routers/api/v1/packages/nuget/package.go @@ -0,0 +1,65 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "sort" + + "code.gitea.io/gitea/models" + nuget_module "code.gitea.io/gitea/modules/packages/nuget" + + "github.com/hashicorp/go-version" + jsoniter "github.com/json-iterator/go" +) + +// Package represents a package with NuGet metadata +type Package struct { + *models.Package + Metadata *nuget_module.Metadata +} + +func intializePackages(packages []*models.Package) ([]*Package, error) { + nugetPackages := make([]*Package, 0, len(packages)) + for _, p := range packages { + np, err := intializePackage(p) + if err != nil { + return nil, err + } + nugetPackages = append(nugetPackages, np) + } + return nugetPackages, nil +} + +func intializePackage(p *models.Package) (*Package, error) { + v, err := version.NewSemver(p.Version) + if err != nil { + return nil, err + } + p.SemVer = v + + var m *nuget_module.Metadata + err = jsoniter.Unmarshal([]byte(p.MetadataRaw), &m) + if err != nil { + return nil, err + } + if m == nil { + m = &nuget_module.Metadata{} + } + return &Package{ + p, + m, + }, nil +} + +func sortPackagesByVersionASC(packages []*Package) []*Package { + sortedPackages := make([]*Package, len(packages)) + copy(sortedPackages, packages) + + sort.Slice(sortedPackages, func(i, j int) bool { + return sortedPackages[i].SemVer.LessThan(sortedPackages[j].SemVer) + }) + + return sortedPackages +} diff --git a/services/packages/packages.go b/services/packages/packages.go index f9b4f18bbbf33..0cf0c11422bc3 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -11,18 +11,26 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" + + jsoniter "github.com/json-iterator/go" ) // CreatePackage creates a new package -func CreatePackage(creator *models.User, repository *models.Repository, packageType models.PackageType, name, version string, metaData interface{}, allowDuplicate bool) (*models.Package, error) { +func CreatePackage(creator *models.User, repository *models.Repository, packageType models.PackageType, name, version string, metadata interface{}, allowDuplicate bool) (*models.Package, error) { + metadataJSON, err := jsoniter.Marshal(metadata) + if err != nil { + log.Error("Error converting metadata to JSON: %v", err) + return nil, err + } + p := &models.Package{ - RepositoryID: repository.ID, - CreatorID: creator.ID, - Type: packageType, - Name: name, - LowerName: strings.ToLower(name), - Version: version, - MetaData: metaData, + RepoID: repository.ID, + CreatorID: creator.ID, + Type: packageType, + Name: name, + LowerName: strings.ToLower(name), + Version: version, + MetadataRaw: string(metadataJSON), } if err := models.TryInsertPackage(p); err != nil { if err == models.ErrDuplicatePackage { @@ -93,3 +101,27 @@ func DeletePackage(repository *models.Repository, packageType models.PackageType return nil } + +// GetPackageFileStream returns the content of the specific package file +func GetPackageFileStream(repository *models.Repository, packageType models.PackageType, name, version, filename string) (io.ReadCloser, *models.PackageFile, error) { + p, err := models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) + if err != nil { + return nil, nil, err + } + + pfs, err := p.GetFiles() + if err != nil { + return nil, nil, err + } + + filename = strings.ToLower(filename) + + for _, pf := range pfs { + if pf.LowerName == filename { + s, err := packages_module.NewContentStore().Get(pf.ID) + return s, pf, err + } + } + + return nil, nil, models.ErrPackageNotExist +} From 2b9d96a55574c2109a1d99b3f4f007cbe6533387 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 13 Jul 2021 19:53:55 +0000 Subject: [PATCH 006/130] Moved service index to api file. --- routers/api/v1/packages/nuget/api.go | 28 ++++++++++++++++++++++++++ routers/api/v1/packages/nuget/nuget.go | 27 +++---------------------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/routers/api/v1/packages/nuget/api.go b/routers/api/v1/packages/nuget/api.go index f7586ab65b83c..c11d84bb83eef 100644 --- a/routers/api/v1/packages/nuget/api.go +++ b/routers/api/v1/packages/nuget/api.go @@ -12,6 +12,34 @@ import ( "github.com/hashicorp/go-version" ) +// ServiceIndexResponse https://docs.microsoft.com/en-us/nuget/api/service-index#resources +type ServiceIndexResponse struct { + Version string `json:"version"` + Resources []ServiceResource `json:"resources"` +} + +// ServiceResource https://docs.microsoft.com/en-us/nuget/api/service-index#resource +type ServiceResource struct { + ID string `json:"@id"` + Type string `json:"@type"` +} + +func createServiceIndexResponse(root string) *ServiceIndexResponse { + return &ServiceIndexResponse{ + Version: "3.0.0", + Resources: []ServiceResource{ + {ID: root + "/query", Type: "SearchQueryService"}, + {ID: root + "/query", Type: "SearchQueryService/3.0.0-beta"}, + {ID: root + "/query", Type: "SearchQueryService/3.0.0-rc"}, + {ID: root + "/registration", Type: "RegistrationsBaseUrl"}, + {ID: root + "/registration", Type: "RegistrationsBaseUrl/3.0.0-beta"}, + {ID: root + "/registration", Type: "RegistrationsBaseUrl/3.0.0-rc"}, + {ID: root + "/package", Type: "PackageBaseAddress/3.0.0"}, + {ID: root, Type: "PackagePublish/2.0.0"}, + }, + } +} + // RegistrationIndexResponse https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#response type RegistrationIndexResponse struct { RegistrationIndexURL string `json:"@id"` diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 6cda36e77c975..c82751dacd02e 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -5,7 +5,6 @@ package nuget import ( - //"errors" "fmt" "net/http" "strings" @@ -24,29 +23,9 @@ import ( func ServiceIndex(ctx *context.APIContext) { repoName := ctx.Repo.Repository.FullName() - type resource struct { - ID string `json:"@id"` - Type string `json:"@type"` - } - - serviceIndex := struct { - Version string `json:"version"` - Resources []resource `json:"resources"` - }{ - Version: "3.0.0", - Resources: []resource{ - {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/query", Type: "SearchQueryService"}, - {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/query", Type: "SearchQueryService/3.0.0-beta"}, - {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/query", Type: "SearchQueryService/3.0.0-rc"}, - {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/registration", Type: "RegistrationsBaseUrl"}, - {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/registration", Type: "RegistrationsBaseUrl/3.0.0-beta"}, - {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/registration", Type: "RegistrationsBaseUrl/3.0.0-rc"}, - {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget/package", Type: "PackageBaseAddress/3.0.0"}, - {ID: setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget", Type: "PackagePublish/2.0.0"}, - }, - } - - ctx.JSON(http.StatusOK, serviceIndex) + resp := createServiceIndexResponse(setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget") + + ctx.JSON(http.StatusOK, resp) } // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages From 99298b74744141ded1a7094f87ab783775281ab4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 14 Jul 2021 23:14:00 +0200 Subject: [PATCH 007/130] Added NPM package registry. --- integrations/api_packages_npm_test.go | 118 ++++++++++++ integrations/api_packages_nuget_test.go | 52 ++++-- models/migrations/v188.go | 13 +- models/package.go | 28 ++- modules/packages/content_store.go | 16 +- modules/packages/npm/creator.go | 211 +++++++++++++++++++++ modules/packages/npm/creator_test.go | 224 +++++++++++++++++++++++ modules/packages/npm/metadata.go | 15 ++ modules/packages/nuget/metadata.go | 42 ++--- modules/packages/nuget/metadata_test.go | 24 +++ routers/api/v1/api.go | 8 + routers/api/v1/packages/npm/api.go | 61 ++++++ routers/api/v1/packages/npm/npm.go | 114 ++++++++++++ routers/api/v1/packages/npm/package.go | 74 ++++++++ routers/api/v1/packages/nuget/api.go | 23 ++- routers/api/v1/packages/nuget/nuget.go | 4 +- routers/api/v1/packages/nuget/package.go | 7 +- services/packages/packages.go | 38 +++- 18 files changed, 992 insertions(+), 80 deletions(-) create mode 100644 integrations/api_packages_npm_test.go create mode 100644 modules/packages/npm/creator.go create mode 100644 modules/packages/npm/creator_test.go create mode 100644 modules/packages/npm/metadata.go create mode 100644 routers/api/v1/packages/npm/api.go create mode 100644 routers/api/v1/packages/npm/npm.go create mode 100644 routers/api/v1/packages/npm/package.go diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go new file mode 100644 index 0000000000000..a8097c90e4aa9 --- /dev/null +++ b/integrations/api_packages_npm_test.go @@ -0,0 +1,118 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/url" + "strings" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/packages/npm" + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestPackageNPM(t *testing.T) { + defer prepareTestEnv(t)() + repository := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) + user := models.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + + packageName := "@scope/test-package" + packageVersion := "1.0.1-pre" + packageAuthor := "KN4CK3R" + packageDescription := "Test Description" + + data := "H4sIAAAAAAAA/ytITM5OTE/VL4DQelnF+XkMVAYGBgZmJiYK2MRBwNDcSIHB2NTMwNDQzMwAqA7IMDUxA9LUdgg2UFpcklgEdAql5kD8ogCnhwio5lJQUMpLzE1VslJQcihOzi9I1S9JLS7RhSYIJR2QgrLUouLM/DyQGkM9Az1D3YIiqExKanFyUWZBCVQ2BKhVwQVJDKwosbQkI78IJO/tZ+LsbRykxFXLNdA+HwWjYBSMgpENACgAbtAACAAA" + upload := `{ + "_id": "` + packageName + `", + "name": "` + packageName + `", + "description": "` + packageDescription + `", + "versions": { + "` + packageVersion + `": { + "name": "` + packageName + `", + "version": "` + packageVersion + `", + "description": "` + packageDescription + `", + "author": { + "name": "` + packageAuthor + `" + }, + "dist": { + "integrity": "sha512-yA4FJsVhetynGfOC1jFf79BuS+jrHbm0fhh+aHzCQkOaOBXKf9oBnC4a6DnLLnEsHQDRLYd00cwj8sCXpC+wIg==", + "shasum": "aaa7eaf852a948b0aa05afeda35b1badca155d90" + } + } + }, + "_attachments": { + "` + packageName + `-` + packageVersion + `.tgz": { + "data": "` + data + `" + } + } + }` + + root := fmt.Sprintf("/api/v1/repos/%s/%s/packages/npm/%s", user.Name, repository.Name, url.QueryEscape(packageName)) + filename := fmt.Sprintf("%s-%s.tgz", strings.Split(packageName, "/")[1], packageVersion) + + t.Run("Upload", func(t *testing.T) { + req := NewRequestWithBody(t, "PUT", root, strings.NewReader(upload)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + ps, err := models.GetPackagesByRepositoryID(repository.ID) + assert.NoError(t, err) + assert.Len(t, ps, 1) + assert.Equal(t, packageName, ps[0].Name) + assert.Equal(t, packageVersion, ps[0].Version) + + pfs, err := ps[0].GetFiles() + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, filename, pfs[0].Name) + assert.Equal(t, int64(192), pfs[0].Size) + }) + + t.Run("UploadExists", func(t *testing.T) { + req := NewRequestWithBody(t, "PUT", root, strings.NewReader(upload)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) + + t.Run("Download", func(t *testing.T) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/-/%s/%s", root, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + b, _ := base64.StdEncoding.DecodeString(data) + assert.Equal(t, b, resp.Body.Bytes()) + }) + + t.Run("PackageMetadata", func(t *testing.T) { + req := NewRequest(t, "GET", root) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result npm.PackageMetadata + DecodeJSON(t, resp, &result) + + assert.Equal(t, packageName, result.ID) + assert.Equal(t, packageName, result.Name) + assert.Equal(t, packageDescription, result.Description) + assert.Contains(t, result.DistTags, "latest") + assert.Equal(t, packageVersion, result.DistTags["latest"]) + assert.Equal(t, packageAuthor, result.Author.Name) + assert.Contains(t, result.Versions, packageVersion) + pmv := result.Versions[packageVersion] + assert.Equal(t, fmt.Sprintf("%s@%s", packageName, packageVersion), pmv.ID) + assert.Equal(t, packageName, pmv.Name) + assert.Equal(t, packageDescription, pmv.Description) + assert.Equal(t, packageAuthor, pmv.Author.Name) + assert.Equal(t, "sha512-yA4FJsVhetynGfOC1jFf79BuS+jrHbm0fhh+aHzCQkOaOBXKf9oBnC4a6DnLLnEsHQDRLYd00cwj8sCXpC+wIg==", pmv.Dist.Integrity) + assert.Equal(t, "aaa7eaf852a948b0aa05afeda35b1badca155d90", pmv.Dist.Shasum) + assert.Equal(t, fmt.Sprintf("%s%s/-/%s/%s", setting.AppURL, root[1:], packageVersion, filename), pmv.Dist.Tarball) + }) +} diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index c5e08a96cc156..a31d87f96f0da 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -34,10 +34,10 @@ func TestPackageNuGet(t *testing.T) { w.Write([]byte(` - `+packageName+` - `+packageVersion+` - `+packageAuthors+` - `+packageDescription+` + ` + packageName + ` + ` + packageVersion + ` + ` + packageAuthors + ` + ` + packageDescription + ` `)) archive.Close() @@ -48,9 +48,35 @@ func TestPackageNuGet(t *testing.T) { t.Run("ServiceIndex", func(t *testing.T) { req := NewRequest(t, "GET", fmt.Sprintf("%s/index.json", url)) req = AddBasicAuthHeader(req, user.Name) - _ = MakeRequest(t, req, http.StatusOK) + resp := MakeRequest(t, req, http.StatusOK) + + var result nuget.ServiceIndexResponse + DecodeJSON(t, resp, &result) - + assert.Equal(t, "3.0.0", result.Version) + assert.NotEmpty(t, result.Resources) + + root := setting.AppURL + url[1:] + for _, r := range result.Resources { + switch r.Type { + case "SearchQueryService": + fallthrough + case "SearchQueryService/3.0.0-beta": + fallthrough + case "SearchQueryService/3.0.0-rc": + assert.Equal(t, root+"/query", r.ID) + case "RegistrationsBaseUrl": + fallthrough + case "RegistrationsBaseUrl/3.0.0-beta": + fallthrough + case "RegistrationsBaseUrl/3.0.0-rc": + assert.Equal(t, root+"/registration", r.ID) + case "PackageBaseAddress/3.0.0": + assert.Equal(t, root+"/package", r.ID) + case "PackagePublish/2.0.0": + assert.Equal(t, root, r.ID) + } + } }) t.Run("Upload", func(t *testing.T) { @@ -86,13 +112,13 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("SearchService", func(t *testing.T) { - cases := []struct{ - Query string - Skip int - Take int - ExpectedTotal int64 + cases := []struct { + Query string + Skip int + Take int + ExpectedTotal int64 ExpectedResults int - } { + }{ {"", 0, 0, 1, 1}, {"", 0, 10, 1, 1}, {"gitea", 0, 10, 0, 0}, @@ -112,7 +138,7 @@ func TestPackageNuGet(t *testing.T) { assert.Len(t, result.Data, c.ExpectedResults, "case %d: unexpected result count", i) } }) - + t.Run("RegistrationService", func(t *testing.T) { indexURL := fmt.Sprintf("%s%s/registration/%s/index.json", setting.AppURL, url[1:], packageName) leafURL := fmt.Sprintf("%s%s/registration/%s/%s.json", setting.AppURL, url[1:], packageName, packageVersion) diff --git a/models/migrations/v188.go b/models/migrations/v188.go index 5f8fb3aea52e5..48c2deae06414 100644 --- a/models/migrations/v188.go +++ b/models/migrations/v188.go @@ -30,11 +30,14 @@ func addPackageTables(x *xorm.Engine) error { } type PackageFile struct { - ID int64 `xorm:"pk autoincr"` - PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - Size int64 - Name string `xorm:"UNIQUE(s) NOT NULL"` - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + ID int64 `xorm:"pk autoincr"` + PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + Size int64 + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + HashSHA1 string `xorm:"hash_sha1"` + HashSHA256 string `xorm:"hash_sha256"` + HashSHA512 string `xorm:"hash_sha512"` CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"updated"` diff --git a/models/package.go b/models/package.go index 0ee54f91ef661..15f13bcb81cbe 100644 --- a/models/package.go +++ b/models/package.go @@ -10,7 +10,6 @@ import ( "code.gitea.io/gitea/modules/timeutil" - "github.com/hashicorp/go-version" "xorm.io/builder" ) @@ -21,6 +20,7 @@ type PackageType int const ( PackageGeneric PackageType = iota PackageNuGet // 1 + PackageNPM // 2 ) var ( @@ -37,10 +37,9 @@ type Package struct { CreatorID int64 Type PackageType `xorm:"UNIQUE(s) INDEX NOT NULL"` Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` - SemVer *version.Version `xorm:"-"` - MetadataRaw string `xorm:"TEXT"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` + MetadataRaw string `xorm:"TEXT"` CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"updated"` @@ -48,11 +47,14 @@ type Package struct { // PackageFile represents files associated with a package type PackageFile struct { - ID int64 `xorm:"pk autoincr"` - PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - Size int64 - Name string `xorm:"UNIQUE(s) NOT NULL"` - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + ID int64 `xorm:"pk autoincr"` + PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + Size int64 + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + HashSHA1 string `xorm:"hash_sha1"` + HashSHA256 string `xorm:"hash_sha256"` + HashSHA512 string `xorm:"hash_sha512"` CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"updated"` @@ -175,6 +177,12 @@ func InsertPackageFile(pf *PackageFile) error { return err } +// UpdatePackageFile updates a package file +func UpdatePackageFile(pf *PackageFile) error { + _, err := x.ID(pf.ID).Update(pf) + return err +} + // DeletePackageFileByID deletes a package file func DeletePackageFileByID(fileID int64) error { _, err := x.ID(fileID).Delete(&PackageFile{}) diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index 5c22c8eaf5119..c1bfb99f36d69 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -23,21 +23,21 @@ func NewContentStore() *ContentStore { } // Get get the package file content -func (s *ContentStore) Get(packageFileID int64) (storage.Object, error) { - return s.store.Open(toRelativePath(packageFileID)) +func (s *ContentStore) Get(packageID, packageFileID int64) (storage.Object, error) { + return s.store.Open(toRelativePath(packageID, packageFileID)) } // Save stores the package file content -func (s *ContentStore) Save(packageFileID int64, r io.Reader, size int64) error { - _, err := s.store.Save(toRelativePath(packageFileID), r, size) +func (s *ContentStore) Save(packageID, packageFileID int64, r io.Reader, size int64) error { + _, err := s.store.Save(toRelativePath(packageID, packageFileID), r, size) return err } // Delete deletes the package file content -func (s *ContentStore) Delete(packageFileID int64) error { - return s.store.Delete(toRelativePath(packageFileID)) +func (s *ContentStore) Delete(packageID, packageFileID int64) error { + return s.store.Delete(toRelativePath(packageID, packageFileID)) } -func toRelativePath(packageFileID int64) string { - return fmt.Sprintf("%d/package", packageFileID) +func toRelativePath(packageID, packageFileID int64) string { + return fmt.Sprintf("%d/%d", packageID, packageFileID) } diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go new file mode 100644 index 0000000000000..b0766f7a92e15 --- /dev/null +++ b/modules/packages/npm/creator.go @@ -0,0 +1,211 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "bytes" + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "errors" + "fmt" + "io" + "regexp" + "strings" + "time" + + "github.com/hashicorp/go-version" + jsoniter "github.com/json-iterator/go" +) + +var ( + // ErrInvalidPackage indicates an invalid package + ErrInvalidPackage = errors.New("The package is invalid") + // ErrInvalidPackageName indicates an invalid name + ErrInvalidPackageName = errors.New("The package name is invalid") + // ErrInvalidPackageVersion indicates an invalid version + ErrInvalidPackageVersion = errors.New("The package version is invalid") + // ErrInvalidAttachment indicates a invalid attachment + ErrInvalidAttachment = errors.New("The package attachment is invalid") + // ErrInvalidIntegrity indicates an integrity validation error + ErrInvalidIntegrity = errors.New("Failed to validate integrity") +) + +var nameMatch = regexp.MustCompile(`\A(?:@([^/~'!\(\)\*]+?)[/])?([^/~'!\(\)\*]+?)\z`) + +// Package represents a NPM package +type Package struct { + Name string + Version string + Metadata Metadata + Filename string + Data []byte +} + +// PackageMetadata https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type PackageMetadata struct { + ID string `json:"_id"` + Name string `json:"name"` + Description string `json:"description"` + DistTags map[string]string `json:"dist-tags,omitempty"` + Versions map[string]*PackageMetadataVersion `json:"versions"` + Readme string `json:"readme,omitempty"` + Maintainers []User `json:"maintainers,omitempty"` + Time map[string]time.Time `json:"time,omitempty"` + Homepage string `json:"homepage,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Repository Repository `json:"repository,omitempty"` + Author User `json:"author"` + ReadmeFilename string `json:"readmeFilename,omitempty"` + Users map[string]bool `json:"users,omitempty"` + License string `json:"license,omitempty"` +} + +// PackageMetadataVersion https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type PackageMetadataVersion struct { + ID string `json:"_id"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Author User `json:"author"` + Homepage string `json:"homepage,omitempty"` + License string `json:"license,omitempty"` + Repository Repository `json:"repository,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevDependencies map[string]string `json:"devDependencies,omitempty"` + Readme string `json:"readme,omitempty"` + Dist PackageDistribution `json:"dist"` + Maintainers []User `json:"maintainers,omitempty"` +} + +// PackageDistribution https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type PackageDistribution struct { + Integrity string `json:"integrity"` + Shasum string `json:"shasum"` + Tarball string `json:"tarball"` + FileCount int `json:"fileCount,omitempty"` + UnpackedSize int `json:"unpackedSize,omitempty"` + NpmSignature string `json:"npm-signature,omitempty"` +} + +// User https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type User struct { + Username string `json:"username,omitempty"` + Name string `json:"name"` + Email string `json:"email,omitempty"` + URL string `json:"url,omitempty"` +} + +// Repository https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type Repository struct { + Type string `json:"type"` + URL string `json:"url"` +} + +// PackageAttachment https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type PackageAttachment struct { + ContentType string `json:"content_type"` + Data string `json:"data"` + Length int `json:"length"` +} + +type packageUpload struct { + PackageMetadata + Attachments map[string]*PackageAttachment `json:"_attachments"` +} + +// ParsePackage parses the content into a NPM package +func ParsePackage(r io.Reader) (*Package, error) { + var upload packageUpload + if err := jsoniter.NewDecoder(r).Decode(&upload); err != nil { + return nil, err + } + + for key, meta := range upload.Versions { + if !validateName(meta.Name) { + return nil, ErrInvalidPackageName + } + + _, err := version.NewSemver(key) + if err != nil { + return nil, ErrInvalidPackageVersion + } + + p := &Package{ + Name: meta.Name, + Version: meta.Version, + Metadata: Metadata{ + Description: meta.Description, + Author: meta.Author.Name, + License: meta.License, + Homepage: meta.Homepage, + Dependencies: meta.Dependencies, + Readme: meta.Readme, + }, + } + + names := strings.SplitN(p.Name, "/", 2) + name := names[0] + if len(names) == 2 { + name = names[1] + } + p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version)) + + attachment := func() *PackageAttachment { + for _, a := range upload.Attachments { + return a + } + return nil + }() + if attachment == nil || len(attachment.Data) == 0 { + return nil, ErrInvalidAttachment + } + + data, err := base64.StdEncoding.DecodeString(attachment.Data) + if err != nil { + return nil, ErrInvalidAttachment + } + p.Data = data + + integrity := strings.SplitN(meta.Dist.Integrity, "-", 2) + if len(integrity) != 2 { + return nil, ErrInvalidIntegrity + } + integrityHash, err := base64.StdEncoding.DecodeString(integrity[1]) + if err != nil { + return nil, ErrInvalidIntegrity + } + var hash []byte + switch integrity[0] { + case "sha1": + tmp := sha1.Sum(data) + hash = tmp[:] + case "sha512": + tmp := sha512.Sum512(data) + hash = tmp[:] + } + if !bytes.Equal(integrityHash, hash) { + return nil, ErrInvalidIntegrity + } + + return p, nil + } + + return nil, ErrInvalidPackage +} + +func validateName(name string) bool { + if strings.TrimSpace(name) != name { + return false + } + if len(name) == 0 || len(name) > 214 { + return false + } + if name[0] == '.' || name[0] == '_' { + return false + } + return nameMatch.MatchString(name) +} diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go new file mode 100644 index 0000000000000..c777bbac91d36 --- /dev/null +++ b/modules/packages/npm/creator_test.go @@ -0,0 +1,224 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "bytes" + "encoding/base64" + "fmt" + "strings" + "testing" + + jsoniter "github.com/json-iterator/go" + "github.com/stretchr/testify/assert" +) + +func TestParsePackage(t *testing.T) { + packageName := "@scope/test-package" + packageVersion := "1.0.1-pre" + packageAuthor := "KN4CK3R" + packageDescription := "Test Description" + data := "H4sIAAAAAAAA/ytITM5OTE/VL4DQelnF+XkMVAYGBgZmJiYK2MRBwNDcSIHB2NTMwNDQzMwAqA7IMDUxA9LUdgg2UFpcklgEdAql5kD8ogCnhwio5lJQUMpLzE1VslJQcihOzi9I1S9JLS7RhSYIJR2QgrLUouLM/DyQGkM9Az1D3YIiqExKanFyUWZBCVQ2BKhVwQVJDKwosbQkI78IJO/tZ+LsbRykxFXLNdA+HwWjYBSMgpENACgAbtAACAAA" + integrity := "sha512-yA4FJsVhetynGfOC1jFf79BuS+jrHbm0fhh+aHzCQkOaOBXKf9oBnC4a6DnLLnEsHQDRLYd00cwj8sCXpC+wIg==" + + t.Run("InvalidUpload", func(t *testing.T) { + p, err := ParsePackage(bytes.NewReader([]byte{0})) + assert.Nil(t, p) + assert.Error(t, err) + }) + + t.Run("InvalidUploadNoData", func(t *testing.T) { + b, _ := jsoniter.Marshal(packageUpload{}) + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackage) + }) + + t.Run("InvalidPackageName", func(t *testing.T) { + name := " test " + b, _ := jsoniter.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: name, + Name: name, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: name, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageName) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + version := "first-version" + b, _ := jsoniter.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageName, + Name: packageName, + Versions: map[string]*PackageMetadataVersion{ + version: { + Name: packageName, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageVersion) + }) + + t.Run("InvalidAttachment", func(t *testing.T) { + b, _ := jsoniter.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageName, + Name: packageName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageName, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + "dummy.tgz": {}, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidAttachment) + }) + + t.Run("InvalidData", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) + b, _ := jsoniter.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageName, + Name: packageName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageName, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: "/", + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidAttachment) + }) + + t.Run("InvalidIntegrity", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) + b, _ := jsoniter.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageName, + Name: packageName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageName, + Dist: PackageDistribution{ + Integrity: "sha512-test==", + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: data, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidIntegrity) + }) + + t.Run("InvalidIntegrity2", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) + b, _ := jsoniter.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageName, + Name: packageName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageName, + Dist: PackageDistribution{ + Integrity: integrity, + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: base64.StdEncoding.EncodeToString([]byte("data")), + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidIntegrity) + }) + + t.Run("Valid", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) + b, _ := jsoniter.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageName, + Name: packageName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageName, + Version: packageVersion, + Description: packageDescription, + Author: User{Name: packageAuthor}, + License: "MIT", + Homepage: "https://gitea.io/", + Readme: packageDescription, + Dependencies: map[string]string{ + "package": "1.2.0", + }, + Dist: PackageDistribution{ + Integrity: integrity, + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: data, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, fmt.Sprintf("%s-%s.tgz", strings.Split(packageName, "/")[1], packageVersion), p.Filename) + b, _ = base64.StdEncoding.DecodeString(data) + assert.Equal(t, b, p.Data) + assert.Equal(t, packageDescription, p.Metadata.Description) + assert.Equal(t, packageDescription, p.Metadata.Readme) + assert.Equal(t, packageAuthor, p.Metadata.Author) + assert.Equal(t, "MIT", p.Metadata.License) + assert.Equal(t, "https://gitea.io/", p.Metadata.Homepage) + assert.Contains(t, p.Metadata.Dependencies, "package") + assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"]) + }) +} diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go new file mode 100644 index 0000000000000..f580c868e9e6d --- /dev/null +++ b/modules/packages/npm/metadata.go @@ -0,0 +1,15 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +// Metadata represents the metadata of a NPM package +type Metadata struct { + Description string `json:"description"` + Author string `json:"author"` + License string `json:"license"` + Homepage string `json:"homepage"` + Dependencies map[string]string `json:"dependencies"` + Readme string `json:"readme"` +} diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 6ca2de0d9cc95..f67ac78c4ed6c 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -33,22 +33,21 @@ const maxNuspecFileSize = 3 * 1024 * 1024 // Metadata represents the metadata of a Nuget package type Metadata struct { - ID string `json:"-"` - Version string `json:"-"` - Description string - Summary string - ReleaseNotes string - Authors string - RequireLicenseAcceptance bool - ProjectURL string `json:"project_url"` - RepositoryURL string `json:"repository_url"` - Dependencies map[string][]Dependency + ID string `json:"-"` + Version string `json:"-"` + Description string `json:"description"` + Summary string `json:"summary"` + ReleaseNotes string `json:"release_notes"` + Authors string `json:"authors"` + ProjectURL string `json:"project_url"` + RepositoryURL string `json:"repository_url"` + Dependencies map[string][]Dependency `json:"dependencies"` } // Dependency represents a dependency of a Nuget package type Dependency struct { - ID string - Version string + ID string `json:"id"` + Version string `json:"version"` } type nuspecPackage struct { @@ -123,16 +122,15 @@ func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { } m := &Metadata{ - ID: p.Metadata.ID, - Version: v.String(), - Description: p.Metadata.Description, - Summary: p.Metadata.Summary, - ReleaseNotes: p.Metadata.ReleaseNotes, - Authors: p.Metadata.Authors, - RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance, - ProjectURL: p.Metadata.ProjectURL, - RepositoryURL: p.Metadata.Repository.URL, - Dependencies: make(map[string][]Dependency), + ID: p.Metadata.ID, + Version: v.String(), + Description: p.Metadata.Description, + Summary: p.Metadata.Summary, + ReleaseNotes: p.Metadata.ReleaseNotes, + Authors: p.Metadata.Authors, + ProjectURL: p.Metadata.ProjectURL, + RepositoryURL: p.Metadata.Repository.URL, + Dependencies: make(map[string][]Dependency), } for _, group := range p.Metadata.Dependencies.Group { deps := make([]Dependency, 0, len(group.Dependency)) diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go index 87c73c53ebfe2..b72b282bab8c3 100644 --- a/modules/packages/nuget/metadata_test.go +++ b/modules/packages/nuget/metadata_test.go @@ -76,11 +76,35 @@ func TestParsePackageMetaData(t *testing.T) { t.Run("InvalidNuspecFile", func(t *testing.T) { data := createArchive("package.nuspec", "") + m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, m) + assert.Error(t, err) + }) + + t.Run("InvalidPackageId", func(t *testing.T) { + data := createArchive("package.nuspec", ` + + + `) + m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) assert.Nil(t, m) assert.ErrorIs(t, err, ErrNuspecInvalidID) }) + t.Run("InvalidPackageVersion", func(t *testing.T) { + data := createArchive("package.nuspec", ` + + + `+id+` + + `) + + m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, m) + assert.ErrorIs(t, err, ErrNuspecInvalidVersion) + }) + t.Run("Valid", func(t *testing.T) { data := createArchive("package.nuspec", nuspecContent) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index ae64cb3b3b8ad..1b8907b90be28 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -80,6 +80,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/notify" "code.gitea.io/gitea/routers/api/v1/org" "code.gitea.io/gitea/routers/api/v1/packages/generic" + "code.gitea.io/gitea/routers/api/v1/packages/npm" "code.gitea.io/gitea/routers/api/v1/packages/nuget" "code.gitea.io/gitea/routers/api/v1/repo" "code.gitea.io/gitea/routers/api/v1/settings" @@ -997,6 +998,13 @@ func Routes() *web.Route { }) }) }, reqBasicAuth()) + m.Group("/npm", func() { + m.Group("/{id}", func() { + m.Get("", npm.PackageMetadata) + m.Put("" /*reqRepoWriter(models.UnitTypePackage),*/, npm.UploadPackage) + m.Get("/-/{version}/{filename}", npm.DownloadPackageContent) + }) + }, reqToken()) }) }, repoAssignment()) }) diff --git a/routers/api/v1/packages/npm/api.go b/routers/api/v1/packages/npm/api.go new file mode 100644 index 0000000000000..a916d97747519 --- /dev/null +++ b/routers/api/v1/packages/npm/api.go @@ -0,0 +1,61 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "net/url" + + npm_module "code.gitea.io/gitea/modules/packages/npm" +) + +func createPackageMetadataResponse(registryURL string, packages []*Package) *npm_module.PackageMetadata { + sortedPackages := sortPackagesByVersionASC(packages) + + versions := make(map[string]*npm_module.PackageMetadataVersion) + for _, p := range sortedPackages { + versions[p.SemVer.String()] = createPackageMetadataVersion(registryURL, p) + } + + latest := sortedPackages[len(sortedPackages)-1] + + distTags := make(map[string]string) + distTags["latest"] = latest.Version + + return &npm_module.PackageMetadata{ + ID: latest.Package.Name, + Name: latest.Package.Name, + DistTags: distTags, + Description: latest.Metadata.Description, + Readme: latest.Metadata.Readme, + Homepage: latest.Metadata.Homepage, + Author: npm_module.User{Name: latest.Metadata.Author}, + License: latest.Metadata.License, + Versions: versions, + } +} + +func createPackageMetadataVersion(registryURL string, p *Package) *npm_module.PackageMetadataVersion { + hashBytes, _ := hex.DecodeString(p.PackageFile.HashSHA512) + + return &npm_module.PackageMetadataVersion{ + ID: fmt.Sprintf("%s@%s", p.Package.Name, p.Package.Version), + Name: p.Package.Name, + Version: p.Package.Version, + Description: p.Metadata.Description, + Author: npm_module.User{Name: p.Metadata.Author}, + Homepage: p.Metadata.Homepage, + License: p.Metadata.License, + Dependencies: p.Metadata.Dependencies, + Readme: p.Metadata.Readme, + Dist: npm_module.PackageDistribution{ + Shasum: p.PackageFile.HashSHA1, + Integrity: "sha512-" + base64.StdEncoding.EncodeToString(hashBytes), + Tarball: fmt.Sprintf("%s/%s/-/%s/%s", registryURL, url.QueryEscape(p.Package.Name), p.Package.Version, p.PackageFile.LowerName), + }, + } +} diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go new file mode 100644 index 0000000000000..99caee7c31ff1 --- /dev/null +++ b/routers/api/v1/packages/npm/npm.go @@ -0,0 +1,114 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "bytes" + "net/http" + "net/url" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + npm_module "code.gitea.io/gitea/modules/packages/npm" + "code.gitea.io/gitea/modules/setting" + + package_service "code.gitea.io/gitea/services/packages" +) + +// PackageMetadata returns the metadata for a single package +func PackageMetadata(ctx *context.APIContext) { + packageName, err := url.QueryUnescape(ctx.Params("id")) + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + + packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageNPM, packageName) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + if len(packages) == 0 { + ctx.Error(http.StatusNotFound, "", err) + return + } + + npmPackages, err := intializePackages(packages) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + resp := createPackageMetadataResponse( + setting.AppURL+"api/v1/repos/"+ctx.Repo.Repository.FullName()+"/packages/npm", + npmPackages, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// DownloadPackageContent serves the content of a package +func DownloadPackageContent(ctx *context.APIContext) { + packageName, err := url.QueryUnescape(ctx.Params("id")) + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + packageVersion := ctx.Params("version") + filename := ctx.Params("filename") + + s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageNPM, packageName, packageVersion, filename) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.Error(http.StatusNotFound, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackage creates a new package +func UploadPackage(ctx *context.APIContext) { + defer ctx.Req.Body.Close() + + npmPackage, err := npm_module.ParsePackage(ctx.Req.Body) + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + + p, err := package_service.CreatePackage( + ctx.User, + ctx.Repo.Repository, + models.PackageNPM, + npmPackage.Name, + npmPackage.Version, + npmPackage.Metadata, + false, + ) + if err != nil { + if err == models.ErrDuplicatePackage { + ctx.Error(http.StatusBadRequest, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + _, err = package_service.AddFileToPackage(p, npmPackage.Filename, int64(len(npmPackage.Data)), bytes.NewReader(npmPackage.Data)) + if err != nil { + if err := models.DeletePackageByID(p.ID); err != nil { + log.Error("Error deleting package by id: %v", err) + } + ctx.Error(http.StatusInternalServerError, "", err) + } + + ctx.PlainText(http.StatusCreated, nil) +} diff --git a/routers/api/v1/packages/npm/package.go b/routers/api/v1/packages/npm/package.go new file mode 100644 index 0000000000000..aa845c58b5a81 --- /dev/null +++ b/routers/api/v1/packages/npm/package.go @@ -0,0 +1,74 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package npm + +import ( + "sort" + + "code.gitea.io/gitea/models" + npm_module "code.gitea.io/gitea/modules/packages/npm" + + "github.com/hashicorp/go-version" + jsoniter "github.com/json-iterator/go" +) + +// Package represents a package with NPM metadata +type Package struct { + *models.Package + *models.PackageFile + SemVer *version.Version + Metadata *npm_module.Metadata +} + +func intializePackages(packages []*models.Package) ([]*Package, error) { + nugetPackages := make([]*Package, 0, len(packages)) + for _, p := range packages { + np, err := intializePackage(p) + if err != nil { + return nil, err + } + nugetPackages = append(nugetPackages, np) + } + return nugetPackages, nil +} + +func intializePackage(p *models.Package) (*Package, error) { + v, err := version.NewSemver(p.Version) + if err != nil { + return nil, err + } + + var m *npm_module.Metadata + err = jsoniter.Unmarshal([]byte(p.MetadataRaw), &m) + if err != nil { + return nil, err + } + if m == nil { + m = &npm_module.Metadata{} + } + + pfs, err := p.GetFiles() + if err != nil { + return nil, err + } + + return &Package{ + Package: p, + PackageFile: pfs[0], + SemVer: v, + Metadata: m, + }, nil +} + +func sortPackagesByVersionASC(packages []*Package) []*Package { + sortedPackages := make([]*Package, len(packages)) + copy(sortedPackages, packages) + + sort.Slice(sortedPackages, func(i, j int) bool { + return sortedPackages[i].SemVer.LessThan(sortedPackages[j].SemVer) + }) + + return sortedPackages +} diff --git a/routers/api/v1/packages/nuget/api.go b/routers/api/v1/packages/nuget/api.go index c11d84bb83eef..b981cc98256b7 100644 --- a/routers/api/v1/packages/nuget/api.go +++ b/routers/api/v1/packages/nuget/api.go @@ -14,7 +14,7 @@ import ( // ServiceIndexResponse https://docs.microsoft.com/en-us/nuget/api/service-index#resources type ServiceIndexResponse struct { - Version string `json:"version"` + Version string `json:"version"` Resources []ServiceResource `json:"resources"` } @@ -120,17 +120,16 @@ func createRegistrationIndexPageItem(l *linkBuilder, p *Package) *RegistrationIn RegistrationLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), PackageContentURL: l.GetPackageDownloadURL(p.Name, p.Version), CatalogEntry: &CatalogEntry{ - CatalogLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), - PackageContentURL: l.GetPackageDownloadURL(p.Name, p.Version), - ID: p.Name, - Version: p.Version, - Description: p.Metadata.Description, - Summary: p.Metadata.Summary, - ReleaseNotes: p.Metadata.ReleaseNotes, - Authors: p.Metadata.Authors, - RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance, - ProjectURL: p.Metadata.ProjectURL, - DependencyGroups: createDependencyGroups(p), + CatalogLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), + PackageContentURL: l.GetPackageDownloadURL(p.Name, p.Version), + ID: p.Name, + Version: p.Version, + Description: p.Metadata.Description, + Summary: p.Metadata.Summary, + ReleaseNotes: p.Metadata.ReleaseNotes, + Authors: p.Metadata.Authors, + ProjectURL: p.Metadata.ProjectURL, + DependencyGroups: createDependencyGroups(p), }, } } diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index c82751dacd02e..f3cde14ce5eb4 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -143,7 +143,7 @@ func DownloadPackageContent(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion, filename) + s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageNuGet, packageName, packageVersion, filename) if err != nil { if err == models.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) @@ -210,7 +210,7 @@ func UploadPackage(ctx *context.APIContext) { ctx.PlainText(http.StatusCreated, nil) } -// DeletePackage heard deletes the package +// DeletePackage hard deletes the package // https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#delete-a-package func DeletePackage(ctx *context.APIContext) { packageName := ctx.Params("id") diff --git a/routers/api/v1/packages/nuget/package.go b/routers/api/v1/packages/nuget/package.go index 8480e28e29bb2..442b134321c18 100644 --- a/routers/api/v1/packages/nuget/package.go +++ b/routers/api/v1/packages/nuget/package.go @@ -17,6 +17,7 @@ import ( // Package represents a package with NuGet metadata type Package struct { *models.Package + SemVer *version.Version Metadata *nuget_module.Metadata } @@ -37,7 +38,6 @@ func intializePackage(p *models.Package) (*Package, error) { if err != nil { return nil, err } - p.SemVer = v var m *nuget_module.Metadata err = jsoniter.Unmarshal([]byte(p.MetadataRaw), &m) @@ -48,8 +48,9 @@ func intializePackage(p *models.Package) (*Package, error) { m = &nuget_module.Metadata{} } return &Package{ - p, - m, + Package: p, + SemVer: v, + Metadata: m, }, nil } diff --git a/services/packages/packages.go b/services/packages/packages.go index 0cf0c11422bc3..b75b2542425e1 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -5,6 +5,10 @@ package packages import ( + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "fmt" "io" "strings" @@ -58,9 +62,33 @@ func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reade return nil, err } - packageStore := packages_module.NewContentStore() - if err := packageStore.Save(pf.ID, r, size); err != nil { - log.Error("Error saving package file: %v", err) + h1 := sha1.New() + h256 := sha256.New() + h512 := sha512.New() + + r = io.TeeReader(r, io.MultiWriter(h1, h256, h512)) + + contentStore := packages_module.NewContentStore() + + err := func() error { + err := contentStore.Save(p.ID, pf.ID, r, size) + if err != nil { + log.Error("Error saving package file in content store: %v", err) + return err + } + + pf.HashSHA1 = fmt.Sprintf("%x", h1.Sum(nil)) + pf.HashSHA256 = fmt.Sprintf("%x", h256.Sum(nil)) + pf.HashSHA512 = fmt.Sprintf("%x", h512.Sum(nil)) + if err = models.UpdatePackageFile(pf); err != nil { + log.Error("Error updating package file: %v", err) + return err + } + return nil + }() + if err != nil { + _ = contentStore.Delete(p.ID, pf.ID) + if err := models.DeletePackageFileByID(pf.ID); err != nil { log.Error("Error deleting package file: %v", err) } @@ -88,7 +116,7 @@ func DeletePackage(repository *models.Repository, packageType models.PackageType contentStore := packages_module.NewContentStore() for _, pf := range pfs { - if err := contentStore.Delete(pf.ID); err != nil { + if err := contentStore.Delete(p.ID, pf.ID); err != nil { log.Error("Error deleting package file: %v", err) return err } @@ -118,7 +146,7 @@ func GetPackageFileStream(repository *models.Repository, packageType models.Pack for _, pf := range pfs { if pf.LowerName == filename { - s, err := packages_module.NewContentStore().Get(pf.ID) + s, err := packages_module.NewContentStore().Get(p.ID, pf.ID) return s, pf, err } } From c7fb9329642298f6c5c73d79690b160c952e5edd Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 17 Jul 2021 12:06:22 +0000 Subject: [PATCH 008/130] Added Maven package registry. --- integrations/api_packages_generic_test.go | 4 +- integrations/api_packages_maven_test.go | 106 ++++++++ integrations/api_packages_npm_test.go | 2 +- integrations/api_packages_nuget_test.go | 4 +- models/migrations/v188.go | 1 + models/package.go | 90 +++++-- modules/context/context.go | 14 +- modules/packages/maven/metadata.go | 83 ++++++ modules/packages/maven/metadata_test.go | 73 +++++ modules/packages/npm/creator.go | 2 +- modules/packages/npm/creator_test.go | 2 +- modules/packages/npm/metadata.go | 2 +- modules/packages/nuget/metadata.go | 4 +- modules/packages/nuget/metadata_test.go | 4 +- routers/api/v1/api.go | 5 + routers/api/v1/packages/generic/generic.go | 8 +- routers/api/v1/packages/maven/api.go | 36 +++ routers/api/v1/packages/maven/maven.go | 295 +++++++++++++++++++++ routers/api/v1/packages/maven/package.go | 59 +++++ routers/api/v1/packages/npm/api.go | 4 +- routers/api/v1/packages/npm/npm.go | 4 +- routers/api/v1/packages/npm/package.go | 6 +- routers/api/v1/packages/nuget/nuget.go | 13 +- routers/api/v1/packages/nuget/package.go | 6 +- services/packages/packages.go | 39 +-- 25 files changed, 798 insertions(+), 68 deletions(-) create mode 100644 integrations/api_packages_maven_test.go create mode 100644 modules/packages/maven/metadata.go create mode 100644 modules/packages/maven/metadata_test.go create mode 100644 routers/api/v1/packages/maven/api.go create mode 100644 routers/api/v1/packages/maven/maven.go create mode 100644 routers/api/v1/packages/maven/package.go diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index 89e099b3fd365..aec78eb3c8fbb 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -33,7 +33,7 @@ func TestPackageGeneric(t *testing.T) { req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) MakeRequest(t, req, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryID(repository.ID) + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageGeneric) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) @@ -62,7 +62,7 @@ func TestPackageGeneric(t *testing.T) { req := NewRequest(t, "DELETE", url) MakeRequest(t, req, http.StatusOK) - ps, err := models.GetPackagesByRepositoryID(repository.ID) + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageGeneric) assert.NoError(t, err) assert.Empty(t, ps) }) diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go new file mode 100644 index 0000000000000..d01e9234200c5 --- /dev/null +++ b/integrations/api_packages_maven_test.go @@ -0,0 +1,106 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/packages/maven" + + jsoniter "github.com/json-iterator/go" + "github.com/stretchr/testify/assert" +) + +func TestPackageMaven(t *testing.T) { + defer prepareTestEnv(t)() + repository := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) + user := models.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + + groupID := "com.gitea" + artifactID := "test-project" + packageName := groupID + "-" + artifactID + packageVersion := "1.0.1" + packageDescription := "Test Description" + + root := fmt.Sprintf("/api/v1/repos/%s/%s/packages/maven/%s/%s", user.Name, repository.Name, strings.ReplaceAll(groupID, ".", "/"), artifactID) + filename := fmt.Sprintf("%s-%s.jar", packageName, packageVersion) + + putFile := func(t *testing.T, path, content string, expectedStatus int) { + req := NewRequestWithBody(t, "PUT", root+path, strings.NewReader(content)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, expectedStatus) + } + + t.Run("Upload", func(t *testing.T) { + putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusCreated) + putFile(t, "/maven-metadata.xml", "test", http.StatusOK) + + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageMaven) + assert.NoError(t, err) + assert.Len(t, ps, 1) + assert.Equal(t, packageName, ps[0].Name) + assert.Equal(t, packageVersion, ps[0].Version) + + pfs, err := ps[0].GetFiles() + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, filename, pfs[0].Name) + assert.Equal(t, int64(4), pfs[0].Size) + }) + + t.Run("UploadExists", func(t *testing.T) { + putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusBadRequest) + }) + + t.Run("Download", func(t *testing.T) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, []byte("test"), resp.Body.Bytes()) + }) + + t.Run("UploadVerifySHA1", func(t *testing.T) { + t.Run("Missmatch", func(t *testing.T) { + putFile(t, fmt.Sprintf("/%s/%s.sha1", packageVersion, filename), "test", http.StatusBadRequest) + }) + t.Run("Valid", func(t *testing.T) { + putFile(t, fmt.Sprintf("/%s/%s.sha1", packageVersion, filename), "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", http.StatusOK) + }) + }) + + t.Run("UploadPOM", func(t *testing.T) { + pomContent := ` + + ` + groupID + ` + ` + artifactID + ` + ` + packageVersion + ` + ` + packageDescription + ` +` + + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageMaven) + assert.NoError(t, err) + assert.Len(t, ps, 1) + + var m *maven.Metadata + err = jsoniter.Unmarshal([]byte(ps[0].MetadataRaw), &m) + assert.NoError(t, err) + assert.Empty(t, m.Description) + + putFile(t, fmt.Sprintf("/%s/%s.pom", packageVersion, filename), pomContent, http.StatusCreated) + + ps, err = models.GetPackagesByRepositoryAndType(repository.ID, models.PackageMaven) + assert.NoError(t, err) + assert.Len(t, ps, 1) + + err = jsoniter.Unmarshal([]byte(ps[0].MetadataRaw), &m) + assert.NoError(t, err) + assert.Equal(t, packageDescription, m.Description) + }) +} diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index a8097c90e4aa9..d49e0e6900c7a 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -63,7 +63,7 @@ func TestPackageNPM(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryID(repository.ID) + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNPM) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index a31d87f96f0da..4d2de4ba0b121 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -84,7 +84,7 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryID(repository.ID) + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNuGet) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) @@ -199,7 +199,7 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - ps, err := models.GetPackagesByRepositoryID(repository.ID) + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNuGet) assert.NoError(t, err) assert.Empty(t, ps) }) diff --git a/models/migrations/v188.go b/models/migrations/v188.go index 48c2deae06414..56ee43c6942f5 100644 --- a/models/migrations/v188.go +++ b/models/migrations/v188.go @@ -35,6 +35,7 @@ func addPackageTables(x *xorm.Engine) error { Size int64 Name string LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + HashMD5 string `xorm:"hash_md5"` HashSHA1 string `xorm:"hash_sha1"` HashSHA256 string `xorm:"hash_sha256"` HashSHA512 string `xorm:"hash_sha512"` diff --git a/models/package.go b/models/package.go index 15f13bcb81cbe..432602a455284 100644 --- a/models/package.go +++ b/models/package.go @@ -21,6 +21,7 @@ const ( PackageGeneric PackageType = iota PackageNuGet // 1 PackageNPM // 2 + PackageMaven // 3 ) var ( @@ -28,6 +29,10 @@ var ( ErrDuplicatePackage = errors.New("Package does exist already") // ErrPackageNotExist indicates a package not exist error ErrPackageNotExist = errors.New("Package does not exist") + // ErrDuplicatePackageFile indicates a duplicated package file error + ErrDuplicatePackageFile = errors.New("Package file does exist already") + // ErrPackageFileNotExist indicates a package file not exist error + ErrPackageFileNotExist = errors.New("Package file does not exist") ) // Package represents a package @@ -52,6 +57,7 @@ type PackageFile struct { Size int64 Name string LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + HashMD5 string `xorm:"hash_md5"` HashSHA1 string `xorm:"hash_sha1"` HashSHA256 string `xorm:"hash_sha256"` HashSHA512 string `xorm:"hash_sha512"` @@ -66,29 +72,55 @@ func (p *Package) GetFiles() ([]*PackageFile, error) { return packageFiles, x.Where("package_id = ?", p.ID).Find(&packageFiles) } +// GetFileByName gets the specific package file by name +func (p *Package) GetFileByName(name string) (*PackageFile, error) { + pf := &PackageFile{ + PackageID: p.ID, + LowerName: strings.ToLower(name), + } + has, err := x.Get(pf) + if err != nil { + return nil, err + } + if !has { + return nil, ErrDuplicatePackage + } + return pf, nil +} + // TryInsertPackage inserts a package // If a package already exists ErrDuplicatePackage is returned -func TryInsertPackage(p *Package) error { +func TryInsertPackage(p *Package) (*Package, error) { sess := x.NewSession() defer sess.Close() if err := sess.Begin(); err != nil { - return err + return nil, err } - has, err := sess.Get(p) - if err != nil { - return err + key := &Package{ + RepoID: p.RepoID, + Type: p.Type, + LowerName: p.LowerName, + Version: p.Version, } + has, err := sess.Get(key) + if err != nil { + return nil, err + } if has { - return ErrDuplicatePackage + return key, ErrDuplicatePackage } - if _, err = sess.Insert(p); err != nil { - return err + return nil, err } + return p, sess.Commit() +} - return sess.Commit() +// UpdatePackage updates a package +func UpdatePackage(p *Package) error { + _, err := x.ID(p.ID).Update(p) + return err } // DeletePackageByID deletes a package and its files by ID @@ -103,7 +135,7 @@ func DeletePackageByID(packageID int64) error { // DeletePackagesByRepositoryID deletes all packages of a repository func DeletePackagesByRepositoryID(repositoryID int64) error { - packages, err := GetPackagesByRepositoryID(repositoryID) + packages, err := GetPackagesByRepository(repositoryID) if err != nil { return err } @@ -117,12 +149,18 @@ func DeletePackagesByRepositoryID(repositoryID int64) error { return nil } -// GetPackagesByRepositoryID returns all packages of a repository -func GetPackagesByRepositoryID(repositoryID int64) ([]*Package, error) { +// GetPackagesByRepository returns all packages of a repository +func GetPackagesByRepository(repositoryID int64) ([]*Package, error) { packages := make([]*Package, 0, 10) return packages, x.Where("repo_id = ?", repositoryID).Find(&packages) } +// GetPackagesByRepositoryAndType returns all packages of a repository with the specific type +func GetPackagesByRepositoryAndType(repositoryID int64, packageType PackageType) ([]*Package, error) { + packages := make([]*Package, 0, 10) + return packages, x.Where("repo_id = ? AND type = ?", repositoryID, packageType).Find(&packages) +} + // GetPackagesByName gets all repository packages with the specific name func GetPackagesByName(repositoryID int64, packageType PackageType, packageName string) ([]*Package, error) { packages := make([]*Package, 0, 10) @@ -171,10 +209,30 @@ func SearchPackages(repositoryID int64, packageType PackageType, query string, s return count, packages, err } -// InsertPackageFile inserts a package file -func InsertPackageFile(pf *PackageFile) error { - _, err := x.Insert(pf) - return err +// TryInsertPackageFile inserts a package file +func TryInsertPackageFile(pf *PackageFile) (*PackageFile, error) { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return nil, err + } + + key := &PackageFile{ + PackageID: pf.PackageID, + LowerName: pf.LowerName, + } + + has, err := sess.Get(key) + if err != nil { + return nil, err + } + if has { + return key, ErrDuplicatePackageFile + } + if _, err = sess.Insert(pf); err != nil { + return nil, err + } + return pf, sess.Commit() } // UpdatePackageFile updates a package file diff --git a/modules/context/context.go b/modules/context/context.go index e575e90e10158..211ac14ec1c0b 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -326,23 +326,25 @@ func (ctx *Context) QueryOptionalBool(key string, defaults ...util.OptionalBool) } // UploadStream returns the request body or the first form file -func (ctx *Context) UploadStream() (io.ReadCloser, error) { +// Only form files need to get closed. +func (ctx *Context) UploadStream() (io.ReadCloser, bool, error) { contentType := strings.ToLower(ctx.Req.Header.Get("Content-Type")) if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") || strings.HasPrefix(contentType, "multipart/form-data") { if err := ctx.Req.ParseMultipartForm(32 * 1024 * 1024); err != nil { - return nil, err + return nil, false, err } if ctx.Req.MultipartForm.File == nil { - return nil, http.ErrMissingFile + return nil, false, http.ErrMissingFile } for _, files := range ctx.Req.MultipartForm.File { if len(files) > 0 { - return files[0].Open() + r, err := files[0].Open() + return r, true, err } } - return nil, http.ErrMissingFile + return nil, false, http.ErrMissingFile } - return ctx.Req.Body, nil + return ctx.Req.Body, false, nil } // HandleText handles HTTP status code diff --git a/modules/packages/maven/metadata.go b/modules/packages/maven/metadata.go new file mode 100644 index 0000000000000..09df4c4e75804 --- /dev/null +++ b/modules/packages/maven/metadata.go @@ -0,0 +1,83 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package maven + +import ( + "encoding/xml" + "io" +) + +// Metadata represents the metadata of a Maven package +type Metadata struct { + GroupID string `json:"group_id"` + ArtifactID string `json:"artifact_id"` + Name string `json:"name"` + Description string `json:"description"` + ProjectURL string `json:"project_url"` + Licenses []string `json:"licenses"` + Dependencies []*Dependency `json:"dependencies"` +} + +// Dependency represents a dependency of a Maven package +type Dependency struct { + GroupID string `json:"group_id"` + ArtifactID string `json:"artifact_id"` + Version string `json:"version"` +} + +type pomStruct struct { + XMLName xml.Name `xml:"project"` + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Name string `xml:"name"` + Description string `xml:"description"` + URL string `xml:"url"` + Licenses []struct { + Name string `xml:"name"` + URL string `xml:"url"` + Distribution string `xml:"distribution"` + } `xml:"licenses>license"` + Dependencies []struct { + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Scope string `xml:"scope"` + } `xml:"dependencies>dependency"` +} + +// ParsePackageMetaData parses the metadata of a pom file +func ParsePackageMetaData(r io.Reader) (*Metadata, error) { + var pom pomStruct + if err := xml.NewDecoder(r).Decode(&pom); err != nil { + return nil, err + } + + licenses := make([]string, 0, len(pom.Licenses)) + for _, l := range pom.Licenses { + if l.Name != "" { + licenses = append(licenses, l.Name) + } + } + + dependencies := make([]*Dependency, 0, len(pom.Dependencies)) + for _, d := range pom.Dependencies { + dependencies = append(dependencies, &Dependency{ + GroupID: d.GroupID, + ArtifactID: d.ArtifactID, + Version: d.Version, + }) + } + + return &Metadata{ + GroupID: pom.GroupID, + ArtifactID: pom.ArtifactID, + Name: pom.Name, + Description: pom.Description, + ProjectURL: pom.URL, + Licenses: licenses, + Dependencies: dependencies, + }, nil +} diff --git a/modules/packages/maven/metadata_test.go b/modules/packages/maven/metadata_test.go new file mode 100644 index 0000000000000..a17d4565603a0 --- /dev/null +++ b/modules/packages/maven/metadata_test.go @@ -0,0 +1,73 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package maven + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + groupID = "org.gitea" + artifactID = "my-project" + version = "1.0.1" + name = "My Gitea Project" + description = "Package Description" + projectURL = "https://gitea.io" + license = "MIT" + dependencyGroupID = "org.gitea.core" + dependencyArtifactID = "git" + dependencyVersion = "5.0.0" +) + +const pomContent = ` + + ` + groupID + ` + ` + artifactID + ` + ` + version + ` + ` + name + ` + ` + description + ` + ` + projectURL + ` + + + ` + license + ` + + + + + ` + dependencyGroupID + ` + ` + dependencyArtifactID + ` + ` + dependencyVersion + ` + + +` + +func TestParsePackageMetaData(t *testing.T) { + t.Run("InvalidFile", func(t *testing.T) { + m, err := ParsePackageMetaData(strings.NewReader("")) + assert.Nil(t, m) + assert.Error(t, err) + }) + + t.Run("Valid", func(t *testing.T) { + m, err := ParsePackageMetaData(strings.NewReader(pomContent)) + assert.NoError(t, err) + assert.NotNil(t, m) + + assert.Equal(t, groupID, m.GroupID) + assert.Equal(t, artifactID, m.ArtifactID) + assert.Equal(t, name, m.Name) + assert.Equal(t, description, m.Description) + assert.Equal(t, projectURL, m.ProjectURL) + assert.Len(t, m.Licenses, 1) + assert.Equal(t, license, m.Licenses[0]) + assert.Len(t, m.Dependencies, 1) + assert.Equal(t, dependencyGroupID, m.Dependencies[0].GroupID) + assert.Equal(t, dependencyArtifactID, m.Dependencies[0].ArtifactID) + assert.Equal(t, dependencyVersion, m.Dependencies[0].Version) + }) +} diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index b0766f7a92e15..8eff6dac1140e 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -141,7 +141,7 @@ func ParsePackage(r io.Reader) (*Package, error) { Description: meta.Description, Author: meta.Author.Name, License: meta.License, - Homepage: meta.Homepage, + ProjectURL: meta.Homepage, Dependencies: meta.Dependencies, Readme: meta.Readme, }, diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index c777bbac91d36..aba8fa9c0057f 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -217,7 +217,7 @@ func TestParsePackage(t *testing.T) { assert.Equal(t, packageDescription, p.Metadata.Readme) assert.Equal(t, packageAuthor, p.Metadata.Author) assert.Equal(t, "MIT", p.Metadata.License) - assert.Equal(t, "https://gitea.io/", p.Metadata.Homepage) + assert.Equal(t, "https://gitea.io/", p.Metadata.ProjectURL) assert.Contains(t, p.Metadata.Dependencies, "package") assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"]) }) diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index f580c868e9e6d..a2e40b5b6c9cb 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -9,7 +9,7 @@ type Metadata struct { Description string `json:"description"` Author string `json:"author"` License string `json:"license"` - Homepage string `json:"homepage"` + ProjectURL string `json:"project_url"` Dependencies map[string]string `json:"dependencies"` Readme string `json:"readme"` } diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index f67ac78c4ed6c..ac6203f049b0d 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -106,9 +106,7 @@ func ParsePackageMetaData(r io.ReaderAt, size int64) (*Metadata, error) { // ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { var p nuspecPackage - dec := xml.NewDecoder(r) - err := dec.Decode(&p) - if err != nil { + if err := xml.NewDecoder(r).Decode(&p); err != nil { return nil, err } diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go index b72b282bab8c3..6e50256dbcf41 100644 --- a/modules/packages/nuget/metadata_test.go +++ b/modules/packages/nuget/metadata_test.go @@ -17,11 +17,11 @@ const ( id = "System.Gitea" semver = "1.0.1" authors = "Gitea Authors" - projectURL = "https://gitea.com" + projectURL = "https://gitea.io" description = "Package Description" summary = "Package Summary" releaseNotes = "Package Release Notes" - repositoryURL = "https://gitea.com/gitea/gitea" + repositoryURL = "https://gitea.io/gitea/gitea" targetFramework = ".NETStandard2.1" dependencyID = "System.Text.Json" dependencyVersion = "5.0.0" diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 1b8907b90be28..195e9756143ac 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -80,6 +80,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/notify" "code.gitea.io/gitea/routers/api/v1/org" "code.gitea.io/gitea/routers/api/v1/packages/generic" + "code.gitea.io/gitea/routers/api/v1/packages/maven" "code.gitea.io/gitea/routers/api/v1/packages/npm" "code.gitea.io/gitea/routers/api/v1/packages/nuget" "code.gitea.io/gitea/routers/api/v1/repo" @@ -982,6 +983,10 @@ func Routes() *web.Route { m.Delete("" /*reqRepoWriter(models.UnitTypePackage),*/, generic.DeletePackage) }) }, reqToken()) + m.Group("/maven", func() { + m.Put("/*" /*reqRepoWriter(models.UnitTypePackage),*/, maven.UploadPackageFile) + m.Get("/*" /*reqRepoWriter(models.UnitTypePackage),*/, maven.DownloadPackageFile) + }, reqToken()) m.Group("/nuget", func() { m.Put("/" /*reqRepoWriter(models.UnitTypePackage),*/, nuget.UploadPackage) m.Get("/index.json", nuget.ServiceIndex) diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 5b65264ed1e3d..3ee1c6f0622db 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -32,7 +32,7 @@ func DownloadPackageContent(ctx *context.APIContext) { s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion, filename) if err != nil { - if err == models.ErrPackageNotExist { + if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -53,12 +53,14 @@ func UploadPackage(ctx *context.APIContext) { return } - upload, err := ctx.UploadStream() + upload, close, err := ctx.UploadStream() if err != nil { ctx.Error(http.StatusBadRequest, "", err) return } - defer upload.Close() + if close { + defer upload.Close() + } buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) if err != nil { diff --git a/routers/api/v1/packages/maven/api.go b/routers/api/v1/packages/maven/api.go new file mode 100644 index 0000000000000..e00c08ceeb8a0 --- /dev/null +++ b/routers/api/v1/packages/maven/api.go @@ -0,0 +1,36 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package maven + +import ( + "encoding/xml" +) + +// MetadataResponse https://maven.apache.org/ref/3.2.5/maven-repository-metadata/repository-metadata.html +type MetadataResponse struct { + XMLName xml.Name `xml:"metadata"` + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Latest string `xml:"versioning>latest"` + Version []string `xml:"versioning>versions>version"` +} + +func createMetadataResponse(packages []*Package) *MetadataResponse { + sortedPackages := sortPackagesByVersionASC(packages) + + versions := make([]string, 0, len(sortedPackages)) + for _, p := range sortedPackages { + versions = append(versions, p.Version) + } + + latest := sortedPackages[len(sortedPackages)-1] + + return &MetadataResponse{ + GroupID: latest.Metadata.GroupID, + ArtifactID: latest.Metadata.ArtifactID, + Latest: latest.Version, + Version: versions, + } +} diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go new file mode 100644 index 0000000000000..8b46f929ba83c --- /dev/null +++ b/routers/api/v1/packages/maven/maven.go @@ -0,0 +1,295 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package maven + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "path/filepath" + "regexp" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/packages" + maven_module "code.gitea.io/gitea/modules/packages/maven" + "code.gitea.io/gitea/modules/util/filebuffer" + + package_service "code.gitea.io/gitea/services/packages" + + jsoniter "github.com/json-iterator/go" +) + +const mavenMetadataFile = "maven-metadata.xml" + +var ( + errInvalidParameters = errors.New("Request parameters are invalid") + illegalCharacters = regexp.MustCompile(`[\\/:"<>|?\*]`) +) + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.APIContext) { + params, err := extractPathParameters(ctx) + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + + if params.IsMeta { + serveMavenMetadata(ctx, params) + } else { + servePackageFile(ctx, params) + } +} + +func serveMavenMetadata(ctx *context.APIContext, params parameters) { + if params.Version == "" { + // /com/foo/project/maven-metadata.xml[.sha1/.md5] + + packageName := params.GroupID + "-" + params.ArtifactID + packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageMaven, packageName) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + if len(packages) == 0 { + ctx.Error(http.StatusNotFound, "", err) + return + } + + mavenPackages, err := intializePackages(packages) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + xmlMetadata, err := xml.Marshal(createMetadataResponse(mavenPackages)) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + xmlMetadataWithHeader := append([]byte(xml.Header), xmlMetadata...) + + switch strings.ToLower(filepath.Ext(params.Filename)) { + case ".sha1": + ctx.PlainText(http.StatusOK, []byte(fmt.Sprintf("%x", sha1.Sum(xmlMetadataWithHeader)))) + case ".md5": + ctx.PlainText(http.StatusOK, []byte(fmt.Sprintf("%x", md5.Sum(xmlMetadataWithHeader)))) + default: + ctx.PlainText(http.StatusOK, xmlMetadataWithHeader) + } + } else { + // /com/foo/project/1-SNAPSHOT/maven-metadata.xml[.sha1/.md5] + + ctx.Error(http.StatusNotFound, "", "") + } +} + +func servePackageFile(ctx *context.APIContext, params parameters) { + packageName := params.GroupID + "-" + params.ArtifactID + + p, err := models.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, models.PackageMaven, packageName, params.Version) + if err == models.ErrPackageNotExist { + ctx.Error(http.StatusNotFound, "", err) + return + } + + filename := params.Filename + + ext := strings.ToLower(filepath.Ext(filename)) + if ext == ".sha1" || ext == ".md5" { + filename = filename[:len(filename)-len(ext)] + } + + pf, err := p.GetFileByName(filename) + if err != nil { + if err == models.ErrPackageFileNotExist { + ctx.Error(http.StatusNotFound, "", "") + return + } + log.Error("Error getting file by name: %v", err) + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + if ext == ".sha1" { + ctx.PlainText(http.StatusOK, []byte(pf.HashSHA1)) + return + } + if ext == ".md5" { + ctx.PlainText(http.StatusOK, []byte(pf.HashMD5)) + return + } + s, err := packages.NewContentStore().Get(p.ID, pf.ID) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackageFile adds a file to the package. If the package does not exist, it gets created. +func UploadPackageFile(ctx *context.APIContext) { + params, err := extractPathParameters(ctx) + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + + log.Trace("Parameters: %+v", params) + + if params.IsMeta { + ctx.PlainText(http.StatusOK, nil) + return + } + + packageName := params.GroupID + "-" + params.ArtifactID + + p, err := package_service.CreatePackage( + ctx.User, + ctx.Repo.Repository, + models.PackageMaven, + packageName, + params.Version, + &maven_module.Metadata{ + GroupID: params.GroupID, + ArtifactID: params.ArtifactID, + }, + true, + ) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + buf, err := filebuffer.CreateFromReader(ctx.Req.Body, 32*1024*1024) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + defer buf.Close() + + ext := filepath.Ext(params.Filename) + + if ext == ".sha1" || ext == ".md5" { + if ext == ".sha1" { + pf, err := p.GetFileByName(params.Filename[:len(params.Filename)-5]) + if err != nil { + if err == models.ErrPackageFileNotExist { + ctx.Error(http.StatusNotFound, "", "") + return + } + log.Error("GetFileByName: %v", err) + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + hash, err := ioutil.ReadAll(buf) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + if pf.HashSHA1 != string(hash) { + ctx.Error(http.StatusBadRequest, "", "hash mismatch") + return + } + } + + ctx.PlainText(http.StatusOK, nil) + return + } + + // If it's the package pom file extract the metadata + if ext == ".pom" { + metadata, err := maven_module.ParsePackageMetaData(buf) + if err != nil { + log.Error("Error parsing package metadata: %v", err) + } + if metadata != nil { + raw, err := jsoniter.Marshal(metadata) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + p.MetadataRaw = string(raw) + if err := models.UpdatePackage(p); err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + } + if _, err := buf.Seek(0, io.SeekStart); err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + } + + _, err = package_service.AddFileToPackage(p, params.Filename, buf.Size(), buf) + if err != nil { + if err == models.ErrDuplicatePackageFile { + ctx.Error(http.StatusBadRequest, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + ctx.PlainText(http.StatusCreated, nil) +} + +type parameters struct { + GroupID string + ArtifactID string + Version string + Filename string + IsMeta bool +} + +func extractPathParameters(ctx *context.APIContext) (parameters, error) { + parts := strings.Split(ctx.Params("*"), "/") + + p := parameters{ + Filename: parts[len(parts)-1], + } + + p.IsMeta = p.Filename == mavenMetadataFile || p.Filename == mavenMetadataFile+".sha1" || p.Filename == mavenMetadataFile+".md5" + + parts = parts[:len(parts)-1] + if len(parts) == 0 { + return p, errInvalidParameters + } + + p.Version = parts[len(parts)-1] + if p.IsMeta && !strings.HasSuffix(p.Version, "-SNAPSHOT") { + p.Version = "" + } else { + parts = parts[:len(parts)-1] + } + + if illegalCharacters.MatchString(p.Version) { + return p, errInvalidParameters + } + + if len(parts) < 2 { + return p, errInvalidParameters + } + + p.ArtifactID = parts[len(parts)-1] + p.GroupID = strings.Join(parts[:len(parts)-1], ".") + + if illegalCharacters.MatchString(p.GroupID) || illegalCharacters.MatchString(p.ArtifactID) { + return p, errInvalidParameters + } + + return p, nil +} diff --git a/routers/api/v1/packages/maven/package.go b/routers/api/v1/packages/maven/package.go new file mode 100644 index 0000000000000..9ef5d0550d622 --- /dev/null +++ b/routers/api/v1/packages/maven/package.go @@ -0,0 +1,59 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package maven + +import ( + "sort" + + "code.gitea.io/gitea/models" + maven_module "code.gitea.io/gitea/modules/packages/maven" + + jsoniter "github.com/json-iterator/go" +) + +// Package represents a package with Maven metadata +type Package struct { + *models.Package + Metadata *maven_module.Metadata +} + +func intializePackages(packages []*models.Package) ([]*Package, error) { + pgs := make([]*Package, 0, len(packages)) + for _, p := range packages { + np, err := intializePackage(p) + if err != nil { + return nil, err + } + pgs = append(pgs, np) + } + return pgs, nil +} + +func intializePackage(p *models.Package) (*Package, error) { + var m *maven_module.Metadata + err := jsoniter.Unmarshal([]byte(p.MetadataRaw), &m) + if err != nil { + return nil, err + } + if m == nil { + m = &maven_module.Metadata{} + } + + return &Package{ + Package: p, + Metadata: m, + }, nil +} + +func sortPackagesByVersionASC(packages []*Package) []*Package { + sortedPackages := make([]*Package, len(packages)) + copy(sortedPackages, packages) + + sort.Slice(sortedPackages, func(i, j int) bool { + return sortedPackages[i].Version < sortedPackages[j].Version + }) + + return sortedPackages +} diff --git a/routers/api/v1/packages/npm/api.go b/routers/api/v1/packages/npm/api.go index a916d97747519..d4e9592b48510 100644 --- a/routers/api/v1/packages/npm/api.go +++ b/routers/api/v1/packages/npm/api.go @@ -32,7 +32,7 @@ func createPackageMetadataResponse(registryURL string, packages []*Package) *npm DistTags: distTags, Description: latest.Metadata.Description, Readme: latest.Metadata.Readme, - Homepage: latest.Metadata.Homepage, + Homepage: latest.Metadata.ProjectURL, Author: npm_module.User{Name: latest.Metadata.Author}, License: latest.Metadata.License, Versions: versions, @@ -48,7 +48,7 @@ func createPackageMetadataVersion(registryURL string, p *Package) *npm_module.Pa Version: p.Package.Version, Description: p.Metadata.Description, Author: npm_module.User{Name: p.Metadata.Author}, - Homepage: p.Metadata.Homepage, + Homepage: p.Metadata.ProjectURL, License: p.Metadata.License, Dependencies: p.Metadata.Dependencies, Readme: p.Metadata.Readme, diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 99caee7c31ff1..2d39bd01612b3 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -62,7 +62,7 @@ func DownloadPackageContent(ctx *context.APIContext) { s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageNPM, packageName, packageVersion, filename) if err != nil { - if err == models.ErrPackageNotExist { + if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -76,8 +76,6 @@ func DownloadPackageContent(ctx *context.APIContext) { // UploadPackage creates a new package func UploadPackage(ctx *context.APIContext) { - defer ctx.Req.Body.Close() - npmPackage, err := npm_module.ParsePackage(ctx.Req.Body) if err != nil { ctx.Error(http.StatusBadRequest, "", err) diff --git a/routers/api/v1/packages/npm/package.go b/routers/api/v1/packages/npm/package.go index aa845c58b5a81..b74f368cea256 100644 --- a/routers/api/v1/packages/npm/package.go +++ b/routers/api/v1/packages/npm/package.go @@ -23,15 +23,15 @@ type Package struct { } func intializePackages(packages []*models.Package) ([]*Package, error) { - nugetPackages := make([]*Package, 0, len(packages)) + pgs := make([]*Package, 0, len(packages)) for _, p := range packages { np, err := intializePackage(p) if err != nil { return nil, err } - nugetPackages = append(nugetPackages, np) + pgs = append(pgs, np) } - return nugetPackages, nil + return pgs, nil } func intializePackage(p *models.Package) (*Package, error) { diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index f3cde14ce5eb4..faaef9f15c1aa 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -6,6 +6,7 @@ package nuget import ( "fmt" + "io" "net/http" "strings" @@ -145,7 +146,7 @@ func DownloadPackageContent(ctx *context.APIContext) { s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageNuGet, packageName, packageVersion, filename) if err != nil { - if err == models.ErrPackageNotExist { + if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -160,12 +161,14 @@ func DownloadPackageContent(ctx *context.APIContext) { // UploadPackage creates a new package with the metadata contained in the uploaded nupgk file // https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#push-a-package func UploadPackage(ctx *context.APIContext) { - upload, err := ctx.UploadStream() + upload, close, err := ctx.UploadStream() if err != nil { ctx.Error(http.StatusBadRequest, "", err) return } - defer upload.Close() + if close { + defer upload.Close() + } buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) if err != nil { @@ -179,6 +182,10 @@ func UploadPackage(ctx *context.APIContext) { ctx.Error(http.StatusInternalServerError, "", err) return } + if _, err := buf.Seek(0, io.SeekStart); err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } p, err := package_service.CreatePackage( ctx.User, diff --git a/routers/api/v1/packages/nuget/package.go b/routers/api/v1/packages/nuget/package.go index 442b134321c18..c4be91e77aa3b 100644 --- a/routers/api/v1/packages/nuget/package.go +++ b/routers/api/v1/packages/nuget/package.go @@ -22,15 +22,15 @@ type Package struct { } func intializePackages(packages []*models.Package) ([]*Package, error) { - nugetPackages := make([]*Package, 0, len(packages)) + pgs := make([]*Package, 0, len(packages)) for _, p := range packages { np, err := intializePackage(p) if err != nil { return nil, err } - nugetPackages = append(nugetPackages, np) + pgs = append(pgs, np) } - return nugetPackages, nil + return pgs, nil } func intializePackage(p *models.Package) (*Package, error) { diff --git a/services/packages/packages.go b/services/packages/packages.go index b75b2542425e1..b8b88fd278ef6 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -5,6 +5,7 @@ package packages import ( + "crypto/md5" "crypto/sha1" "crypto/sha256" "crypto/sha512" @@ -27,6 +28,8 @@ func CreatePackage(creator *models.User, repository *models.Repository, packageT return nil, err } + log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", creator.ID, repository.ID, packageType, name, version, metadata, allowDuplicate) + p := &models.Package{ RepoID: repository.ID, CreatorID: creator.ID, @@ -36,7 +39,7 @@ func CreatePackage(creator *models.User, repository *models.Repository, packageT Version: version, MetadataRaw: string(metadataJSON), } - if err := models.TryInsertPackage(p); err != nil { + if p, err = models.TryInsertPackage(p); err != nil { if err == models.ErrDuplicatePackage { if allowDuplicate { return p, nil @@ -51,32 +54,40 @@ func CreatePackage(creator *models.User, repository *models.Repository, packageT // AddFileToPackage adds a new file to package and stores its content func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reader) (*models.PackageFile, error) { + log.Trace("Creating package file: %v, %v, %s", p.ID, size, filename) + pf := &models.PackageFile{ PackageID: p.ID, Size: size, Name: filename, LowerName: strings.ToLower(filename), } - if err := models.InsertPackageFile(pf); err != nil { + var err error + if pf, err = models.TryInsertPackageFile(pf); err != nil { + if err == models.ErrDuplicatePackageFile { + return nil, err + } log.Error("Error inserting package file: %v", err) return nil, err } + md5 := md5.New() h1 := sha1.New() h256 := sha256.New() h512 := sha512.New() - r = io.TeeReader(r, io.MultiWriter(h1, h256, h512)) + r = io.TeeReader(r, io.MultiWriter(md5, h1, h256, h512)) contentStore := packages_module.NewContentStore() - err := func() error { + err = func() error { err := contentStore.Save(p.ID, pf.ID, r, size) if err != nil { log.Error("Error saving package file in content store: %v", err) return err } + pf.HashMD5 = fmt.Sprintf("%x", md5.Sum(nil)) pf.HashSHA1 = fmt.Sprintf("%x", h1.Sum(nil)) pf.HashSHA256 = fmt.Sprintf("%x", h256.Sum(nil)) pf.HashSHA512 = fmt.Sprintf("%x", h512.Sum(nil)) @@ -99,6 +110,8 @@ func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reade // DeletePackage deletes a package and all associated files func DeletePackage(repository *models.Repository, packageType models.PackageType, name, version string) error { + log.Trace("Deleting package: %v, %v, %s, %s", repository.ID, packageType, name, version) + p, err := models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) if err != nil { if err == models.ErrPackageNotExist { @@ -117,7 +130,7 @@ func DeletePackage(repository *models.Repository, packageType models.PackageType contentStore := packages_module.NewContentStore() for _, pf := range pfs { if err := contentStore.Delete(p.ID, pf.ID); err != nil { - log.Error("Error deleting package file: %v", err) + log.Error("Error deleting package file [%s]: %v", pf.Name, err) return err } } @@ -132,24 +145,18 @@ func DeletePackage(repository *models.Repository, packageType models.PackageType // GetPackageFileStream returns the content of the specific package file func GetPackageFileStream(repository *models.Repository, packageType models.PackageType, name, version, filename string) (io.ReadCloser, *models.PackageFile, error) { + log.Trace("Getting package file stream: %v, %v, %s, %s, %s", repository.ID, packageType, name, version, filename) + p, err := models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) if err != nil { return nil, nil, err } - pfs, err := p.GetFiles() + pf, err := p.GetFileByName(filename) if err != nil { return nil, nil, err } - filename = strings.ToLower(filename) - - for _, pf := range pfs { - if pf.LowerName == filename { - s, err := packages_module.NewContentStore().Get(p.ID, pf.ID) - return s, pf, err - } - } - - return nil, nil, models.ErrPackageNotExist + s, err := packages_module.NewContentStore().Get(p.ID, pf.ID) + return s, pf, err } From 8aaf412c346bbe4d2e31ac6ece00cd38687f9c42 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 18 Jul 2021 14:35:06 +0000 Subject: [PATCH 009/130] Added PyPI package registry. --- integrations/api_packages_pypi_test.go | 144 +++++++++++++++++++++ models/package.go | 1 + modules/packages/pypi/metadata.go | 15 +++ routers/api/v1/api.go | 6 + routers/api/v1/packages/generic/generic.go | 3 + routers/api/v1/packages/npm/npm.go | 1 + routers/api/v1/packages/nuget/nuget.go | 1 + routers/api/v1/packages/pypi/package.go | 53 ++++++++ routers/api/v1/packages/pypi/pypi.go | 143 ++++++++++++++++++++ templates/api/packages/pypi/simple.tmpl | 15 +++ 10 files changed, 382 insertions(+) create mode 100644 integrations/api_packages_pypi_test.go create mode 100644 modules/packages/pypi/metadata.go create mode 100644 routers/api/v1/packages/pypi/package.go create mode 100644 routers/api/v1/packages/pypi/pypi.go create mode 100644 templates/api/packages/pypi/simple.tmpl diff --git a/integrations/api_packages_pypi_test.go b/integrations/api_packages_pypi_test.go new file mode 100644 index 0000000000000..5c3b9d4015bae --- /dev/null +++ b/integrations/api_packages_pypi_test.go @@ -0,0 +1,144 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + "regexp" + "strings" + "testing" + + "code.gitea.io/gitea/models" + + "github.com/stretchr/testify/assert" +) + +func TestPackagePyPI(t *testing.T) { + defer prepareTestEnv(t)() + repository := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) + user := models.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + + packageName := "test-package" + packageVersion := "1.0.1" + packageAuthor := "KN4CK3R" + packageDescription := "Test Description" + + content := "test" + hashSHA256 := "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + root := fmt.Sprintf("/api/v1/repos/%s/%s/packages/pypi", user.Name, repository.Name) + + uploadFile := func(t *testing.T, filename, content string, expectedStatus int) { + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, _ := writer.CreateFormFile("content", filename) + _, _ = io.Copy(part, strings.NewReader(content)) + + writer.WriteField("name", packageName) + writer.WriteField("version", packageVersion) + writer.WriteField("author", packageAuthor) + writer.WriteField("summary", packageDescription) + writer.WriteField("description", packageDescription) + writer.WriteField("sha256_digest", hashSHA256) + writer.WriteField("requires_python", "3.6") + + _ = writer.Close() + + req := NewRequestWithBody(t, "POST", root, body) + req.Header.Add("Content-Type", writer.FormDataContentType()) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, expectedStatus) + } + + t.Run("Upload", func(t *testing.T) { + filename := "test.whl" + uploadFile(t, filename, content, http.StatusCreated) + + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackagePyPI) + assert.NoError(t, err) + assert.Len(t, ps, 1) + assert.Equal(t, packageName, ps[0].Name) + assert.Equal(t, packageVersion, ps[0].Version) + + pfs, err := ps[0].GetFiles() + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, filename, pfs[0].Name) + assert.Equal(t, int64(4), pfs[0].Size) + }) + + t.Run("UploadAddFile", func(t *testing.T) { + filename := "test.tar.gz" + uploadFile(t, filename, content, http.StatusCreated) + + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackagePyPI) + assert.NoError(t, err) + assert.Len(t, ps, 1) + assert.Equal(t, packageName, ps[0].Name) + assert.Equal(t, packageVersion, ps[0].Version) + + pf, err := ps[0].GetFileByName(filename) + assert.NoError(t, err) + assert.NotNil(t, pf) + assert.Equal(t, filename, pf.Name) + assert.Equal(t, int64(4), pf.Size) + + pfs, err := ps[0].GetFiles() + assert.NoError(t, err) + assert.Len(t, pfs, 2) + }) + + t.Run("UploadHashMismatch", func(t *testing.T) { + filename := "test2.whl" + uploadFile(t, filename, "dummy", http.StatusBadRequest) + }) + + t.Run("UploadExists", func(t *testing.T) { + uploadFile(t, "test.whl", content, http.StatusBadRequest) + uploadFile(t, "test.tar.gz", content, http.StatusBadRequest) + }) + + t.Run("Download", func(t *testing.T) { + downloadFile := func(filename string) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/files/%s/%s/%s", root, packageName, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, []byte(content), resp.Body.Bytes()) + } + + downloadFile("test.whl") + downloadFile("test.tar.gz") + }) + + t.Run("PackageMetadata", func(t *testing.T) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + htmlDoc := NewHTMLParser(t, resp.Body) + nodes := htmlDoc.doc.Find("a").Nodes + assert.Len(t, nodes, 2) + + hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256-%s`, root, packageName, packageVersion, hashSHA256)) + + for _, a := range nodes { + for _, att := range a.Attr { + switch att.Key { + case "href": + assert.Regexp(t, hrefMatcher, att.Val) + case "data-requires-python": + assert.Equal(t, "3.6", att.Val) + default: + t.Fail() + } + } + } + }) +} diff --git a/models/package.go b/models/package.go index 432602a455284..13dcdbb401d68 100644 --- a/models/package.go +++ b/models/package.go @@ -22,6 +22,7 @@ const ( PackageNuGet // 1 PackageNPM // 2 PackageMaven // 3 + PackagePyPI // 4 ) var ( diff --git a/modules/packages/pypi/metadata.go b/modules/packages/pypi/metadata.go new file mode 100644 index 0000000000000..e3998af794789 --- /dev/null +++ b/modules/packages/pypi/metadata.go @@ -0,0 +1,15 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pypi + +// Metadata represents the metadata of a PyPI package +type Metadata struct { + Author string `json:"author"` + Description string `json:"description"` + Summary string `json:"summary"` + ProjectURL string `json:"project_url"` + License string `json:"license"` + RequiresPython string `json:"requires_python"` +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 195e9756143ac..6b69f14671cce 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -83,6 +83,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/packages/maven" "code.gitea.io/gitea/routers/api/v1/packages/npm" "code.gitea.io/gitea/routers/api/v1/packages/nuget" + "code.gitea.io/gitea/routers/api/v1/packages/pypi" "code.gitea.io/gitea/routers/api/v1/repo" "code.gitea.io/gitea/routers/api/v1/settings" _ "code.gitea.io/gitea/routers/api/v1/swagger" // for swagger generation @@ -1010,6 +1011,11 @@ func Routes() *web.Route { m.Get("/-/{version}/{filename}", npm.DownloadPackageContent) }) }, reqToken()) + m.Group("/pypi", func() { + m.Post("/" /*reqRepoWriter(models.UnitTypePackage),*/, pypi.UploadPackageFile) + m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageContent) + m.Get("/simple/{id}", pypi.PackageMetadata) + }, reqBasicAuth()) }) }, repoAssignment()) }) diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 3ee1c6f0622db..89da2a19f43c2 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -96,6 +96,7 @@ func UploadPackage(ctx *context.APIContext) { log.Error("Error deleting package by id: %v", err) } ctx.Error(http.StatusInternalServerError, "", "") + return } ctx.PlainText(http.StatusCreated, nil) @@ -116,7 +117,9 @@ func DeletePackage(ctx *context.APIContext) { return } ctx.Error(http.StatusInternalServerError, "", "") + return } + ctx.PlainText(http.StatusOK, nil) } func sanitizeParameters(ctx *context.APIContext) (string, string, string, error) { diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 2d39bd01612b3..440f5445c4519 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -106,6 +106,7 @@ func UploadPackage(ctx *context.APIContext) { log.Error("Error deleting package by id: %v", err) } ctx.Error(http.StatusInternalServerError, "", err) + return } ctx.PlainText(http.StatusCreated, nil) diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index faaef9f15c1aa..2c40f41bf28c2 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -212,6 +212,7 @@ func UploadPackage(ctx *context.APIContext) { log.Error("Error deleting package by id: %v", err) } ctx.Error(http.StatusInternalServerError, "", err) + return } ctx.PlainText(http.StatusCreated, nil) diff --git a/routers/api/v1/packages/pypi/package.go b/routers/api/v1/packages/pypi/package.go new file mode 100644 index 0000000000000..8050d4f94d5b7 --- /dev/null +++ b/routers/api/v1/packages/pypi/package.go @@ -0,0 +1,53 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pypi + +import ( + "code.gitea.io/gitea/models" + pypi_module "code.gitea.io/gitea/modules/packages/pypi" + + jsoniter "github.com/json-iterator/go" +) + +// Package represents a package with NPM metadata +type Package struct { + *models.Package + Files []*models.PackageFile + Metadata *pypi_module.Metadata +} + +func intializePackages(packages []*models.Package) ([]*Package, error) { + pgs := make([]*Package, 0, len(packages)) + for _, p := range packages { + np, err := intializePackage(p) + if err != nil { + return nil, err + } + pgs = append(pgs, np) + } + return pgs, nil +} + +func intializePackage(p *models.Package) (*Package, error) { + var m *pypi_module.Metadata + err := jsoniter.Unmarshal([]byte(p.MetadataRaw), &m) + if err != nil { + return nil, err + } + if m == nil { + m = &pypi_module.Metadata{} + } + + pfs, err := p.GetFiles() + if err != nil { + return nil, err + } + + return &Package{ + Package: p, + Files: pfs, + Metadata: m, + }, nil +} diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go new file mode 100644 index 0000000000000..1cfb156ce4397 --- /dev/null +++ b/routers/api/v1/packages/pypi/pypi.go @@ -0,0 +1,143 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pypi + +import ( + "crypto/sha256" + "fmt" + "io" + "net/http" + "regexp" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + pypi_module "code.gitea.io/gitea/modules/packages/pypi" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" + + package_service "code.gitea.io/gitea/services/packages" +) + +// https://www.python.org/dev/peps/pep-0503/#normalized-names +var normalizer = strings.NewReplacer(".", "-", "_", "-") +var nameMatcher = regexp.MustCompile(`\A[a-z0-9\.\-_]+\z`) + +// https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions +var versionMatcher = regexp.MustCompile(`^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$`) + +// PackageMetadata returns the metadata for a single package +func PackageMetadata(ctx *context.APIContext) { + packageName := normalizer.Replace(ctx.Params("id")) + + packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackagePyPI, packageName) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + if len(packages) == 0 { + ctx.Error(http.StatusNotFound, "", err) + return + } + + pypiPackages, err := intializePackages(packages) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + ctx.Data["RegistryURL"] = setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/pypi" + ctx.Data["Package"] = pypiPackages[0] + ctx.Data["Packages"] = pypiPackages + ctx.Render = templates.HTMLRenderer() + ctx.HTML(http.StatusOK, "api/packages/pypi/simple") +} + +// DownloadPackageContent serves the content of a package +func DownloadPackageContent(ctx *context.APIContext) { + packageName := normalizer.Replace(ctx.Params("id")) + packageVersion := ctx.Params("version") + filename := ctx.Params("filename") + + s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackagePyPI, packageName, packageVersion, filename) + if err != nil { + if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { + ctx.Error(http.StatusNotFound, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackageFile adds a file to the package. If the package does not exist, it gets created. +func UploadPackageFile(ctx *context.APIContext) { + file, fileHeader, err := ctx.Req.FormFile("content") + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + defer file.Close() + + h256 := sha256.New() + if _, err := io.Copy(h256, file); err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + if !strings.EqualFold(ctx.Req.FormValue("sha256_digest"), fmt.Sprintf("%x", h256.Sum(nil))) { + ctx.Error(http.StatusBadRequest, "", "hash mismatch") + return + } + + if _, err := file.Seek(0, io.SeekStart); err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + packageName := normalizer.Replace(ctx.Req.FormValue("name")) + packageVersion := ctx.Req.FormValue("version") + if !nameMatcher.MatchString(packageName) || !versionMatcher.MatchString(packageVersion) { + ctx.Error(http.StatusBadRequest, "", "invalid name or version") + return + } + + metadata := &pypi_module.Metadata{ + Author: ctx.Req.FormValue("author"), + Description: ctx.Req.FormValue("description"), + Summary: ctx.Req.FormValue("summary"), + ProjectURL: ctx.Req.FormValue("home_page"), + License: ctx.Req.FormValue("license"), + RequiresPython: ctx.Req.FormValue("requires_python"), + } + + p, err := package_service.CreatePackage( + ctx.User, + ctx.Repo.Repository, + models.PackagePyPI, + packageName, + packageVersion, + metadata, + true, + ) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + _, err = package_service.AddFileToPackage(p, fileHeader.Filename, fileHeader.Size, file) + if err != nil { + if err == models.ErrDuplicatePackageFile { + ctx.Error(http.StatusBadRequest, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + ctx.PlainText(http.StatusCreated, nil) +} diff --git a/templates/api/packages/pypi/simple.tmpl b/templates/api/packages/pypi/simple.tmpl new file mode 100644 index 0000000000000..f3ed056505179 --- /dev/null +++ b/templates/api/packages/pypi/simple.tmpl @@ -0,0 +1,15 @@ + + + + Links for {{.Package.Name}} + + +

Links for {{.Package.Name}}

+ {{range .Packages}} + {{$p := .}} + {{range .Files}} + {{.Name}}
+ {{end}} + {{end}} + + \ No newline at end of file From 06790f91d00a2f9fc0fef41d99aa2c2e95d3dbba Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 19 Jul 2021 21:32:34 +0000 Subject: [PATCH 010/130] Summary is deprecated. --- modules/packages/nuget/metadata.go | 7 +++---- modules/packages/nuget/metadata_test.go | 3 --- routers/api/v1/packages/nuget/api.go | 4 ---- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index ac6203f049b0d..7704ddd20d164 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -36,7 +36,6 @@ type Metadata struct { ID string `json:"-"` Version string `json:"-"` Description string `json:"description"` - Summary string `json:"summary"` ReleaseNotes string `json:"release_notes"` Authors string `json:"authors"` ProjectURL string `json:"project_url"` @@ -58,7 +57,6 @@ type nuspecPackage struct { RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` ProjectURL string `xml:"projectUrl"` Description string `xml:"description"` - Summary string `xml:"summary"` ReleaseNotes string `xml:"releaseNotes"` Repository struct { URL string `xml:"url,attr"` @@ -123,7 +121,6 @@ func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { ID: p.Metadata.ID, Version: v.String(), Description: p.Metadata.Description, - Summary: p.Metadata.Summary, ReleaseNotes: p.Metadata.ReleaseNotes, Authors: p.Metadata.Authors, ProjectURL: p.Metadata.ProjectURL, @@ -141,7 +138,9 @@ func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { Version: dep.Version, }) } - m.Dependencies[group.TargetFramework] = deps + if len(deps) > 0 { + m.Dependencies[group.TargetFramework] = deps + } } return m, nil } diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go index 6e50256dbcf41..40a00a190e4f9 100644 --- a/modules/packages/nuget/metadata_test.go +++ b/modules/packages/nuget/metadata_test.go @@ -19,7 +19,6 @@ const ( authors = "Gitea Authors" projectURL = "https://gitea.io" description = "Package Description" - summary = "Package Summary" releaseNotes = "Package Release Notes" repositoryURL = "https://gitea.io/gitea/gitea" targetFramework = ".NETStandard2.1" @@ -36,7 +35,6 @@ const nuspecContent = ` true ` + projectURL + ` ` + description + ` - ` + summary + ` ` + releaseNotes + ` @@ -124,7 +122,6 @@ func TestParseNuspecMetaData(t *testing.T) { assert.Equal(t, authors, m.Authors) assert.Equal(t, projectURL, m.ProjectURL) assert.Equal(t, description, m.Description) - assert.Equal(t, summary, m.Summary) assert.Equal(t, releaseNotes, m.ReleaseNotes) assert.Equal(t, repositoryURL, m.RepositoryURL) assert.Len(t, m.Dependencies, 1) diff --git a/routers/api/v1/packages/nuget/api.go b/routers/api/v1/packages/nuget/api.go index b981cc98256b7..1e8c372a54a42 100644 --- a/routers/api/v1/packages/nuget/api.go +++ b/routers/api/v1/packages/nuget/api.go @@ -71,7 +71,6 @@ type CatalogEntry struct { ID string `json:"id"` Version string `json:"version"` Description string `json:"description"` - Summary string `json:"summary"` ReleaseNotes string `json:"releaseNotes"` Authors string `json:"authors"` RequireLicenseAcceptance bool `json:"requireLicenseAcceptance"` @@ -125,7 +124,6 @@ func createRegistrationIndexPageItem(l *linkBuilder, p *Package) *RegistrationIn ID: p.Name, Version: p.Version, Description: p.Metadata.Description, - Summary: p.Metadata.Summary, ReleaseNotes: p.Metadata.ReleaseNotes, Authors: p.Metadata.Authors, ProjectURL: p.Metadata.ProjectURL, @@ -202,7 +200,6 @@ type SearchResult struct { Version string `json:"version"` Versions []*SearchResultVersion `json:"versions"` Description string `json:"description"` - Summary string `json:"summary"` Authors string `json:"authors"` ProjectURL string `json:"projectURL"` RegistrationIndexURL string `json:"registration"` @@ -258,7 +255,6 @@ func createSearchResult(l *linkBuilder, packages []*Package) *SearchResult { Version: latest.Version, Versions: versions, Description: latest.Metadata.Description, - Summary: latest.Metadata.Summary, Authors: latest.Metadata.Authors, ProjectURL: latest.Metadata.ProjectURL, RegistrationIndexURL: l.GetRegistrationIndexURL(latest.Name), From 338dee3eb9180c9d172495c9f01823f47f4392ac Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Jul 2021 19:21:45 +0000 Subject: [PATCH 011/130] Changed npm name. --- integrations/api_packages_npm_test.go | 2 +- models/package.go | 18 ++++++- modules/packages/pypi/metadata.go | 13 ++--- routers/api/v1/packages/generic/generic.go | 4 +- routers/api/v1/packages/npm/npm.go | 6 +-- routers/api/v1/packages/nuget/nuget.go | 4 +- routers/api/v1/packages/pypi/pypi.go | 2 +- services/packages/packages.go | 60 ++++++++++++++++++++-- 8 files changed, 89 insertions(+), 20 deletions(-) diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index d49e0e6900c7a..cc3443d1a19ec 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -63,7 +63,7 @@ func TestPackageNPM(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNPM) + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNpm) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) diff --git a/models/package.go b/models/package.go index 13dcdbb401d68..c882363514e3e 100644 --- a/models/package.go +++ b/models/package.go @@ -20,11 +20,27 @@ type PackageType int const ( PackageGeneric PackageType = iota PackageNuGet // 1 - PackageNPM // 2 + PackageNpm // 2 PackageMaven // 3 PackagePyPI // 4 ) +func (pt PackageType) String() string { + switch pt { + case PackageGeneric: + return "Generic" + case PackageNuGet: + return "NuGet" + case PackageNpm: + return "npm" + case PackageMaven: + return "Maven" + case PackagePyPI: + return "PyPI" + } + return "" +} + var ( // ErrDuplicatePackage indicates a duplicated package error ErrDuplicatePackage = errors.New("Package does exist already") diff --git a/modules/packages/pypi/metadata.go b/modules/packages/pypi/metadata.go index e3998af794789..27fc585975644 100644 --- a/modules/packages/pypi/metadata.go +++ b/modules/packages/pypi/metadata.go @@ -6,10 +6,11 @@ package pypi // Metadata represents the metadata of a PyPI package type Metadata struct { - Author string `json:"author"` - Description string `json:"description"` - Summary string `json:"summary"` - ProjectURL string `json:"project_url"` - License string `json:"license"` - RequiresPython string `json:"requires_python"` + Author string `json:"author"` + Description string `json:"description"` + LongDescription string `json:"long_description"` + Summary string `json:"summary"` + ProjectURL string `json:"project_url"` + License string `json:"license"` + RequiresPython string `json:"requires_python"` } diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 89da2a19f43c2..0c742f0abf4c0 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -30,7 +30,7 @@ func DownloadPackageContent(ctx *context.APIContext) { return } - s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion, filename) + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion, filename) if err != nil { if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) @@ -110,7 +110,7 @@ func DeletePackage(ctx *context.APIContext) { return } - err = package_service.DeletePackage(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) + err = package_service.DeletePackageByNameAndVersion(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) if err != nil { if err == models.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 440f5445c4519..75365943dc02d 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -26,7 +26,7 @@ func PackageMetadata(ctx *context.APIContext) { return } - packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageNPM, packageName) + packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageNpm, packageName) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return @@ -60,7 +60,7 @@ func DownloadPackageContent(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageNPM, packageName, packageVersion, filename) + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, models.PackageNpm, packageName, packageVersion, filename) if err != nil { if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) @@ -85,7 +85,7 @@ func UploadPackage(ctx *context.APIContext) { p, err := package_service.CreatePackage( ctx.User, ctx.Repo.Repository, - models.PackageNPM, + models.PackageNpm, npmPackage.Name, npmPackage.Version, npmPackage.Metadata, diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 2c40f41bf28c2..fa18f8e3380d4 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -144,7 +144,7 @@ func DownloadPackageContent(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackageNuGet, packageName, packageVersion, filename) + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, models.PackageNuGet, packageName, packageVersion, filename) if err != nil { if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) @@ -224,7 +224,7 @@ func DeletePackage(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") - err := package_service.DeletePackage(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) + err := package_service.DeletePackageByNameAndVersion(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) if err != nil { if err == models.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index 1cfb156ce4397..f2a8bd5fe6ec3 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -61,7 +61,7 @@ func DownloadPackageContent(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetPackageFileStream(ctx.Repo.Repository, models.PackagePyPI, packageName, packageVersion, filename) + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, models.PackagePyPI, packageName, packageVersion, filename) if err != nil { if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) diff --git a/services/packages/packages.go b/services/packages/packages.go index b8b88fd278ef6..8550d58ca27db 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -108,8 +108,8 @@ func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reade return pf, nil } -// DeletePackage deletes a package and all associated files -func DeletePackage(repository *models.Repository, packageType models.PackageType, name, version string) error { +// DeletePackageByNameAndVersion deletes a package and all associated files +func DeletePackageByNameAndVersion(repository *models.Repository, packageType models.PackageType, name, version string) error { log.Trace("Deleting package: %v, %v, %s, %s", repository.ID, packageType, name, version) p, err := models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) @@ -121,6 +121,30 @@ func DeletePackage(repository *models.Repository, packageType models.PackageType return err } + return deletePackage(p) +} + +// DeletePackageByID deletes a package and all associated files +func DeletePackageByID(repository *models.Repository, packageID int64) error { + log.Trace("Deleting package: %v, %v", repository.ID, packageID) + + p, err := models.GetPackageByID(packageID) + if err != nil { + if err == models.ErrPackageNotExist { + return err + } + log.Error("Error getting package: %v", err) + return err + } + + if p.RepoID != repository.ID { + return models.ErrPackageNotExist + } + + return deletePackage(p) +} + +func deletePackage(p *models.Package) error { pfs, err := p.GetFiles() if err != nil { log.Error("Error getting package files: %v", err) @@ -143,15 +167,43 @@ func DeletePackage(repository *models.Repository, packageType models.PackageType return nil } -// GetPackageFileStream returns the content of the specific package file -func GetPackageFileStream(repository *models.Repository, packageType models.PackageType, name, version, filename string) (io.ReadCloser, *models.PackageFile, error) { +// GetFileStreamByPackageNameAndVersion returns the content of the specific package file +func GetFileStreamByPackageNameAndVersion(repository *models.Repository, packageType models.PackageType, name, version, filename string) (io.ReadCloser, *models.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s", repository.ID, packageType, name, version, filename) p, err := models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) if err != nil { + if err == models.ErrPackageNotExist { + return nil, nil, err + } + log.Error("Error getting package: %v", err) + return nil, nil, err + } + + return getPackageFileStream(p, filename) +} + +// GetFileStreamByPackageID returns the content of the specific package file +func GetFileStreamByPackageID(repository *models.Repository, packageID int64, filename string) (io.ReadCloser, *models.PackageFile, error) { + log.Trace("Getting package file stream: %v, %v, %s", repository.ID, packageID, filename) + + p, err := models.GetPackageByID(packageID) + if err != nil { + if err == models.ErrPackageNotExist { + return nil, nil, err + } + log.Error("Error getting package: %v", err) return nil, nil, err } + if p.RepoID != repository.ID { + return nil, nil, models.ErrPackageNotExist + } + + return getPackageFileStream(p, filename) +} + +func getPackageFileStream(p *models.Package, filename string) (io.ReadCloser, *models.PackageFile, error) { pf, err := p.GetFileByName(filename) if err != nil { return nil, nil, err From 0544b22e0df0a53c1ad2a6565a24f784bbf149c9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Jul 2021 19:38:44 +0000 Subject: [PATCH 012/130] Sanitize project url. --- modules/packages/maven/metadata.go | 6 ++++++ modules/packages/npm/creator.go | 6 ++++++ modules/packages/nuget/metadata.go | 6 ++++++ routers/api/v1/packages/pypi/pypi.go | 19 +++++++++++++------ 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/modules/packages/maven/metadata.go b/modules/packages/maven/metadata.go index 09df4c4e75804..6e0dd88a6add8 100644 --- a/modules/packages/maven/metadata.go +++ b/modules/packages/maven/metadata.go @@ -7,6 +7,8 @@ package maven import ( "encoding/xml" "io" + + "code.gitea.io/gitea/modules/validation" ) // Metadata represents the metadata of a Maven package @@ -55,6 +57,10 @@ func ParsePackageMetaData(r io.Reader) (*Metadata, error) { return nil, err } + if !validation.IsValidURL(pom.URL) { + pom.URL = "" + } + licenses := make([]string, 0, len(pom.Licenses)) for _, l := range pom.Licenses { if l.Name != "" { diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 8eff6dac1140e..25fd30c57431e 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -16,6 +16,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/validation" + "github.com/hashicorp/go-version" jsoniter "github.com/json-iterator/go" ) @@ -134,6 +136,10 @@ func ParsePackage(r io.Reader) (*Package, error) { return nil, ErrInvalidPackageVersion } + if !validation.IsValidURL(meta.Homepage) { + meta.Homepage = "" + } + p := &Package{ Name: meta.Name, Version: meta.Version, diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 7704ddd20d164..7d2c3e293cbc5 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -13,6 +13,8 @@ import ( "regexp" "strings" + "code.gitea.io/gitea/modules/validation" + "github.com/hashicorp/go-version" ) @@ -117,6 +119,10 @@ func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { return nil, ErrNuspecInvalidVersion } + if !validation.IsValidURL(p.Metadata.ProjectURL) { + p.Metadata.ProjectURL = "" + } + m := &Metadata{ ID: p.Metadata.ID, Version: v.String(), diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index f2a8bd5fe6ec3..7c4e6e938804b 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -17,6 +17,7 @@ import ( pypi_module "code.gitea.io/gitea/modules/packages/pypi" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" + "code.gitea.io/gitea/modules/validation" package_service "code.gitea.io/gitea/services/packages" ) @@ -106,13 +107,19 @@ func UploadPackageFile(ctx *context.APIContext) { return } + projectURL := ctx.Req.FormValue("home_page") + if !validation.IsValidURL(projectURL) { + projectURL = "" + } + metadata := &pypi_module.Metadata{ - Author: ctx.Req.FormValue("author"), - Description: ctx.Req.FormValue("description"), - Summary: ctx.Req.FormValue("summary"), - ProjectURL: ctx.Req.FormValue("home_page"), - License: ctx.Req.FormValue("license"), - RequiresPython: ctx.Req.FormValue("requires_python"), + Author: ctx.Req.FormValue("author"), + Description: ctx.Req.FormValue("description"), + LongDescription: ctx.Req.FormValue("long_description"), + Summary: ctx.Req.FormValue("summary"), + ProjectURL: projectURL, + License: ctx.Req.FormValue("license"), + RequiresPython: ctx.Req.FormValue("requires_python"), } p, err := package_service.CreatePackage( From 5a3c5612f016bea7a2dc5f4befb96aaa70b4a0e3 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Jul 2021 20:38:15 +0000 Subject: [PATCH 013/130] Allow only scoped packages. --- modules/packages/npm/creator.go | 9 ++++---- modules/packages/npm/creator_test.go | 34 ++++++++++++++++------------ modules/packages/npm/metadata.go | 2 ++ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 25fd30c57431e..33b99dc87bf2d 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -35,7 +35,7 @@ var ( ErrInvalidIntegrity = errors.New("Failed to validate integrity") ) -var nameMatch = regexp.MustCompile(`\A(?:@([^/~'!\(\)\*]+?)[/])?([^/~'!\(\)\*]+?)\z`) +var nameMatch = regexp.MustCompile(`\A(@[^\/~'!\(\)\*]+?)[\/]([^_.][^\/~'!\(\)\*]+)\z`) // Package represents a NPM package type Package struct { @@ -136,6 +136,8 @@ func ParsePackage(r io.Reader) (*Package, error) { return nil, ErrInvalidPackageVersion } + nameParts := strings.SplitN(meta.Name, "/", 2) + if !validation.IsValidURL(meta.Homepage) { meta.Homepage = "" } @@ -144,6 +146,8 @@ func ParsePackage(r io.Reader) (*Package, error) { Name: meta.Name, Version: meta.Version, Metadata: Metadata{ + Scope: nameParts[0], + Name: nameParts[1], Description: meta.Description, Author: meta.Author.Name, License: meta.License, @@ -210,8 +214,5 @@ func validateName(name string) bool { if len(name) == 0 || len(name) > 214 { return false } - if name[0] == '.' || name[0] == '_' { - return false - } return nameMatch.MatchString(name) } diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index aba8fa9c0057f..6abedf35913ff 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -37,22 +37,28 @@ func TestParsePackage(t *testing.T) { }) t.Run("InvalidPackageName", func(t *testing.T) { - name := " test " - b, _ := jsoniter.Marshal(packageUpload{ - PackageMetadata: PackageMetadata{ - ID: name, - Name: name, - Versions: map[string]*PackageMetadataVersion{ - packageVersion: { - Name: name, + test := func(t *testing.T, name string) { + b, _ := jsoniter.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: name, + Name: name, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: name, + }, }, }, - }, - }) - - p, err := ParsePackage(bytes.NewReader(b)) - assert.Nil(t, p) - assert.ErrorIs(t, err, ErrInvalidPackageName) + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageName) + } + + test(t, " test ") + test(t, "invalid/scope") + test(t, "@invalid/_name") + test(t, "@invalid/.name") }) t.Run("InvalidPackageVersion", func(t *testing.T) { diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index a2e40b5b6c9cb..f0c9586ccc516 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -6,6 +6,8 @@ package npm // Metadata represents the metadata of a NPM package type Metadata struct { + Scope string `json:"scope"` + Name string `json:"name"` Description string `json:"description"` Author string `json:"author"` License string `json:"license"` From ea36a75f9eb0bd6ff98dc6d565de86e6140b1359 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Jul 2021 22:16:42 +0000 Subject: [PATCH 014/130] Added user interface. --- models/package.go | 67 +++++++ models/repo_unit.go | 2 +- models/unit.go | 14 ++ modules/context/context.go | 1 + modules/context/repo.go | 1 + modules/packages/npm/creator_test.go | 2 +- modules/setting/setting.go | 2 + modules/templates/helper.go | 12 +- options/locale/locale_en-US.ini | 35 ++++ routers/api/v1/api.go | 16 +- routers/web/repo/packages.go | 171 ++++++++++++++++++ routers/web/repo/setting.go | 9 + routers/web/web.go | 11 ++ services/forms/repo_form.go | 1 + templates/repo/header.tmpl | 6 + templates/repo/packages/content/generic.tmpl | 12 ++ templates/repo/packages/content/maven.tmpl | 59 ++++++ templates/repo/packages/content/npm.tmpl | 38 ++++ templates/repo/packages/content/nuget.tmpl | 36 ++++ templates/repo/packages/content/pypi.tmpl | 24 +++ templates/repo/packages/list.tmpl | 55 ++++++ templates/repo/packages/metadata/generic.tmpl | 0 templates/repo/packages/metadata/maven.tmpl | 10 + templates/repo/packages/metadata/npm.tmpl | 5 + templates/repo/packages/metadata/nuget.tmpl | 4 + templates/repo/packages/metadata/pypi.tmpl | 6 + templates/repo/packages/view.tmpl | 79 ++++++++ templates/repo/settings/options.tmpl | 13 ++ 28 files changed, 680 insertions(+), 11 deletions(-) create mode 100644 routers/web/repo/packages.go create mode 100644 templates/repo/packages/content/generic.tmpl create mode 100644 templates/repo/packages/content/maven.tmpl create mode 100644 templates/repo/packages/content/npm.tmpl create mode 100644 templates/repo/packages/content/nuget.tmpl create mode 100644 templates/repo/packages/content/pypi.tmpl create mode 100644 templates/repo/packages/list.tmpl create mode 100644 templates/repo/packages/metadata/generic.tmpl create mode 100644 templates/repo/packages/metadata/maven.tmpl create mode 100644 templates/repo/packages/metadata/npm.tmpl create mode 100644 templates/repo/packages/metadata/nuget.tmpl create mode 100644 templates/repo/packages/metadata/pypi.tmpl create mode 100644 templates/repo/packages/view.tmpl diff --git a/models/package.go b/models/package.go index c882363514e3e..c2de2e5c404c5 100644 --- a/models/package.go +++ b/models/package.go @@ -8,6 +8,7 @@ import ( "errors" "strings" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" @@ -57,6 +58,7 @@ type Package struct { ID int64 `xorm:"pk autoincr"` RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` CreatorID int64 + Creator *User `xorm:"-"` Type PackageType `xorm:"UNIQUE(s) INDEX NOT NULL"` Name string LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` @@ -67,6 +69,16 @@ type Package struct { UpdatedUnix timeutil.TimeStamp `xorm:"updated"` } +// LoadCreator loads poster +func (p *Package) LoadCreator() error { + if p.Creator == nil { + var err error + p.Creator, err = getUserByID(x, p.CreatorID) + return err + } + return nil +} + // PackageFile represents files associated with a package type PackageFile struct { ID int64 `xorm:"pk autoincr"` @@ -166,6 +178,61 @@ func DeletePackagesByRepositoryID(repositoryID int64) error { return nil } +// PackageSearchOptions are options for GetLatestPackagesGrouped +type PackageSearchOptions struct { + RepoID int64 + Page int + Query string + Type string +} + +// GetLatestPackagesGrouped returns a list of all packages in their latest version of the repository +func GetLatestPackagesGrouped(opts PackageSearchOptions) ([]*Package, int64, error) { + var cond builder.Cond = builder.Eq{"package.repo_id": opts.RepoID} + cond = cond.And(builder.Expr("p2.id IS NULL")) + + switch opts.Type { + case "generic": + cond = cond.And(builder.Eq{"package.type": PackageGeneric}) + case "nuget": + cond = cond.And(builder.Eq{"package.type": PackageNuGet}) + case "npm": + cond = cond.And(builder.Eq{"package.type": PackageNpm}) + case "maven": + cond = cond.And(builder.Eq{"package.type": PackageMaven}) + case "pypi": + cond = cond.And(builder.Eq{"package.type": PackagePyPI}) + } + + if opts.Query != "" { + cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.Query)}) + } + + sess := x.Where(cond). + Table("package"). + Join("left", "package p2", "package.repo_id = p2.repo_id AND package.type = p2.type AND package.lower_name = p2.lower_name AND package.version < p2.version") + + if opts.Page > 0 { + sess = sess.Limit(setting.UI.PackagesPagingNum, (opts.Page-1)*setting.UI.PackagesPagingNum) + } + + packages := make([]*Package, 0, setting.UI.PackagesPagingNum) + count, err := sess.FindAndCount(&packages) + return packages, count, err +} + +// GetPackageByID returns the package with the specific id +func GetPackageByID(packageID int64) (*Package, error) { + p := &Package{} + has, err := x.ID(packageID).Get(p) + if err != nil { + return nil, err + } else if !has { + return nil, ErrPackageNotExist + } + return p, nil +} + // GetPackagesByRepository returns all packages of a repository func GetPackagesByRepository(repositoryID int64) ([]*Package, error) { packages := make([]*Package, 0, 10) diff --git a/models/repo_unit.go b/models/repo_unit.go index d8060d16a03c9..338b9ea4157ab 100644 --- a/models/repo_unit.go +++ b/models/repo_unit.go @@ -153,7 +153,7 @@ func (r *RepoUnit) BeforeSet(colName string, val xorm.Cell) { switch colName { case "type": switch UnitType(Cell2Int64(val)) { - case UnitTypeCode, UnitTypeReleases, UnitTypeWiki, UnitTypeProjects: + case UnitTypeCode, UnitTypeReleases, UnitTypeWiki, UnitTypeProjects, UnitTypePackages: r.Config = new(UnitConfig) case UnitTypeExternalWiki: r.Config = new(ExternalWikiConfig) diff --git a/models/unit.go b/models/unit.go index 939deba574824..6b2f04004ba8f 100644 --- a/models/unit.go +++ b/models/unit.go @@ -25,6 +25,7 @@ const ( UnitTypeExternalWiki // 6 ExternalWiki UnitTypeExternalTracker // 7 ExternalTracker UnitTypeProjects // 8 Kanban board + UnitTypePackages // 9 Packages ) // Value returns integer value for unit type @@ -50,6 +51,8 @@ func (u UnitType) String() string { return "UnitTypeExternalTracker" case UnitTypeProjects: return "UnitTypeProjects" + case UnitTypePackages: + return "UnitTypePackages" } return fmt.Sprintf("Unknown UnitType %d", u) } @@ -72,6 +75,7 @@ var ( UnitTypeExternalWiki, UnitTypeExternalTracker, UnitTypeProjects, + UnitTypePackages, } // DefaultRepoUnits contains the default unit types @@ -82,6 +86,7 @@ var ( UnitTypeReleases, UnitTypeWiki, UnitTypeProjects, + UnitTypePackages, } // NotAllowedDefaultRepoUnits contains units that can't be default @@ -255,6 +260,14 @@ var ( 5, } + UnitPackages = Unit{ + UnitTypePackages, + "repo.packages", + "/packages", + "repo.packages.desc", + 6, + } + // Units contains all the units Units = map[UnitType]Unit{ UnitTypeCode: UnitCode, @@ -265,6 +278,7 @@ var ( UnitTypeWiki: UnitWiki, UnitTypeExternalWiki: UnitExternalWiki, UnitTypeProjects: UnitProjects, + UnitTypePackages: UnitPackages, } ) diff --git a/modules/context/context.go b/modules/context/context.go index 211ac14ec1c0b..e48a84bd78138 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -782,6 +782,7 @@ func Contexter() func(next http.Handler) http.Handler { ctx.Data["UnitWikiGlobalDisabled"] = models.UnitTypeWiki.UnitGlobalDisabled() ctx.Data["UnitIssuesGlobalDisabled"] = models.UnitTypeIssues.UnitGlobalDisabled() ctx.Data["UnitPullsGlobalDisabled"] = models.UnitTypePullRequests.UnitGlobalDisabled() + ctx.Data["UnitPackagesGlobalDisabled"] = models.UnitTypePackages.UnitGlobalDisabled() ctx.Data["UnitProjectsGlobalDisabled"] = models.UnitTypeProjects.UnitGlobalDisabled() ctx.Data["i18n"] = locale diff --git a/modules/context/repo.go b/modules/context/repo.go index ea8323bdfc22b..07a3b64f99390 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -875,6 +875,7 @@ func UnitTypes() func(ctx *Context) { ctx.Data["UnitTypeExternalWiki"] = models.UnitTypeExternalWiki ctx.Data["UnitTypeExternalTracker"] = models.UnitTypeExternalTracker ctx.Data["UnitTypeProjects"] = models.UnitTypeProjects + ctx.Data["UnitTypePackages"] = models.UnitTypePackages } } diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index 6abedf35913ff..c294b695adaf0 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -49,7 +49,7 @@ func TestParsePackage(t *testing.T) { }, }, }) - + p, err := ParsePackage(bytes.NewReader(b)) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidPackageName) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index e3da5796e4268..cf9bf79eb3936 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -198,6 +198,7 @@ var ( MembersPagingNum int FeedMaxCommitNum int FeedPagingNum int + PackagesPagingNum int GraphMaxCommitNum int CodeCommentLines int ReactionMaxUserNum int @@ -250,6 +251,7 @@ var ( MembersPagingNum: 20, FeedMaxCommitNum: 5, FeedPagingNum: 20, + PackagesPagingNum: 20, GraphMaxCommitNum: 100, CodeCommentLines: 4, ReactionMaxUserNum: 10, diff --git a/modules/templates/helper.go b/modules/templates/helper.go index f9b2dafd22a14..2369b74ae368f 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -30,6 +30,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/svg" @@ -164,7 +165,16 @@ func NewFuncMap() []template.FuncMap { "RenderEmojiPlain": emoji.ReplaceAliases, "ReactionToEmoji": ReactionToEmoji, "RenderNote": RenderNote, - "IsMultilineCommitMessage": IsMultilineCommitMessage, + "RenderMarkdownToHtml": func(input string) template.HTML { + output, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + }, input) + if err != nil { + log.Error("RenderString: %v", err) + } + return template.HTML(output) + }, + "IsMultilineCommitMessage": IsMultilineCommitMessage, "ThemeColorMetaTag": func() string { return setting.UI.ThemeColorMetaTag }, diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 191cb5de6764c..4eec0a93c4812 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1492,6 +1492,40 @@ milestones.filter_sort.most_complete = Most complete milestones.filter_sort.most_issues = Most issues milestones.filter_sort.least_issues = Least issues +packages = Packages +packages.desc = Manage repository packages. +packages.filter.package_type = Type +packages.filter.package_type.all = All +packages.published_by = Published %[1]s by %[3]s +packages.installation = Installation +packages.about = About this package +packages.dependencies = Dependencies +packages.details = Details +packages.details.author = Author +packages.details.project_site = Project Site +packages.details.license = License +packages.assets = Assets +packages.versions = Versions +packages.versions.on = on +packages.delete = Delete package +packages.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? +packages.delete.success = The package has been deleted. +packages.generic.use = Download the package, for example with curl: +packages.generic.documentation = For more information on the generic registry, see the documentation. +packages.maven.registry = Setup this registry in your projects pom.xml file: +packages.maven.use = To use the package include the following in the dependencies block in the pom.xml file: +packages.maven.use_2 = Alternatively use this Maven command: +packages.maven.documentation = For more information on the Maven registry, see the documentation. +packages.nuget.registry = Setup this registry with the following command: +packages.nuget.use = Install the package with the following command: +packages.nuget.documentation = For more information on the NuGet registry, see the documentation. +packages.npm.registry = Setup this registry in your projects .npmrc file: +packages.npm.use = To install the package use the following command: +packages.npm.documentation = For more information on the npm registry, see the documentation. +packages.pypi.requires = Requires Python +packages.pypi.use = To install the package use the following command: +packages.pypi.documentation = For more information on the PyPI registry, see the documentation. + signing.will_sign = This commit will be signed with key '%s' signing.wont_sign.error = There was an error whilst checking if the commit could be signed signing.wont_sign.nokey = There is no key available to sign this commit @@ -1664,6 +1698,7 @@ settings.pulls.allow_rebase_merge_commit = Enable Rebasing with explicit merge c settings.pulls.allow_squash_commits = Enable Squashing to Merge Commits settings.pulls.allow_manual_merge = Enable Mark PR as manually merged settings.pulls.enable_autodetect_manual_merge = Enable autodetect manual merge (Note: In some special cases, misjudgments can occur) +settings.packages_desc = Enable Repository Packages Registry settings.projects_desc = Enable Repository Projects settings.admin_settings = Administrator Settings settings.admin_enable_health_check = Enable Repository Health Checks (git fsck) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 6b69f14671cce..f9de2ed23ee9f 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -980,16 +980,16 @@ func Routes() *web.Route { m.Group("/generic", func() { m.Group("/{packagename}/{packageversion}/{filename}", func() { m.Get("", generic.DownloadPackageContent) - m.Put("" /*reqRepoWriter(models.UnitTypePackage),*/, generic.UploadPackage) - m.Delete("" /*reqRepoWriter(models.UnitTypePackage),*/, generic.DeletePackage) + m.Put("", reqRepoWriter(models.UnitTypePackages), generic.UploadPackage) + m.Delete("", reqRepoWriter(models.UnitTypePackages), generic.DeletePackage) }) }, reqToken()) m.Group("/maven", func() { - m.Put("/*" /*reqRepoWriter(models.UnitTypePackage),*/, maven.UploadPackageFile) - m.Get("/*" /*reqRepoWriter(models.UnitTypePackage),*/, maven.DownloadPackageFile) + m.Put("/*", reqRepoWriter(models.UnitTypePackages), maven.UploadPackageFile) + m.Get("/*", reqRepoWriter(models.UnitTypePackages), maven.DownloadPackageFile) }, reqToken()) m.Group("/nuget", func() { - m.Put("/" /*reqRepoWriter(models.UnitTypePackage),*/, nuget.UploadPackage) + m.Put("/", reqRepoWriter(models.UnitTypePackages), nuget.UploadPackage) m.Get("/index.json", nuget.ServiceIndex) m.Get("/query", nuget.SearchService) m.Group("/registration/{id}", func() { @@ -999,7 +999,7 @@ func Routes() *web.Route { m.Group("/package/{id}", func() { m.Get("/index.json", nuget.EnumeratePackageVersions) m.Group("/{version}", func() { - m.Delete("/" /*reqRepoWriter(models.UnitTypePackage),*/, nuget.DeletePackage) + m.Delete("/", reqRepoWriter(models.UnitTypePackages), nuget.DeletePackage) m.Get("/{filename}", nuget.DownloadPackageContent) }) }) @@ -1007,12 +1007,12 @@ func Routes() *web.Route { m.Group("/npm", func() { m.Group("/{id}", func() { m.Get("", npm.PackageMetadata) - m.Put("" /*reqRepoWriter(models.UnitTypePackage),*/, npm.UploadPackage) + m.Put("", reqRepoWriter(models.UnitTypePackages), npm.UploadPackage) m.Get("/-/{version}/{filename}", npm.DownloadPackageContent) }) }, reqToken()) m.Group("/pypi", func() { - m.Post("/" /*reqRepoWriter(models.UnitTypePackage),*/, pypi.UploadPackageFile) + m.Post("/", reqRepoWriter(models.UnitTypePackages), pypi.UploadPackageFile) m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageContent) m.Get("/simple/{id}", pypi.PackageMetadata) }, reqBasicAuth()) diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go new file mode 100644 index 0000000000000..d0168193e6a7f --- /dev/null +++ b/routers/web/repo/packages.go @@ -0,0 +1,171 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repo + +import ( + "net/http" + "sort" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/packages/maven" + "code.gitea.io/gitea/modules/packages/npm" + "code.gitea.io/gitea/modules/packages/nuget" + "code.gitea.io/gitea/modules/packages/pypi" + "code.gitea.io/gitea/modules/setting" + + package_service "code.gitea.io/gitea/services/packages" + + jsoniter "github.com/json-iterator/go" +) + +const ( + tplPackages base.TplName = "repo/packages/list" + tplPackagesView base.TplName = "repo/packages/view" +) + +// MustEnablePackages checks if packages are enabled +func MustEnablePackages(ctx *context.Context) { + if models.UnitTypePackages.UnitGlobalDisabled() || !ctx.Repo.CanRead(models.UnitTypePackages) { + ctx.NotFound("MustEnablePackages", nil) + } +} + +// Packages displays a list of all packages in the repository +func Packages(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("repo.packages") + + query := ctx.QueryTrim("q") + packageType := ctx.QueryTrim("package_type") + page := ctx.QueryInt("page") + if page <= 1 { + page = 1 + } + + repo := ctx.Repo.Repository + + packages, count, err := models.GetLatestPackagesGrouped(models.PackageSearchOptions{ + RepoID: repo.ID, + Query: query, + Page: page, + Type: packageType, + }) + if err != nil { + ctx.ServerError("GetLatestPackagesGrouped", err) + return + } + + for _, p := range packages { + if err := p.LoadCreator(); err != nil { + ctx.ServerError("LoadCreator", err) + return + } + } + + ctx.Data["Packages"] = packages + ctx.Data["IsPackagesPage"] = true + ctx.Data["Query"] = query + ctx.Data["PackageType"] = packageType + + pager := context.NewPagination(int(count), setting.UI.PackagesPagingNum, page, 5) + pager.AddParam(ctx, "q", "Query") + pager.AddParam(ctx, "package_type", "PackageType") + ctx.Data["Page"] = pager + + ctx.HTML(http.StatusOK, tplPackages) +} + +// ViewPackage displays a single package +func ViewPackage(ctx *context.Context) { + p, err := models.GetPackageByID(ctx.ParamsInt64(":id")) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.NotFound("", nil) + } else { + ctx.ServerError("GetPackageByID", err) + } + return + } + if p.RepoID != ctx.Repo.Repository.ID { + ctx.NotFound("", nil) + return + } + if err := p.LoadCreator(); err != nil { + ctx.ServerError("LoadCreator", err) + return + } + ctx.Data["Package"] = p + + var metadata interface{} + switch p.Type { + case models.PackageNuGet: + metadata = &nuget.Metadata{} + case models.PackageNpm: + metadata = &npm.Metadata{} + case models.PackageMaven: + metadata = &maven.Metadata{} + case models.PackagePyPI: + metadata = &pypi.Metadata{} + } + if metadata != nil { + if err := jsoniter.Unmarshal([]byte(p.MetadataRaw), &metadata); err != nil { + ctx.ServerError("Unmarshal", err) + return + } + } + ctx.Data["Metadata"] = metadata + + files, err := p.GetFiles() + if err != nil { + ctx.ServerError("GetFiles", err) + return + } + ctx.Data["Files"] = files + + otherVersions, err := models.GetPackagesByName(ctx.Repo.Repository.ID, p.Type, p.LowerName) + if err != nil { + ctx.ServerError("GetPackagesByName", err) + return + } + sort.Slice(otherVersions, func(i, j int) bool { + return otherVersions[i].Version > otherVersions[j].Version + }) + ctx.Data["OtherVersions"] = otherVersions + + ctx.Data["Repo"] = ctx.Repo.Repository + ctx.Data["CanWritePackages"] = ctx.Repo.Permission.CanWrite(models.UnitTypePackages) + ctx.Data["PageIsPackages"] = true + + ctx.HTML(http.StatusOK, tplPackagesView) +} + +// DeletePackagePost deletes a package +func DeletePackagePost(ctx *context.Context) { + err := package_service.DeletePackageByID(ctx.Repo.Repository, ctx.ParamsInt64(":id")) + if err != nil { + ctx.Flash.Error(err.Error()) + } else { + ctx.Flash.Success(ctx.Tr("repo.packages.delete.success")) + } + + ctx.Redirect(ctx.Repo.RepoLink + "/packages") +} + +// DownloadPackageFile serves the content of a package file +func DownloadPackageFile(ctx *context.Context) { + s, pf, err := package_service.GetFileStreamByPackageID(ctx.Repo.Repository, ctx.ParamsInt64(":id"), ctx.Params(":filename")) + if err != nil { + if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { + ctx.NotFound("", err) + return + } + ctx.ServerError("GetFileStreamByPackageID", err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 5e8c2c5276251..26db4ca29a8f6 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -402,6 +402,15 @@ func SettingsPost(ctx *context.Context) { } } + if form.EnablePackages && !models.UnitTypePackages.UnitGlobalDisabled() { + units = append(units, models.RepoUnit{ + RepoID: repo.ID, + Type: models.UnitTypePackages, + }) + } else if !models.UnitTypePackages.UnitGlobalDisabled() { + deleteUnitTypes = append(deleteUnitTypes, models.UnitTypePackages) + } + if form.EnableProjects && !models.UnitTypeProjects.UnitGlobalDisabled() { units = append(units, models.RepoUnit{ RepoID: repo.ID, diff --git a/routers/web/web.go b/routers/web/web.go index 7a47f479c0da3..0a121ee0fcb4f 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -482,6 +482,8 @@ func RegisterRoutes(m *web.Route) { reqRepoIssuesOrPullsReader := context.RequireRepoReaderOr(models.UnitTypeIssues, models.UnitTypePullRequests) reqRepoProjectsReader := context.RequireRepoReader(models.UnitTypeProjects) reqRepoProjectsWriter := context.RequireRepoWriter(models.UnitTypeProjects) + reqRepoPackagesReader := context.RequireRepoReader(models.UnitTypePackages) + reqRepoPackagesWriter := context.RequireRepoWriter(models.UnitTypePackages) // ***** START: Organization ***** m.Group("/org", func() { @@ -837,6 +839,15 @@ func RegisterRoutes(m *web.Route) { m.Get("/milestones", reqRepoIssuesOrPullsReader, repo.Milestones) }, context.RepoRef()) + m.Group("/packages", func() { + m.Get("", repo.Packages) + m.Group("/{id}", func() { + m.Get("/", repo.ViewPackage) + m.Post("/", reqRepoPackagesWriter, repo.DeletePackagePost) + m.Get("/files/{filename}", repo.DownloadPackageFile) + }) + }, reqRepoPackagesReader, repo.MustEnablePackages) + m.Group("/projects", func() { m.Get("", repo.Projects) m.Get("/{id}", repo.ViewProject) diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 71a83a8be36e2..45b94012a9166 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -141,6 +141,7 @@ type RepoSettingForm struct { TrackerURLFormat string TrackerIssueStyle string EnableCloseIssuesViaCommitInAnyBranch bool + EnablePackages bool EnableProjects bool EnablePulls bool PullsIgnoreWhitespace bool diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index 49a651e6c5a86..fabb703744216 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -132,6 +132,12 @@ {{end}} + {{ if and (not .UnitPackagesGlobalDisabled) (.Permission.CanRead $.UnitTypePackages)}} + + {{svg "octicon-package"}} {{.i18n.Tr "repo.packages"}} + + {{ end }} + {{ if and (not .UnitProjectsGlobalDisabled) (.Permission.CanRead $.UnitTypeProjects)}} {{svg "octicon-project"}} {{.i18n.Tr "repo.project_board"}} diff --git a/templates/repo/packages/content/generic.tmpl b/templates/repo/packages/content/generic.tmpl new file mode 100644 index 0000000000000..4c3969ff4799f --- /dev/null +++ b/templates/repo/packages/content/generic.tmpl @@ -0,0 +1,12 @@ +{{if eq .Package.Type 0}} +

{{.i18n.Tr "repo.packages.installation"}}

+
+
+ +
+ +
+
+ {{.i18n.Tr "repo.packages.generic.documentation" | Safe}} +
+{{end}} \ No newline at end of file diff --git a/templates/repo/packages/content/maven.tmpl b/templates/repo/packages/content/maven.tmpl new file mode 100644 index 0000000000000..2ffced4813f73 --- /dev/null +++ b/templates/repo/packages/content/maven.tmpl @@ -0,0 +1,59 @@ +{{if eq .Package.Type 3}} +

{{.i18n.Tr "repo.packages.installation"}}

+
+
+ +
+ +
+
+
+ +
+ +
+ +
+ +
+
+ {{.i18n.Tr "repo.packages.maven.documentation" | Safe}} +
+ + {{if .Metadata.Description}} +

{{.i18n.Tr "repo.packages.about"}}

+
+ {{.Metadata.Description}} +
+ {{end}} + + {{if .Metadata.Dependencies}} +

{{.i18n.Tr "repo.packages.dependencies"}}

+
+ {{range .Metadata.Dependencies}} +
{{.GroupID}}:{{.ArtifactID}} ({{.Version}})
+ {{end}} +
+ {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/content/npm.tmpl b/templates/repo/packages/content/npm.tmpl new file mode 100644 index 0000000000000..8c0b5f66b2afa --- /dev/null +++ b/templates/repo/packages/content/npm.tmpl @@ -0,0 +1,38 @@ +{{if eq .Package.Type 2}} +

{{.i18n.Tr "repo.packages.installation"}}

+
+
+ +
+ +
+ +
+ +
+
+ {{.i18n.Tr "repo.packages.npm.documentation" | Safe}} +
+ + {{if or .Metadata.Description .Metadata.Readme}} +

{{.i18n.Tr "repo.packages.about"}}

+
+ {{if .Metadata.Readme}} +
+ {{RenderMarkdownToHtml .Metadata.Readme}} +
+ {{else if .Metadata.Description}} + {{.Metadata.Description}} + {{end}} +
+ {{end}} + + {{if .Metadata.Dependencies}} +

{{.i18n.Tr "repo.packages.dependencies"}}

+
+ {{range $dependency, $version := .Metadata.Dependencies}} +
{{$dependency}} ({{$version}})
+ {{end}} +
+ {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/content/nuget.tmpl b/templates/repo/packages/content/nuget.tmpl new file mode 100644 index 0000000000000..57a41aeeb5d4a --- /dev/null +++ b/templates/repo/packages/content/nuget.tmpl @@ -0,0 +1,36 @@ +{{if eq .Package.Type 1}} +

{{.i18n.Tr "repo.packages.installation"}}

+
+
+ +
+ +
+ +
+ +
+
+ {{.i18n.Tr "repo.packages.nuget.documentation" | Safe}} +
+ + {{if or .Metadata.Description .Metadata.ReleaseNotes}} +

{{.i18n.Tr "repo.packages.about"}}

+
+ {{if .Metadata.Description}}{{.Metadata.Description}}{{end}} + {{if .Metadata.ReleaseNotes}}{{Str2html .Metadata.ReleaseNotes}}{{end}} +
+ {{end}} + + {{if .Metadata.Dependencies}} +

{{.i18n.Tr "repo.packages.dependencies"}}

+
+ {{range $framework, $dependencies := .Metadata.Dependencies}} +
{{$framework}}
+ {{range $dependencies}} +
{{.ID}} ({{.Version}})
+ {{end}} + {{end}} +
+ {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/content/pypi.tmpl b/templates/repo/packages/content/pypi.tmpl new file mode 100644 index 0000000000000..dfbc6a8529e19 --- /dev/null +++ b/templates/repo/packages/content/pypi.tmpl @@ -0,0 +1,24 @@ +{{if eq .Package.Type 4}} +

{{.i18n.Tr "repo.packages.installation"}}

+
+
+ +
+ +
+
+ {{.i18n.Tr "repo.packages.pypi.documentation" | Safe}} +
+ + {{if or .Metadata.Description .Metadata.LongDescription .Metadata.Summary}} +

{{.i18n.Tr "repo.packages.about"}}

+
+

{{if .Metadata.Summary}}{{.Metadata.Summary}}{{end}}

+ {{if .Metadata.LongDescription}} + {{RenderMarkdownToHtml .Metadata.LongDescription}} + {{else if .Metadata.Description}} + {{RenderMarkdownToHtml .Metadata.Description}} + {{end}} +
+ {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/list.tmpl b/templates/repo/packages/list.tmpl new file mode 100644 index 0000000000000..cb342ab31ee8e --- /dev/null +++ b/templates/repo/packages/list.tmpl @@ -0,0 +1,55 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
+ {{template "base/alert" .}} +
+
+
+
+ + + +
+
+
+
+
+
+ + {{.i18n.Tr "repo.packages.filter.package_type"}} + {{svg "octicon-triangle-down" 14 "dropdown icon"}} + +
+ {{.i18n.Tr "repo.packages.filter.package_type.all"}} + Generic + NuGet + npm + Maven + PyPI +
+
+
+
+
+
+ {{range .Packages}} +
  • +
    +
    + {{.Name}} + {{.Version}} + {{.Type.String}} +
    +
    + {{ $timeStr := TimeSinceUnix .CreatedUnix $.Lang }} + {{$.i18n.Tr "repo.packages.published_by" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) | Safe}} +
    +
    +
    • + {{end}} + {{template "base/paginate" .}} +
    +
    +
    +{{template "base/footer" .}} \ No newline at end of file diff --git a/templates/repo/packages/metadata/generic.tmpl b/templates/repo/packages/metadata/generic.tmpl new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/templates/repo/packages/metadata/maven.tmpl b/templates/repo/packages/metadata/maven.tmpl new file mode 100644 index 0000000000000..0ea9adfd5b8d4 --- /dev/null +++ b/templates/repo/packages/metadata/maven.tmpl @@ -0,0 +1,10 @@ +{{if eq .Package.Type 3}} + {{if .Metadata.Name}}
    {{svg "octicon-note" 16 "mr-3"}} {{.Metadata.Name}}
    {{end}} + {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .Metadata.Licenses}} +
    {{.i18n.Tr "repo.packages.details.license"}}:
    + {{range .Metadata.Licenses}} +
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    + {{end}} + {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/metadata/npm.tmpl b/templates/repo/packages/metadata/npm.tmpl new file mode 100644 index 0000000000000..0c37cc968cad4 --- /dev/null +++ b/templates/repo/packages/metadata/npm.tmpl @@ -0,0 +1,5 @@ +{{if eq .Package.Type 2}} + {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Author}}
    {{end}} + {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/metadata/nuget.tmpl b/templates/repo/packages/metadata/nuget.tmpl new file mode 100644 index 0000000000000..557d0d3e0ca4d --- /dev/null +++ b/templates/repo/packages/metadata/nuget.tmpl @@ -0,0 +1,4 @@ +{{if eq .Package.Type 1}} + {{if .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Authors}}
    {{end}} + {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/metadata/pypi.tmpl b/templates/repo/packages/metadata/pypi.tmpl new file mode 100644 index 0000000000000..281cdbfe58649 --- /dev/null +++ b/templates/repo/packages/metadata/pypi.tmpl @@ -0,0 +1,6 @@ +{{if eq .Package.Type 4}} + {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Author}}
    {{end}} + {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} + {{if .Metadata.RequiresPython}}
    {{.i18n.Tr "repo.packages.python.requires"}}: {{.Metadata.RequiresPython}}
    {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/view.tmpl b/templates/repo/packages/view.tmpl new file mode 100644 index 0000000000000..54534dfd6fc3d --- /dev/null +++ b/templates/repo/packages/view.tmpl @@ -0,0 +1,79 @@ +{{template "base/head" .}} +
    + {{template "repo/header" .}} +
    +
    +
    +
    +
    +

    {{.Package.Name}} ({{.Package.Version}})

    +
    + {{ $timeStr := TimeSinceUnix .Package.CreatedUnix $.Lang }} +
    {{.i18n.Tr "repo.packages.published_by" $timeStr .Package.Creator.HomeLink (.Package.Creator.GetDisplayName | Escape) | Safe}} {{svg "octicon-package"}} {{.Package.Type.String}}
    +
    +
    + +
    + {{template "repo/packages/content/generic" .}} + {{template "repo/packages/content/nuget" .}} + {{template "repo/packages/content/npm" .}} + {{template "repo/packages/content/maven" .}} + {{template "repo/packages/content/pypi" .}} +
    + +
    +
    + {{.i18n.Tr "repo.packages.details"}} +
    {{svg "octicon-calendar" 16 "mr-3"}} {{.Package.CreatedUnix.FormatDate}}
    + {{template "repo/packages/metadata/generic" .}} + {{template "repo/packages/metadata/nuget" .}} + {{template "repo/packages/metadata/npm" .}} + {{template "repo/packages/metadata/maven" .}} + {{template "repo/packages/metadata/pypi" .}} + +
    + {{.i18n.Tr "repo.packages.assets"}} + {{range .Files}} + + {{end}} + + {{if .OtherVersions}} +
    + {{.i18n.Tr "repo.packages.versions"}} + {{range .OtherVersions}} +
    {{.Version}} {{$.i18n.Tr "repo.packages.versions.on"}} {{.CreatedUnix.FormatDate}}
    + {{end}} + {{end}} + + {{if .CanWritePackages}} +
    +
    + +
    + + {{end}} +
    +
    +
    +
    +
    +
    +{{template "base/footer" .}} \ No newline at end of file diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index eb76a3b720064..f1356a52ce6b2 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -374,6 +374,19 @@
    + {{$isPackagesEnabled := .Repository.UnitEnabled $.UnitTypePackages}} +
    + + {{if .UnitTypePackages.UnitGlobalDisabled}} +
    + {{else}} +
    + {{end}} + + +
    +
    + {{$isProjectsEnabled := .Repository.UnitEnabled $.UnitTypeProjects}}
    From 8c6715b03558186a662241366adf9263f46f0446 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Jul 2021 22:18:24 +0000 Subject: [PATCH 015/130] Changed method name. --- routers/api/v1/api.go | 8 ++++---- routers/api/v1/packages/generic/generic.go | 4 ++-- routers/api/v1/packages/npm/npm.go | 4 ++-- routers/api/v1/packages/nuget/nuget.go | 4 ++-- routers/api/v1/packages/pypi/pypi.go | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index f9de2ed23ee9f..0f3e62e6fc2df 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -979,7 +979,7 @@ func Routes() *web.Route { m.Group("/packages", func() { m.Group("/generic", func() { m.Group("/{packagename}/{packageversion}/{filename}", func() { - m.Get("", generic.DownloadPackageContent) + m.Get("", generic.DownloadPackageFile) m.Put("", reqRepoWriter(models.UnitTypePackages), generic.UploadPackage) m.Delete("", reqRepoWriter(models.UnitTypePackages), generic.DeletePackage) }) @@ -1000,7 +1000,7 @@ func Routes() *web.Route { m.Get("/index.json", nuget.EnumeratePackageVersions) m.Group("/{version}", func() { m.Delete("/", reqRepoWriter(models.UnitTypePackages), nuget.DeletePackage) - m.Get("/{filename}", nuget.DownloadPackageContent) + m.Get("/{filename}", nuget.DownloadPackageFile) }) }) }, reqBasicAuth()) @@ -1008,12 +1008,12 @@ func Routes() *web.Route { m.Group("/{id}", func() { m.Get("", npm.PackageMetadata) m.Put("", reqRepoWriter(models.UnitTypePackages), npm.UploadPackage) - m.Get("/-/{version}/{filename}", npm.DownloadPackageContent) + m.Get("/-/{version}/{filename}", npm.DownloadPackageFile) }) }, reqToken()) m.Group("/pypi", func() { m.Post("/", reqRepoWriter(models.UnitTypePackages), pypi.UploadPackageFile) - m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageContent) + m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) m.Get("/simple/{id}", pypi.PackageMetadata) }, reqBasicAuth()) }) diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 0c742f0abf4c0..70763916a2fcc 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -22,8 +22,8 @@ import ( var packageNameRegex = regexp.MustCompile(`\A[A-Za-z0-9\.\_\-\+]+\z`) var filenameRegex = packageNameRegex -// DownloadPackageContent serves the specific generic package. -func DownloadPackageContent(ctx *context.APIContext) { +// DownloadPackageFile serves the specific generic package. +func DownloadPackageFile(ctx *context.APIContext) { packageName, packageVersion, filename, err := sanitizeParameters(ctx) if err != nil { ctx.Error(http.StatusBadRequest, "", err) diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 75365943dc02d..5f5508fa9df8c 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -50,8 +50,8 @@ func PackageMetadata(ctx *context.APIContext) { ctx.JSON(http.StatusOK, resp) } -// DownloadPackageContent serves the content of a package -func DownloadPackageContent(ctx *context.APIContext) { +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.APIContext) { packageName, err := url.QueryUnescape(ctx.Params("id")) if err != nil { ctx.Error(http.StatusBadRequest, "", err) diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index fa18f8e3380d4..85e9fe6db86d9 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -138,8 +138,8 @@ func EnumeratePackageVersions(ctx *context.APIContext) { ctx.JSON(http.StatusOK, resp) } -// DownloadPackageContent https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg -func DownloadPackageContent(ctx *context.APIContext) { +// DownloadPackageFile https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg +func DownloadPackageFile(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") filename := ctx.Params("filename") diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index 7c4e6e938804b..76888cd7ea665 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -56,8 +56,8 @@ func PackageMetadata(ctx *context.APIContext) { ctx.HTML(http.StatusOK, "api/packages/pypi/simple") } -// DownloadPackageContent serves the content of a package -func DownloadPackageContent(ctx *context.APIContext) { +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.APIContext) { packageName := normalizer.Replace(ctx.Params("id")) packageVersion := ctx.Params("version") filename := ctx.Params("filename") From 4a57e223a1c50813a1761580da806f35faa1110b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 20 Jul 2021 22:26:52 +0000 Subject: [PATCH 016/130] Added missing migration file. --- models/migrations/v189.go | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 models/migrations/v189.go diff --git a/models/migrations/v189.go b/models/migrations/v189.go new file mode 100644 index 0000000000000..56ee43c6942f5 --- /dev/null +++ b/models/migrations/v189.go @@ -0,0 +1,48 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func addPackageTables(x *xorm.Engine) error { + type Package struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatorID int64 + Type int `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` + MetadataRaw string `xorm:"TEXT"` + + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + } + + if err := x.Sync2(new(Package)); err != nil { + return err + } + + type PackageFile struct { + ID int64 `xorm:"pk autoincr"` + PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + Size int64 + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + HashMD5 string `xorm:"hash_md5"` + HashSHA1 string `xorm:"hash_sha1"` + HashSHA256 string `xorm:"hash_sha256"` + HashSHA512 string `xorm:"hash_sha512"` + + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + } + + return x.Sync2(new(PackageFile)) +} From b8ac3e3ad94006bb3c8989c54c45fe54eeeec044 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 21 Jul 2021 07:16:00 +0200 Subject: [PATCH 017/130] Set page info. --- routers/web/repo/packages.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index d0168193e6a7f..a3cef8f8b68ff 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -37,6 +37,7 @@ func MustEnablePackages(ctx *context.Context) { // Packages displays a list of all packages in the repository func Packages(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.packages") + ctx.Data["IsPackagesPage"] = true query := ctx.QueryTrim("q") packageType := ctx.QueryTrim("package_type") @@ -66,7 +67,6 @@ func Packages(ctx *context.Context) { } ctx.Data["Packages"] = packages - ctx.Data["IsPackagesPage"] = true ctx.Data["Query"] = query ctx.Data["PackageType"] = packageType @@ -97,6 +97,9 @@ func ViewPackage(ctx *context.Context) { ctx.ServerError("LoadCreator", err) return } + + ctx.Data["Title"] = p.Name + ctx.Data["IsPackagesPage"] = true ctx.Data["Package"] = p var metadata interface{} From 1386b27f7a29e756bdec29d2155c59a912c40087 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 21 Jul 2021 21:35:09 +0000 Subject: [PATCH 018/130] Added documentation. --- docs/content/doc/developers.en-us.md | 2 +- docs/content/doc/developers.zh-tw.md | 2 +- docs/content/doc/features/comparison.en-us.md | 3 +- docs/content/doc/packages.en-us.md | 12 ++ docs/content/doc/packages/generic.en-us.md | 82 ++++++++++++++ docs/content/doc/packages/maven.en-us.md | 103 ++++++++++++++++++ docs/content/doc/packages/npm.en-us.md | 81 ++++++++++++++ docs/content/doc/packages/nuget.en-us.md | 90 +++++++++++++++ docs/content/doc/packages/pypi.en-us.md | 80 ++++++++++++++ docs/content/doc/translation.de-de.md | 2 +- docs/content/doc/translation.en-us.md | 2 +- docs/content/doc/translation.zh-tw.md | 2 +- 12 files changed, 455 insertions(+), 6 deletions(-) create mode 100644 docs/content/doc/packages.en-us.md create mode 100644 docs/content/doc/packages/generic.en-us.md create mode 100644 docs/content/doc/packages/maven.en-us.md create mode 100644 docs/content/doc/packages/npm.en-us.md create mode 100644 docs/content/doc/packages/nuget.en-us.md create mode 100644 docs/content/doc/packages/pypi.en-us.md diff --git a/docs/content/doc/developers.en-us.md b/docs/content/doc/developers.en-us.md index c24a23dfae4a6..917049e5dfa1e 100644 --- a/docs/content/doc/developers.en-us.md +++ b/docs/content/doc/developers.en-us.md @@ -8,6 +8,6 @@ draft: false menu: sidebar: name: "Developers" - weight: 50 + weight: 55 identifier: "developers" --- diff --git a/docs/content/doc/developers.zh-tw.md b/docs/content/doc/developers.zh-tw.md index e2fbd4a34f51d..c9ce6634ada83 100644 --- a/docs/content/doc/developers.zh-tw.md +++ b/docs/content/doc/developers.zh-tw.md @@ -8,6 +8,6 @@ draft: false menu: sidebar: name: "開發人員" - weight: 50 + weight: 55 identifier: "developers" --- diff --git a/docs/content/doc/features/comparison.en-us.md b/docs/content/doc/features/comparison.en-us.md index a703766855f4b..a6f9f66250599 100644 --- a/docs/content/doc/features/comparison.en-us.md +++ b/docs/content/doc/features/comparison.en-us.md @@ -48,7 +48,8 @@ _Symbols used in table:_ | Integrated Git-powered wiki | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ (cloud only) | ✘ | | Deploy Tokens | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | Repository Tokens with write rights | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Built-in Container Registry | [✘](https://github.com/go-gitea/gitea/issues/2316) | ✘ | ✘ | ✓ | ✓ | ✘ | ✘ | +| Built-in Container Registry | [✘](https://github.com/go-gitea/gitea/issues/2316) | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | +| Built-in Package Registry | ✓ | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | | External git mirroring | ✓ | ✓ | ✘ | ✘ | ✓ | ✓ | ✓ | | FIDO U2F (2FA) | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ✘ | | Built-in CI/CD | ✘ | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | diff --git a/docs/content/doc/packages.en-us.md b/docs/content/doc/packages.en-us.md new file mode 100644 index 0000000000000..e613b6b250e27 --- /dev/null +++ b/docs/content/doc/packages.en-us.md @@ -0,0 +1,12 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "Package Registry" +slug: "packages" +toc: false +draft: false +menu: + sidebar: + name: "Package Registry" + weight: 45 + identifier: "packages" +--- diff --git a/docs/content/doc/packages/generic.en-us.md b/docs/content/doc/packages/generic.en-us.md new file mode 100644 index 0000000000000..cb40c5e5b00e3 --- /dev/null +++ b/docs/content/doc/packages/generic.en-us.md @@ -0,0 +1,82 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "Generic Packages Repository" +slug: "generic" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Generic" + weight: 10 + identifier: "generic" +--- + +# Generic Packages Repository + +Publish generic files, like release binaries or other output, in your project’s Package Registry. + +**Table of Contents** + +{{< toc >}} + +## Authenticate to the package registry + +To authenticate to the Package Registry, you need to provide [custom HTTP headers or use HTTP Basic authentication]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). + +## Publish a package + +To publish a generic package perform a HTTP PUT operation with the package content in the request body. +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +``` +PUT https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/generic/{package_name}/{package_version}/{file_name} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | +| `package_name` | The package name. It can contain only lowercase letters (`a-z`), uppercase letter (`A-Z`), numbers (`0-9`), dots (`.`), hyphens (`-`), or underscores (`_`). | +| `package_version` | The package version as described in the [SemVer](https://semver.org/) spec. | +| `file_name` | The filename. It can contain only lowercase letters (`a-z`), uppercase letter (`A-Z`), numbers (`0-9`), dots (`.`), hyphens (`-`), or underscores (`_`). | + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_password_or_token \ + --upload-file path/to/file.bin \ + "https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/generic/test_package/1.0.0/file.bin" +``` + +The server reponds with the following HTTP Status codes. + +| HTTP Status Code | Meaning | +| ----------------- | ------- | +| `201 Created` | The package has been published. | +| `400 Bad Request` | The package name and/or version are invalid or a package with the same name and version already exist. | + +## Download a package + +To download a generic package perform a HTTP GET operation. + +``` +GET https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/generic/{package_name}/{package_version}/{file_name} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | +| `package_name` | The package name. | +| `package_version` | The package version. | +| `file_name` | The filename. | + +The file content is served in the response body. The response content type is `application/octet-stream`. + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_token_or_password \ + "https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/generic/test_package/1.0.0/file.bin" +``` diff --git a/docs/content/doc/packages/maven.en-us.md b/docs/content/doc/packages/maven.en-us.md new file mode 100644 index 0000000000000..173acb16194a9 --- /dev/null +++ b/docs/content/doc/packages/maven.en-us.md @@ -0,0 +1,103 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "Maven Packages Repository" +slug: "maven" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Maven" + weight: 40 + identifier: "maven" +--- + +# Maven Packages Repository + +Publish [Maven](https://maven.apache.org) packages in your project’s Package Registry. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Maven package registry, you can use [Maven](https://maven.apache.org/install.html) or [Gradle](https://gradle.org/install/). +The following examples use `Maven`. + +## Configuring the package registry + +To register the project’s package registry you first need to add your access token to the [`settings.xml`](https://maven.apache.org/settings.html) file: + +```xml + + + + gitea + + + + Authorization + token {access_token} + + + + + + +``` + +Afterwards add the following sections to your project `pom.xml` file: + +```xml + + + gitea + https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/maven + + + + + gitea + https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/maven + + + gitea + https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/maven + + +``` + +| Parameter | Description | +| -------------- | ----------- | +| `access_token` | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | + +## Publish a package + +To publish a package simply run: + +```shell +mvn deploy +``` + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a Maven package from the package registry, add a new dependency to your project `pom.xml` file: + +```xml + + com.test.package + test_project + 1.0.0 + +``` + +Afterwards run: + +```shell +mvn install +``` \ No newline at end of file diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md new file mode 100644 index 0000000000000..3a823a47c0b7e --- /dev/null +++ b/docs/content/doc/packages/npm.en-us.md @@ -0,0 +1,81 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "npm Packages Repository" +slug: "npm" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "npm" + weight: 30 + identifier: "npm" +--- + +# npm Packages Repository + +Publish [npm](https://www.npmjs.com/) packages in your project’s Package Registry. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the npm package registry, you need [Node.js and npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm/) or an other tool like [Yarn](https://classic.yarnpkg.com/en/docs/install). + +Only [scoped](https://docs.npmjs.com/misc/scope/) packages are supported. + +The following examples use the `npm` tool with the scope `@test`. + +## Configuring the package registry + +To register the project’s package registry you need to configure a new package source. + +```shell +npm config set {scope}:registry https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/npm +npm config set -- '//gitea.example.com/api/v1/repos/{owner}/{repository}/packages/npm/:_authToken' "{token}" +``` + +| Parameter | Description | +| ------------ | ----------- | +| `scope` | The scope of the packages. | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | +| `token` | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). | + +For example: + +```shell +npm config set @test:registry https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/npm +npm config set -- '//gitea.example.com/api/v1/repos/testuser/test-repository/packages/npm/:_authToken' "personal_access_token" +``` + +## Publish a package + +Publish a package by running the following command in your project: + +```shell +npm publish +``` + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a package from the package registry, execute the following command: + +```shell +npm install {scope}/{package_name} +``` + +| Parameter | Description | +| -------------- | ----------- | +| `scope` | The scope of the packages. | +| `package_name` | The package name. | + +For example: + +```shell +npm install @test/test_package +``` \ No newline at end of file diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md new file mode 100644 index 0000000000000..8aaef7d661908 --- /dev/null +++ b/docs/content/doc/packages/nuget.en-us.md @@ -0,0 +1,90 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "NuGet Packages Repository" +slug: "nuget" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "NuGet" + weight: 20 + identifier: "nuget" +--- + +# NuGet Packages Repository + +Publish [NuGet](https://www.nuget.org/) packages in your project’s Package Registry. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the NuGet package registry, you can use command-line interface (CLI) tools as well as NuGet features in various IDEs like Visual Studio. +More informations about NuGet clients can be found in [the official documentation](https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools). +The following examples use the `dotnet nuget` tool. + +## Configuring the package registry + +To register the project’s package registry you need to configure a new NuGet feed source: + +```shell +dotnet nuget add source --name {source_name} --username {username} --password {password} https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/nuget/index.json +``` + +| Parameter | Description | +| ------------- | ----------- | +| `source_name` | The desired source name. | +| `username` | Your Gitea username. | +| `password` | Your Gitea password or a personal access token. | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | + +For example: + +```shell +dotnet nuget add source --name gitea --username testuser --password password123 https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/nuget/index.json +``` + +## Publish a package + +Publish a package by running the following command: + +```shell +dotnet nuget push --source {source_name} {package_file} +``` + +| Parameter | Description | +| -------------- | ----------- | +| `source_name` | The desired source name. | +| `package_file` | Path to the package `.nupkg` file. | + +For example: + +```shell +dotnet nuget push --source gitea test_package.1.0.0.nupkg +``` + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a NuGet package from the package registry, execute the following command: + +```shell +dotnet add package --source {source_name} --version {package_version} {package_name} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `source_name` | The desired source name. | +| `package_name` | The package name. | +| `package_version` | The package version. | + +For example: + +```shell +dotnet add package --source gitea --version 1.0.0 test_package +``` \ No newline at end of file diff --git a/docs/content/doc/packages/pypi.en-us.md b/docs/content/doc/packages/pypi.en-us.md new file mode 100644 index 0000000000000..33372ee553c56 --- /dev/null +++ b/docs/content/doc/packages/pypi.en-us.md @@ -0,0 +1,80 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "PyPI Packages Repository" +slug: "pypi" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "PyPI" + weight: 50 + identifier: "pypi" +--- + +# PyPI Packages Repository + +Publish [PyPI](https://pypi.org/) packages in your project’s Package Registry. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the PyPI package registry, you need to use the tools [pip](https://pypi.org/project/pip/) to consume and [twine](https://pypi.org/project/twine/) to publish packages. + +## Configuring the package registry + +To register the project’s package registry you need to edit your local `~/.pypirc` file. Add + +```ini +[distutils] +index-servers = gitea + +[gitea] +repository = https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/pypi +username = {username} +password = {password} +``` + +| Placeholder | Description | +| ------------ | ----------- | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | +| `username` | Your Gitea username. | +| `password` | Your Gitea password or a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). | + +## Publish a package + +Publish a package by running the following command: + +```shell +python3 -m twine upload --repository gitea /path/to/files/* +``` + +The package files have the extensions `.tar.gz` and `.whl`. + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a PyPI package from the package registry, execute the following command: + +```shell +pip install --index-url https://{username}:{password}@gitea.example.com/api/v1/repos/{owner}/{repository}/packages/pypi/simple --no-deps {package_name} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `username` | Your Gitea username. | +| `password` | Your Gitea password or a personal access token. | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | +| `package_name` | The package name. | + +For example: + +```shell +pip install --index-url https://testuser:password123@gitea.example.com/api/v1/repos/testuser/test-repository/packages/pypi/simple --no-deps test_package +``` diff --git a/docs/content/doc/translation.de-de.md b/docs/content/doc/translation.de-de.md index 585783a706256..3470faa59b4c4 100644 --- a/docs/content/doc/translation.de-de.md +++ b/docs/content/doc/translation.de-de.md @@ -8,6 +8,6 @@ draft: false menu: sidebar: name: "Übersetzung" - weight: 45 + weight: 50 identifier: "translation" --- diff --git a/docs/content/doc/translation.en-us.md b/docs/content/doc/translation.en-us.md index 208eb32ab8aa4..c281088503aec 100644 --- a/docs/content/doc/translation.en-us.md +++ b/docs/content/doc/translation.en-us.md @@ -8,6 +8,6 @@ draft: false menu: sidebar: name: "Translation" - weight: 45 + weight: 50 identifier: "translation" --- diff --git a/docs/content/doc/translation.zh-tw.md b/docs/content/doc/translation.zh-tw.md index ca820c093c697..5374e87e891c8 100644 --- a/docs/content/doc/translation.zh-tw.md +++ b/docs/content/doc/translation.zh-tw.md @@ -8,6 +8,6 @@ draft: false menu: sidebar: name: "翻譯" - weight: 45 + weight: 50 identifier: "translation" --- From 8ba72d71aded9bedc4a61a39bde92fde19ecf257 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 21 Jul 2021 21:50:46 +0000 Subject: [PATCH 019/130] Added documentation links. --- options/locale/locale_en-US.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index c4e76e28f00e2..38ee5636fe445 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1524,20 +1524,20 @@ packages.delete = Delete package packages.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? packages.delete.success = The package has been deleted. packages.generic.use = Download the package, for example with curl: -packages.generic.documentation = For more information on the generic registry, see the documentation. +packages.generic.documentation = For more information on the generic registry, see the documentation. packages.maven.registry = Setup this registry in your projects pom.xml file: packages.maven.use = To use the package include the following in the dependencies block in the pom.xml file: packages.maven.use_2 = Alternatively use this Maven command: -packages.maven.documentation = For more information on the Maven registry, see the documentation. +packages.maven.documentation = For more information on the Maven registry, see the documentation. packages.nuget.registry = Setup this registry with the following command: packages.nuget.use = Install the package with the following command: -packages.nuget.documentation = For more information on the NuGet registry, see the documentation. +packages.nuget.documentation = For more information on the NuGet registry, see the documentation. packages.npm.registry = Setup this registry in your projects .npmrc file: packages.npm.use = To install the package use the following command: -packages.npm.documentation = For more information on the npm registry, see the documentation. +packages.npm.documentation = For more information on the npm registry, see the documentation. packages.pypi.requires = Requires Python packages.pypi.use = To install the package use the following command: -packages.pypi.documentation = For more information on the PyPI registry, see the documentation. +packages.pypi.documentation = For more information on the PyPI registry, see the documentation. signing.will_sign = This commit will be signed with key '%s' signing.wont_sign.error = There was an error whilst checking if the commit could be signed From cab94bbafddc43e16f9e772b2b74437775081d9d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 22 Jul 2021 15:38:18 +0000 Subject: [PATCH 020/130] Fixed wrong error message. --- models/package.go | 2 +- routers/web/repo/packages.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/package.go b/models/package.go index c2de2e5c404c5..e4df42e77b091 100644 --- a/models/package.go +++ b/models/package.go @@ -112,7 +112,7 @@ func (p *Package) GetFileByName(name string) (*PackageFile, error) { return nil, err } if !has { - return nil, ErrDuplicatePackage + return nil, ErrPackageFileNotExist } return pf, nil } diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index a3cef8f8b68ff..2467527c1949e 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -97,7 +97,7 @@ func ViewPackage(ctx *context.Context) { ctx.ServerError("LoadCreator", err) return } - + ctx.Data["Title"] = p.Name ctx.Data["IsPackagesPage"] = true ctx.Data["Package"] = p From f16296c0202ced1a48b33eb8bcb9e6d257130335 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 22 Jul 2021 16:04:26 +0000 Subject: [PATCH 021/130] Lint template files. --- templates/api/packages/pypi/simple.tmpl | 2 +- templates/repo/packages/content/generic.tmpl | 2 +- templates/repo/packages/content/maven.tmpl | 2 +- templates/repo/packages/content/npm.tmpl | 2 +- templates/repo/packages/content/nuget.tmpl | 2 +- templates/repo/packages/content/pypi.tmpl | 2 +- templates/repo/packages/list.tmpl | 2 +- templates/repo/packages/metadata/maven.tmpl | 2 +- templates/repo/packages/metadata/npm.tmpl | 2 +- templates/repo/packages/metadata/nuget.tmpl | 2 +- templates/repo/packages/metadata/pypi.tmpl | 2 +- templates/repo/packages/view.tmpl | 9 +-------- 12 files changed, 12 insertions(+), 19 deletions(-) diff --git a/templates/api/packages/pypi/simple.tmpl b/templates/api/packages/pypi/simple.tmpl index f3ed056505179..3815b56e6f33b 100644 --- a/templates/api/packages/pypi/simple.tmpl +++ b/templates/api/packages/pypi/simple.tmpl @@ -12,4 +12,4 @@ {{end}} {{end}} - \ No newline at end of file + diff --git a/templates/repo/packages/content/generic.tmpl b/templates/repo/packages/content/generic.tmpl index 4c3969ff4799f..c3d398fb90db9 100644 --- a/templates/repo/packages/content/generic.tmpl +++ b/templates/repo/packages/content/generic.tmpl @@ -9,4 +9,4 @@
    {{.i18n.Tr "repo.packages.generic.documentation" | Safe}}
    -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/content/maven.tmpl b/templates/repo/packages/content/maven.tmpl index 2ffced4813f73..fd558fabd6925 100644 --- a/templates/repo/packages/content/maven.tmpl +++ b/templates/repo/packages/content/maven.tmpl @@ -56,4 +56,4 @@ {{end}} {{end}} -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/content/npm.tmpl b/templates/repo/packages/content/npm.tmpl index 8c0b5f66b2afa..5298bbca5a4f6 100644 --- a/templates/repo/packages/content/npm.tmpl +++ b/templates/repo/packages/content/npm.tmpl @@ -35,4 +35,4 @@ {{end}} {{end}} -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/content/nuget.tmpl b/templates/repo/packages/content/nuget.tmpl index 57a41aeeb5d4a..291f94df0385d 100644 --- a/templates/repo/packages/content/nuget.tmpl +++ b/templates/repo/packages/content/nuget.tmpl @@ -33,4 +33,4 @@ {{end}} {{end}} -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/content/pypi.tmpl b/templates/repo/packages/content/pypi.tmpl index dfbc6a8529e19..15d61d4b34492 100644 --- a/templates/repo/packages/content/pypi.tmpl +++ b/templates/repo/packages/content/pypi.tmpl @@ -21,4 +21,4 @@ {{end}} {{end}} -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/list.tmpl b/templates/repo/packages/list.tmpl index cb342ab31ee8e..60338166afbd7 100644 --- a/templates/repo/packages/list.tmpl +++ b/templates/repo/packages/list.tmpl @@ -52,4 +52,4 @@ -{{template "base/footer" .}} \ No newline at end of file +{{template "base/footer" .}} diff --git a/templates/repo/packages/metadata/maven.tmpl b/templates/repo/packages/metadata/maven.tmpl index 0ea9adfd5b8d4..246d5889a5c19 100644 --- a/templates/repo/packages/metadata/maven.tmpl +++ b/templates/repo/packages/metadata/maven.tmpl @@ -7,4 +7,4 @@
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} {{end}} -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/metadata/npm.tmpl b/templates/repo/packages/metadata/npm.tmpl index 0c37cc968cad4..702fc94d6ed97 100644 --- a/templates/repo/packages/metadata/npm.tmpl +++ b/templates/repo/packages/metadata/npm.tmpl @@ -2,4 +2,4 @@ {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Author}}
    {{end}} {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/metadata/nuget.tmpl b/templates/repo/packages/metadata/nuget.tmpl index 557d0d3e0ca4d..2ef835826a3ad 100644 --- a/templates/repo/packages/metadata/nuget.tmpl +++ b/templates/repo/packages/metadata/nuget.tmpl @@ -1,4 +1,4 @@ {{if eq .Package.Type 1}} {{if .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Authors}}
    {{end}} {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/metadata/pypi.tmpl b/templates/repo/packages/metadata/pypi.tmpl index 281cdbfe58649..113b0c92bac92 100644 --- a/templates/repo/packages/metadata/pypi.tmpl +++ b/templates/repo/packages/metadata/pypi.tmpl @@ -3,4 +3,4 @@ {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} {{if .Metadata.RequiresPython}}
    {{.i18n.Tr "repo.packages.python.requires"}}: {{.Metadata.RequiresPython}}
    {{end}} -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/view.tmpl b/templates/repo/packages/view.tmpl index 54534dfd6fc3d..72b6ad2f8810f 100644 --- a/templates/repo/packages/view.tmpl +++ b/templates/repo/packages/view.tmpl @@ -12,7 +12,6 @@
    {{.i18n.Tr "repo.packages.published_by" $timeStr .Package.Creator.HomeLink (.Package.Creator.GetDisplayName | Escape) | Safe}} {{svg "octicon-package"}} {{.Package.Type.String}}
    -
    {{template "repo/packages/content/generic" .}} {{template "repo/packages/content/nuget" .}} @@ -20,7 +19,6 @@ {{template "repo/packages/content/maven" .}} {{template "repo/packages/content/pypi" .}}
    -
    {{.i18n.Tr "repo.packages.details"}} @@ -30,13 +28,11 @@ {{template "repo/packages/metadata/npm" .}} {{template "repo/packages/metadata/maven" .}} {{template "repo/packages/metadata/pypi" .}} -
    {{.i18n.Tr "repo.packages.assets"}} {{range .Files}} {{end}} - {{if .OtherVersions}}
    {{.i18n.Tr "repo.packages.versions"}} @@ -44,7 +40,6 @@
    {{.Version}} {{$.i18n.Tr "repo.packages.versions.on"}} {{.CreatedUnix.FormatDate}}
    {{end}} {{end}} - {{if .CanWritePackages}}
    @@ -58,10 +53,8 @@
    {{.i18n.Tr "repo.packages.delete.notice" .Package.Name .Package.Version}}
    -
    {{.CsrfTokenHtml}} -
    {{.i18n.Tr "cancel"}}
    @@ -76,4 +69,4 @@
    -{{template "base/footer" .}} \ No newline at end of file +{{template "base/footer" .}} From 814a942cd47d425525b8a99d8fa649d5ba687be6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 7 Aug 2021 10:49:42 +0000 Subject: [PATCH 022/130] Fixed merge errors. --- routers/api/v1/packages/nuget/nuget.go | 6 +++--- routers/web/repo/packages.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 85e9fe6db86d9..29902a67fb1b2 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -31,9 +31,9 @@ func ServiceIndex(ctx *context.APIContext) { // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages func SearchService(ctx *context.APIContext) { - query := ctx.QueryTrim("q") - skip := ctx.QueryInt("skip") - take := ctx.QueryInt("take") + query := ctx.FormTrim("q") + skip := ctx.FormInt("skip") + take := ctx.FormInt("take") total, packages, err := models.SearchPackages(ctx.Repo.Repository.ID, models.PackageNuGet, query, skip, take) if err != nil { diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index 2467527c1949e..f84e101f2700a 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -39,9 +39,9 @@ func Packages(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.packages") ctx.Data["IsPackagesPage"] = true - query := ctx.QueryTrim("q") - packageType := ctx.QueryTrim("package_type") - page := ctx.QueryInt("page") + query := ctx.FormTrim("q") + packageType := ctx.FormTrim("package_type") + page := ctx.FormInt("page") if page <= 1 { page = 1 } From 94177549648b6e6d5a36735ad1b0950b3e880b4b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 7 Aug 2021 11:27:57 +0000 Subject: [PATCH 023/130] Fixed unit test storage path. --- models/unit_tests.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/unit_tests.go b/models/unit_tests.go index f8d6819333611..ac65a23de4cb2 100644 --- a/models/unit_tests.go +++ b/models/unit_tests.go @@ -76,6 +76,8 @@ func MainTest(m *testing.M, pathToGiteaRoot string) { setting.RepoArchive.Storage.Path = filepath.Join(setting.AppDataPath, "repo-archive") + setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages") + if err = storage.Init(); err != nil { fatalTestError("storage.Init: %v\n", err) } From 02708f88bdc4c8a47ef22a64df1e2e1e6beaace4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 8 Aug 2021 18:45:39 +0000 Subject: [PATCH 024/130] Switch to json module. --- integrations/api_packages_maven_test.go | 6 +++--- modules/packages/npm/creator.go | 4 ++-- modules/packages/npm/creator_test.go | 19 ++++++++++--------- routers/api/v1/packages/maven/maven.go | 5 ++--- routers/api/v1/packages/maven/package.go | 5 ++--- routers/api/v1/packages/npm/package.go | 4 ++-- routers/api/v1/packages/nuget/package.go | 4 ++-- routers/api/v1/packages/pypi/package.go | 5 ++--- routers/web/repo/packages.go | 5 ++--- services/packages/packages.go | 5 ++--- 10 files changed, 29 insertions(+), 33 deletions(-) diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go index d01e9234200c5..0d39428f25ed3 100644 --- a/integrations/api_packages_maven_test.go +++ b/integrations/api_packages_maven_test.go @@ -11,9 +11,9 @@ import ( "testing" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/maven" - jsoniter "github.com/json-iterator/go" "github.com/stretchr/testify/assert" ) @@ -89,7 +89,7 @@ func TestPackageMaven(t *testing.T) { assert.Len(t, ps, 1) var m *maven.Metadata - err = jsoniter.Unmarshal([]byte(ps[0].MetadataRaw), &m) + err = json.Unmarshal([]byte(ps[0].MetadataRaw), &m) assert.NoError(t, err) assert.Empty(t, m.Description) @@ -99,7 +99,7 @@ func TestPackageMaven(t *testing.T) { assert.NoError(t, err) assert.Len(t, ps, 1) - err = jsoniter.Unmarshal([]byte(ps[0].MetadataRaw), &m) + err = json.Unmarshal([]byte(ps[0].MetadataRaw), &m) assert.NoError(t, err) assert.Equal(t, packageDescription, m.Description) }) diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 33b99dc87bf2d..5639169ac8a43 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -16,10 +16,10 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/validation" "github.com/hashicorp/go-version" - jsoniter "github.com/json-iterator/go" ) var ( @@ -122,7 +122,7 @@ type packageUpload struct { // ParsePackage parses the content into a NPM package func ParsePackage(r io.Reader) (*Package, error) { var upload packageUpload - if err := jsoniter.NewDecoder(r).Decode(&upload); err != nil { + if err := json.NewDecoder(r).Decode(&upload); err != nil { return nil, err } diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index c294b695adaf0..04946cf310641 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -11,7 +11,8 @@ import ( "strings" "testing" - jsoniter "github.com/json-iterator/go" + "code.gitea.io/gitea/modules/json" + "github.com/stretchr/testify/assert" ) @@ -30,7 +31,7 @@ func TestParsePackage(t *testing.T) { }) t.Run("InvalidUploadNoData", func(t *testing.T) { - b, _ := jsoniter.Marshal(packageUpload{}) + b, _ := json.Marshal(packageUpload{}) p, err := ParsePackage(bytes.NewReader(b)) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidPackage) @@ -38,7 +39,7 @@ func TestParsePackage(t *testing.T) { t.Run("InvalidPackageName", func(t *testing.T) { test := func(t *testing.T, name string) { - b, _ := jsoniter.Marshal(packageUpload{ + b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ ID: name, Name: name, @@ -63,7 +64,7 @@ func TestParsePackage(t *testing.T) { t.Run("InvalidPackageVersion", func(t *testing.T) { version := "first-version" - b, _ := jsoniter.Marshal(packageUpload{ + b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ ID: packageName, Name: packageName, @@ -81,7 +82,7 @@ func TestParsePackage(t *testing.T) { }) t.Run("InvalidAttachment", func(t *testing.T) { - b, _ := jsoniter.Marshal(packageUpload{ + b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ ID: packageName, Name: packageName, @@ -103,7 +104,7 @@ func TestParsePackage(t *testing.T) { t.Run("InvalidData", func(t *testing.T) { filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) - b, _ := jsoniter.Marshal(packageUpload{ + b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ ID: packageName, Name: packageName, @@ -127,7 +128,7 @@ func TestParsePackage(t *testing.T) { t.Run("InvalidIntegrity", func(t *testing.T) { filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) - b, _ := jsoniter.Marshal(packageUpload{ + b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ ID: packageName, Name: packageName, @@ -154,7 +155,7 @@ func TestParsePackage(t *testing.T) { t.Run("InvalidIntegrity2", func(t *testing.T) { filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) - b, _ := jsoniter.Marshal(packageUpload{ + b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ ID: packageName, Name: packageName, @@ -181,7 +182,7 @@ func TestParsePackage(t *testing.T) { t.Run("Valid", func(t *testing.T) { filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) - b, _ := jsoniter.Marshal(packageUpload{ + b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ ID: packageName, Name: packageName, diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 8b46f929ba83c..c47b7700f3b30 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -19,14 +19,13 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/packages" maven_module "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/util/filebuffer" package_service "code.gitea.io/gitea/services/packages" - - jsoniter "github.com/json-iterator/go" ) const mavenMetadataFile = "maven-metadata.xml" @@ -217,7 +216,7 @@ func UploadPackageFile(ctx *context.APIContext) { log.Error("Error parsing package metadata: %v", err) } if metadata != nil { - raw, err := jsoniter.Marshal(metadata) + raw, err := json.Marshal(metadata) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return diff --git a/routers/api/v1/packages/maven/package.go b/routers/api/v1/packages/maven/package.go index 9ef5d0550d622..6ac7000bd33cb 100644 --- a/routers/api/v1/packages/maven/package.go +++ b/routers/api/v1/packages/maven/package.go @@ -8,9 +8,8 @@ import ( "sort" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/json" maven_module "code.gitea.io/gitea/modules/packages/maven" - - jsoniter "github.com/json-iterator/go" ) // Package represents a package with Maven metadata @@ -33,7 +32,7 @@ func intializePackages(packages []*models.Package) ([]*Package, error) { func intializePackage(p *models.Package) (*Package, error) { var m *maven_module.Metadata - err := jsoniter.Unmarshal([]byte(p.MetadataRaw), &m) + err := json.Unmarshal([]byte(p.MetadataRaw), &m) if err != nil { return nil, err } diff --git a/routers/api/v1/packages/npm/package.go b/routers/api/v1/packages/npm/package.go index b74f368cea256..ad46d04576ba2 100644 --- a/routers/api/v1/packages/npm/package.go +++ b/routers/api/v1/packages/npm/package.go @@ -8,10 +8,10 @@ import ( "sort" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/json" npm_module "code.gitea.io/gitea/modules/packages/npm" "github.com/hashicorp/go-version" - jsoniter "github.com/json-iterator/go" ) // Package represents a package with NPM metadata @@ -41,7 +41,7 @@ func intializePackage(p *models.Package) (*Package, error) { } var m *npm_module.Metadata - err = jsoniter.Unmarshal([]byte(p.MetadataRaw), &m) + err = json.Unmarshal([]byte(p.MetadataRaw), &m) if err != nil { return nil, err } diff --git a/routers/api/v1/packages/nuget/package.go b/routers/api/v1/packages/nuget/package.go index c4be91e77aa3b..94332fc9fc803 100644 --- a/routers/api/v1/packages/nuget/package.go +++ b/routers/api/v1/packages/nuget/package.go @@ -8,10 +8,10 @@ import ( "sort" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/json" nuget_module "code.gitea.io/gitea/modules/packages/nuget" "github.com/hashicorp/go-version" - jsoniter "github.com/json-iterator/go" ) // Package represents a package with NuGet metadata @@ -40,7 +40,7 @@ func intializePackage(p *models.Package) (*Package, error) { } var m *nuget_module.Metadata - err = jsoniter.Unmarshal([]byte(p.MetadataRaw), &m) + err = json.Unmarshal([]byte(p.MetadataRaw), &m) if err != nil { return nil, err } diff --git a/routers/api/v1/packages/pypi/package.go b/routers/api/v1/packages/pypi/package.go index 8050d4f94d5b7..ae9692314608d 100644 --- a/routers/api/v1/packages/pypi/package.go +++ b/routers/api/v1/packages/pypi/package.go @@ -6,9 +6,8 @@ package pypi import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/json" pypi_module "code.gitea.io/gitea/modules/packages/pypi" - - jsoniter "github.com/json-iterator/go" ) // Package represents a package with NPM metadata @@ -32,7 +31,7 @@ func intializePackages(packages []*models.Package) ([]*Package, error) { func intializePackage(p *models.Package) (*Package, error) { var m *pypi_module.Metadata - err := jsoniter.Unmarshal([]byte(p.MetadataRaw), &m) + err := json.Unmarshal([]byte(p.MetadataRaw), &m) if err != nil { return nil, err } diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index f84e101f2700a..4604c256031f1 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/packages/nuget" @@ -18,8 +19,6 @@ import ( "code.gitea.io/gitea/modules/setting" package_service "code.gitea.io/gitea/services/packages" - - jsoniter "github.com/json-iterator/go" ) const ( @@ -114,7 +113,7 @@ func ViewPackage(ctx *context.Context) { metadata = &pypi.Metadata{} } if metadata != nil { - if err := jsoniter.Unmarshal([]byte(p.MetadataRaw), &metadata); err != nil { + if err := json.Unmarshal([]byte(p.MetadataRaw), &metadata); err != nil { ctx.ServerError("Unmarshal", err) return } diff --git a/services/packages/packages.go b/services/packages/packages.go index 8550d58ca27db..8cfdbc1ad663e 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -14,15 +14,14 @@ import ( "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" - - jsoniter "github.com/json-iterator/go" ) // CreatePackage creates a new package func CreatePackage(creator *models.User, repository *models.Repository, packageType models.PackageType, name, version string, metadata interface{}, allowDuplicate bool) (*models.Package, error) { - metadataJSON, err := jsoniter.Marshal(metadata) + metadataJSON, err := json.Marshal(metadata) if err != nil { log.Error("Error converting metadata to JSON: %v", err) return nil, err From cb360c03c5650c76697f5b331962135d54a85026 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 9 Aug 2021 16:06:45 +0000 Subject: [PATCH 025/130] Added suggestions. --- models/package.go | 7 ++--- templates/repo/packages/content/maven.tmpl | 8 +++--- templates/repo/packages/content/npm.tmpl | 8 +++--- templates/repo/packages/content/nuget.tmpl | 10 ++++--- templates/repo/packages/metadata/maven.tmpl | 8 +++--- templates/repo/packages/metadata/npm.tmpl | 6 ++--- templates/repo/packages/metadata/nuget.tmpl | 4 +-- templates/repo/packages/metadata/pypi.tmpl | 8 +++--- templates/repo/packages/view.tmpl | 29 +++++++++++++++------ 9 files changed, 54 insertions(+), 34 deletions(-) diff --git a/models/package.go b/models/package.go index e4df42e77b091..d14216b3e4def 100644 --- a/models/package.go +++ b/models/package.go @@ -270,9 +270,10 @@ func GetPackageByNameAndVersion(repositoryID int64, packageType PackageType, pac // SearchPackages searches for packages by name and can be used to navigate through the package list func SearchPackages(repositoryID int64, packageType PackageType, query string, skip, take int) (int64, []*Package, error) { - cond := builder.NewCond() - cond = cond.And(builder.Eq{"repo_id": repositoryID}) - cond = cond.And(builder.Eq{"type": packageType}) + var cond builder.Cond = builder.Eq{ + "repo_id": repositoryID, + "type": packageType, + } if query != "" { cond = cond.And(builder.Like{"lower_name", strings.ToLower(query)}) } diff --git a/templates/repo/packages/content/maven.tmpl b/templates/repo/packages/content/maven.tmpl index fd558fabd6925..0a81641ac48bb 100644 --- a/templates/repo/packages/content/maven.tmpl +++ b/templates/repo/packages/content/maven.tmpl @@ -51,9 +51,11 @@ {{if .Metadata.Dependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    - {{range .Metadata.Dependencies}} -
    {{.GroupID}}:{{.ArtifactID}} ({{.Version}})
    - {{end}} +
    + {{range .Metadata.Dependencies}} +
    {{.GroupID}}:{{.ArtifactID}} ({{.Version}})
    + {{end}} +
    {{end}} {{end}} diff --git a/templates/repo/packages/content/npm.tmpl b/templates/repo/packages/content/npm.tmpl index 5298bbca5a4f6..a756fa2b19e55 100644 --- a/templates/repo/packages/content/npm.tmpl +++ b/templates/repo/packages/content/npm.tmpl @@ -30,9 +30,11 @@ {{if .Metadata.Dependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    - {{range $dependency, $version := .Metadata.Dependencies}} -
    {{$dependency}} ({{$version}})
    - {{end}} +
    + {{range $dependency, $version := .Metadata.Dependencies}} +
    {{$dependency}} ({{$version}})
    + {{end}} +
    {{end}} {{end}} diff --git a/templates/repo/packages/content/nuget.tmpl b/templates/repo/packages/content/nuget.tmpl index 291f94df0385d..e4d931402149e 100644 --- a/templates/repo/packages/content/nuget.tmpl +++ b/templates/repo/packages/content/nuget.tmpl @@ -26,10 +26,12 @@

    {{.i18n.Tr "repo.packages.dependencies"}}

    {{range $framework, $dependencies := .Metadata.Dependencies}} -
    {{$framework}}
    - {{range $dependencies}} -
    {{.ID}} ({{.Version}})
    - {{end}} +
    +
    {{$framework}}
    + {{range $dependencies}} +
    {{.ID}} ({{.Version}})
    + {{end}} +
    {{end}}
    {{end}} diff --git a/templates/repo/packages/metadata/maven.tmpl b/templates/repo/packages/metadata/maven.tmpl index 246d5889a5c19..b7015be23c34e 100644 --- a/templates/repo/packages/metadata/maven.tmpl +++ b/templates/repo/packages/metadata/maven.tmpl @@ -1,10 +1,10 @@ {{if eq .Package.Type 3}} - {{if .Metadata.Name}}
    {{svg "octicon-note" 16 "mr-3"}} {{.Metadata.Name}}
    {{end}} - {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .Metadata.Name}}
    {{svg "octicon-note" 16 "mr-3"}} {{.Metadata.Name}}
    {{end}} + {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} {{if .Metadata.Licenses}} -
    {{.i18n.Tr "repo.packages.details.license"}}:
    +
    {{.i18n.Tr "repo.packages.details.license"}}:
    {{range .Metadata.Licenses}} -
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    +
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} {{end}} {{end}} diff --git a/templates/repo/packages/metadata/npm.tmpl b/templates/repo/packages/metadata/npm.tmpl index 702fc94d6ed97..381aab099e0df 100644 --- a/templates/repo/packages/metadata/npm.tmpl +++ b/templates/repo/packages/metadata/npm.tmpl @@ -1,5 +1,5 @@ {{if eq .Package.Type 2}} - {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Author}}
    {{end}} - {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} + {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Author}}
    {{end}} + {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/nuget.tmpl b/templates/repo/packages/metadata/nuget.tmpl index 2ef835826a3ad..5e9c0da13d380 100644 --- a/templates/repo/packages/metadata/nuget.tmpl +++ b/templates/repo/packages/metadata/nuget.tmpl @@ -1,4 +1,4 @@ {{if eq .Package.Type 1}} - {{if .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Authors}}
    {{end}} - {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Authors}}
    {{end}} + {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/pypi.tmpl b/templates/repo/packages/metadata/pypi.tmpl index 113b0c92bac92..25ec8202bcc08 100644 --- a/templates/repo/packages/metadata/pypi.tmpl +++ b/templates/repo/packages/metadata/pypi.tmpl @@ -1,6 +1,6 @@ {{if eq .Package.Type 4}} - {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Author}}
    {{end}} - {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} - {{if .Metadata.RequiresPython}}
    {{.i18n.Tr "repo.packages.python.requires"}}: {{.Metadata.RequiresPython}}
    {{end}} + {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Author}}
    {{end}} + {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} + {{if .Metadata.RequiresPython}}
    {{.i18n.Tr "repo.packages.pypi.requires"}}: {{.Metadata.RequiresPython}}
    {{end}} {{end}} diff --git a/templates/repo/packages/view.tmpl b/templates/repo/packages/view.tmpl index 72b6ad2f8810f..b55ce57339eec 100644 --- a/templates/repo/packages/view.tmpl +++ b/templates/repo/packages/view.tmpl @@ -22,23 +22,36 @@
    {{.i18n.Tr "repo.packages.details"}} -
    {{svg "octicon-calendar" 16 "mr-3"}} {{.Package.CreatedUnix.FormatDate}}
    - {{template "repo/packages/metadata/generic" .}} - {{template "repo/packages/metadata/nuget" .}} - {{template "repo/packages/metadata/npm" .}} - {{template "repo/packages/metadata/maven" .}} - {{template "repo/packages/metadata/pypi" .}} +
    +
    + {{svg "octicon-calendar" 16 "mr-3"}} {{.Package.CreatedUnix.FormatDate}} +
    + {{template "repo/packages/metadata/generic" .}} + {{template "repo/packages/metadata/nuget" .}} + {{template "repo/packages/metadata/npm" .}} + {{template "repo/packages/metadata/maven" .}} + {{template "repo/packages/metadata/pypi" .}} +
    {{.i18n.Tr "repo.packages.assets"}} +
    {{range .Files}} - +
    + {{.Name}} +
    {{end}} +
    {{if .OtherVersions}}
    {{.i18n.Tr "repo.packages.versions"}} +
    {{range .OtherVersions}} -
    {{.Version}} {{$.i18n.Tr "repo.packages.versions.on"}} {{.CreatedUnix.FormatDate}}
    +
    + {{.Version}} + {{$.i18n.Tr "repo.packages.versions.on"}} {{.CreatedUnix.FormatDate}} +
    {{end}} +
    {{end}} {{if .CanWritePackages}}
    From 7264b7e060f682cebd4ba7f1069936d46b85cad2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 10 Aug 2021 20:13:00 +0200 Subject: [PATCH 026/130] Added package webhook. --- models/package.go | 4 +- models/webhook.go | 11 ++++ models/webhook_test.go | 1 + modules/convert/package.go | 51 +++++++++++++++++++ modules/notification/base/notifier.go | 3 ++ modules/notification/base/null.go | 8 +++ modules/notification/notification.go | 14 +++++ modules/notification/webhook/webhook.go | 30 ++++++++++- modules/structs/hook.go | 25 +++++++++ modules/structs/package.go | 38 ++++++++++++++ options/locale/locale_en-US.ini | 2 + routers/api/v1/packages/generic/generic.go | 2 +- routers/api/v1/packages/nuget/nuget.go | 2 +- routers/web/repo/packages.go | 2 +- routers/web/repo/webhook.go | 1 + services/forms/repo_form.go | 1 + services/packages/packages.go | 16 ++++-- templates/repo/settings/webhook/settings.tmpl | 10 ++++ 18 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 modules/convert/package.go create mode 100644 modules/structs/package.go diff --git a/models/package.go b/models/package.go index d14216b3e4def..b8c526df6eb86 100644 --- a/models/package.go +++ b/models/package.go @@ -69,7 +69,7 @@ type Package struct { UpdatedUnix timeutil.TimeStamp `xorm:"updated"` } -// LoadCreator loads poster +// LoadCreator loads the creator func (p *Package) LoadCreator() error { if p.Creator == nil { var err error @@ -272,7 +272,7 @@ func GetPackageByNameAndVersion(repositoryID int64, packageType PackageType, pac func SearchPackages(repositoryID int64, packageType PackageType, query string, skip, take int) (int64, []*Package, error) { var cond builder.Cond = builder.Eq{ "repo_id": repositoryID, - "type": packageType, + "type": packageType, } if query != "" { cond = cond.And(builder.Like{"lower_name", strings.ToLower(query)}) diff --git a/models/webhook.go b/models/webhook.go index 138ba26bde090..5dc68751dd5f7 100644 --- a/models/webhook.go +++ b/models/webhook.go @@ -97,6 +97,7 @@ type HookEvents struct { PullRequestSync bool `json:"pull_request_sync"` Repository bool `json:"repository"` Release bool `json:"release"` + Package bool `json:"package"` } // HookEvent represents events that will delivery hook. @@ -297,6 +298,12 @@ func (w *Webhook) HasRepositoryEvent() bool { (w.ChooseEvents && w.HookEvents.Repository) } +// HasPackageEvent returns if hook enabled package event. +func (w *Webhook) HasPackageEvent() bool { + return w.SendEverything || + (w.ChooseEvents && w.HookEvents.Package) +} + // EventCheckers returns event checkers func (w *Webhook) EventCheckers() []struct { Has func() bool @@ -326,6 +333,7 @@ func (w *Webhook) EventCheckers() []struct { {w.HasPullRequestSyncEvent, HookEventPullRequestSync}, {w.HasRepositoryEvent, HookEventRepository}, {w.HasReleaseEvent, HookEventRelease}, + {w.HasPackageEvent, HookEventPackage}, } } @@ -595,6 +603,7 @@ const ( HookEventPullRequestSync HookEventType = "pull_request_sync" HookEventRepository HookEventType = "repository" HookEventRelease HookEventType = "release" + HookEventPackage HookEventType = "package" ) // Event returns the HookEventType as an event string @@ -625,6 +634,8 @@ func (h HookEventType) Event() string { return "repository" case HookEventRelease: return "release" + case HookEventPackage: + return "package" } return "" } diff --git a/models/webhook_test.go b/models/webhook_test.go index 84520666c0b22..e4a26c12f2d7f 100644 --- a/models/webhook_test.go +++ b/models/webhook_test.go @@ -69,6 +69,7 @@ func TestWebhook_EventsArray(t *testing.T) { "pull_request", "pull_request_assign", "pull_request_label", "pull_request_milestone", "pull_request_comment", "pull_request_review_approved", "pull_request_review_rejected", "pull_request_review_comment", "pull_request_sync", "repository", "release", + "package", }, (&Webhook{ HookEvent: &HookEvent{SendEverything: true}, diff --git a/modules/convert/package.go b/modules/convert/package.go new file mode 100644 index 0000000000000..3e4f8a52fe875 --- /dev/null +++ b/modules/convert/package.go @@ -0,0 +1,51 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package convert + +import ( + "code.gitea.io/gitea/models" + api "code.gitea.io/gitea/modules/structs" +) + +// ToPackage convert a models.Package to api.Package +func ToPackage(p *models.Package) *api.Package { + if err := p.LoadCreator(); err != nil { + return &api.Package{} + } + files, err := p.GetFiles() + if err != nil { + return &api.Package{} + } + + apiFiles := make([]*api.PackageFile, 0, len(files)) + for _, file := range files { + apiFiles = append(apiFiles, ToPackageFile(file)) + } + return &api.Package{ + ID: p.ID, + Creator: ToUser(p.Creator, nil), + Type: p.Type.String(), + Name: p.Name, + Version: p.Version, + Files: apiFiles, + CreatedAt: p.CreatedUnix.AsTime(), + UpdatedAt: p.CreatedUnix.AsTime(), + } +} + +// ToPackageFile converts models.PackageFile to api.PackageFile +func ToPackageFile(pf *models.PackageFile) *api.PackageFile { + return &api.PackageFile{ + ID: pf.ID, + Size: pf.Size, + Name: pf.Name, + HashMD5: pf.HashMD5, + HashSHA1: pf.HashSHA1, + HashSHA256: pf.HashSHA256, + HashSHA512: pf.HashSHA512, + CreatedAt: pf.CreatedUnix.AsTime(), + UpdatedAt: pf.UpdatedUnix.AsTime(), + } +} diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index 8f8aa659b45d9..6654af7443cbe 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -59,4 +59,7 @@ type Notifier interface { NotifySyncDeleteRef(doer *models.User, repo *models.Repository, refType, refFullName string) NotifyRepoPendingTransfer(doer, newOwner *models.User, repo *models.Repository) + + NotifyPackageCreate(repo *models.Repository, p *models.Package) + NotifyPackageDelete(doer *models.User, repo *models.Repository, p *models.Package) } diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go index 32fe259bca8d7..311dcc2e4a40a 100644 --- a/modules/notification/base/null.go +++ b/modules/notification/base/null.go @@ -170,3 +170,11 @@ func (*NullNotifier) NotifySyncDeleteRef(doer *models.User, repo *models.Reposit // NotifyRepoPendingTransfer places a place holder function func (*NullNotifier) NotifyRepoPendingTransfer(doer, newOwner *models.User, repo *models.Repository) { } + +// NotifyPackageCreate places a place holder function +func (*NullNotifier) NotifyPackageCreate(repo *models.Repository, p *models.Package) { +} + +// NotifyPackageDelete places a place holder function +func (*NullNotifier) NotifyPackageDelete(doer *models.User, repo *models.Repository, p *models.Package) { +} diff --git a/modules/notification/notification.go b/modules/notification/notification.go index b574f3ccda225..a06e332a12c6d 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -297,3 +297,17 @@ func NotifyRepoPendingTransfer(doer, newOwner *models.User, repo *models.Reposit notifier.NotifyRepoPendingTransfer(doer, newOwner, repo) } } + +// NotifyPackageCreate notifies creation of a package to notifiers +func NotifyPackageCreate(repo *models.Repository, p *models.Package) { + for _, notifier := range notifiers { + notifier.NotifyPackageCreate(repo, p) + } +} + +// NotifyPackageDelete notifies deletion of a package to notifiers +func NotifyPackageDelete(doer *models.User, repo *models.Repository, p *models.Package) { + for _, notifier := range notifiers { + notifier.NotifyPackageDelete(doer, repo, p) + } +} diff --git a/modules/notification/webhook/webhook.go b/modules/notification/webhook/webhook.go index acdb91efe373e..1c784ec1b1134 100644 --- a/modules/notification/webhook/webhook.go +++ b/modules/notification/webhook/webhook.go @@ -470,7 +470,6 @@ func (m *webhookNotifier) NotifyDeleteComment(doer *models.User, comment *models if err != nil { log.Error("PrepareWebhooks [comment_id: %d]: %v", comment.ID, err) } - } func (m *webhookNotifier) NotifyIssueChangeLabels(doer *models.User, issue *models.Issue, @@ -819,3 +818,32 @@ func (m *webhookNotifier) NotifySyncCreateRef(pusher *models.User, repo *models. func (m *webhookNotifier) NotifySyncDeleteRef(pusher *models.User, repo *models.Repository, refType, refFullName string) { m.NotifyDeleteRef(pusher, repo, refType, refFullName) } + +func (m *webhookNotifier) NotifyPackageCreate(repo *models.Repository, p *models.Package) { + if err := p.LoadCreator(); err != nil { + log.Error("LoadCreator: %v", err) + } + + notifyPackage(p.Creator, repo, p, api.HookPackageCreated) +} + +func (m *webhookNotifier) NotifyPackageDelete(doer *models.User, repo *models.Repository, p *models.Package) { + notifyPackage(doer, repo, p, api.HookPackageDeleted) +} + +func notifyPackage(sender *models.User, repo *models.Repository, p *models.Package, action api.HookPackageAction) { + org := repo.MustOwner() + if !org.IsOrganization() { + org = nil + } + + if err := webhook_services.PrepareWebhooks(repo, models.HookEventPackage, &api.PackagePayload{ + Action: action, + Repository: convert.ToRepo(repo, models.AccessModeNone), + Package: convert.ToPackage(p), + Organization: convert.ToUser(org, nil), + Sender: convert.ToUser(sender, nil), + }); err != nil { + log.Error("PrepareWebhooks: %v", err) + } +} diff --git a/modules/structs/hook.go b/modules/structs/hook.go index 163fb5c94d619..89df03ce81130 100644 --- a/modules/structs/hook.go +++ b/modules/structs/hook.go @@ -112,6 +112,7 @@ var ( _ Payloader = &PullRequestPayload{} _ Payloader = &RepositoryPayload{} _ Payloader = &ReleasePayload{} + _ Payloader = &PackagePayload{} ) // _________ __ @@ -427,3 +428,27 @@ type RepositoryPayload struct { func (p *RepositoryPayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } + +// HookPackageAction an action that happens to a package +type HookPackageAction string + +const ( + // HookPackageCreated created + HookPackageCreated HookPackageAction = "created" + // HookPackageDeleted deleted + HookPackageDeleted HookPackageAction = "deleted" +) + +// PackagePayload represents a package payload +type PackagePayload struct { + Action HookPackageAction `json:"action"` + Repository *Repository `json:"repository"` + Package *Package `json:"package"` + Organization *User `json:"organization"` + Sender *User `json:"sender"` +} + +// JSONPayload implements Payload +func (p *PackagePayload) JSONPayload() ([]byte, error) { + return json.MarshalIndent(p, "", " ") +} diff --git a/modules/structs/package.go b/modules/structs/package.go new file mode 100644 index 0000000000000..3dab2c710ef6e --- /dev/null +++ b/modules/structs/package.go @@ -0,0 +1,38 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package structs + +import ( + "time" +) + +// Package represents a package +type Package struct { + ID int64 `json:"id"` + Creator *User `json:"creator"` + Type string `json:"type"` + Name string `json:"name"` + Version string `json:"version"` + Files []*PackageFile `json:"files"` + // swagger:strfmt date-time + CreatedAt time.Time `json:"created_at"` + // swagger:strfmt date-time + UpdatedAt time.Time `json:"updated_at"` +} + +// PackageFile represents a package file +type PackageFile struct { + ID int64 `json:"id"` + Size int64 + Name string `json:"name"` + HashMD5 string `json:"md5"` + HashSHA1 string `json:"sha1"` + HashSHA256 string `json:"sha256"` + HashSHA512 string `json:"sha512"` + // swagger:strfmt date-time + CreatedAt time.Time `json:"created_at"` + // swagger:strfmt date-time + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e994e34f82b9b..2a6bcc3e50369 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1864,6 +1864,8 @@ settings.event_pull_request_review = Pull Request Reviewed settings.event_pull_request_review_desc = Pull request approved, rejected, or review comment. settings.event_pull_request_sync = Pull Request Synchronized settings.event_pull_request_sync_desc = Pull request synchronized. +settings.event_package = Package +settings.event_package_desc = Package created or deleted in a repository. settings.branch_filter = Branch filter settings.branch_filter_desc = Branch whitelist for push, branch creation and branch deletion events, specified as glob pattern. If empty or *, events for all branches are reported. See github.com/gobwas/glob documentation for syntax. Examples: master, {master,release*}. settings.active = Active diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 70763916a2fcc..5d62acf2c6610 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -110,7 +110,7 @@ func DeletePackage(ctx *context.APIContext) { return } - err = package_service.DeletePackageByNameAndVersion(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) + err = package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) if err != nil { if err == models.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 29902a67fb1b2..9bccbd022b52e 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -224,7 +224,7 @@ func DeletePackage(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") - err := package_service.DeletePackageByNameAndVersion(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) + err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) if err != nil { if err == models.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index 4604c256031f1..4b4ba7f022edb 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -146,7 +146,7 @@ func ViewPackage(ctx *context.Context) { // DeletePackagePost deletes a package func DeletePackagePost(ctx *context.Context) { - err := package_service.DeletePackageByID(ctx.Repo.Repository, ctx.ParamsInt64(":id")) + err := package_service.DeletePackageByID(ctx.User, ctx.Repo.Repository, ctx.ParamsInt64(":id")) if err != nil { ctx.Flash.Error(err.Error()) } else { diff --git a/routers/web/repo/webhook.go b/routers/web/repo/webhook.go index 79f47470f25a5..3c29725cde83e 100644 --- a/routers/web/repo/webhook.go +++ b/routers/web/repo/webhook.go @@ -177,6 +177,7 @@ func ParseHookEvent(form forms.WebhookForm) *models.HookEvent { PullRequestReview: form.PullRequestReview, PullRequestSync: form.PullRequestSync, Repository: form.Repository, + Package: form.Package, }, BranchFilter: form.BranchFilter, } diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index ee454b3622e07..0f61f4230160b 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -236,6 +236,7 @@ type WebhookForm struct { PullRequestReview bool PullRequestSync bool Repository bool + Package bool Active bool BranchFilter string `binding:"GlobPattern"` } diff --git a/services/packages/packages.go b/services/packages/packages.go index 8cfdbc1ad663e..032e8bf70afbf 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/notification" packages_module "code.gitea.io/gitea/modules/packages" ) @@ -48,6 +49,9 @@ func CreatePackage(creator *models.User, repository *models.Repository, packageT log.Error("Error inserting package: %v", err) return nil, err } + + notification.NotifyPackageCreate(repository, p) + return p, nil } @@ -108,7 +112,7 @@ func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reade } // DeletePackageByNameAndVersion deletes a package and all associated files -func DeletePackageByNameAndVersion(repository *models.Repository, packageType models.PackageType, name, version string) error { +func DeletePackageByNameAndVersion(doer *models.User, repository *models.Repository, packageType models.PackageType, name, version string) error { log.Trace("Deleting package: %v, %v, %s, %s", repository.ID, packageType, name, version) p, err := models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) @@ -120,11 +124,11 @@ func DeletePackageByNameAndVersion(repository *models.Repository, packageType mo return err } - return deletePackage(p) + return deletePackage(doer, repository, p) } // DeletePackageByID deletes a package and all associated files -func DeletePackageByID(repository *models.Repository, packageID int64) error { +func DeletePackageByID(doer *models.User, repository *models.Repository, packageID int64) error { log.Trace("Deleting package: %v, %v", repository.ID, packageID) p, err := models.GetPackageByID(packageID) @@ -140,10 +144,10 @@ func DeletePackageByID(repository *models.Repository, packageID int64) error { return models.ErrPackageNotExist } - return deletePackage(p) + return deletePackage(doer, repository, p) } -func deletePackage(p *models.Package) error { +func deletePackage(doer *models.User, repository *models.Repository, p *models.Package) error { pfs, err := p.GetFiles() if err != nil { log.Error("Error getting package files: %v", err) @@ -163,6 +167,8 @@ func deletePackage(p *models.Package) error { return err } + notification.NotifyPackageDelete(doer, repository, p) + return nil } diff --git a/templates/repo/settings/webhook/settings.tmpl b/templates/repo/settings/webhook/settings.tmpl index 934794b539b07..8220d3f8e4b19 100644 --- a/templates/repo/settings/webhook/settings.tmpl +++ b/templates/repo/settings/webhook/settings.tmpl @@ -87,6 +87,16 @@
    + +
    +
    +
    + + + {{.i18n.Tr "repo.settings.event_package_desc"}} +
    +
    +
    From 762590029e49239fb055fd2fcadf54bbe30bacef Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 11 Aug 2021 15:33:05 +0000 Subject: [PATCH 027/130] Add package api. --- models/package.go | 60 ++--- modules/convert/package.go | 9 - modules/structs/package.go | 11 +- routers/api/v1/api.go | 6 + routers/api/v1/packages/nuget/nuget.go | 18 +- routers/api/v1/repo/package.go | 227 +++++++++++++++++++ routers/api/v1/swagger/repo.go | 21 ++ routers/web/repo/packages.go | 5 +- templates/swagger/v1_json.tmpl | 300 +++++++++++++++++++++++++ 9 files changed, 601 insertions(+), 56 deletions(-) create mode 100644 routers/api/v1/repo/package.go diff --git a/models/package.go b/models/package.go index b8c526df6eb86..8b7f3ca7b92ce 100644 --- a/models/package.go +++ b/models/package.go @@ -8,7 +8,6 @@ import ( "errors" "strings" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" @@ -181,15 +180,13 @@ func DeletePackagesByRepositoryID(repositoryID int64) error { // PackageSearchOptions are options for GetLatestPackagesGrouped type PackageSearchOptions struct { RepoID int64 - Page int - Query string Type string + Query string + ListOptions } -// GetLatestPackagesGrouped returns a list of all packages in their latest version of the repository -func GetLatestPackagesGrouped(opts PackageSearchOptions) ([]*Package, int64, error) { +func (opts PackageSearchOptions) toConds() builder.Cond { var cond builder.Cond = builder.Eq{"package.repo_id": opts.RepoID} - cond = cond.And(builder.Expr("p2.id IS NULL")) switch opts.Type { case "generic": @@ -208,15 +205,32 @@ func GetLatestPackagesGrouped(opts PackageSearchOptions) ([]*Package, int64, err cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.Query)}) } + return cond +} + +// GetPackages returns a list of all packages of the repository +func GetPackages(opts PackageSearchOptions) ([]*Package, int64, error) { + sess := x.Where(opts.toConds()) + + sess = opts.setSessionPagination(sess) + + packages := make([]*Package, 0, opts.PageSize) + count, err := sess.FindAndCount(&packages) + return packages, count, err +} + +// GetLatestPackagesGrouped returns a list of all packages in their latest version of the repository +func GetLatestPackagesGrouped(opts PackageSearchOptions) ([]*Package, int64, error) { + cond := opts.toConds(). + And(builder.Expr("p2.id IS NULL")) + sess := x.Where(cond). Table("package"). Join("left", "package p2", "package.repo_id = p2.repo_id AND package.type = p2.type AND package.lower_name = p2.lower_name AND package.version < p2.version") - if opts.Page > 0 { - sess = sess.Limit(setting.UI.PackagesPagingNum, (opts.Page-1)*setting.UI.PackagesPagingNum) - } + sess = opts.setSessionPagination(sess) - packages := make([]*Package, 0, setting.UI.PackagesPagingNum) + packages := make([]*Package, 0, opts.PageSize) count, err := sess.FindAndCount(&packages) return packages, count, err } @@ -268,32 +282,6 @@ func GetPackageByNameAndVersion(repositoryID int64, packageType PackageType, pac return p, nil } -// SearchPackages searches for packages by name and can be used to navigate through the package list -func SearchPackages(repositoryID int64, packageType PackageType, query string, skip, take int) (int64, []*Package, error) { - var cond builder.Cond = builder.Eq{ - "repo_id": repositoryID, - "type": packageType, - } - if query != "" { - cond = cond.And(builder.Like{"lower_name", strings.ToLower(query)}) - } - - if take <= 0 || take > 100 { - take = 100 - } - - sess := x.Where(cond) - if skip > 0 { - sess = sess.Limit(take, skip) - } else { - sess = sess.Limit(take) - } - - packages := make([]*Package, 0, take) - count, err := sess.FindAndCount(&packages) - return count, packages, err -} - // TryInsertPackageFile inserts a package file func TryInsertPackageFile(pf *PackageFile) (*PackageFile, error) { sess := x.NewSession() diff --git a/modules/convert/package.go b/modules/convert/package.go index 3e4f8a52fe875..6d0b82ce23d13 100644 --- a/modules/convert/package.go +++ b/modules/convert/package.go @@ -14,22 +14,13 @@ func ToPackage(p *models.Package) *api.Package { if err := p.LoadCreator(); err != nil { return &api.Package{} } - files, err := p.GetFiles() - if err != nil { - return &api.Package{} - } - apiFiles := make([]*api.PackageFile, 0, len(files)) - for _, file := range files { - apiFiles = append(apiFiles, ToPackageFile(file)) - } return &api.Package{ ID: p.ID, Creator: ToUser(p.Creator, nil), Type: p.Type.String(), Name: p.Name, Version: p.Version, - Files: apiFiles, CreatedAt: p.CreatedUnix.AsTime(), UpdatedAt: p.CreatedUnix.AsTime(), } diff --git a/modules/structs/package.go b/modules/structs/package.go index 3dab2c710ef6e..0ed7232c3f750 100644 --- a/modules/structs/package.go +++ b/modules/structs/package.go @@ -10,12 +10,11 @@ import ( // Package represents a package type Package struct { - ID int64 `json:"id"` - Creator *User `json:"creator"` - Type string `json:"type"` - Name string `json:"name"` - Version string `json:"version"` - Files []*PackageFile `json:"files"` + ID int64 `json:"id"` + Creator *User `json:"creator"` + Type string `json:"type"` + Name string `json:"name"` + Version string `json:"version"` // swagger:strfmt date-time CreatedAt time.Time `json:"created_at"` // swagger:strfmt date-time diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 7c8380dc46741..13db69e377ab8 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1020,6 +1020,12 @@ func Routes() *web.Route { m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) m.Get("/simple/{id}", pypi.PackageMetadata) }, reqBasicAuth()) + m.Group("/{id}", func() { + m.Get("/", repo.GetPackage) + m.Delete("/", repo.DeletePackage) + m.Get("/files", repo.ListPackageFiles) + }, reqAnyRepoReader()) + m.Get("/", reqAnyRepoReader(), repo.ListPackages) }) }, repoAssignment()) }) diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 9bccbd022b52e..5f7788c9470d0 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -31,11 +31,21 @@ func ServiceIndex(ctx *context.APIContext) { // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages func SearchService(ctx *context.APIContext) { - query := ctx.FormTrim("q") skip := ctx.FormInt("skip") take := ctx.FormInt("take") - - total, packages, err := models.SearchPackages(ctx.Repo.Repository.ID, models.PackageNuGet, query, skip, take) + if take <= 0 { + take = setting.API.DefaultPagingNum + } + + packages, count, err := models.GetPackages(models.PackageSearchOptions{ + RepoID: ctx.Repo.Repository.ID, + Type: "nuget", + Query: ctx.FormTrim("q"), + ListOptions: models.ListOptions{ + Page: skip / take, + PageSize: take, + }, + }) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return @@ -49,7 +59,7 @@ func SearchService(ctx *context.APIContext) { resp := createSearchResultResponse( &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, - total, + count, nugetPackages, ) diff --git a/routers/api/v1/repo/package.go b/routers/api/v1/repo/package.go new file mode 100644 index 0000000000000..167872513c2fa --- /dev/null +++ b/routers/api/v1/repo/package.go @@ -0,0 +1,227 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repo + +import ( + "fmt" + "net/http" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/convert" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/routers/api/v1/utils" + package_service "code.gitea.io/gitea/services/packages" +) + +// ListPackages gets all packages of a repository +func ListPackages(ctx *context.APIContext) { + // swagger:operation GET /repos/{owner}/{repo}/packages repository repoListPackages + // --- + // summary: Gets all packages of a repository + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: page + // in: query + // description: page number of results to return (1-based) + // type: integer + // - name: limit + // in: query + // description: page size of results + // type: integer + // - name: package_type + // in: query + // description: page size of results + // schema: + // type: string + // enum: [generic, nuget, npm, maven, pypi] + // - name: q + // in: query + // description: name filter + // type: string + // responses: + // "200": + // "$ref": "#/responses/PackageList" + + listOptions := utils.GetListOptions(ctx) + + packageType := ctx.FormTrim("package_type") + query := ctx.FormTrim("q") + + repo := ctx.Repo.Repository + + packages, count, err := models.GetPackages(models.PackageSearchOptions{ + RepoID: repo.ID, + Type: packageType, + Query: query, + ListOptions: listOptions, + }) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetPackages", err) + return + } + + apiPackages := make([]*api.Package, 0, len(packages)) + for _, p := range packages { + if err := p.LoadCreator(); err != nil { + ctx.Error(http.StatusInternalServerError, "LoadCreator", err) + return + } + apiPackages = append(apiPackages, convert.ToPackage(p)) + } + + ctx.SetLinkHeader(int(count), listOptions.PageSize) + ctx.Header().Set("X-Total-Count", fmt.Sprint(count)) + ctx.Header().Set("Access-Control-Expose-Headers", "X-Total-Count, Link") + ctx.JSON(http.StatusOK, apiPackages) +} + +// GetPackage gets a package +func GetPackage(ctx *context.APIContext) { + // swagger:operation GET /repos/{owner}/{repo}/packages/{id} repository repoGetPackage + // --- + // summary: Gets a package + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: id + // in: path + // description: id of the package + // type: integer + // format: int64 + // required: true + // responses: + // "200": + // "$ref": "#/responses/Package" + // "404": + // "$ref": "#/responses/notFound" + + p, err := models.GetPackageByID(ctx.ParamsInt64(":id")) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.NotFound() + } else { + ctx.Error(http.StatusInternalServerError, "GetPackageByID", err) + } + return + } + + ctx.JSON(http.StatusOK, convert.ToPackage(p)) +} + +// DeletePackage delete a package from a repository +func DeletePackage(ctx *context.APIContext) { + // swagger:operation DELETE /repos/{owner}/{repo}/packages/{id} repository repoDeletePackage + // --- + // summary: Delete a package from a repository + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: id + // in: path + // description: id of the package to delete + // type: integer + // format: int64 + // required: true + // responses: + // "204": + // "$ref": "#/responses/empty" + // "404": + // "$ref": "#/responses/notFound" + + err := package_service.DeletePackageByID(ctx.User, ctx.Repo.Repository, ctx.ParamsInt64(":id")) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.NotFound() + } else { + ctx.Error(http.StatusInternalServerError, "DeletePackageByID", err) + } + return + } + ctx.Status(http.StatusNoContent) +} + +// ListPackageFiles gets all files of a package +func ListPackageFiles(ctx *context.APIContext) { + // swagger:operation GET /repos/{owner}/{repo}/packages/{id}/files repository repoListPackageFiles + // --- + // summary: Gets all files of a package + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: id + // in: path + // description: id of the package to delete + // type: integer + // format: int64 + // required: true + // responses: + // "200": + // "$ref": "#/responses/PackageFileList" + // "404": + // "$ref": "#/responses/notFound" + + p, err := models.GetPackageByID(ctx.ParamsInt64(":id")) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.NotFound() + } else { + ctx.Error(http.StatusInternalServerError, "GetPackageByID", err) + } + return + } + + files, err := p.GetFiles() + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetFiles", err) + return + } + + apiPackageFiles := make([]*api.PackageFile, 0, len(files)) + for _, pf := range files { + apiPackageFiles = append(apiPackageFiles, convert.ToPackageFile(pf)) + } + + ctx.JSON(http.StatusOK, apiPackageFiles) +} diff --git a/routers/api/v1/swagger/repo.go b/routers/api/v1/swagger/repo.go index d539bcb9feabb..e9d9b7f946d27 100644 --- a/routers/api/v1/swagger/repo.go +++ b/routers/api/v1/swagger/repo.go @@ -316,3 +316,24 @@ type swaggerCombinedStatus struct { // in: body Body api.CombinedStatus `json:"body"` } + +// Package +// swagger:response Package +type swaggerResponsePackage struct { + // in:body + Body api.Package `json:"body"` +} + +// PackageList +// swagger:response PackageList +type swaggerResponsePackageList struct { + // in:body + Body []api.Package `json:"body"` +} + +// PackageFileList +// swagger:response PackageFileList +type swaggerResponsePackageFileList struct { + // in:body + Body []api.PackageFile `json:"body"` +} diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index 4b4ba7f022edb..b3f6ae000b436 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -50,8 +50,11 @@ func Packages(ctx *context.Context) { packages, count, err := models.GetLatestPackagesGrouped(models.PackageSearchOptions{ RepoID: repo.ID, Query: query, - Page: page, Type: packageType, + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.PackagesPagingNum, + }, }) if err != nil { ctx.ServerError("GetLatestPackagesGrouped", err) diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index a1d92abec7a86..7d9b6ba38dc6f 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -7005,6 +7005,199 @@ } } }, + "/repos/{owner}/{repo}/packages": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "Gets all packages of a repository", + "operationId": "repoListPackages", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "page number of results to return (1-based)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "page size of results", + "name": "limit", + "in": "query" + }, + { + "description": "page size of results", + "name": "package_type", + "in": "query", + "schema": { + "type": "string", + "enum": [ + "generic", + "nuget", + "npm", + "maven", + "pypi" + ] + } + }, + { + "type": "string", + "description": "name filter", + "name": "q", + "in": "query" + } + ], + "responses": { + "200": { + "$ref": "#/responses/PackageList" + } + } + } + }, + "/repos/{owner}/{repo}/packages/{id}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "Gets a package", + "operationId": "repoGetPackage", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "integer", + "format": "int64", + "description": "id of the package", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/Package" + }, + "404": { + "$ref": "#/responses/notFound" + } + } + }, + "delete": { + "tags": [ + "repository" + ], + "summary": "Delete a package from a repository", + "operationId": "repoDeletePackage", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "integer", + "format": "int64", + "description": "id of the package to delete", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "$ref": "#/responses/empty" + }, + "404": { + "$ref": "#/responses/notFound" + } + } + } + }, + "/repos/{owner}/{repo}/packages/{id}/files": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "Gets all package files of a package", + "operationId": "repoListPackageFiles", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "integer", + "format": "int64", + "description": "id of the package to delete", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/PackageFileList" + }, + "404": { + "$ref": "#/responses/notFound" + } + } + } + }, "/repos/{owner}/{repo}/pulls": { "get": { "produces": [ @@ -15637,6 +15830,89 @@ }, "x-go-package": "code.gitea.io/gitea/modules/structs" }, + "Package": { + "description": "Package represents a package", + "type": "object", + "properties": { + "created_at": { + "type": "string", + "format": "date-time", + "x-go-name": "CreatedAt" + }, + "creator": { + "$ref": "#/definitions/User" + }, + "id": { + "type": "integer", + "format": "int64", + "x-go-name": "ID" + }, + "name": { + "type": "string", + "x-go-name": "Name" + }, + "type": { + "type": "string", + "x-go-name": "Type" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "x-go-name": "UpdatedAt" + }, + "version": { + "type": "string", + "x-go-name": "Version" + } + }, + "x-go-package": "code.gitea.io/gitea/modules/structs" + }, + "PackageFile": { + "description": "PackageFile represents a package file", + "type": "object", + "properties": { + "Size": { + "type": "integer", + "format": "int64" + }, + "created_at": { + "type": "string", + "format": "date-time", + "x-go-name": "CreatedAt" + }, + "id": { + "type": "integer", + "format": "int64", + "x-go-name": "ID" + }, + "md5": { + "type": "string", + "x-go-name": "HashMD5" + }, + "name": { + "type": "string", + "x-go-name": "Name" + }, + "sha1": { + "type": "string", + "x-go-name": "HashSHA1" + }, + "sha256": { + "type": "string", + "x-go-name": "HashSHA256" + }, + "sha512": { + "type": "string", + "x-go-name": "HashSHA512" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "x-go-name": "UpdatedAt" + } + }, + "x-go-package": "code.gitea.io/gitea/modules/structs" + }, "PayloadCommit": { "description": "PayloadCommit represents a commit", "type": "object", @@ -17463,6 +17739,30 @@ } } }, + "Package": { + "description": "Package", + "schema": { + "$ref": "#/definitions/Package" + } + }, + "PackageFileList": { + "description": "PackageFileList", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/PackageFile" + } + } + }, + "PackageList": { + "description": "PackageList", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Package" + } + } + }, "PublicKey": { "description": "PublicKey", "schema": { From 53a47deef4cd7a28d41ab08f3376b85448938e11 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 11 Aug 2021 15:57:50 +0000 Subject: [PATCH 028/130] Fixed swagger file. --- templates/swagger/v1_json.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 15513c19f211c..f7000759dbe5c 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -7208,7 +7208,7 @@ "tags": [ "repository" ], - "summary": "Gets all package files of a package", + "summary": "Gets all files of a package", "operationId": "repoListPackageFiles", "parameters": [ { From e6075abf73be137487cfde2a861f9e042e4a8dfb Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 11 Aug 2021 16:15:06 +0000 Subject: [PATCH 029/130] Fixed enum and comments. --- routers/api/v1/repo/package.go | 11 +++++------ templates/swagger/v1_json.tmpl | 26 ++++++++++++-------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/routers/api/v1/repo/package.go b/routers/api/v1/repo/package.go index 167872513c2fa..0a7cdaafd1fed 100644 --- a/routers/api/v1/repo/package.go +++ b/routers/api/v1/repo/package.go @@ -44,10 +44,9 @@ func ListPackages(ctx *context.APIContext) { // type: integer // - name: package_type // in: query - // description: page size of results - // schema: - // type: string - // enum: [generic, nuget, npm, maven, pypi] + // description: package type filter + // type: string + // enum: [generic, nuget, npm, maven, pypi] // - name: q // in: query // description: name filter @@ -150,7 +149,7 @@ func DeletePackage(ctx *context.APIContext) { // required: true // - name: id // in: path - // description: id of the package to delete + // description: id of the package // type: integer // format: int64 // required: true @@ -192,7 +191,7 @@ func ListPackageFiles(ctx *context.APIContext) { // required: true // - name: id // in: path - // description: id of the package to delete + // description: id of the package // type: integer // format: int64 // required: true diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index f7000759dbe5c..301388db523c4 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -7089,19 +7089,17 @@ "in": "query" }, { - "description": "page size of results", + "enum": [ + "generic", + "nuget", + "npm", + "maven", + "pypi" + ], + "type": "string", + "description": "package type filter", "name": "package_type", - "in": "query", - "schema": { - "type": "string", - "enum": [ - "generic", - "nuget", - "npm", - "maven", - "pypi" - ] - } + "in": "query" }, { "type": "string", @@ -7184,7 +7182,7 @@ { "type": "integer", "format": "int64", - "description": "id of the package to delete", + "description": "id of the package", "name": "id", "in": "path", "required": true @@ -7228,7 +7226,7 @@ { "type": "integer", "format": "int64", - "description": "id of the package to delete", + "description": "id of the package", "name": "id", "in": "path", "required": true From 068b1b1cf88967a316abdc4f5dd4b38ce525d712 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 11 Aug 2021 19:13:00 +0200 Subject: [PATCH 030/130] Fixed NuGet pagination. --- models/commit_status.go | 6 ++-- models/gpg_key.go | 2 +- models/issue_comment.go | 2 +- models/issue_label.go | 4 +-- models/issue_milestone.go | 2 +- models/issue_stopwatch.go | 2 +- models/issue_watch.go | 2 +- models/list_options.go | 7 ++--- models/notification.go | 2 +- models/oauth2_application.go | 2 +- models/org.go | 6 ++-- models/org_team.go | 6 ++-- models/package.go | 22 ++++++------- models/pull_list.go | 2 +- models/release.go | 2 +- models/repo.go | 4 +-- models/repo_watch.go | 2 +- models/review.go | 2 +- models/session_paginator.go | 43 ++++++++++++++++++++++++++ models/ssh_key.go | 2 +- models/ssh_key_deploy.go | 2 +- models/ssh_key_principals.go | 2 +- models/star.go | 2 +- models/token.go | 2 +- models/topic.go | 2 +- models/user.go | 10 +++--- models/webhook.go | 4 +-- routers/api/v1/packages/nuget/nuget.go | 14 +++------ routers/api/v1/repo/package.go | 10 +++--- routers/web/repo/packages.go | 4 +-- 30 files changed, 104 insertions(+), 70 deletions(-) create mode 100644 models/session_paginator.go diff --git a/models/commit_status.go b/models/commit_status.go index c27582024280f..79a7686ee90c2 100644 --- a/models/commit_status.go +++ b/models/commit_status.go @@ -97,7 +97,7 @@ func GetCommitStatuses(repo *Repository, sha string, opts *CommitStatusOptions) } countSession := listCommitStatusesStatement(repo, sha, opts) - countSession = opts.setSessionPagination(countSession) + countSession = opts.SetSessionPagination(countSession) maxResults, err := countSession.Count(new(CommitStatus)) if err != nil { log.Error("Count PRs: %v", err) @@ -106,7 +106,7 @@ func GetCommitStatuses(repo *Repository, sha string, opts *CommitStatusOptions) statuses := make([]*CommitStatus, 0, opts.PageSize) findSession := listCommitStatusesStatement(repo, sha, opts) - findSession = opts.setSessionPagination(findSession) + findSession = opts.SetSessionPagination(findSession) sortCommitStatusesSession(findSession, opts.SortType) return statuses, maxResults, findSession.Find(&statuses) } @@ -149,7 +149,7 @@ func getLatestCommitStatus(e Engine, repoID int64, sha string, listOptions ListO Select("max( id ) as id"). GroupBy("context_hash").OrderBy("max( id ) desc") - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) err := sess.Find(&ids) if err != nil { diff --git a/models/gpg_key.go b/models/gpg_key.go index 74ffb82a545b5..41bc784b908e2 100644 --- a/models/gpg_key.go +++ b/models/gpg_key.go @@ -64,7 +64,7 @@ func ListGPGKeys(uid int64, listOptions ListOptions) ([]*GPGKey, error) { func listGPGKeys(e Engine, uid int64, listOptions ListOptions) ([]*GPGKey, error) { sess := e.Table(&GPGKey{}).Where("owner_id=? AND primary_key_id=''", uid) if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) } keys := make([]*GPGKey, 0, 2) diff --git a/models/issue_comment.go b/models/issue_comment.go index 755041efd7ed1..8788c304ee488 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -1007,7 +1007,7 @@ func findComments(e Engine, opts FindCommentsOptions) ([]*Comment, error) { } if opts.Page != 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) } // WARNING: If you change this order you will need to fix createCodeComment diff --git a/models/issue_label.go b/models/issue_label.go index d1ff4692366d0..60cbec90077f9 100644 --- a/models/issue_label.go +++ b/models/issue_label.go @@ -433,7 +433,7 @@ func getLabelsByRepoID(e Engine, repoID int64, sortType string, listOptions List } if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) } return labels, sess.Find(&labels) @@ -545,7 +545,7 @@ func getLabelsByOrgID(e Engine, orgID int64, sortType string, listOptions ListOp } if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) } return labels, sess.Find(&labels) diff --git a/models/issue_milestone.go b/models/issue_milestone.go index 5e934cde0a02f..cd6ca41531db7 100644 --- a/models/issue_milestone.go +++ b/models/issue_milestone.go @@ -399,7 +399,7 @@ func GetMilestones(opts GetMilestonesOption) (MilestoneList, error) { } if opts.Page != 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) } switch opts.SortType { diff --git a/models/issue_stopwatch.go b/models/issue_stopwatch.go index 8cdad94fd4ae1..e1323ffc6cc07 100644 --- a/models/issue_stopwatch.go +++ b/models/issue_stopwatch.go @@ -45,7 +45,7 @@ func GetUserStopwatches(userID int64, listOptions ListOptions) ([]*Stopwatch, er sws := make([]*Stopwatch, 0, 8) sess := x.Where("stopwatch.user_id = ?", userID) if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) } err := sess.Find(&sws) diff --git a/models/issue_watch.go b/models/issue_watch.go index a3cbbf2c1d965..ac4ec638deb9e 100644 --- a/models/issue_watch.go +++ b/models/issue_watch.go @@ -111,7 +111,7 @@ func getIssueWatchers(e Engine, issueID int64, listOptions ListOptions) (IssueWa Join("INNER", "`user`", "`user`.id = `issue_watch`.user_id") if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) watches := make([]*IssueWatch, 0, listOptions.PageSize) return watches, sess.Find(&watches) } diff --git a/models/list_options.go b/models/list_options.go index ff02933f9bac4..1d465fbe77c16 100644 --- a/models/list_options.go +++ b/models/list_options.go @@ -16,18 +16,15 @@ type ListOptions struct { Page int // start from 1 } -func (opts *ListOptions) getPaginatedSession() *xorm.Session { +func (opts *ListOptions) GetPaginatedSession() *xorm.Session { opts.setDefaultValues() return x.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) } -func (opts *ListOptions) setSessionPagination(sess *xorm.Session) *xorm.Session { +func (opts *ListOptions) SetSessionPagination(sess *xorm.Session) *xorm.Session { opts.setDefaultValues() - if opts.PageSize <= 0 { - return sess - } return sess.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) } diff --git a/models/notification.go b/models/notification.go index c4c7728ad9f6b..e6fdaecd4e66a 100644 --- a/models/notification.go +++ b/models/notification.go @@ -110,7 +110,7 @@ func (opts *FindNotificationOptions) ToCond() builder.Cond { func (opts *FindNotificationOptions) ToSession(e Engine) *xorm.Session { sess := e.Where(opts.ToCond()) if opts.Page != 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) } return sess } diff --git a/models/oauth2_application.go b/models/oauth2_application.go index 2aa9fbd3d9217..81bc9636bf2b6 100644 --- a/models/oauth2_application.go +++ b/models/oauth2_application.go @@ -275,7 +275,7 @@ func ListOAuth2Applications(uid int64, listOptions ListOptions) ([]*OAuth2Applic Desc("id") if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) apps := make([]*OAuth2Application, 0, listOptions.PageSize) return apps, sess.Find(&apps) diff --git a/models/org.go b/models/org.go index 58fb26b1bb511..cbe4173bbb802 100644 --- a/models/org.go +++ b/models/org.go @@ -65,7 +65,7 @@ func (org *User) getTeams(e Engine) error { // GetTeams returns paginated teams that belong to organization. func (org *User) GetTeams(opts *SearchTeamOptions) error { if opts.Page != 0 { - return org.getTeams(opts.getPaginatedSession()) + return org.getTeams(opts.GetPaginatedSession()) } return org.getTeams(x) @@ -529,7 +529,7 @@ func GetOrgUsersByUserID(uid int64, opts *SearchOrganizationsOptions) ([]*OrgUse } if opts.PageSize != 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) } err := sess. @@ -549,7 +549,7 @@ func getOrgUsersByOrgID(e Engine, opts *FindOrgMembersOpts) ([]*OrgUser, error) sess.And("is_public = ?", true) } if opts.ListOptions.PageSize > 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) ous := make([]*OrgUser, 0, opts.PageSize) return ous, sess.Find(&ous) diff --git a/models/org_team.go b/models/org_team.go index 6226772b9a828..e357c99d95bdc 100644 --- a/models/org_team.go +++ b/models/org_team.go @@ -168,7 +168,7 @@ func (t *Team) GetRepositories(opts *SearchTeamOptions) error { return t.getRepositories(x) } - return t.getRepositories(opts.getPaginatedSession()) + return t.getRepositories(opts.GetPaginatedSession()) } func (t *Team) getMembers(e Engine) (err error) { @@ -182,7 +182,7 @@ func (t *Team) GetMembers(opts *SearchMembersOptions) (err error) { return t.getMembers(x) } - return t.getMembers(opts.getPaginatedSession()) + return t.getMembers(opts.GetPaginatedSession()) } // AddMember adds new membership of the team to the organization, @@ -795,7 +795,7 @@ func getUserTeams(e Engine, userID int64, listOptions ListOptions) (teams []*Tea Join("INNER", "team_user", "team_user.team_id = team.id"). Where("team_user.uid=?", userID) if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) } return teams, sess.Find(&teams) } diff --git a/models/package.go b/models/package.go index 8b7f3ca7b92ce..45851373d3ebd 100644 --- a/models/package.go +++ b/models/package.go @@ -179,13 +179,13 @@ func DeletePackagesByRepositoryID(repositoryID int64) error { // PackageSearchOptions are options for GetLatestPackagesGrouped type PackageSearchOptions struct { - RepoID int64 - Type string - Query string - ListOptions + RepoID int64 + Type string + Query string + Paginator SessionPaginator } -func (opts PackageSearchOptions) toConds() builder.Cond { +func (opts *PackageSearchOptions) toConds() builder.Cond { var cond builder.Cond = builder.Eq{"package.repo_id": opts.RepoID} switch opts.Type { @@ -209,18 +209,18 @@ func (opts PackageSearchOptions) toConds() builder.Cond { } // GetPackages returns a list of all packages of the repository -func GetPackages(opts PackageSearchOptions) ([]*Package, int64, error) { +func GetPackages(opts *PackageSearchOptions) ([]*Package, int64, error) { sess := x.Where(opts.toConds()) - sess = opts.setSessionPagination(sess) + sess = opts.Paginator.SetSessionPagination(sess) - packages := make([]*Package, 0, opts.PageSize) + packages := make([]*Package, 0, 10) count, err := sess.FindAndCount(&packages) return packages, count, err } // GetLatestPackagesGrouped returns a list of all packages in their latest version of the repository -func GetLatestPackagesGrouped(opts PackageSearchOptions) ([]*Package, int64, error) { +func GetLatestPackagesGrouped(opts *PackageSearchOptions) ([]*Package, int64, error) { cond := opts.toConds(). And(builder.Expr("p2.id IS NULL")) @@ -228,9 +228,9 @@ func GetLatestPackagesGrouped(opts PackageSearchOptions) ([]*Package, int64, err Table("package"). Join("left", "package p2", "package.repo_id = p2.repo_id AND package.type = p2.type AND package.lower_name = p2.lower_name AND package.version < p2.version") - sess = opts.setSessionPagination(sess) + sess = opts.Paginator.SetSessionPagination(sess) - packages := make([]*Package, 0, opts.PageSize) + packages := make([]*Package, 0, 10) count, err := sess.FindAndCount(&packages) return packages, count, err } diff --git a/models/pull_list.go b/models/pull_list.go index 2f685e19f5eaa..e9caa0fd4108e 100644 --- a/models/pull_list.go +++ b/models/pull_list.go @@ -100,7 +100,7 @@ func PullRequests(baseRepoID int64, opts *PullRequestsOptions) ([]*PullRequest, log.Error("listPullRequestStatement: %v", err) return nil, maxResults, err } - findSession = opts.setSessionPagination(findSession) + findSession = opts.SetSessionPagination(findSession) prs := make([]*PullRequest, 0, opts.PageSize) return prs, maxResults, findSession.Find(&prs) } diff --git a/models/release.go b/models/release.go index 1ce88a8210c9d..cefd1a3eb2732 100644 --- a/models/release.go +++ b/models/release.go @@ -208,7 +208,7 @@ func GetReleasesByRepoID(repoID int64, opts FindReleasesOptions) ([]*Release, er Where(opts.toConds(repoID)) if opts.PageSize != 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) } rels := make([]*Release, 0, opts.PageSize) diff --git a/models/repo.go b/models/repo.go index 93827f6a8842c..0c42528d26eed 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1736,7 +1736,7 @@ func GetUserRepositories(opts *SearchRepoOptions) ([]*Repository, int64, error) sess.Where(cond).OrderBy(opts.OrderBy.String()) repos := make([]*Repository, 0, opts.PageSize) - return repos, count, opts.setSessionPagination(sess).Find(&repos) + return repos, count, opts.SetSessionPagination(sess).Find(&repos) } // GetUserMirrorRepositories returns a list of mirror repositories of given user. @@ -2027,7 +2027,7 @@ func (repo *Repository) GetForks(listOptions ListOptions) ([]*Repository, error) return forks, x.Find(&forks, &Repository{ForkID: repo.ID}) } - sess := listOptions.getPaginatedSession() + sess := listOptions.GetPaginatedSession() forks := make([]*Repository, 0, listOptions.PageSize) return forks, sess.Find(&forks, &Repository{ForkID: repo.ID}) } diff --git a/models/repo_watch.go b/models/repo_watch.go index 656696b34f2e8..75ae43f312fc2 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -165,7 +165,7 @@ func (repo *Repository) GetWatchers(opts ListOptions) ([]*User, error) { Join("LEFT", "watch", "`user`.id=`watch`.user_id"). And("`watch`.mode<>?", RepoWatchModeDont) if opts.Page > 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) users := make([]*User, 0, opts.PageSize) return users, sess.Find(&users) diff --git a/models/review.go b/models/review.go index acb54d970fdc8..8f5e848977e2b 100644 --- a/models/review.go +++ b/models/review.go @@ -195,7 +195,7 @@ func findReviews(e Engine, opts FindReviewOptions) ([]*Review, error) { reviews := make([]*Review, 0, 10) sess := e.Where(opts.toCond()) if opts.Page > 0 { - sess = opts.ListOptions.setSessionPagination(sess) + sess = opts.ListOptions.SetSessionPagination(sess) } return reviews, sess. Asc("created_unix"). diff --git a/models/session_paginator.go b/models/session_paginator.go new file mode 100644 index 0000000000000..b6d7cadb56276 --- /dev/null +++ b/models/session_paginator.go @@ -0,0 +1,43 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "code.gitea.io/gitea/modules/setting" + + "xorm.io/xorm" +) + +type SessionPaginator interface { + GetPaginatedSession() *xorm.Session + SetSessionPagination(sess *xorm.Session) *xorm.Session +} + +// AbsoluteSessionPaginator to paginate results +type AbsoluteSessionPaginator struct { + Skip int + Take int +} + +func (opts *AbsoluteSessionPaginator) GetPaginatedSession() *xorm.Session { + opts.setDefaultValues() + + return x.Limit(opts.Take, opts.Skip) +} + +func (opts *AbsoluteSessionPaginator) SetSessionPagination(sess *xorm.Session) *xorm.Session { + opts.setDefaultValues() + + return sess.Limit(opts.Take, opts.Skip) +} + +func (opts *AbsoluteSessionPaginator) setDefaultValues() { + if opts.Take <= 0 { + opts.Take = setting.API.DefaultPagingNum + } + if opts.Take > setting.API.MaxResponseItems { + opts.Take = setting.API.MaxResponseItems + } +} diff --git a/models/ssh_key.go b/models/ssh_key.go index 6cda4f1658fb2..21684c718fd99 100644 --- a/models/ssh_key.go +++ b/models/ssh_key.go @@ -195,7 +195,7 @@ func SearchPublicKey(uid int64, fingerprint string) ([]*PublicKey, error) { func ListPublicKeys(uid int64, listOptions ListOptions) ([]*PublicKey, error) { sess := x.Where("owner_id = ? AND type != ?", uid, KeyTypePrincipal) if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) keys := make([]*PublicKey, 0, listOptions.PageSize) return keys, sess.Find(&keys) diff --git a/models/ssh_key_deploy.go b/models/ssh_key_deploy.go index 3189bcf456a64..6f327d407726d 100644 --- a/models/ssh_key_deploy.go +++ b/models/ssh_key_deploy.go @@ -272,7 +272,7 @@ func ListDeployKeys(repoID int64, listOptions ListOptions) ([]*DeployKey, error) func listDeployKeys(e Engine, repoID int64, listOptions ListOptions) ([]*DeployKey, error) { sess := e.Where("repo_id = ?", repoID) if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) keys := make([]*DeployKey, 0, listOptions.PageSize) return keys, sess.Find(&keys) diff --git a/models/ssh_key_principals.go b/models/ssh_key_principals.go index 3459e43c8b042..0b84ef07cabff 100644 --- a/models/ssh_key_principals.go +++ b/models/ssh_key_principals.go @@ -114,7 +114,7 @@ func CheckPrincipalKeyString(user *User, content string) (_ string, err error) { func ListPrincipalKeys(uid int64, listOptions ListOptions) ([]*PublicKey, error) { sess := x.Where("owner_id = ? AND type = ?", uid, KeyTypePrincipal) if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) keys := make([]*PublicKey, 0, listOptions.PageSize) return keys, sess.Find(&keys) diff --git a/models/star.go b/models/star.go index 2d9496caf504d..0f0f31dc504e4 100644 --- a/models/star.go +++ b/models/star.go @@ -73,7 +73,7 @@ func (repo *Repository) GetStargazers(opts ListOptions) ([]*User, error) { sess := x.Where("star.repo_id = ?", repo.ID). Join("LEFT", "star", "`user`.id = star.uid") if opts.Page > 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) users := make([]*User, 0, opts.PageSize) return users, sess.Find(&users) diff --git a/models/token.go b/models/token.go index 357afe44a7c0b..33e39c7b5ccdb 100644 --- a/models/token.go +++ b/models/token.go @@ -106,7 +106,7 @@ func ListAccessTokens(opts ListAccessTokensOptions) ([]*AccessToken, error) { sess = sess.Desc("id") if opts.Page != 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) tokens := make([]*AccessToken, 0, opts.PageSize) return tokens, sess.Find(&tokens) diff --git a/models/topic.go b/models/topic.go index 19c572fefebf6..79afec5b9361c 100644 --- a/models/topic.go +++ b/models/topic.go @@ -190,7 +190,7 @@ func FindTopics(opts *FindTopicOptions) (topics []*Topic, err error) { sess.Join("INNER", "repo_topic", "repo_topic.topic_id = topic.id") } if opts.PageSize != 0 && opts.Page != 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) } return topics, sess.Desc("topic.repo_count").Find(&topics) } diff --git a/models/user.go b/models/user.go index c68417a2c3265..1aa38ed9d3ed8 100644 --- a/models/user.go +++ b/models/user.go @@ -332,7 +332,7 @@ func (u *User) GetFollowers(listOptions ListOptions) ([]*User, error) { Join("LEFT", "follow", "`user`.id=follow.user_id") if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) users := make([]*User, 0, listOptions.PageSize) return users, sess.Find(&users) @@ -354,7 +354,7 @@ func (u *User) GetFollowing(listOptions ListOptions) ([]*User, error) { Join("LEFT", "follow", "`user`.id=follow.follow_id") if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) users := make([]*User, 0, listOptions.PageSize) return users, sess.Find(&users) @@ -1677,7 +1677,7 @@ func SearchUsers(opts *SearchUserOptions) (users []*User, _ int64, _ error) { sess := x.Where(cond).OrderBy(opts.OrderBy.String()) if opts.Page != 0 { - sess = opts.setSessionPagination(sess) + sess = opts.SetSessionPagination(sess) } users = make([]*User, 0, opts.PageSize) @@ -1693,7 +1693,7 @@ func GetStarredRepos(userID int64, private bool, listOptions ListOptions) ([]*Re } if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) repos := make([]*Repository, 0, listOptions.PageSize) return repos, sess.Find(&repos) @@ -1713,7 +1713,7 @@ func GetWatchedRepos(userID int64, private bool, listOptions ListOptions) ([]*Re } if listOptions.Page != 0 { - sess = listOptions.setSessionPagination(sess) + sess = listOptions.SetSessionPagination(sess) repos := make([]*Repository, 0, listOptions.PageSize) return repos, sess.Find(&repos) diff --git a/models/webhook.go b/models/webhook.go index 5dc68751dd5f7..97f8f1f153e0e 100644 --- a/models/webhook.go +++ b/models/webhook.go @@ -413,7 +413,7 @@ func GetWebhooksByRepoID(repoID int64, listOptions ListOptions) ([]*Webhook, err return webhooks, x.Find(&webhooks, &Webhook{RepoID: repoID}) } - sess := listOptions.getPaginatedSession() + sess := listOptions.GetPaginatedSession() webhooks := make([]*Webhook, 0, listOptions.PageSize) return webhooks, sess.Find(&webhooks, &Webhook{RepoID: repoID}) @@ -439,7 +439,7 @@ func GetWebhooksByOrgID(orgID int64, listOptions ListOptions) ([]*Webhook, error return ws, x.Find(&ws, &Webhook{OrgID: orgID}) } - sess := listOptions.getPaginatedSession() + sess := listOptions.GetPaginatedSession() ws := make([]*Webhook, 0, listOptions.PageSize) return ws, sess.Find(&ws, &Webhook{OrgID: orgID}) } diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 5f7788c9470d0..6bf1a92cbc673 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -31,19 +31,13 @@ func ServiceIndex(ctx *context.APIContext) { // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages func SearchService(ctx *context.APIContext) { - skip := ctx.FormInt("skip") - take := ctx.FormInt("take") - if take <= 0 { - take = setting.API.DefaultPagingNum - } - - packages, count, err := models.GetPackages(models.PackageSearchOptions{ + packages, count, err := models.GetPackages(&models.PackageSearchOptions{ RepoID: ctx.Repo.Repository.ID, Type: "nuget", Query: ctx.FormTrim("q"), - ListOptions: models.ListOptions{ - Page: skip / take, - PageSize: take, + Paginator: &models.AbsoluteSessionPaginator{ + Skip: ctx.FormInt("skip"), + Take: ctx.FormInt("take"), }, }) if err != nil { diff --git a/routers/api/v1/repo/package.go b/routers/api/v1/repo/package.go index 0a7cdaafd1fed..9e7035ce7f8e8 100644 --- a/routers/api/v1/repo/package.go +++ b/routers/api/v1/repo/package.go @@ -62,11 +62,11 @@ func ListPackages(ctx *context.APIContext) { repo := ctx.Repo.Repository - packages, count, err := models.GetPackages(models.PackageSearchOptions{ - RepoID: repo.ID, - Type: packageType, - Query: query, - ListOptions: listOptions, + packages, count, err := models.GetPackages(&models.PackageSearchOptions{ + RepoID: repo.ID, + Type: packageType, + Query: query, + Paginator: &listOptions, }) if err != nil { ctx.Error(http.StatusInternalServerError, "GetPackages", err) diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index b3f6ae000b436..49421e76d4be9 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -47,11 +47,11 @@ func Packages(ctx *context.Context) { repo := ctx.Repo.Repository - packages, count, err := models.GetLatestPackagesGrouped(models.PackageSearchOptions{ + packages, count, err := models.GetLatestPackagesGrouped(&models.PackageSearchOptions{ RepoID: repo.ID, Query: query, Type: packageType, - ListOptions: models.ListOptions{ + Paginator: &models.ListOptions{ Page: page, PageSize: setting.UI.PackagesPagingNum, }, From c515aa6aca30107e981e6a6c886e872dab045d9c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 11 Aug 2021 19:14:00 +0200 Subject: [PATCH 031/130] Print test names. --- integrations/api_packages_generic_test.go | 12 ++++++++++++ integrations/api_packages_maven_test.go | 14 +++++++++++++ integrations/api_packages_npm_test.go | 8 ++++++++ integrations/api_packages_nuget_test.go | 24 +++++++++++++++++++++++ integrations/api_packages_pypi_test.go | 12 ++++++++++++ 5 files changed, 70 insertions(+) diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index aec78eb3c8fbb..1d42b1e22d053 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -30,6 +30,8 @@ func TestPackageGeneric(t *testing.T) { url := fmt.Sprintf("/api/v1/repos/%s/%s/packages/generic/%s/%s/%s?token=%s", user.Name, repository.Name, packageName, packageVersion, filename, token) t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) MakeRequest(t, req, http.StatusCreated) @@ -47,11 +49,15 @@ func TestPackageGeneric(t *testing.T) { }) t.Run("UploadExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) MakeRequest(t, req, http.StatusBadRequest) }) t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", url) resp := MakeRequest(t, req, http.StatusOK) @@ -59,6 +65,8 @@ func TestPackageGeneric(t *testing.T) { }) t.Run("Delete", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "DELETE", url) MakeRequest(t, req, http.StatusOK) @@ -68,11 +76,15 @@ func TestPackageGeneric(t *testing.T) { }) t.Run("DownloadNotExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", url) MakeRequest(t, req, http.StatusNotFound) }) t.Run("DeleteNotExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "DELETE", url) MakeRequest(t, req, http.StatusNotFound) }) diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go index 0d39428f25ed3..1ae4dff50f1b0 100644 --- a/integrations/api_packages_maven_test.go +++ b/integrations/api_packages_maven_test.go @@ -38,6 +38,8 @@ func TestPackageMaven(t *testing.T) { } t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusCreated) putFile(t, "/maven-metadata.xml", "test", http.StatusOK) @@ -55,10 +57,14 @@ func TestPackageMaven(t *testing.T) { }) t.Run("UploadExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusBadRequest) }) t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) @@ -67,15 +73,23 @@ func TestPackageMaven(t *testing.T) { }) t.Run("UploadVerifySHA1", func(t *testing.T) { + defer PrintCurrentTest(t)() + t.Run("Missmatch", func(t *testing.T) { + defer PrintCurrentTest(t)() + putFile(t, fmt.Sprintf("/%s/%s.sha1", packageVersion, filename), "test", http.StatusBadRequest) }) t.Run("Valid", func(t *testing.T) { + defer PrintCurrentTest(t)() + putFile(t, fmt.Sprintf("/%s/%s.sha1", packageVersion, filename), "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", http.StatusOK) }) }) t.Run("UploadPOM", func(t *testing.T) { + defer PrintCurrentTest(t)() + pomContent := ` ` + groupID + ` diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index cc3443d1a19ec..5aa955479c4d3 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -59,6 +59,8 @@ func TestPackageNPM(t *testing.T) { filename := fmt.Sprintf("%s-%s.tgz", strings.Split(packageName, "/")[1], packageVersion) t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequestWithBody(t, "PUT", root, strings.NewReader(upload)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) @@ -77,12 +79,16 @@ func TestPackageNPM(t *testing.T) { }) t.Run("UploadExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequestWithBody(t, "PUT", root, strings.NewReader(upload)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusBadRequest) }) t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", fmt.Sprintf("%s/-/%s/%s", root, packageVersion, filename)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) @@ -92,6 +98,8 @@ func TestPackageNPM(t *testing.T) { }) t.Run("PackageMetadata", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", root) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index 4d2de4ba0b121..a2620a8255e41 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -46,6 +46,8 @@ func TestPackageNuGet(t *testing.T) { url := fmt.Sprintf("/api/v1/repos/%s/%s/packages/nuget", user.Name, repository.Name) t.Run("ServiceIndex", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", fmt.Sprintf("%s/index.json", url)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) @@ -80,6 +82,8 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) @@ -98,12 +102,16 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("UploadExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusBadRequest) }) t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.nupkg", url, packageName, packageVersion, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) @@ -112,6 +120,8 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("SearchService", func(t *testing.T) { + defer PrintCurrentTest(t)() + cases := []struct { Query string Skip int @@ -140,11 +150,15 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("RegistrationService", func(t *testing.T) { + defer PrintCurrentTest(t)() + indexURL := fmt.Sprintf("%s%s/registration/%s/index.json", setting.AppURL, url[1:], packageName) leafURL := fmt.Sprintf("%s%s/registration/%s/%s.json", setting.AppURL, url[1:], packageName, packageVersion) contentURL := fmt.Sprintf("%s%s/package/%s/%s/%s.%s.nupkg", setting.AppURL, url[1:], packageName, packageVersion, packageName, packageVersion) t.Run("RegistrationIndex", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", fmt.Sprintf("%s/registration/%s/index.json", url, packageName)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) @@ -169,6 +183,8 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("RegistrationLeaf", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", fmt.Sprintf("%s/registration/%s/%s.json", url, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) @@ -183,6 +199,8 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("PackageService", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/index.json", url, packageName)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) @@ -195,6 +213,8 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("Delete", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/package/%s/%s", url, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) @@ -205,12 +225,16 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("DownloadNotExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.nupkg", url, packageName, packageVersion, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusNotFound) }) t.Run("DeleteNotExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/package/%s/%s", url, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusNotFound) diff --git a/integrations/api_packages_pypi_test.go b/integrations/api_packages_pypi_test.go index 5c3b9d4015bae..73c4d423b5f36 100644 --- a/integrations/api_packages_pypi_test.go +++ b/integrations/api_packages_pypi_test.go @@ -57,6 +57,8 @@ func TestPackagePyPI(t *testing.T) { } t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + filename := "test.whl" uploadFile(t, filename, content, http.StatusCreated) @@ -74,6 +76,8 @@ func TestPackagePyPI(t *testing.T) { }) t.Run("UploadAddFile", func(t *testing.T) { + defer PrintCurrentTest(t)() + filename := "test.tar.gz" uploadFile(t, filename, content, http.StatusCreated) @@ -95,16 +99,22 @@ func TestPackagePyPI(t *testing.T) { }) t.Run("UploadHashMismatch", func(t *testing.T) { + defer PrintCurrentTest(t)() + filename := "test2.whl" uploadFile(t, filename, "dummy", http.StatusBadRequest) }) t.Run("UploadExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + uploadFile(t, "test.whl", content, http.StatusBadRequest) uploadFile(t, "test.tar.gz", content, http.StatusBadRequest) }) t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + downloadFile := func(filename string) { req := NewRequest(t, "GET", fmt.Sprintf("%s/files/%s/%s/%s", root, packageName, packageVersion, filename)) req = AddBasicAuthHeader(req, user.Name) @@ -118,6 +128,8 @@ func TestPackagePyPI(t *testing.T) { }) t.Run("PackageMetadata", func(t *testing.T) { + defer PrintCurrentTest(t)() + req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) From 55c6b2b367ff785fc55f247415908ff3bb3f462a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 11 Aug 2021 19:57:00 +0200 Subject: [PATCH 032/130] Added api tests. --- integrations/api_packages_generic_test.go | 2 +- integrations/api_packages_maven_test.go | 2 +- integrations/api_packages_npm_test.go | 2 +- integrations/api_packages_nuget_test.go | 2 +- integrations/api_packages_pypi_test.go | 2 +- integrations/api_packages_test.go | 104 ++++++++++++++++++++++ 6 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 integrations/api_packages_test.go diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index 1d42b1e22d053..1959ae47ecaaa 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go index 1ae4dff50f1b0..38dbcf908ff29 100644 --- a/integrations/api_packages_maven_test.go +++ b/integrations/api_packages_maven_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index 5aa955479c4d3..b7ee8b7cd744a 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index a2620a8255e41..84e1fc1b7e0f3 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/integrations/api_packages_pypi_test.go b/integrations/api_packages_pypi_test.go index 73c4d423b5f36..44e9cf58344cd 100644 --- a/integrations/api_packages_pypi_test.go +++ b/integrations/api_packages_pypi_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/integrations/api_packages_test.go b/integrations/api_packages_test.go new file mode 100644 index 0000000000000..ab0d8f81cd509 --- /dev/null +++ b/integrations/api_packages_test.go @@ -0,0 +1,104 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "bytes" + "fmt" + "net/http" + "testing" + + "code.gitea.io/gitea/models" + api "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/assert" +) + +func TestPackageAPI(t *testing.T) { + defer prepareTestEnv(t)() + repository := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 15}).(*models.Repository) + user := models.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + session := loginUser(t, user.Name) + token := getTokenForLoggedInUser(t, session) + + packageName := "test-package" + packageVersion := "1.0.3" + filename := "file.bin" + + url := fmt.Sprintf("/api/v1/repos/%s/%s/packages/generic/%s/%s/%s?token=%s", user.Name, repository.Name, packageName, packageVersion, filename, token) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{})) + MakeRequest(t, req, http.StatusCreated) + + var packageID int64 + + t.Run("ListPackages", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages?token=%s", user.Name, repository.Name, token)) + resp := MakeRequest(t, req, http.StatusOK) + + var packages []*api.Package + DecodeJSON(t, resp, &packages) + + assert.Len(t, packages, 1) + assert.Equal(t, "Generic", packages[0].Type) + assert.Equal(t, packageName, packages[0].Name) + assert.Equal(t, packageVersion, packages[0].Version) + assert.NotNil(t, packages[0].Creator) + assert.Equal(t, user.Name, packages[0].Creator.UserName) + + packageID = packages[0].ID + }) + + t.Run("GetPackage", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d?token=%s", user.Name, repository.Name, 123456, token)) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d?token=%s", user.Name, repository.Name, packageID, token)) + resp := MakeRequest(t, req, http.StatusOK) + + var p *api.Package + DecodeJSON(t, resp, &p) + + assert.Equal(t, "Generic", p.Type) + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.NotNil(t, p.Creator) + assert.Equal(t, user.Name, p.Creator.UserName) + }) + + t.Run("ListPackageFiles", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d/files?token=%s", user.Name, repository.Name, 123456, token)) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d/files?token=%s", user.Name, repository.Name, packageID, token)) + resp := MakeRequest(t, req, http.StatusOK) + + var files []*api.PackageFile + DecodeJSON(t, resp, &files) + + assert.Len(t, files, 1) + assert.Equal(t, int64(0), files[0].Size) + assert.Equal(t, filename, files[0].Name) + assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", files[0].HashMD5) + assert.Equal(t, "da39a3ee5e6b4b0d3255bfef95601890afd80709", files[0].HashSHA1) + assert.Equal(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", files[0].HashSHA256) + assert.Equal(t, "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", files[0].HashSHA512) + }) + + t.Run("DeletePackage", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d?token=%s", user.Name, repository.Name, 123456, token)) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d?token=%s", user.Name, repository.Name, packageID, token)) + MakeRequest(t, req, http.StatusNoContent) + }) +} From 6fadfe92cce1d415936911ddd63cacc860bd138a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 15 Aug 2021 17:16:44 +0000 Subject: [PATCH 033/130] Fixed access level. --- routers/api/v1/api.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c913b1ab4f508..c49773063c518 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -991,7 +991,7 @@ func Routes() *web.Route { }, reqToken()) m.Group("/maven", func() { m.Put("/*", reqRepoWriter(models.UnitTypePackages), maven.UploadPackageFile) - m.Get("/*", reqRepoWriter(models.UnitTypePackages), maven.DownloadPackageFile) + m.Get("/*", maven.DownloadPackageFile) }, reqToken()) m.Group("/nuget", func() { m.Put("/", reqRepoWriter(models.UnitTypePackages), nuget.UploadPackage) @@ -1023,11 +1023,11 @@ func Routes() *web.Route { }, reqBasicAuth()) m.Group("/{id}", func() { m.Get("/", repo.GetPackage) - m.Delete("/", repo.DeletePackage) + m.Delete("/", reqRepoWriter(models.UnitTypePackages), repo.DeletePackage) m.Get("/files", repo.ListPackageFiles) - }, reqAnyRepoReader()) - m.Get("/", reqAnyRepoReader(), repo.ListPackages) - }) + }) + m.Get("/", repo.ListPackages) + }, reqRepoReader(models.UnitTypePackages)) }, repoAssignment()) }) From e60969b5ad73baa67f8ac4969d91231e7463fdd6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 15 Aug 2021 19:06:38 +0000 Subject: [PATCH 034/130] Fix User unmarshal. --- docs/content/doc/packages/npm.en-us.md | 4 ++-- modules/packages/npm/creator.go | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index 3a823a47c0b7e..c676ad23cba74 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -33,7 +33,7 @@ The following examples use the `npm` tool with the scope `@test`. To register the project’s package registry you need to configure a new package source. ```shell -npm config set {scope}:registry https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/npm +npm config set {scope}:registry https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/npm/ npm config set -- '//gitea.example.com/api/v1/repos/{owner}/{repository}/packages/npm/:_authToken' "{token}" ``` @@ -47,7 +47,7 @@ npm config set -- '//gitea.example.com/api/v1/repos/{owner}/{repository}/package For example: ```shell -npm config set @test:registry https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/npm +npm config set @test:registry https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/npm/ npm config set -- '//gitea.example.com/api/v1/repos/testuser/test-repository/packages/npm/:_authToken' "personal_access_token" ``` diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 5639169ac8a43..bcfaa769fa650 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -101,6 +101,31 @@ type User struct { URL string `json:"url,omitempty"` } +// UnmarshalJSON is needed because User objects can be strings or objects +func (u *User) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + if err := json.Unmarshal(data, &u.Name); err != nil { + return err + } + case '{': + var tmp struct { + Username string `json:"username"` + Name string `json:"name"` + Email string `json:"email"` + URL string `json:"url"` + } + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + u.Username = tmp.Username + u.Name = tmp.Name + u.Email = tmp.Email + u.URL = tmp.URL + } + return nil +} + // Repository https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version type Repository struct { Type string `json:"type"` From 4ced95341c3570be9e40c33764662c05164056e3 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 23 Aug 2021 19:56:28 +0000 Subject: [PATCH 035/130] Added RubyGems package registry. --- docs/content/doc/packages/rubygems.en-us.md | 122 +++++++ integrations/api_packages_rubygems_test.go | 204 ++++++++++++ models/package.go | 39 ++- modules/context/context.go | 37 +-- modules/packages/rubygems/marshal.go | 311 ++++++++++++++++++ modules/packages/rubygems/marshal_test.go | 99 ++++++ modules/packages/rubygems/metadata.go | 215 ++++++++++++ modules/packages/rubygems/metadata_test.go | 89 +++++ options/locale/locale_en-US.ini | 6 + routers/api/v1/api.go | 12 + routers/api/v1/packages/nuget/nuget.go | 2 +- routers/api/v1/packages/rubygems/package.go | 75 +++++ routers/api/v1/packages/rubygems/rubygems.go | 229 +++++++++++++ routers/web/repo/packages.go | 3 + services/packages/packages.go | 7 +- templates/repo/packages/content/maven.tmpl | 14 +- templates/repo/packages/content/npm.tmpl | 14 +- templates/repo/packages/content/nuget.tmpl | 24 +- templates/repo/packages/content/pypi.tmpl | 7 +- templates/repo/packages/content/rubygems.tmpl | 76 +++++ templates/repo/packages/list.tmpl | 1 + templates/repo/packages/metadata/maven.tmpl | 7 +- templates/repo/packages/metadata/npm.tmpl | 4 +- templates/repo/packages/metadata/nuget.tmpl | 2 +- templates/repo/packages/metadata/pypi.tmpl | 7 +- .../repo/packages/metadata/rubygems.tmpl | 5 + templates/repo/packages/view.tmpl | 2 + 27 files changed, 1552 insertions(+), 61 deletions(-) create mode 100644 docs/content/doc/packages/rubygems.en-us.md create mode 100644 integrations/api_packages_rubygems_test.go create mode 100644 modules/packages/rubygems/marshal.go create mode 100644 modules/packages/rubygems/marshal_test.go create mode 100644 modules/packages/rubygems/metadata.go create mode 100644 modules/packages/rubygems/metadata_test.go create mode 100644 routers/api/v1/packages/rubygems/package.go create mode 100644 routers/api/v1/packages/rubygems/rubygems.go create mode 100644 templates/repo/packages/content/rubygems.tmpl create mode 100644 templates/repo/packages/metadata/rubygems.tmpl diff --git a/docs/content/doc/packages/rubygems.en-us.md b/docs/content/doc/packages/rubygems.en-us.md new file mode 100644 index 0000000000000..16e4125b9dac0 --- /dev/null +++ b/docs/content/doc/packages/rubygems.en-us.md @@ -0,0 +1,122 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "RubyGems Packages Repository" +slug: "rubygems" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "RubyGems" + weight: 60 + identifier: "rubygems" +--- + +# RubyGems Packages Repository + +Publish [RubyGems](https://guides.rubygems.org/) packages in your project’s Package Registry. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the RubyGems package registry, you need to use the [gem](https://guides.rubygems.org/command-reference/) command line tool to consume and publish packages. + +## Configuring the package registry + +To register the project’s package registry edit the `~/.gem/credentials` file and add: + +```ini +--- +https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/rubygems: Bearer {token} +``` + +| Parameter | Description | +| ------------- | ----------- | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | +| `token` | Your personal access token. | + +For example: + +```ini +--- +https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/rubygems: Bearer 3bd626f84b01cd26b873931eace1e430a5773cc4 +``` + +## Publish a package + +Publish a package by running the following command: + +```shell +gem push --host {host} {package_file} +``` + +| Parameter | Description | +| -------------- | ----------- | +| `host` | URL to the package registry. | +| `package_file` | Path to the package `.gem` file. | + +For example: + +```shell +gem push --host https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/rubygems test_package-1.0.0.gem +``` + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a package from the package registry you can use [Bundler](https://bundler.io) or `gem`. + +### Bundler + +Add a new `source` block to your `Gemfile`: + +``` +source "https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/rubygems" do + gem "{package_name}" +end +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | +| `package_name` | The package name. | + +For example: + +``` +source "https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/rubygems" do + gem "test_package" +end +``` + +Afterwards run the following command: + +```shell +bundle install +``` + +### gem + +Execute the following command: + +```shell +gem install --host https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/rubygems {package_name} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `owner` | The owner of the repository. | +| `repository` | The name of the repository. | +| `package_name` | The package name. | + +For example: + +```shell +gem install --host https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/rubygems test_package +``` \ No newline at end of file diff --git a/integrations/api_packages_rubygems_test.go b/integrations/api_packages_rubygems_test.go new file mode 100644 index 0000000000000..a81d8dc5cbf8d --- /dev/null +++ b/integrations/api_packages_rubygems_test.go @@ -0,0 +1,204 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "bytes" + "encoding/base64" + "fmt" + "mime/multipart" + "net/http" + "testing" + + "code.gitea.io/gitea/models" + + "github.com/stretchr/testify/assert" +) + +func TestPackageRubyGems(t *testing.T) { + defer prepareTestEnv(t)() + repository := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) + user := models.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + + packageName := "gitea" + packageVersion := "1.0.5" + packageFilename := "gitea-1.0.5.gem" + + gemContent, _ := base64.StdEncoding.DecodeString(`bWV0YWRhdGEuZ3oAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAwMDA0NDQAMDAwMDAw +MAAwMDAwMDAwADAwMDAwMDAxMDQxADE0MTEwNzcyMzY2ADAxMzQ0MQAgMAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB1c3RhcgAwMHdoZWVsAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAd2hlZWwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwMDAwMDAwADAwMDAw +MDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAf +iwgA9vQjYQID1VVNb9QwEL37V5he9pRsmlJAFlQckCoOXAriQIUix5nNmsYf2JOqKwS/nYmz2d3Q +qqCCKpFdadfjmfdm5nmcLMv4k9DXm6Wrv4BCcQ5GiPcelF5pJVE7y6w0IHirESS7hhDJJu4I+jhu +Mc53Tsd5kZ8y30lcuWAEH2KY7HHtQhQs4+cJkwwuwNdeB6JhtbaNDoLTL1MQsFJrqQnr8jNrJJJH +WZTHWfEiK094UYj0zYvp4Z9YAx5sA1ZpSCS3M30zeWwo2bG60FvUBjIKJts2GwMW76r0Yr9NzjN3 +YhwsGX2Ozl4dpcWwvK9d43PQtDIv9igvHwSyIIwFmXHjqTqxLY8MPkCADmQk80p2EfZ6VbM6/ue6 +/1D0Bq7/qeA/zh6W82leHmhFWUHn/JbsEfT6q7QbiCpoj8l0QcEUFLmX6kq2wBEiMjBSd+Pwt7T5 +Ot0kuXYMbkD1KOuOBnWYb7hBsAP4bhlkFRqnqpWefMZ/pHCn6+WIFGq2dgY8EQq+RvRRLJcTyZJ1 +WhHqGPTu7QdmACXdJFLwb9+ZdxErbSPKrqsMxJhAWCJ1qaqRdtu6yktcT/STsamG0qp7rsa5EL/K +MBua30uw4ynzExqYWRJDfx8/kQWN3PwsDh2jYLr1W+pZcAmCs9splvnz/Flesqhbq21bXcGG/OLh ++2fv/JTF3hgZyCW9OaZjxoZjdnBGfgKpxZyJ1QYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGF0 +YS50YXIuZ3oAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAwMDA0NDQAMDAwMDAwMAAw +MDAwMDAwADAwMDAwMDAwMjQyADE0MTEwNzcyMzY2ADAxMzM2MQAgMAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB1c3RhcgAwMHdoZWVsAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAd2hlZWwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwMDAwMDAwADAwMDAwMDAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAfiwgA +9vQjYQID7M/NCsMgDABgz32KrA/QxersK/Q17ExXIcyhlr7+HLv1sJ02KPhBCPk5JOyn881nsl2c +xI+gRDRaC3zbZ8RBCamlxGHolTFlX11kLwDFH6wp21hO2RYi/rD3bb5/7iCubFOCMbBtABzNkIjn +bvGlAnisOUE7EnOALUR2p7b06e6aV4iqqqrquJ4AAAD//wMA+sA/NQAIAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGNoZWNr +c3Vtcy55YW1sLmd6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwMDAwNDQ0ADAwMDAwMDAAMDAw +MDAwMAAwMDAwMDAwMDQ1MAAxNDExMDc3MjM2NgAwMTQ2MTIAIDAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAdXN0YXIAMDB3aGVlbAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAHdoZWVsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMDAwMDAwMAAwMDAwMDAwAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAH4sIAPb0 +I2ECA2WQOa4UQAxE8znFXGCQ21vbPyMj5wRuL0Qk6EecnmZCyKyy9FSvXq/X4/u3ryj68Xg+f/Zn +VHzGlx+/P57qvU4XxWalBKftSXOgCjNYkdRycrC5Axem+W4HqS12PNEv7836jF9vnlHxwSyxKY+y +go0cPblyHzkrZ4HF1GSVhe7mOOoasXNk2fnbUxb+19Pp9tobD/QlJKMX7y204PREh6nQ5hG9Alw6 +x4TnmtA+aekGfm6wAseog2LSgpR4Q7cYnAH3K4qAQa6A6JCC1gpuY7P+9YxE5SZ+j0eVGbaBTwBQ +iIqRUyyzLCoFCBdYNWxniapTavD97blXTzFvgoVoAsKBAtlU48cdaOmeZDpwV01OtcGwjscfeUrY +B9QBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) + + root := fmt.Sprintf("/api/v1/repos/%s/%s/packages/rubygems", user.Name, repository.Name) + + uploadFile := func(t *testing.T, expectedStatus int) { + req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/api/v1/gems", root), bytes.NewReader(gemContent)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, expectedStatus) + } + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + uploadFile(t, http.StatusCreated) + + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageRubyGems) + assert.NoError(t, err) + assert.Len(t, ps, 1) + assert.Equal(t, packageName, ps[0].Name) + assert.Equal(t, packageVersion, ps[0].Version) + + pfs, err := ps[0].GetFiles() + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, packageFilename, pfs[0].Name) + assert.Equal(t, int64(4608), pfs[0].Size) + }) + + t.Run("UploadExists", func(t *testing.T) { + defer PrintCurrentTest(t)() + + uploadFile(t, http.StatusBadRequest) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/gems/%s", root, packageFilename)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, gemContent, resp.Body.Bytes()) + }) + + t.Run("DownloadGemspec", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/quick/Marshal.4.8/%sspec.rz", root, packageFilename)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + b, _ := base64.StdEncoding.DecodeString(`eJxi4Si1EndPzbWyCi5ITc5My0xOLMnMz2M8zMIRLeGpxGWsZ6RnzGbF5hqSyempxJWeWZKayGbN +EBJqJQjWFZZaVJyZnxfN5qnEZahnoGcKkjTwVBJyB6lUKEhMzk5MTwULGngqcRaVJlWCONEMBp5K +DGAWSKc7zFhPJamg0qRK99TcYphehZLU4hKInFhGSUlBsZW+PtgZepn5+iDxECRzDUDGcfh6hoA4 +gAAAAP//MS06Gw==`) + assert.Equal(t, b, resp.Body.Bytes()) + }) + + t.Run("EnumeratePackages", func(t *testing.T) { + defer PrintCurrentTest(t)() + + enumeratePackages := func(t *testing.T, endpoint string, expectedContent []byte) { + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s", root, endpoint)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, expectedContent, resp.Body.Bytes()) + } + + b, _ := base64.StdEncoding.DecodeString(`H4sICAAAAAAA/3NwZWNzLjQuOABi4Yhmi+bwVOJKzyxJTWSzYnMNCbUSdE/NtbIKSy0qzszPi2bzVOIy1DPQM2WzZgjxVOIsKk2qBDEBAQAA///xOEYKOwAAAA==`) + enumeratePackages(t, "specs.4.8.gz", b) + b, _ = base64.StdEncoding.DecodeString(`H4sICAAAAAAA/2xhdGVzdF9zcGVjcy40LjgAYuGIZovm8FTiSs8sSU1ks2JzDQm1EnRPzbWyCkstKs7Mz4tm81TiMtQz0DNls2YI8VTiLCpNqgQxAQEAAP//8ThGCjsAAAA=`) + enumeratePackages(t, "latest_specs.4.8.gz", b) + b, _ = base64.StdEncoding.DecodeString(`H4sICAAAAAAA/3ByZXJlbGVhc2Vfc3BlY3MuNC44AGLhiGYABAAA//9snXr5BAAAAA==`) + enumeratePackages(t, "prerelease_specs.4.8.gz", b) + }) + + t.Run("Delete", func(t *testing.T) { + defer PrintCurrentTest(t)() + + body := bytes.Buffer{} + writer := multipart.NewWriter(&body) + writer.WriteField("gem_name", packageName) + writer.WriteField("version", packageVersion) + writer.Close() + + req := NewRequestWithBody(t, "DELETE", fmt.Sprintf("%s/api/v1/gems/yank", root), &body) + req.Header.Add("Content-Type", writer.FormDataContentType()) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageRubyGems) + assert.NoError(t, err) + assert.Empty(t, ps) + }) +} diff --git a/models/package.go b/models/package.go index 45851373d3ebd..82cba7dce5590 100644 --- a/models/package.go +++ b/models/package.go @@ -18,11 +18,12 @@ type PackageType int // Note: new type must append to the end of list to maintain compatibility. const ( - PackageGeneric PackageType = iota - PackageNuGet // 1 - PackageNpm // 2 - PackageMaven // 3 - PackagePyPI // 4 + PackageGeneric PackageType = iota + PackageNuGet // 1 + PackageNpm // 2 + PackageMaven // 3 + PackagePyPI // 4 + PackageRubyGems // 5 ) func (pt PackageType) String() string { @@ -37,6 +38,8 @@ func (pt PackageType) String() string { return "Maven" case PackagePyPI: return "PyPI" + case PackageRubyGems: + return "RubyGems" } return "" } @@ -199,6 +202,8 @@ func (opts *PackageSearchOptions) toConds() builder.Cond { cond = cond.And(builder.Eq{"package.type": PackageMaven}) case "pypi": cond = cond.And(builder.Eq{"package.type": PackagePyPI}) + case "rubygems": + cond = cond.And(builder.Eq{"package.type": PackageRubyGems}) } if opts.Query != "" { @@ -212,7 +217,9 @@ func (opts *PackageSearchOptions) toConds() builder.Cond { func GetPackages(opts *PackageSearchOptions) ([]*Package, int64, error) { sess := x.Where(opts.toConds()) - sess = opts.Paginator.SetSessionPagination(sess) + if opts.Paginator != nil { + sess = opts.Paginator.SetSessionPagination(sess) + } packages := make([]*Package, 0, 10) count, err := sess.FindAndCount(&packages) @@ -228,7 +235,9 @@ func GetLatestPackagesGrouped(opts *PackageSearchOptions) ([]*Package, int64, er Table("package"). Join("left", "package p2", "package.repo_id = p2.repo_id AND package.type = p2.type AND package.lower_name = p2.lower_name AND package.version < p2.version") - sess = opts.Paginator.SetSessionPagination(sess) + if opts.Paginator != nil { + sess = opts.Paginator.SetSessionPagination(sess) + } packages := make([]*Package, 0, 10) count, err := sess.FindAndCount(&packages) @@ -282,6 +291,22 @@ func GetPackageByNameAndVersion(repositoryID int64, packageType PackageType, pac return p, nil } +// GetPackagesByFilename gets a repository packages by filename +func GetPackagesByFilename(repositoryID int64, packageType PackageType, packageFilename string) ([]*Package, error) { + var cond builder.Cond = builder.Eq{ + "package.repo_id": repositoryID, + "package.type": packageType, + "package_file.lower_name": strings.ToLower(packageFilename), + } + + packages := make([]*Package, 0, 10) + return packages, x. + Table("package"). + Where(cond). + Join("INNER", "package_file", "package.id = package_file.package_id"). + Find(&packages) +} + // TryInsertPackageFile inserts a package file func TryInsertPackageFile(pf *PackageFile) (*PackageFile, error) { sess := x.NewSession() diff --git a/modules/context/context.go b/modules/context/context.go index 64e9854d190ef..cf4755c5109d2 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -316,6 +316,18 @@ func (ctx *Context) HandleText(status int, title string) { ctx.PlainText(status, []byte(title)) } +// SetServeHeaders sets necessary content serve headers +func (ctx *Context) SetServeHeaders(filename string) { + ctx.Resp.Header().Set("Content-Description", "File Transfer") + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+filename) + ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") + ctx.Resp.Header().Set("Expires", "0") + ctx.Resp.Header().Set("Cache-Control", "must-revalidate") + ctx.Resp.Header().Set("Pragma", "public") + ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") +} + // ServeContent serves content to http request func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interface{}) { modtime := time.Now() @@ -325,14 +337,7 @@ func (ctx *Context) ServeContent(name string, r io.ReadSeeker, params ...interfa modtime = v } } - ctx.Resp.Header().Set("Content-Description", "File Transfer") - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name) - ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") - ctx.Resp.Header().Set("Expires", "0") - ctx.Resp.Header().Set("Cache-Control", "must-revalidate") - ctx.Resp.Header().Set("Pragma", "public") - ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") + ctx.SetServeHeaders(name) http.ServeContent(ctx.Resp, ctx.Req, name, modtime, r) } @@ -353,25 +358,13 @@ func (ctx *Context) ServeFile(file string, names ...string) { } else { name = path.Base(file) } - ctx.Resp.Header().Set("Content-Description", "File Transfer") - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name) - ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") - ctx.Resp.Header().Set("Expires", "0") - ctx.Resp.Header().Set("Cache-Control", "must-revalidate") - ctx.Resp.Header().Set("Pragma", "public") + ctx.SetServeHeaders(name) http.ServeFile(ctx.Resp, ctx.Req, file) } // ServeStream serves file via io stream func (ctx *Context) ServeStream(rd io.Reader, name string) { - ctx.Resp.Header().Set("Content-Description", "File Transfer") - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+name) - ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") - ctx.Resp.Header().Set("Expires", "0") - ctx.Resp.Header().Set("Cache-Control", "must-revalidate") - ctx.Resp.Header().Set("Pragma", "public") + ctx.SetServeHeaders(name) _, err := io.Copy(ctx.Resp, rd) if err != nil { ctx.ServerError("Download file failed", err) diff --git a/modules/packages/rubygems/marshal.go b/modules/packages/rubygems/marshal.go new file mode 100644 index 0000000000000..2c45042fa8c01 --- /dev/null +++ b/modules/packages/rubygems/marshal.go @@ -0,0 +1,311 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "bufio" + "bytes" + "errors" + "io" + "reflect" +) + +const ( + majorVersion = 4 + minorVersion = 8 + + typeNil = '0' + typeTrue = 'T' + typeFalse = 'F' + typeFixnum = 'i' + typeString = '"' + typeSymbol = ':' + typeSymbolLink = ';' + typeArray = '[' + typeIVar = 'I' + typeUserMarshal = 'U' + typeUserDef = 'u' + typeObject = 'o' +) + +var ( + // ErrUnsupportedType indicates an unsupported type + ErrUnsupportedType = errors.New("Type is unsupported") + // ErrInvalidIntRange indicates an invalid number range + ErrInvalidIntRange = errors.New("Number is not in valid range") +) + +// RubyUserMarshal is a Ruby object that has a marshal_load function. +type RubyUserMarshal struct { + Name string + Value interface{} +} + +// RubyUserDef is a Ruby object that has a _load function. +type RubyUserDef struct { + Name string + Value interface{} +} + +// RubyObject is a default Ruby object. +type RubyObject struct { + Name string + Member map[string]interface{} +} + +// MarshalEncoder mimics Rubys Marshal class. +// Note: Only supports types used by the RubyGems package registry. +type MarshalEncoder struct { + w *bufio.Writer + symbols map[string]int +} + +// NewMarshalEncoder creates a new MarshalEncoder +func NewMarshalEncoder(w io.Writer) *MarshalEncoder { + return &MarshalEncoder{ + w: bufio.NewWriter(w), + symbols: map[string]int{}, + } +} + +// Encode encodes the given type +func (e *MarshalEncoder) Encode(v interface{}) error { + if _, err := e.w.Write([]byte{majorVersion, minorVersion}); err != nil { + return err + } + + if err := e.marshal(v); err != nil { + return err + } + + return e.w.Flush() +} + +func (e *MarshalEncoder) marshal(v interface{}) error { + if v == nil { + return e.marshalNil() + } + + val := reflect.ValueOf(v) + typ := reflect.TypeOf(v) + + if typ.Kind() == reflect.Ptr { + val = val.Elem() + typ = typ.Elem() + } + + switch typ.Kind() { + case reflect.Bool: + return e.marshalBool(val.Bool()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32: + return e.marshalInt(val.Int()) + case reflect.String: + return e.marshalString(val.String()) + case reflect.Slice, reflect.Array: + return e.marshalArray(val) + } + + switch typ.Name() { + case "RubyUserMarshal": + return e.marshalUserMarshal(val.Interface().(RubyUserMarshal)) + case "RubyUserDef": + return e.marshalUserDef(val.Interface().(RubyUserDef)) + case "RubyObject": + return e.marshalObject(val.Interface().(RubyObject)) + } + + return ErrUnsupportedType +} + +func (e *MarshalEncoder) marshalNil() error { + return e.w.WriteByte(typeNil) +} + +func (e *MarshalEncoder) marshalBool(b bool) error { + if b { + return e.w.WriteByte(typeTrue) + } + return e.w.WriteByte(typeFalse) +} + +func (e *MarshalEncoder) marshalInt(i int64) error { + if err := e.w.WriteByte(typeFixnum); err != nil { + return err + } + + return e.marshalIntInternal(i) +} + +func (e *MarshalEncoder) marshalIntInternal(i int64) error { + if i == 0 { + return e.w.WriteByte(0) + } else if 0 < i && i < 123 { + return e.w.WriteByte(byte(i + 5)) + } else if -124 < i && i <= -1 { + return e.w.WriteByte(byte(i - 5)) + } + + var len int + if 122 < i && i <= 0xff { + len = 1 + } else if 0xff < i && i <= 0xffff { + len = 2 + } else if 0xffff < i && i <= 0xffffff { + len = 3 + } else if 0xffffff < i && i <= 0x3fffffff { + len = 4 + } else if -0x100 <= i && i < -123 { + len = -1 + } else if -0x10000 <= i && i < -0x100 { + len = -2 + } else if -0x1000000 <= i && i < -0x100000 { + len = -3 + } else if -0x40000000 <= i && i < -0x1000000 { + len = -4 + } else { + return ErrInvalidIntRange + } + + if err := e.w.WriteByte(byte(len)); err != nil { + return err + } + if len < 0 { + len = -len + } + + for c := 0; c < len; c++ { + if err := e.w.WriteByte(byte(i >> uint(8*c) & 0xff)); err != nil { + return err + } + } + + return nil +} + +func (e *MarshalEncoder) marshalString(str string) error { + if err := e.w.WriteByte(typeIVar); err != nil { + return err + } + + if err := e.marshalRawString(str); err != nil { + return err + } + + if err := e.marshalIntInternal(1); err != nil { + return err + } + + if err := e.marshalSymbol("E"); err != nil { + return err + } + + return e.marshalBool(true) +} + +func (e *MarshalEncoder) marshalRawString(str string) error { + if err := e.w.WriteByte(typeString); err != nil { + return err + } + + if err := e.marshalIntInternal(int64(len(str))); err != nil { + return err + } + + _, err := e.w.WriteString(str) + return err +} + +func (e *MarshalEncoder) marshalSymbol(str string) error { + if index, ok := e.symbols[str]; ok { + if err := e.w.WriteByte(typeSymbolLink); err != nil { + return err + } + return e.marshalIntInternal(int64(index)) + } + + e.symbols[str] = len(e.symbols) + + if err := e.w.WriteByte(typeSymbol); err != nil { + return err + } + + if err := e.marshalIntInternal(int64(len(str))); err != nil { + return err + } + + _, err := e.w.WriteString(str) + return err +} + +func (e *MarshalEncoder) marshalArray(arr reflect.Value) error { + if err := e.w.WriteByte(typeArray); err != nil { + return err + } + + len := arr.Len() + + if err := e.marshalIntInternal(int64(len)); err != nil { + return err + } + + for i := 0; i < len; i++ { + if err := e.marshal(arr.Index(i).Interface()); err != nil { + return err + } + } + return nil +} + +func (e *MarshalEncoder) marshalUserMarshal(userMarshal RubyUserMarshal) error { + if err := e.w.WriteByte(typeUserMarshal); err != nil { + return err + } + + if err := e.marshalSymbol(userMarshal.Name); err != nil { + return err + } + + return e.marshal(userMarshal.Value) +} + +func (e *MarshalEncoder) marshalUserDef(userDef RubyUserDef) error { + var buf bytes.Buffer + if err := NewMarshalEncoder(&buf).Encode(userDef.Value); err != nil { + return err + } + + if err := e.w.WriteByte(typeUserDef); err != nil { + return err + } + if err := e.marshalSymbol(userDef.Name); err != nil { + return err + } + if err := e.marshalIntInternal(int64(buf.Len())); err != nil { + return err + } + _, err := e.w.Write(buf.Bytes()) + return err +} + +func (e *MarshalEncoder) marshalObject(obj RubyObject) error { + if err := e.w.WriteByte(typeObject); err != nil { + return err + } + if err := e.marshalSymbol(obj.Name); err != nil { + return err + } + if err := e.marshalIntInternal(int64(len(obj.Member))); err != nil { + return err + } + for k, v := range obj.Member { + if err := e.marshalSymbol(k); err != nil { + return err + } + if err := e.marshal(v); err != nil { + return err + } + } + return nil +} diff --git a/modules/packages/rubygems/marshal_test.go b/modules/packages/rubygems/marshal_test.go new file mode 100644 index 0000000000000..e5963ebcd632c --- /dev/null +++ b/modules/packages/rubygems/marshal_test.go @@ -0,0 +1,99 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMinimalEncoder(t *testing.T) { + cases := []struct { + Value interface{} + Expected []byte + Error error + }{ + { + Value: nil, + Expected: []byte{4, 8, 0x30}, + }, + { + Value: true, + Expected: []byte{4, 8, 'T'}, + }, + { + Value: false, + Expected: []byte{4, 8, 'F'}, + }, + { + Value: 0, + Expected: []byte{4, 8, 'i', 0}, + }, + { + Value: 1, + Expected: []byte{4, 8, 'i', 6}, + }, + { + Value: -1, + Expected: []byte{4, 8, 'i', 0xfa}, + }, + { + Value: 0x1fffffff, + Expected: []byte{4, 8, 'i', 4, 0xff, 0xff, 0xff, 0x1f}, + }, + { + Value: 0x41000000, + Error: ErrInvalidIntRange, + }, + { + Value: "test", + Expected: []byte{4, 8, 'I', '"', 9, 't', 'e', 's', 't', 6, ':', 6, 'E', 'T'}, + }, + { + Value: []int{1, 2}, + Expected: []byte{4, 8, '[', 7, 'i', 6, 'i', 7}, + }, + { + Value: &RubyUserMarshal{ + Name: "Test", + Value: 4, + }, + Expected: []byte{4, 8, 'U', ':', 9, 'T', 'e', 's', 't', 'i', 9}, + }, + { + Value: &RubyUserDef{ + Name: "Test", + Value: 4, + }, + Expected: []byte{4, 8, 'u', ':', 9, 'T', 'e', 's', 't', 9, 4, 8, 'i', 9}, + }, + { + Value: &RubyObject{ + Name: "Test", + Member: map[string]interface{}{ + "test": 4, + }, + }, + Expected: []byte{4, 8, 'o', ':', 9, 'T', 'e', 's', 't', 6, ':', 9, 't', 'e', 's', 't', 'i', 9}, + }, + { + Value: &struct { + Name string + }{ + "test", + }, + Error: ErrUnsupportedType, + }, + } + + for i, c := range cases { + var b bytes.Buffer + err := NewMarshalEncoder(&b).Encode(c.Value) + assert.ErrorIs(t, err, c.Error) + assert.Equal(t, c.Expected, b.Bytes(), "case %d", i) + } +} diff --git a/modules/packages/rubygems/metadata.go b/modules/packages/rubygems/metadata.go new file mode 100644 index 0000000000000..4fe90850bf2f7 --- /dev/null +++ b/modules/packages/rubygems/metadata.go @@ -0,0 +1,215 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "archive/tar" + "compress/gzip" + "errors" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/validation" + + "gopkg.in/yaml.v2" +) + +var ( + // ErrMissingMetadataFile indicates a missing metadata.gz file + ErrMissingMetadataFile = errors.New("Metadata file is missing") + // ErrInvalidName indicates an invalid id in the metadata.gz file + ErrInvalidName = errors.New("Metadata file contains an invalid name") + // ErrInvalidVersion indicates an invalid version in the metadata.gz file + ErrInvalidVersion = errors.New("Metadata file contains an invalid version") +) + +var versionMatcher = regexp.MustCompile(`\A[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z`) + +// Metadata represents the metadata of a RubyGems package +type Metadata struct { + Name string `json:"-"` + Version string `json:"-"` + Platform string `json:"platform"` + Description string `json:"description"` + Summary string `json:"summary"` + Authors []string `json:"authors"` + Licenses []string `json:"licenses"` + RequiredRubyVersion []VersionRequirement `json:"required_ruby_version"` + RequiredRubygemsVersion []VersionRequirement `json:"required_rubygems_version"` + ProjectURL string `json:"project_url"` + RuntimeDependencies []Dependency `json:"runtime_dependencies"` + DevelopmentDependencies []Dependency `json:"development_dependencies"` +} + +// VersionRequirement represents a version restriction +type VersionRequirement struct { + Restriction string `json:"restriction"` + Version string `json:"version"` +} + +// Dependency represents a dependency of a RubyGems package +type Dependency struct { + Name string `json:"name"` + Version []VersionRequirement `json:"version"` +} + +type gemspec struct { + Name string `yaml:"name"` + Version struct { + Version string `yaml:"version"` + } `yaml:"version"` + Platform string `yaml:"platform"` + Authors []string `yaml:"authors"` + Autorequire interface{} `yaml:"autorequire"` + Bindir string `yaml:"bindir"` + CertChain []interface{} `yaml:"cert_chain"` + Date string `yaml:"date"` + Dependencies []struct { + Name string `yaml:"name"` + Requirement requirement `yaml:"requirement"` + Type string `yaml:"type"` + Prerelease bool `yaml:"prerelease"` + VersionRequirements requirement `yaml:"version_requirements"` + } `yaml:"dependencies"` + Description string `yaml:"description"` + Email string `yaml:"email"` + Executables []string `yaml:"executables"` + Extensions []interface{} `yaml:"extensions"` + ExtraRdocFiles []string `yaml:"extra_rdoc_files"` + Files []string `yaml:"files"` + Homepage string `yaml:"homepage"` + Licenses []string `yaml:"licenses"` + Metadata struct { + BugTrackerURI string `yaml:"bug_tracker_uri"` + ChangelogURI string `yaml:"changelog_uri"` + DocumentationURI string `yaml:"documentation_uri"` + SourceCodeURI string `yaml:"source_code_uri"` + } `yaml:"metadata"` + PostInstallMessage interface{} `yaml:"post_install_message"` + RdocOptions []interface{} `yaml:"rdoc_options"` + RequirePaths []string `yaml:"require_paths"` + RequiredRubyVersion requirement `yaml:"required_ruby_version"` + RequiredRubygemsVersion requirement `yaml:"required_rubygems_version"` + Requirements []interface{} `yaml:"requirements"` + RubygemsVersion string `yaml:"rubygems_version"` + SigningKey interface{} `yaml:"signing_key"` + SpecificationVersion int `yaml:"specification_version"` + Summary string `yaml:"summary"` + TestFiles []interface{} `yaml:"test_files"` +} + +type requirement struct { + Requirements [][]interface{} `yaml:"requirements"` +} + +// AsVersionRequirement converts into []VersionRequirement +func (r requirement) AsVersionRequirement() []VersionRequirement { + requirements := make([]VersionRequirement, 0, len(r.Requirements)) + for _, req := range r.Requirements { + if len(req) != 2 { + continue + } + restriction, ok := req[0].(string) + if !ok { + continue + } + vm, ok := req[1].(map[interface{}]interface{}) + if !ok { + continue + } + versionInt, ok := vm["version"] + if !ok { + continue + } + version, ok := versionInt.(string) + if !ok || version == "0" { + continue + } + + requirements = append(requirements, VersionRequirement{ + Restriction: restriction, + Version: version, + }) + } + return requirements +} + +// ParsePackageMetaData parses the metadata of a Gem package file +func ParsePackageMetaData(r io.Reader) (*Metadata, error) { + archive := tar.NewReader(r) + for { + hdr, err := archive.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hdr.Name == "metadata.gz" { + return parseMetadataFile(archive) + } + } + + return nil, ErrMissingMetadataFile +} + +func parseMetadataFile(r io.Reader) (*Metadata, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer zr.Close() + + var spec gemspec + if err := yaml.NewDecoder(zr).Decode(&spec); err != nil { + return nil, err + } + + if len(spec.Name) == 0 || strings.Contains(spec.Name, "/") { + return nil, ErrInvalidName + } + + if !versionMatcher.MatchString(spec.Version.Version) { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(spec.Homepage) { + spec.Homepage = "" + } + if !validation.IsValidURL(spec.Metadata.SourceCodeURI) { + spec.Metadata.SourceCodeURI = "" + } + + m := &Metadata{ + Name: spec.Name, + Version: spec.Version.Version, + Platform: spec.Platform, + Description: spec.Description, + Summary: spec.Summary, + Authors: spec.Authors, + Licenses: spec.Licenses, + ProjectURL: spec.Homepage, + RequiredRubyVersion: spec.RequiredRubyVersion.AsVersionRequirement(), + RequiredRubygemsVersion: spec.RequiredRubygemsVersion.AsVersionRequirement(), + DevelopmentDependencies: make([]Dependency, 0, 5), + RuntimeDependencies: make([]Dependency, 0, 5), + } + + for _, gemdep := range spec.Dependencies { + dep := Dependency{ + Name: gemdep.Name, + Version: gemdep.Requirement.AsVersionRequirement(), + } + if gemdep.Type == ":runtime" { + m.RuntimeDependencies = append(m.RuntimeDependencies, dep) + } else { + m.DevelopmentDependencies = append(m.DevelopmentDependencies, dep) + } + } + + return m, nil +} diff --git a/modules/packages/rubygems/metadata_test.go b/modules/packages/rubygems/metadata_test.go new file mode 100644 index 0000000000000..cf674f1306e4c --- /dev/null +++ b/modules/packages/rubygems/metadata_test.go @@ -0,0 +1,89 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "archive/tar" + "bytes" + "encoding/base64" + "io" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParsePackageMetaData(t *testing.T) { + createArchive := func(filename string, content []byte) io.Reader { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + hdr := &tar.Header{ + Name: filename, + Mode: 0600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + tw.Close() + return &buf + } + + t.Run("MissingMetadataFile", func(t *testing.T) { + data := createArchive("dummy.txt", []byte{0}) + + m, err := ParsePackageMetaData(data) + assert.ErrorIs(t, err, ErrMissingMetadataFile) + assert.Nil(t, m) + }) + + t.Run("Valid", func(t *testing.T) { + content, _ := base64.StdEncoding.DecodeString("H4sICHC/I2EEAG1ldGFkYXRhAAEeAOH/bmFtZTogZwp2ZXJzaW9uOgogIHZlcnNpb246IDEKWw35Tx4AAAA=") + data := createArchive("metadata.gz", content) + + m, err := ParsePackageMetaData(data) + assert.NoError(t, err) + assert.NotNil(t, m) + }) +} + +func TestParseMetadataFile(t *testing.T) { + content, _ := base64.StdEncoding.DecodeString(`H4sIAMe7I2ECA9VVTW/UMBC9+1eYXvaUbJpSQBZUHJAqDlwK4kCFIseZzZrGH9iTqisEv52Js9nd +0KqggiqRXWnX45n3ZuZ5nCzL+JPQ15ulq7+AQnEORoj3HpReaSVRO8usNCB4qxEku4YQySbuCPo4 +bjHOd07HeZGfMt9JXLlgBB9imOxx7UIULOPnCZMMLsDXXgeiYbW2jQ6C0y9TELBSa6kJ6/IzaySS +R1mUx1nxIitPeFGI9M2L6eGfWAMebANWaUgktzN9M3lsKNmxutBb1AYyCibbNhsDFu+q9GK/Tc4z +d2IcLBl9js5eHaXFsLyvXeNz0LQyL/YoLx8EsiCMBZlx46k6sS2PDD5AgA5kJPNKdhH2elWzOv7n +uv9Q9Aau/6ngP84elvNpXh5oRVlB5/yW7BH0+qu0G4gqaI/JdEHBFBS5l+pKtsARIjIwUnfj8Le0 ++TrdJLl2DG5A9SjrjgZ1mG+4QbAD+G4ZZBUap6qVnnzGf6Rwp+vliBRqtnYGPBEKvkb0USyXE8mS +dVoR6hj07u0HZgAl3SRS8G/fmXcRK20jyq6rDMSYQFgidamqkXbbuspLXE/0k7GphtKqe67GuRC/ +yjAbmt9LsOMp8xMamFkSQ38fP5EFjdz8LA4do2C69VvqWXAJgrPbKZb58/xZXrKoW6ttW13Bhvzi +4ftn7/yUxd4YGcglvTmmY8aGY3ZwRn4CqcWcidUGAAA=`) + m, err := parseMetadataFile(bytes.NewReader(content)) + assert.NoError(t, err) + assert.NotNil(t, m) + + assert.Equal(t, "gitea", m.Name) + assert.Equal(t, "1.0.5", m.Version) + assert.Equal(t, "ruby", m.Platform) + assert.Equal(t, "Gitea package", m.Summary) + assert.Equal(t, "RubyGems package test", m.Description) + assert.Equal(t, []string{"Gitea"}, m.Authors) + assert.Equal(t, "https://gitea.io/", m.ProjectURL) + assert.Equal(t, []string{"MIT"}, m.Licenses) + assert.Empty(t, m.RequiredRubygemsVersion) + assert.Len(t, m.RequiredRubyVersion, 1) + assert.Equal(t, ">=", m.RequiredRubyVersion[0].Restriction) + assert.Equal(t, "2.3.0", m.RequiredRubyVersion[0].Version) + assert.Len(t, m.RuntimeDependencies, 1) + assert.Equal(t, "runtime-dep", m.RuntimeDependencies[0].Name) + assert.Len(t, m.RuntimeDependencies[0].Version, 2) + assert.Equal(t, ">=", m.RuntimeDependencies[0].Version[0].Restriction) + assert.Equal(t, "1.2.0", m.RuntimeDependencies[0].Version[0].Version) + assert.Equal(t, "<", m.RuntimeDependencies[0].Version[1].Restriction) + assert.Equal(t, "2.0", m.RuntimeDependencies[0].Version[1].Version) + assert.Len(t, m.DevelopmentDependencies, 1) + assert.Equal(t, "dev-dep", m.DevelopmentDependencies[0].Name) + assert.Len(t, m.DevelopmentDependencies[0].Version, 1) + assert.Equal(t, "~>", m.DevelopmentDependencies[0].Version[0].Restriction) + assert.Equal(t, "5.2", m.DevelopmentDependencies[0].Version[0].Version) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 2cc59e579e391..67403122eb543 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1516,6 +1516,7 @@ packages.filter.package_type.all = All packages.published_by = Published %[1]s by %[3]s packages.installation = Installation packages.about = About this package +packages.requirements = Requirements packages.dependencies = Dependencies packages.details = Details packages.details.author = Author @@ -1542,6 +1543,11 @@ packages.npm.documentation = For more information on the npm registry, see the documentation. +packages.rubygems.use_1 = Install from the command line +packages.rubygems.use_2 = Install via Gemfile +packages.rubygems.required.ruby = Requires Ruby version +packages.rubygems.required.rubygems = Requires RubyGem version +packages.rubygems.documentation = For more information on the RubyGems registry, see the documentation. signing.will_sign = This commit will be signed with key '%s' signing.wont_sign.error = There was an error whilst checking if the commit could be signed diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c49773063c518..46127f969a957 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -84,6 +84,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/packages/npm" "code.gitea.io/gitea/routers/api/v1/packages/nuget" "code.gitea.io/gitea/routers/api/v1/packages/pypi" + "code.gitea.io/gitea/routers/api/v1/packages/rubygems" "code.gitea.io/gitea/routers/api/v1/repo" "code.gitea.io/gitea/routers/api/v1/settings" _ "code.gitea.io/gitea/routers/api/v1/swagger" // for swagger generation @@ -1021,6 +1022,17 @@ func Routes() *web.Route { m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) m.Get("/simple/{id}", pypi.PackageMetadata) }, reqBasicAuth()) + m.Group("/rubygems", func() { + m.Get("/specs.4.8.gz", rubygems.EnumeratePackages) + m.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) + m.Get("/prerelease_specs.4.8.gz", rubygems.EnumeratePackagesPreRelease) + m.Get("/quick/Marshal.4.8/{filename}", rubygems.ServePackageSpecification) + m.Get("/gems/{filename}", rubygems.DownloadPackageFile) + m.Group("/api/v1/gems", func() { + m.Post("/", rubygems.UploadPackageFile) + m.Delete("/yank", rubygems.DeletePackage) + }, reqRepoWriter(models.UnitTypePackages)) + }, reqToken()) m.Group("/{id}", func() { m.Get("/", repo.GetPackage) m.Delete("/", reqRepoWriter(models.UnitTypePackages), repo.DeletePackage) diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 6bf1a92cbc673..beda583bb14d4 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -228,7 +228,7 @@ func DeletePackage(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") - err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) + err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, models.PackageNuGet, packageName, packageVersion) if err != nil { if err == models.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) diff --git a/routers/api/v1/packages/rubygems/package.go b/routers/api/v1/packages/rubygems/package.go new file mode 100644 index 0000000000000..cf978c621a45d --- /dev/null +++ b/routers/api/v1/packages/rubygems/package.go @@ -0,0 +1,75 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/json" + rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" +) + +// Package represents a package with RubyGems metadata +type Package struct { + *models.Package + Metadata *rubygems_module.Metadata +} + +func intializePackages(packages []*models.Package) ([]*Package, error) { + pgs := make([]*Package, 0, len(packages)) + for _, p := range packages { + np, err := intializePackage(p) + if err != nil { + return nil, err + } + pgs = append(pgs, np) + } + return pgs, nil +} + +func intializePackage(p *models.Package) (*Package, error) { + var m *rubygems_module.Metadata + if err := json.Unmarshal([]byte(p.MetadataRaw), &m); err != nil { + return nil, err + } + if m == nil { + m = &rubygems_module.Metadata{} + } + + return &Package{ + Package: p, + Metadata: m, + }, nil +} + +// AsSpecification creates a Ruby Gem::Specification object used by ServePackageSpecification +func (p *Package) AsSpecification() *rubygems_module.RubyUserDef { + return &rubygems_module.RubyUserDef{ + Name: "Gem::Specification", + Value: []interface{}{ + "3.2.3", // @rubygems_version + 4, // @specification_version, + p.Name, + &rubygems_module.RubyUserMarshal{ + Name: "Gem::Version", + Value: []string{p.Version}, + }, + nil, // date + p.Metadata.Summary, // @summary + nil, // @required_ruby_version + nil, // @required_rubygems_version + p.Metadata.Platform, // @original_platform + []interface{}{}, // @dependencies + nil, // rubyforge_project + "", // @email + p.Metadata.Authors, + p.Metadata.Description, + p.Metadata.ProjectURL, + true, // has_rdoc + p.Metadata.Platform, // @new_platform + nil, + p.Metadata.Licenses, + }, + } +} diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/v1/packages/rubygems/rubygems.go new file mode 100644 index 0000000000000..5483dba9fbe0a --- /dev/null +++ b/routers/api/v1/packages/rubygems/rubygems.go @@ -0,0 +1,229 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package rubygems + +import ( + "compress/gzip" + "compress/zlib" + "fmt" + "io" + "net/http" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" + "code.gitea.io/gitea/modules/util/filebuffer" + + package_service "code.gitea.io/gitea/services/packages" +) + +// EnumeratePackages serves the package list +func EnumeratePackages(ctx *context.APIContext) { + packages, err := models.GetPackagesByRepositoryAndType(ctx.Repo.Repository.ID, models.PackageRubyGems) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + enumeratePackages(ctx, "specs.4.8", packages) +} + +// EnumeratePackagesLatest serves the list of the lastest version of every package +func EnumeratePackagesLatest(ctx *context.APIContext) { + packages, _, err := models.GetLatestPackagesGrouped(&models.PackageSearchOptions{ + RepoID: ctx.Repo.Repository.ID, + Type: "rubygems", + }) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + enumeratePackages(ctx, "latest_specs.4.8", packages) +} + +// EnumeratePackagesPreRelease is not supported and serves an empty list +func EnumeratePackagesPreRelease(ctx *context.APIContext) { + enumeratePackages(ctx, "prerelease_specs.4.8", []*models.Package{}) +} + +func enumeratePackages(ctx *context.APIContext, filename string, packages []*models.Package) { + rubygemsPackages, err := intializePackages(packages) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + specs := make([]interface{}, 0, len(rubygemsPackages)) + for _, p := range rubygemsPackages { + specs = append(specs, []interface{}{ + p.Name, + &rubygems_module.RubyUserMarshal{ + Name: "Gem::Version", + Value: []string{p.Version}, + }, + p.Metadata.Platform, + }) + } + + ctx.SetServeHeaders(filename + ".gz") + + zw := gzip.NewWriter(ctx.Resp) + defer zw.Close() + + zw.Name = filename + + if err := rubygems_module.NewMarshalEncoder(zw).Encode(specs); err != nil { + ctx.ServerError("Download file failed", err) + } +} + +// ServePackageSpecification serves the compressed Gemspec file of a package +func ServePackageSpecification(ctx *context.APIContext) { + filename := ctx.Params("filename") + + if !strings.HasSuffix(filename, ".gemspec.rz") { + ctx.Error(http.StatusBadRequest, "", nil) + return + } + + packages, err := models.GetPackagesByFilename(ctx.Repo.Repository.ID, models.PackageRubyGems, filename[:len(filename)-10]+"gem") + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + if len(packages) != 1 { + ctx.Error(http.StatusNotFound, "", nil) + return + } + + p, err := intializePackage(packages[0]) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + ctx.SetServeHeaders(filename) + + zw := zlib.NewWriter(ctx.Resp) + defer zw.Close() + + spec := p.AsSpecification() + + if err := rubygems_module.NewMarshalEncoder(zw).Encode(spec); err != nil { + ctx.ServerError("Download file failed", err) + } +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.APIContext) { + filename := ctx.Params("filename") + + packages, err := models.GetPackagesByFilename(ctx.Repo.Repository.ID, models.PackageRubyGems, filename) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + if len(packages) != 1 { + ctx.Error(http.StatusNotFound, "", nil) + return + } + + s, pf, err := package_service.GetPackageFileStream(packages[0], filename) + if err != nil { + if err == models.ErrPackageFileNotExist { + ctx.Error(http.StatusNotFound, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackageFile adds a file to the package. If the package does not exist, it gets created. +func UploadPackageFile(ctx *context.APIContext) { + upload, close, err := ctx.UploadStream() + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + if close { + defer upload.Close() + } + + buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + defer buf.Close() + + meta, err := rubygems_module.ParsePackageMetaData(buf) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + if _, err := buf.Seek(0, io.SeekStart); err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + p, err := package_service.CreatePackage( + ctx.User, + ctx.Repo.Repository, + models.PackageRubyGems, + meta.Name, + meta.Version, + meta, + false, + ) + if err != nil { + if err == models.ErrDuplicatePackage { + ctx.Error(http.StatusBadRequest, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + var filename string + if len(meta.Platform) == 0 || meta.Platform == "ruby" { + filename = strings.ToLower(fmt.Sprintf("%s-%s.gem", meta.Name, meta.Version)) + } else { + filename = strings.ToLower(fmt.Sprintf("%s-%s-%s.gem", meta.Name, meta.Version, meta.Platform)) + } + _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) + if err != nil { + if err := models.DeletePackageByID(p.ID); err != nil { + log.Error("Error deleting package by id: %v", err) + } + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + ctx.PlainText(http.StatusCreated, nil) +} + +// DeletePackage deletes a package +func DeletePackage(ctx *context.APIContext) { + packageName := ctx.FormString("gem_name") + packageVersion := ctx.FormString("version") + + err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, models.PackageRubyGems, packageName, packageVersion) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.Error(http.StatusNotFound, "", err) + return + } + ctx.Error(http.StatusInternalServerError, "", "") + } +} diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index 49421e76d4be9..f66fff9c25332 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/packages/nuget" "code.gitea.io/gitea/modules/packages/pypi" + "code.gitea.io/gitea/modules/packages/rubygems" "code.gitea.io/gitea/modules/setting" package_service "code.gitea.io/gitea/services/packages" @@ -114,6 +115,8 @@ func ViewPackage(ctx *context.Context) { metadata = &maven.Metadata{} case models.PackagePyPI: metadata = &pypi.Metadata{} + case models.PackageRubyGems: + metadata = &rubygems.Metadata{} } if metadata != nil { if err := json.Unmarshal([]byte(p.MetadataRaw), &metadata); err != nil { diff --git a/services/packages/packages.go b/services/packages/packages.go index 032e8bf70afbf..9abfc5f3d6661 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -185,7 +185,7 @@ func GetFileStreamByPackageNameAndVersion(repository *models.Repository, package return nil, nil, err } - return getPackageFileStream(p, filename) + return GetPackageFileStream(p, filename) } // GetFileStreamByPackageID returns the content of the specific package file @@ -205,10 +205,11 @@ func GetFileStreamByPackageID(repository *models.Repository, packageID int64, fi return nil, nil, models.ErrPackageNotExist } - return getPackageFileStream(p, filename) + return GetPackageFileStream(p, filename) } -func getPackageFileStream(p *models.Package, filename string) (io.ReadCloser, *models.PackageFile, error) { +// GetPackageFileStream returns the cotent of the specific package file +func GetPackageFileStream(p *models.Package, filename string) (io.ReadCloser, *models.PackageFile, error) { pf, err := p.GetFileByName(filename) if err != nil { return nil, nil, err diff --git a/templates/repo/packages/content/maven.tmpl b/templates/repo/packages/content/maven.tmpl index 0a81641ac48bb..cfe721f9bc70e 100644 --- a/templates/repo/packages/content/maven.tmpl +++ b/templates/repo/packages/content/maven.tmpl @@ -51,10 +51,16 @@ {{if .Metadata.Dependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    -
    - {{range .Metadata.Dependencies}} -
    {{.GroupID}}:{{.ArtifactID}} ({{.Version}})
    - {{end}} +
    + {{range .Metadata.Dependencies}} +
    + {{svg "octicon-package-dependencies" 16 ""}} +
    +
    {{.GroupID}}:{{.ArtifactID}}
    +
    {{.Version}}
    +
    +
    + {{end}}
    {{end}} diff --git a/templates/repo/packages/content/npm.tmpl b/templates/repo/packages/content/npm.tmpl index a756fa2b19e55..47557feeb7be5 100644 --- a/templates/repo/packages/content/npm.tmpl +++ b/templates/repo/packages/content/npm.tmpl @@ -30,10 +30,16 @@ {{if .Metadata.Dependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    -
    - {{range $dependency, $version := .Metadata.Dependencies}} -
    {{$dependency}} ({{$version}})
    - {{end}} +
    + {{range $dependency, $version := .Metadata.Dependencies}} +
    + {{svg "octicon-package-dependencies" 16 ""}} +
    +
    {{$dependency}}
    +
    {{$version}}
    +
    +
    + {{end}}
    {{end}} diff --git a/templates/repo/packages/content/nuget.tmpl b/templates/repo/packages/content/nuget.tmpl index e4d931402149e..989338b12fcd6 100644 --- a/templates/repo/packages/content/nuget.tmpl +++ b/templates/repo/packages/content/nuget.tmpl @@ -25,14 +25,26 @@ {{if .Metadata.Dependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    - {{range $framework, $dependencies := .Metadata.Dependencies}} -
    -
    {{$framework}}
    - {{range $dependencies}} -
    {{.ID}} ({{.Version}})
    +
    + {{range $framework, $dependencies := .Metadata.Dependencies}} +
    +
    +
    {{$framework}}
    +
    + {{range $dependencies}} +
    + {{svg "octicon-package-dependencies" 16 ""}} +
    +
    {{.ID}}
    +
    {{.Version}}
    +
    +
    + {{end}} +
    +
    +
    {{end}}
    - {{end}}
    {{end}} {{end}} diff --git a/templates/repo/packages/content/pypi.tmpl b/templates/repo/packages/content/pypi.tmpl index 15d61d4b34492..5c8ed26df9b01 100644 --- a/templates/repo/packages/content/pypi.tmpl +++ b/templates/repo/packages/content/pypi.tmpl @@ -9,7 +9,6 @@
    {{.i18n.Tr "repo.packages.pypi.documentation" | Safe}}
    - {{if or .Metadata.Description .Metadata.LongDescription .Metadata.Summary}}

    {{.i18n.Tr "repo.packages.about"}}

    @@ -21,4 +20,10 @@ {{end}}
    {{end}} + {{if .Metadata.RequiresPython}} +

    {{.i18n.Tr "repo.packages.requirements"}}

    +
    + {{.i18n.Tr "repo.packages.pypi.requires"}}: {{.Metadata.RequiresPython}} +
    + {{end}} {{end}} diff --git a/templates/repo/packages/content/rubygems.tmpl b/templates/repo/packages/content/rubygems.tmpl new file mode 100644 index 0000000000000..85a0d307aa737 --- /dev/null +++ b/templates/repo/packages/content/rubygems.tmpl @@ -0,0 +1,76 @@ +{{if eq .Package.Type 5}} +

    {{.i18n.Tr "repo.packages.installation"}}

    +
    +
    + +
    + +
    + +
    + +
    +
    + {{.i18n.Tr "repo.packages.rubygems.documentation" | Safe}} +
    + {{if .Metadata.Description}} +

    {{.i18n.Tr "repo.packages.about"}}

    +
    + {{if .Metadata.Description}} + {{.Metadata.Description}} + {{end}} +
    + {{end}} + {{if or .Metadata.RequiredRubyVersion .Metadata.RequiredRubygemsVersion}} +

    {{.i18n.Tr "repo.packages.requirements"}}

    +
    + {{if .Metadata.RequiredRubyVersion}}

    {{.i18n.Tr "repo.packages.rubygems.required.ruby"}}: {{range $i, $v := .Metadata.RequiredRubyVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}} + {{if .Metadata.RequiredRubygemsVersion}}

    {{.i18n.Tr "repo.packages.rubygems.required.rubygems"}}: {{range $i, $v := .Metadata.RequiredRubygemsVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}} +
    + {{end}} + {{if or .Metadata.RuntimeDependencies .Metadata.DevelopmentDependencies}} +

    {{.i18n.Tr "repo.packages.dependencies"}}

    +
    +
    + {{if .Metadata.RuntimeDependencies}} +
    +
    +
    Runtime Dependencies
    +
    + {{range .Metadata.RuntimeDependencies}} +
    + {{svg "octicon-package-dependencies" 16 ""}} +
    +
    {{.Name}}
    +
    {{range $i, $v := .Version}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}
    +
    +
    + {{end}} +
    +
    +
    + {{end}} + {{if .Metadata.DevelopmentDependencies}} +
    +
    +
    Development Dependencies
    +
    + {{range .Metadata.DevelopmentDependencies}} +
    + {{svg "octicon-package-dependencies" 16 ""}} +
    +
    {{.Name}}
    +
    {{range $i, $v := .Version}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}
    +
    +
    + {{end}} +
    +
    +
    + {{end}} +
    +
    + {{end}} +{{end}} diff --git a/templates/repo/packages/list.tmpl b/templates/repo/packages/list.tmpl index 60338166afbd7..8ea24fd73fa98 100644 --- a/templates/repo/packages/list.tmpl +++ b/templates/repo/packages/list.tmpl @@ -27,6 +27,7 @@ npm Maven PyPI + RubyGems
    diff --git a/templates/repo/packages/metadata/maven.tmpl b/templates/repo/packages/metadata/maven.tmpl index b7015be23c34e..c9566b398f5cc 100644 --- a/templates/repo/packages/metadata/maven.tmpl +++ b/templates/repo/packages/metadata/maven.tmpl @@ -1,10 +1,5 @@ {{if eq .Package.Type 3}} {{if .Metadata.Name}}
    {{svg "octicon-note" 16 "mr-3"}} {{.Metadata.Name}}
    {{end}} {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{if .Metadata.Licenses}} -
    {{.i18n.Tr "repo.packages.details.license"}}:
    - {{range .Metadata.Licenses}} -
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    - {{end}} - {{end}} + {{range .Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/npm.tmpl b/templates/repo/packages/metadata/npm.tmpl index 381aab099e0df..a05a84356f69a 100644 --- a/templates/repo/packages/metadata/npm.tmpl +++ b/templates/repo/packages/metadata/npm.tmpl @@ -1,5 +1,5 @@ {{if eq .Package.Type 2}} - {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Author}}
    {{end}} + {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.Metadata.Author}}
    {{end}} {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} + {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.Metadata.License}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/nuget.tmpl b/templates/repo/packages/metadata/nuget.tmpl index 5e9c0da13d380..d9cb79ec388c3 100644 --- a/templates/repo/packages/metadata/nuget.tmpl +++ b/templates/repo/packages/metadata/nuget.tmpl @@ -1,4 +1,4 @@ {{if eq .Package.Type 1}} - {{if .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Authors}}
    {{end}} + {{if .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.Metadata.Authors}}
    {{end}} {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/pypi.tmpl b/templates/repo/packages/metadata/pypi.tmpl index 25ec8202bcc08..316f379255a5e 100644 --- a/templates/repo/packages/metadata/pypi.tmpl +++ b/templates/repo/packages/metadata/pypi.tmpl @@ -1,6 +1,5 @@ {{if eq .Package.Type 4}} - {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.author"}}: {{.Metadata.Author}}
    {{end}} + {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.Metadata.Author}}
    {{end}} {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.license"}}: {{.Metadata.License}}
    {{end}} - {{if .Metadata.RequiresPython}}
    {{.i18n.Tr "repo.packages.pypi.requires"}}: {{.Metadata.RequiresPython}}
    {{end}} -{{end}} + {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.Metadata.License}}
    {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/metadata/rubygems.tmpl b/templates/repo/packages/metadata/rubygems.tmpl new file mode 100644 index 0000000000000..3474204a80c03 --- /dev/null +++ b/templates/repo/packages/metadata/rubygems.tmpl @@ -0,0 +1,5 @@ +{{if eq .Package.Type 5}} + {{range .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.}}
    {{end}} + {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{range .Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} +{{end}} \ No newline at end of file diff --git a/templates/repo/packages/view.tmpl b/templates/repo/packages/view.tmpl index b55ce57339eec..239c183f62918 100644 --- a/templates/repo/packages/view.tmpl +++ b/templates/repo/packages/view.tmpl @@ -18,6 +18,7 @@ {{template "repo/packages/content/npm" .}} {{template "repo/packages/content/maven" .}} {{template "repo/packages/content/pypi" .}} + {{template "repo/packages/content/rubygems" .}}
    @@ -31,6 +32,7 @@ {{template "repo/packages/metadata/npm" .}} {{template "repo/packages/metadata/maven" .}} {{template "repo/packages/metadata/pypi" .}} + {{template "repo/packages/metadata/rubygems" .}}
    {{.i18n.Tr "repo.packages.assets"}} From c060e4a7d39a0a6bb93b7aae04aac9ac6a26924f Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 23 Aug 2021 20:10:00 +0000 Subject: [PATCH 036/130] Fix lint. --- templates/repo/packages/metadata/pypi.tmpl | 2 +- templates/repo/packages/metadata/rubygems.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/repo/packages/metadata/pypi.tmpl b/templates/repo/packages/metadata/pypi.tmpl index 316f379255a5e..c8215bffe73b0 100644 --- a/templates/repo/packages/metadata/pypi.tmpl +++ b/templates/repo/packages/metadata/pypi.tmpl @@ -2,4 +2,4 @@ {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.Metadata.Author}}
    {{end}} {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.Metadata.License}}
    {{end}} -{{end}} \ No newline at end of file +{{end}} diff --git a/templates/repo/packages/metadata/rubygems.tmpl b/templates/repo/packages/metadata/rubygems.tmpl index 3474204a80c03..f3937d7324ebf 100644 --- a/templates/repo/packages/metadata/rubygems.tmpl +++ b/templates/repo/packages/metadata/rubygems.tmpl @@ -2,4 +2,4 @@ {{range .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.}}
    {{end}} {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} {{range .Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} -{{end}} \ No newline at end of file +{{end}} From cd1bc10fa35ea050de089554e6e03dde9fceee24 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 26 Aug 2021 18:06:00 +0200 Subject: [PATCH 037/130] Implemented io.Writer. --- modules/util/filebuffer/file_backed_buffer.go | 128 ++++++++++++------ 1 file changed, 90 insertions(+), 38 deletions(-) diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index 136f7d661e008..3339f373bf072 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -9,51 +9,94 @@ import ( "errors" "io" "io/ioutil" - "math" "os" ) +const maxInt = int(^uint(0) >> 1) // taken from bytes.Buffer + var ( // ErrInvalidMemorySize occurs if the memory size is not in a valid range ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") + // ErrWriteAfterRead occurs if Write is called after a read operation + ErrWriteAfterRead = errors.New("Write is unsupported after a read operation") ) -// FileBackedBuffer implements io.ReadSeekCloser and io.ReaderAt +type readAtSeeker interface { + io.ReadSeeker + io.ReaderAt +} + +// FileBackedBuffer uses a memory buffer with a fixed size. +// If more data is written a temporary file is used instead. +// It implements io.ReadWriteCloser, io.ReadSeekCloser and io.ReaderAt type FileBackedBuffer struct { - size int64 - buffer *bytes.Reader - tmpFile *os.File + maxMemorySize int64 + size int64 + buffer bytes.Buffer + file *os.File + reader readAtSeeker } -// CreateFromReader creates a file backed buffer which uses a buffer with a fixed memory size. -// If more data is available a temporary file is used to store the data. -func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) { - if maxMemorySize < 0 || maxMemorySize == math.MaxInt32 { +// New creates a file backed buffer with a specific maximum memory size +func New(maxMemorySize int) (*FileBackedBuffer, error) { + if maxMemorySize < 0 || maxMemorySize > maxInt { return nil, ErrInvalidMemorySize } - var buf bytes.Buffer - n, err := io.CopyN(&buf, r, int64(maxMemorySize+1)) - if err == io.EOF { - return &FileBackedBuffer{ - size: n, - buffer: bytes.NewReader(buf.Bytes()), - }, nil - } - file, err := ioutil.TempFile("", "buffer-") + return &FileBackedBuffer{ + maxMemorySize: int64(maxMemorySize), + }, nil +} + +// CreateFromReader creates a file backed buffer and copies the provided reader data into it. +func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) { + b, err := New(maxMemorySize) if err != nil { return nil, err } - n, err = io.Copy(file, io.MultiReader(&buf, r)) + _, err = io.Copy(b, r) if err != nil { return nil, err } - return &FileBackedBuffer{ - size: n, - tmpFile: file, - }, nil + return b, nil +} + +// Write implements io.Writer +func (b *FileBackedBuffer) Write(p []byte) (int, error) { + if b.reader != nil { + return 0, ErrWriteAfterRead + } + + var n int + var err error + + if b.file != nil { + n, err = b.file.Write(p) + } else { + if b.size+int64(len(p)) > b.maxMemorySize { + b.file, err = ioutil.TempFile("", "buffer-") + if err != nil { + return 0, err + } + + _, err = io.Copy(b.file, &b.buffer) + if err != nil { + return 0, err + } + + return b.Write(p) + } + + n, err = b.buffer.Write(p) + } + + if err != nil { + return n, err + } + b.size += int64(n) + return n, nil } // Size returns the byte size of the buffered data @@ -61,35 +104,44 @@ func (b *FileBackedBuffer) Size() int64 { return b.size } +func (b *FileBackedBuffer) switchToReader() { + if b.reader != nil { + return + } + + if b.file != nil { + b.reader = b.file + } else { + b.reader = bytes.NewReader(b.buffer.Bytes()) + } +} + // Read implements io.Reader func (b *FileBackedBuffer) Read(p []byte) (int, error) { - if b.tmpFile != nil { - return b.tmpFile.Read(p) - } - return b.buffer.Read(p) + b.switchToReader() + + return b.reader.Read(p) } // ReadAt implements io.ReaderAt func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) { - if b.tmpFile != nil { - return b.tmpFile.ReadAt(p, off) - } - return b.buffer.ReadAt(p, off) + b.switchToReader() + + return b.reader.ReadAt(p, off) } // Seek implements io.Seeker func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { - if b.tmpFile != nil { - return b.tmpFile.Seek(offset, whence) - } - return b.buffer.Seek(offset, whence) + b.switchToReader() + + return b.reader.Seek(offset, whence) } // Close implements io.Closer func (b *FileBackedBuffer) Close() error { - if b.tmpFile != nil { - err := b.tmpFile.Close() - os.Remove(b.tmpFile.Name()) + if b.file != nil { + err := b.file.Close() + os.Remove(b.file.Name()) return err } return nil From 2c9fa64828f38e452906739a6c42fa3185869aeb Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 1 Sep 2021 18:48:00 +0200 Subject: [PATCH 038/130] Added support for sha256/sha512 checksum files. --- routers/api/v1/packages/maven/maven.go | 66 ++++++++++++++++---------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index c47b7700f3b30..67dfdcf807098 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -105,7 +105,7 @@ func servePackageFile(ctx *context.APIContext, params parameters) { filename := params.Filename ext := strings.ToLower(filepath.Ext(filename)) - if ext == ".sha1" || ext == ".md5" { + if isChecksumExtension(ext) { filename = filename[:len(filename)-len(ext)] } @@ -120,14 +120,22 @@ func servePackageFile(ctx *context.APIContext, params parameters) { return } - if ext == ".sha1" { - ctx.PlainText(http.StatusOK, []byte(pf.HashSHA1)) - return - } - if ext == ".md5" { - ctx.PlainText(http.StatusOK, []byte(pf.HashMD5)) + if isChecksumExtension(ext) { + var hash string + switch ext { + case ".sha512": + hash = pf.HashSHA512 + case ".sha256": + hash = pf.HashSHA256 + case ".sha1": + hash = pf.HashSHA1 + case ".md5": + hash = pf.HashMD5 + } + ctx.PlainText(http.StatusOK, []byte(hash)) return } + s, err := packages.NewContentStore().Get(p.ID, pf.ID) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) @@ -180,29 +188,31 @@ func UploadPackageFile(ctx *context.APIContext) { ext := filepath.Ext(params.Filename) - if ext == ".sha1" || ext == ".md5" { - if ext == ".sha1" { - pf, err := p.GetFileByName(params.Filename[:len(params.Filename)-5]) - if err != nil { - if err == models.ErrPackageFileNotExist { - ctx.Error(http.StatusNotFound, "", "") - return - } - log.Error("GetFileByName: %v", err) - ctx.Error(http.StatusInternalServerError, "", err) + // Do not upload checksum files but compare the hashes. + if isChecksumExtension(ext) { + pf, err := p.GetFileByName(params.Filename[:len(params.Filename)-5]) + if err != nil { + if err == models.ErrPackageFileNotExist { + ctx.Error(http.StatusNotFound, "", "") return } + log.Error("GetFileByName: %v", err) + ctx.Error(http.StatusInternalServerError, "", err) + return + } - hash, err := ioutil.ReadAll(buf) - if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) - return - } + hash, err := ioutil.ReadAll(buf) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } - if pf.HashSHA1 != string(hash) { - ctx.Error(http.StatusBadRequest, "", "hash mismatch") - return - } + if (ext == ".md5" && pf.HashMD5 != string(hash)) || + (ext == ".sha1" && pf.HashSHA1 != string(hash)) || + (ext == ".sha256" && pf.HashSHA256 != string(hash)) || + (ext == ".sha512" && pf.HashSHA512 != string(hash)) { + ctx.Error(http.StatusBadRequest, "", "hash mismatch") + return } ctx.PlainText(http.StatusOK, nil) @@ -246,6 +256,10 @@ func UploadPackageFile(ctx *context.APIContext) { ctx.PlainText(http.StatusCreated, nil) } +func isChecksumExtension(ext string) bool { + return ext == ".sha512" || ext == ".sha256" || ext == ".sha1" || ext == ".md5" +} + type parameters struct { GroupID string ArtifactID string From 41efc7c29ac720628c149dcf25d368f6742440af Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 2 Sep 2021 13:09:12 +0000 Subject: [PATCH 039/130] Improved maven-metadata.xml support. --- routers/api/v1/packages/maven/api.go | 15 ++- routers/api/v1/packages/maven/maven.go | 125 +++++++++++++---------- routers/api/v1/packages/maven/package.go | 5 +- 3 files changed, 89 insertions(+), 56 deletions(-) diff --git a/routers/api/v1/packages/maven/api.go b/routers/api/v1/packages/maven/api.go index e00c08ceeb8a0..60eeaff3f5372 100644 --- a/routers/api/v1/packages/maven/api.go +++ b/routers/api/v1/packages/maven/api.go @@ -6,6 +6,7 @@ package maven import ( "encoding/xml" + "strings" ) // MetadataResponse https://maven.apache.org/ref/3.2.5/maven-repository-metadata/repository-metadata.html @@ -13,24 +14,34 @@ type MetadataResponse struct { XMLName xml.Name `xml:"metadata"` GroupID string `xml:"groupId"` ArtifactID string `xml:"artifactId"` + Release string `xml:"versioning>release,omitempty"` Latest string `xml:"versioning>latest"` Version []string `xml:"versioning>versions>version"` } func createMetadataResponse(packages []*Package) *MetadataResponse { - sortedPackages := sortPackagesByVersionASC(packages) + sortedPackages := sortPackagesByCreationASC(packages) + + var release *Package versions := make([]string, 0, len(sortedPackages)) for _, p := range sortedPackages { + if !strings.HasSuffix(p.Version, "-SNAPSHOT") { + release = p + } versions = append(versions, p.Version) } latest := sortedPackages[len(sortedPackages)-1] - return &MetadataResponse{ + resp := &MetadataResponse{ GroupID: latest.Metadata.GroupID, ArtifactID: latest.Metadata.ArtifactID, Latest: latest.Version, Version: versions, } + if release != nil { + resp.Release = release.Version + } + return resp } diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 67dfdcf807098..019511a476f9a 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -7,6 +7,8 @@ package maven import ( "crypto/md5" "crypto/sha1" + "crypto/sha256" + "crypto/sha512" "encoding/xml" "errors" "fmt" @@ -28,7 +30,13 @@ import ( package_service "code.gitea.io/gitea/services/packages" ) -const mavenMetadataFile = "maven-metadata.xml" +const ( + mavenMetadataFile = "maven-metadata.xml" + extensionMD5 = ".md5" + extensionSHA1 = ".sha1" + extensionSHA256 = ".sha256" + extensionSHA512 = ".sha512" +) var ( errInvalidParameters = errors.New("Request parameters are invalid") @@ -43,7 +51,7 @@ func DownloadPackageFile(ctx *context.APIContext) { return } - if params.IsMeta { + if params.IsMeta && params.Version == "" { serveMavenMetadata(ctx, params) } else { servePackageFile(ctx, params) @@ -51,46 +59,54 @@ func DownloadPackageFile(ctx *context.APIContext) { } func serveMavenMetadata(ctx *context.APIContext, params parameters) { - if params.Version == "" { - // /com/foo/project/maven-metadata.xml[.sha1/.md5] + // /com/foo/project/maven-metadata.xml[.md5/.sha1/.sha256/.sha512] - packageName := params.GroupID + "-" + params.ArtifactID - packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageMaven, packageName) - if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) - return - } - if len(packages) == 0 { - ctx.Error(http.StatusNotFound, "", err) - return - } + packageName := params.GroupID + "-" + params.ArtifactID + packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageMaven, packageName) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + if len(packages) == 0 { + ctx.Error(http.StatusNotFound, "", "") + return + } - mavenPackages, err := intializePackages(packages) - if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) - return - } + mavenPackages, err := intializePackages(packages) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } - xmlMetadata, err := xml.Marshal(createMetadataResponse(mavenPackages)) - if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) - return - } - xmlMetadataWithHeader := append([]byte(xml.Header), xmlMetadata...) - - switch strings.ToLower(filepath.Ext(params.Filename)) { - case ".sha1": - ctx.PlainText(http.StatusOK, []byte(fmt.Sprintf("%x", sha1.Sum(xmlMetadataWithHeader)))) - case ".md5": - ctx.PlainText(http.StatusOK, []byte(fmt.Sprintf("%x", md5.Sum(xmlMetadataWithHeader)))) - default: - ctx.PlainText(http.StatusOK, xmlMetadataWithHeader) - } - } else { - // /com/foo/project/1-SNAPSHOT/maven-metadata.xml[.sha1/.md5] + xmlMetadata, err := xml.Marshal(createMetadataResponse(mavenPackages)) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + xmlMetadataWithHeader := append([]byte(xml.Header), xmlMetadata...) - ctx.Error(http.StatusNotFound, "", "") + ext := strings.ToLower(filepath.Ext(params.Filename)) + if isChecksumExtension(ext) { + var hash []byte + switch ext { + case extensionMD5: + tmp := md5.Sum(xmlMetadataWithHeader) + hash = tmp[:] + case extensionSHA1: + tmp := sha1.Sum(xmlMetadataWithHeader) + hash = tmp[:] + case extensionSHA256: + tmp := sha256.Sum256(xmlMetadataWithHeader) + hash = tmp[:] + case extensionSHA512: + tmp := sha512.Sum512(xmlMetadataWithHeader) + hash = tmp[:] + } + ctx.PlainText(http.StatusOK, []byte(fmt.Sprintf("%x", hash))) + return } + + ctx.PlainText(http.StatusOK, xmlMetadataWithHeader) } func servePackageFile(ctx *context.APIContext, params parameters) { @@ -123,14 +139,14 @@ func servePackageFile(ctx *context.APIContext, params parameters) { if isChecksumExtension(ext) { var hash string switch ext { - case ".sha512": - hash = pf.HashSHA512 - case ".sha256": - hash = pf.HashSHA256 - case ".sha1": - hash = pf.HashSHA1 - case ".md5": + case extensionMD5: hash = pf.HashMD5 + case extensionSHA1: + hash = pf.HashSHA1 + case extensionSHA256: + hash = pf.HashSHA256 + case extensionSHA512: + hash = pf.HashSHA512 } ctx.PlainText(http.StatusOK, []byte(hash)) return @@ -155,7 +171,8 @@ func UploadPackageFile(ctx *context.APIContext) { log.Trace("Parameters: %+v", params) - if params.IsMeta { + // Ignore the package index //maven-metadata.xml + if params.IsMeta && params.Version == "" { ctx.PlainText(http.StatusOK, nil) return } @@ -190,7 +207,7 @@ func UploadPackageFile(ctx *context.APIContext) { // Do not upload checksum files but compare the hashes. if isChecksumExtension(ext) { - pf, err := p.GetFileByName(params.Filename[:len(params.Filename)-5]) + pf, err := p.GetFileByName(params.Filename[:len(params.Filename)-len(ext)]) if err != nil { if err == models.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", "") @@ -207,10 +224,10 @@ func UploadPackageFile(ctx *context.APIContext) { return } - if (ext == ".md5" && pf.HashMD5 != string(hash)) || - (ext == ".sha1" && pf.HashSHA1 != string(hash)) || - (ext == ".sha256" && pf.HashSHA256 != string(hash)) || - (ext == ".sha512" && pf.HashSHA512 != string(hash)) { + if (ext == extensionMD5 && pf.HashMD5 != string(hash)) || + (ext == extensionSHA1 && pf.HashSHA1 != string(hash)) || + (ext == extensionSHA256 && pf.HashSHA256 != string(hash)) || + (ext == extensionSHA512 && pf.HashSHA512 != string(hash)) { ctx.Error(http.StatusBadRequest, "", "hash mismatch") return } @@ -257,7 +274,7 @@ func UploadPackageFile(ctx *context.APIContext) { } func isChecksumExtension(ext string) bool { - return ext == ".sha512" || ext == ".sha256" || ext == ".sha1" || ext == ".md5" + return ext == extensionMD5 || ext == extensionSHA1 || ext == extensionSHA256 || ext == extensionSHA512 } type parameters struct { @@ -275,7 +292,11 @@ func extractPathParameters(ctx *context.APIContext) (parameters, error) { Filename: parts[len(parts)-1], } - p.IsMeta = p.Filename == mavenMetadataFile || p.Filename == mavenMetadataFile+".sha1" || p.Filename == mavenMetadataFile+".md5" + p.IsMeta = p.Filename == mavenMetadataFile || + p.Filename == mavenMetadataFile+extensionMD5 || + p.Filename == mavenMetadataFile+extensionSHA1 || + p.Filename == mavenMetadataFile+extensionSHA256 || + p.Filename == mavenMetadataFile+extensionSHA512 parts = parts[:len(parts)-1] if len(parts) == 0 { diff --git a/routers/api/v1/packages/maven/package.go b/routers/api/v1/packages/maven/package.go index 6ac7000bd33cb..02dc69e5dffa4 100644 --- a/routers/api/v1/packages/maven/package.go +++ b/routers/api/v1/packages/maven/package.go @@ -46,12 +46,13 @@ func intializePackage(p *models.Package) (*Package, error) { }, nil } -func sortPackagesByVersionASC(packages []*Package) []*Package { +// Maven and Gradle order packages by their creation timestamp and not by their version string +func sortPackagesByCreationASC(packages []*Package) []*Package { sortedPackages := make([]*Package, len(packages)) copy(sortedPackages, packages) sort.Slice(sortedPackages, func(i, j int) bool { - return sortedPackages[i].Version < sortedPackages[j].Version + return sortedPackages[i].CreatedUnix < sortedPackages[j].CreatedUnix }) return sortedPackages From 1b666dc11d714730234422c9b1e48191757c5239 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 6 Sep 2021 18:24:04 +0000 Subject: [PATCH 040/130] Added support for symbol package uploads. --- modules/packages/nuget/metadata.go | 27 +++++- modules/packages/nuget/metadata_test.go | 64 ++++++++++---- routers/api/v1/api.go | 1 + routers/api/v1/packages/nuget/api.go | 1 + routers/api/v1/packages/nuget/nuget.go | 107 +++++++++++++++++++----- 5 files changed, 162 insertions(+), 38 deletions(-) diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 7d2c3e293cbc5..2ac99e3fda121 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -29,12 +29,23 @@ var ( ErrNuspecInvalidVersion = errors.New("Nuspec file contains an invalid version") ) +// PackageType specifies the package type the metadata describes +type PackageType int + +const ( + // DependencyPackage represents a package (*.nupkg) + DependencyPackage PackageType = iota + 1 + // SymbolsPackage represents a symbol package (*.snupkg) + SymbolsPackage +) + var idmatch = regexp.MustCompile(`\A\w+(?:[.-]\w+)*\z`) const maxNuspecFileSize = 3 * 1024 * 1024 // Metadata represents the metadata of a Nuget package type Metadata struct { + PackageType PackageType `json:"-"` ID string `json:"-"` Version string `json:"-"` Description string `json:"description"` @@ -60,7 +71,12 @@ type nuspecPackage struct { ProjectURL string `xml:"projectUrl"` Description string `xml:"description"` ReleaseNotes string `xml:"releaseNotes"` - Repository struct { + PackageTypes struct { + PackageType []struct { + Name string `xml:"name,attr"` + } `xml:"packageType"` + } `xml:"packageTypes"` + Repository struct { URL string `xml:"url,attr"` } `xml:"repository"` Dependencies struct { @@ -123,7 +139,16 @@ func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { p.Metadata.ProjectURL = "" } + packageType := DependencyPackage + for _, pt := range p.Metadata.PackageTypes.PackageType { + if pt.Name == "SymbolsPackage" { + packageType = SymbolsPackage + break + } + } + m := &Metadata{ + PackageType: packageType, ID: p.Metadata.ID, Version: v.String(), Description: p.Metadata.Description, diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go index 40a00a190e4f9..e4418c3cd2e84 100644 --- a/modules/packages/nuget/metadata_test.go +++ b/modules/packages/nuget/metadata_test.go @@ -45,6 +45,21 @@ const nuspecContent = ` ` +const symbolsNuspecContent = ` + + + ` + id + ` + ` + semver + ` + ` + description + ` + + + + + + + +` + func TestParsePackageMetaData(t *testing.T) { createArchive := func(name, content string) []byte { var buf bytes.Buffer @@ -113,21 +128,36 @@ func TestParsePackageMetaData(t *testing.T) { } func TestParseNuspecMetaData(t *testing.T) { - m, err := ParseNuspecMetaData(strings.NewReader(nuspecContent)) - assert.NoError(t, err) - assert.NotNil(t, m) - - assert.Equal(t, id, m.ID) - assert.Equal(t, semver, m.Version) - assert.Equal(t, authors, m.Authors) - assert.Equal(t, projectURL, m.ProjectURL) - assert.Equal(t, description, m.Description) - assert.Equal(t, releaseNotes, m.ReleaseNotes) - assert.Equal(t, repositoryURL, m.RepositoryURL) - assert.Len(t, m.Dependencies, 1) - assert.Contains(t, m.Dependencies, targetFramework) - deps := m.Dependencies[targetFramework] - assert.Len(t, deps, 1) - assert.Equal(t, dependencyID, deps[0].ID) - assert.Equal(t, dependencyVersion, deps[0].Version) + t.Run("Dependency Package", func(t *testing.T) { + m, err := ParseNuspecMetaData(strings.NewReader(nuspecContent)) + assert.NoError(t, err) + assert.NotNil(t, m) + assert.Equal(t, DependencyPackage, m.PackageType) + + assert.Equal(t, id, m.ID) + assert.Equal(t, semver, m.Version) + assert.Equal(t, authors, m.Authors) + assert.Equal(t, projectURL, m.ProjectURL) + assert.Equal(t, description, m.Description) + assert.Equal(t, releaseNotes, m.ReleaseNotes) + assert.Equal(t, repositoryURL, m.RepositoryURL) + assert.Len(t, m.Dependencies, 1) + assert.Contains(t, m.Dependencies, targetFramework) + deps := m.Dependencies[targetFramework] + assert.Len(t, deps, 1) + assert.Equal(t, dependencyID, deps[0].ID) + assert.Equal(t, dependencyVersion, deps[0].Version) + }) + + t.Run("Symbols Package", func(t *testing.T) { + m, err := ParseNuspecMetaData(strings.NewReader(symbolsNuspecContent)) + assert.NoError(t, err) + assert.NotNil(t, m) + assert.Equal(t, SymbolsPackage, m.PackageType) + + assert.Equal(t, id, m.ID) + assert.Equal(t, semver, m.Version) + assert.Equal(t, description, m.Description) + assert.Empty(t, m.Dependencies) + }) } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 46127f969a957..90c0daa77c206 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1009,6 +1009,7 @@ func Routes() *web.Route { m.Get("/{filename}", nuget.DownloadPackageFile) }) }) + m.Put("/symbolpackage", nuget.UploadSymbolPackage) }, reqBasicAuth()) m.Group("/npm", func() { m.Group("/{id}", func() { diff --git a/routers/api/v1/packages/nuget/api.go b/routers/api/v1/packages/nuget/api.go index 1e8c372a54a42..18515283ffb79 100644 --- a/routers/api/v1/packages/nuget/api.go +++ b/routers/api/v1/packages/nuget/api.go @@ -36,6 +36,7 @@ func createServiceIndexResponse(root string) *ServiceIndexResponse { {ID: root + "/registration", Type: "RegistrationsBaseUrl/3.0.0-rc"}, {ID: root + "/package", Type: "PackageBaseAddress/3.0.0"}, {ID: root, Type: "PackagePublish/2.0.0"}, + {ID: root + "/symbolpackage", Type: "SymbolPackagePublish/4.9.0"}, }, } } diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index beda583bb14d4..47ee0ce1d1421 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -165,43 +165,74 @@ func DownloadPackageFile(ctx *context.APIContext) { // UploadPackage creates a new package with the metadata contained in the uploaded nupgk file // https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#push-a-package func UploadPackage(ctx *context.APIContext) { - upload, close, err := ctx.UploadStream() - if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + meta, buf, closables := processUploadedFile(ctx, nuget_module.DependencyPackage) + defer func() { + for _, c := range closables { + c.Close() + } + }() + if meta == nil { return } - if close { - defer upload.Close() - } - buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) + p, err := package_service.CreatePackage( + ctx.User, + ctx.Repo.Repository, + models.PackageNuGet, + meta.ID, + meta.Version, + meta, + false, + ) if err != nil { + if err == models.ErrDuplicatePackage { + ctx.Error(http.StatusBadRequest, "", err) + return + } ctx.Error(http.StatusInternalServerError, "", err) return } - defer buf.Close() - meta, err := nuget_module.ParsePackageMetaData(buf, buf.Size()) + filename := strings.ToLower(fmt.Sprintf("%s.%s.nupkg", meta.ID, meta.Version)) + _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) if err != nil { + if err := models.DeletePackageByID(p.ID); err != nil { + log.Error("Error deleting package by id: %v", err) + } ctx.Error(http.StatusInternalServerError, "", err) return } - if _, err := buf.Seek(0, io.SeekStart); err != nil { + + ctx.PlainText(http.StatusCreated, nil) +} + +// UploadSymbolPackage adds a symbol package to an existing package +// https://docs.microsoft.com/en-us/nuget/api/symbol-package-publish-resource +func UploadSymbolPackage(ctx *context.APIContext) { + meta, buf, closables := processUploadedFile(ctx, nuget_module.SymbolsPackage) + defer func() { + for _, c := range closables { + c.Close() + } + }() + if meta == nil { + return + } + + p, err := models.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, models.PackageNuGet, meta.ID, meta.Version) + if err != nil { + if err == models.ErrPackageNotExist { + ctx.Error(http.StatusNotFound, "", err) + return + } ctx.Error(http.StatusInternalServerError, "", err) return } - p, err := package_service.CreatePackage( - ctx.User, - ctx.Repo.Repository, - models.PackageNuGet, - meta.ID, - meta.Version, - meta, - false, - ) + filename := strings.ToLower(fmt.Sprintf("%s.%s.snupkg", meta.ID, meta.Version)) + _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) if err != nil { - if err == models.ErrDuplicatePackage { + if err == models.ErrDuplicatePackageFile { ctx.Error(http.StatusBadRequest, "", err) return } @@ -222,6 +253,42 @@ func UploadPackage(ctx *context.APIContext) { ctx.PlainText(http.StatusCreated, nil) } +func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.PackageType) (*nuget_module.Metadata, *filebuffer.FileBackedBuffer, []io.Closer) { + closables := make([]io.Closer, 0, 2) + + upload, close, err := ctx.UploadStream() + if err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return nil, nil, closables + } + + if close { + closables = append(closables, upload) + } + + buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return nil, nil, closables + } + closables = append(closables, buf) + + meta, err := nuget_module.ParsePackageMetaData(buf, buf.Size()) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return nil, nil, closables + } + if meta.PackageType != expectedType { + ctx.Error(http.StatusBadRequest, "", err) + return nil, nil, closables + } + if _, err := buf.Seek(0, io.SeekStart); err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return nil, nil, closables + } + return meta, buf, closables +} + // DeletePackage hard deletes the package // https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#delete-a-package func DeletePackage(ctx *context.APIContext) { From 3e462afd87cc2dd3da5e74c05bf7e5017aac8edf Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 6 Sep 2021 19:03:50 +0000 Subject: [PATCH 041/130] Added tests. --- integrations/api_packages_nuget_test.go | 97 +++++++++++++++++++------ routers/api/v1/packages/nuget/nuget.go | 10 --- 2 files changed, 75 insertions(+), 32 deletions(-) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index 84e1fc1b7e0f3..b128264343d95 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -8,6 +8,7 @@ import ( "archive/zip" "bytes" "fmt" + "io" "net/http" "testing" @@ -82,31 +83,77 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("Upload", func(t *testing.T) { - defer PrintCurrentTest(t)() + t.Run("DependencyPackage", func(t *testing.T) { + defer PrintCurrentTest(t)() - req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) - req = AddBasicAuthHeader(req, user.Name) - MakeRequest(t, req, http.StatusCreated) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNuGet) - assert.NoError(t, err) - assert.Len(t, ps, 1) - assert.Equal(t, packageName, ps[0].Name) - assert.Equal(t, packageVersion, ps[0].Version) + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNuGet) + assert.NoError(t, err) + assert.Len(t, ps, 1) + assert.Equal(t, packageName, ps[0].Name) + assert.Equal(t, packageVersion, ps[0].Version) - pfs, err := ps[0].GetFiles() - assert.NoError(t, err) - assert.Len(t, pfs, 1) - assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name) - assert.Equal(t, int64(len(content)), pfs[0].Size) - }) + pfs, err := ps[0].GetFiles() + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name) + assert.Equal(t, int64(len(content)), pfs[0].Size) - t.Run("UploadExists", func(t *testing.T) { - defer PrintCurrentTest(t)() + req = NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) - req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) - req = AddBasicAuthHeader(req, user.Name) - MakeRequest(t, req, http.StatusBadRequest) + t.Run("SymbolPackage", func(t *testing.T) { + defer PrintCurrentTest(t)() + + createPackage := func(id, packageType string) io.Reader { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create("package.nuspec") + w.Write([]byte(` + + + ` + id + ` + ` + packageVersion + ` + ` + packageAuthors + ` + ` + packageDescription + ` + + + `)) + archive.Close() + return &buf + } + + req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/symbolpackage", url), createPackage("unknown-package", "SymbolsPackage")) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/symbolpackage", url), createPackage(packageName, "DummyPackage")) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + + req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/symbolpackage", url), createPackage(packageName, "SymbolsPackage")) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNuGet) + assert.NoError(t, err) + assert.Len(t, ps, 1) + + pfs, err := ps[0].GetFiles() + assert.NoError(t, err) + assert.Len(t, pfs, 2) + assert.Equal(t, fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion), pfs[1].Name) + assert.Equal(t, int64(368), pfs[1].Size) + + req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/symbolpackage", url), createPackage(packageName, "SymbolsPackage")) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) }) t.Run("Download", func(t *testing.T) { @@ -117,6 +164,10 @@ func TestPackageNuGet(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) assert.Equal(t, content, resp.Body.Bytes()) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.snupkg", url, packageName, packageVersion, packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) }) t.Run("SearchService", func(t *testing.T) { @@ -150,8 +201,6 @@ func TestPackageNuGet(t *testing.T) { }) t.Run("RegistrationService", func(t *testing.T) { - defer PrintCurrentTest(t)() - indexURL := fmt.Sprintf("%s%s/registration/%s/index.json", setting.AppURL, url[1:], packageName) leafURL := fmt.Sprintf("%s%s/registration/%s/%s.json", setting.AppURL, url[1:], packageName, packageVersion) contentURL := fmt.Sprintf("%s%s/package/%s/%s/%s.%s.nupkg", setting.AppURL, url[1:], packageName, packageVersion, packageName, packageVersion) @@ -230,6 +279,10 @@ func TestPackageNuGet(t *testing.T) { req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.nupkg", url, packageName, packageVersion, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.snupkg", url, packageName, packageVersion, packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusNotFound) }) t.Run("DeleteNotExists", func(t *testing.T) { diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 47ee0ce1d1421..01da9ad1ea6c2 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -240,16 +240,6 @@ func UploadSymbolPackage(ctx *context.APIContext) { return } - filename := strings.ToLower(fmt.Sprintf("%s.%s.nupkg", meta.ID, meta.Version)) - _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) - if err != nil { - if err := models.DeletePackageByID(p.ID); err != nil { - log.Error("Error deleting package by id: %v", err) - } - ctx.Error(http.StatusInternalServerError, "", err) - return - } - ctx.PlainText(http.StatusCreated, nil) } From 39a4115b53f88bd2414cb85cbf7cbff076236b0d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 6 Sep 2021 21:01:47 +0000 Subject: [PATCH 042/130] Added overview docs. --- docs/content/doc/packages/overview.en-us.md | 67 +++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 docs/content/doc/packages/overview.en-us.md diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md new file mode 100644 index 0000000000000..3995dc86396e4 --- /dev/null +++ b/docs/content/doc/packages/overview.en-us.md @@ -0,0 +1,67 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "Package Registry" +slug: "overview" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Overview" + weight: 1 + identifier: "overview" +--- + +# Package Registry + +The Package Registry can be used as a public or private registry for common package managers. + +These package types are supported: + +- [Generic]({{< relref "doc/packages/generic.en-us.md" >}}) +- [NuGet]({{< relref "doc/packages/nuget.en-us.md" >}}) +- [npm]({{< relref "doc/packages/npm.en-us.md" >}}) +- [Maven]({{< relref "doc/packages/maven.en-us.md" >}}) +- [PyPI]({{< relref "doc/packages/pypi.en-us.md" >}}) +- [RubyGems]({{< relref "doc/packages/rubygems.en-us.md" >}}) + +**Table of Contents** + +{{< toc >}} + +## View packages + +You can view the packages of a repository on the repository page. + +1. Go to the repoistory. +1. Go to **Packages** in the navigation bar. + +To view more details about a package, select the name of the package. + +## Download a package + +To download a package from your repository: + +1. Go to **Packages** in the navigation bar. +1. Select the name of the package to view the details. +1. In the **Assets** section, select the name of the package file you want to download. + +## Delete a package + +You cannot edit a package after you published it in the Package Registry. Instead, you +must delete and recreate it. + +To delete a package from your repository: + +1. Go to **Packages** in the navigation bar. +1. Select the name of the package to view the details. +1. Click **Delete package** to permanently delete the package. + +## Disable the Package Registry + +The Package Registry is automatically enabled. To disable it for a single repository: + +1. Go to **Settings** in the navigation bar. +1. Disable **Enable Repository Packages Registry**. + +Previously published packages are not deleted by disabling the Package Registry. From fb4f03940efe364720a82fd2fdfdea8fb3b6eb53 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 7 Sep 2021 19:53:06 +0000 Subject: [PATCH 043/130] Added npm dependencies and keywords. --- integrations/api_packages_npm_test.go | 2 +- integrations/api_packages_nuget_test.go | 2 +- modules/packages/npm/creator.go | 54 ++++++++++--------- modules/packages/npm/metadata.go | 22 ++++---- options/locale/locale_en-US.ini | 7 +++ routers/api/v1/packages/npm/package.go | 2 +- routers/api/v1/packages/pypi/package.go | 2 +- templates/repo/packages/content/npm.tmpl | 24 +++++---- .../packages/content/npm_dependencies.tmpl | 18 +++++++ templates/repo/packages/content/rubygems.tmpl | 38 +------------ .../content/rubygems_dependencies.tmpl | 18 +++++++ 11 files changed, 106 insertions(+), 83 deletions(-) create mode 100644 templates/repo/packages/content/npm_dependencies.tmpl create mode 100644 templates/repo/packages/content/rubygems_dependencies.tmpl diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index b7ee8b7cd744a..83bab146157e0 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -19,7 +19,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPackageNPM(t *testing.T) { +func TestPackageNpm(t *testing.T) { defer prepareTestEnv(t)() repository := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) user := models.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index b128264343d95..9be8315511912 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -127,7 +127,7 @@ func TestPackageNuGet(t *testing.T) { archive.Close() return &buf } - + req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/symbolpackage", url), createPackage("unknown-package", "SymbolsPackage")) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusNotFound) diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index bcfaa769fa650..72f5fb842640f 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -37,7 +37,7 @@ var ( var nameMatch = regexp.MustCompile(`\A(@[^\/~'!\(\)\*]+?)[\/]([^_.][^\/~'!\(\)\*]+)\z`) -// Package represents a NPM package +// Package represents a npm package type Package struct { Name string Version string @@ -67,20 +67,22 @@ type PackageMetadata struct { // PackageMetadataVersion https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version type PackageMetadataVersion struct { - ID string `json:"_id"` - Name string `json:"name"` - Version string `json:"version"` - Description string `json:"description"` - Author User `json:"author"` - Homepage string `json:"homepage,omitempty"` - License string `json:"license,omitempty"` - Repository Repository `json:"repository,omitempty"` - Keywords []string `json:"keywords,omitempty"` - Dependencies map[string]string `json:"dependencies,omitempty"` - DevDependencies map[string]string `json:"devDependencies,omitempty"` - Readme string `json:"readme,omitempty"` - Dist PackageDistribution `json:"dist"` - Maintainers []User `json:"maintainers,omitempty"` + ID string `json:"_id"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Author User `json:"author"` + Homepage string `json:"homepage,omitempty"` + License string `json:"license,omitempty"` + Repository Repository `json:"repository,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevDependencies map[string]string `json:"devDependencies,omitempty"` + PeerDependencies map[string]string `json:"peerDependencies,omitempty"` + OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` + Readme string `json:"readme,omitempty"` + Dist PackageDistribution `json:"dist"` + Maintainers []User `json:"maintainers,omitempty"` } // PackageDistribution https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version @@ -144,7 +146,7 @@ type packageUpload struct { Attachments map[string]*PackageAttachment `json:"_attachments"` } -// ParsePackage parses the content into a NPM package +// ParsePackage parses the content into a npm package func ParsePackage(r io.Reader) (*Package, error) { var upload packageUpload if err := json.NewDecoder(r).Decode(&upload); err != nil { @@ -171,14 +173,18 @@ func ParsePackage(r io.Reader) (*Package, error) { Name: meta.Name, Version: meta.Version, Metadata: Metadata{ - Scope: nameParts[0], - Name: nameParts[1], - Description: meta.Description, - Author: meta.Author.Name, - License: meta.License, - ProjectURL: meta.Homepage, - Dependencies: meta.Dependencies, - Readme: meta.Readme, + Scope: nameParts[0], + Name: nameParts[1], + Description: meta.Description, + Author: meta.Author.Name, + License: meta.License, + ProjectURL: meta.Homepage, + Keywords: meta.Keywords, + Dependencies: meta.Dependencies, + DevelopmentDependencies: meta.DevDependencies, + PeerDependencies: meta.PeerDependencies, + OptionalDependencies: meta.OptionalDependencies, + Readme: meta.Readme, }, } diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index f0c9586ccc516..cb6589ae7895a 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -4,14 +4,18 @@ package npm -// Metadata represents the metadata of a NPM package +// Metadata represents the metadata of a npm package type Metadata struct { - Scope string `json:"scope"` - Name string `json:"name"` - Description string `json:"description"` - Author string `json:"author"` - License string `json:"license"` - ProjectURL string `json:"project_url"` - Dependencies map[string]string `json:"dependencies"` - Readme string `json:"readme"` + Scope string `json:"scope"` + Name string `json:"name"` + Description string `json:"description"` + Author string `json:"author"` + License string `json:"license"` + ProjectURL string `json:"project_url"` + Keywords []string `json:"keywords"` + Dependencies map[string]string `json:"dependencies"` + DevelopmentDependencies map[string]string `json:"development_dependencies"` + PeerDependencies map[string]string `json:"peer_dependencies"` + OptionalDependencies map[string]string `json:"optional_dependencies"` + Readme string `json:"readme"` } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3b52097261799..68571dac3a8cd 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1521,6 +1521,7 @@ packages.installation = Installation packages.about = About this package packages.requirements = Requirements packages.dependencies = Dependencies +packages.keywords = Keywords packages.details = Details packages.details.author = Author packages.details.project_site = Project Site @@ -1543,11 +1544,17 @@ packages.nuget.documentation = For more information on the NuGet registry, see < packages.npm.registry = Setup this registry in your projects .npmrc file: packages.npm.use = To install the package use the following command: packages.npm.documentation = For more information on the npm registry, see the documentation. +packages.npm.dependencies = Dependencies +packages.npm.dependencies.development = Development Dependencies +packages.npm.dependencies.peer = Peer Dependencies +packages.npm.dependencies.optional = Optional Dependencies packages.pypi.requires = Requires Python packages.pypi.use = To install the package use the following command: packages.pypi.documentation = For more information on the PyPI registry, see the documentation. packages.rubygems.use_1 = Install from the command line packages.rubygems.use_2 = Install via Gemfile +packages.npm.dependencies.runtime = Runtime Dependencies +packages.npm.dependencies.development = Development Dependencies packages.rubygems.required.ruby = Requires Ruby version packages.rubygems.required.rubygems = Requires RubyGem version packages.rubygems.documentation = For more information on the RubyGems registry, see the documentation. diff --git a/routers/api/v1/packages/npm/package.go b/routers/api/v1/packages/npm/package.go index ad46d04576ba2..90a8b3c251288 100644 --- a/routers/api/v1/packages/npm/package.go +++ b/routers/api/v1/packages/npm/package.go @@ -14,7 +14,7 @@ import ( "github.com/hashicorp/go-version" ) -// Package represents a package with NPM metadata +// Package represents a package with npm metadata type Package struct { *models.Package *models.PackageFile diff --git a/routers/api/v1/packages/pypi/package.go b/routers/api/v1/packages/pypi/package.go index ae9692314608d..812bbadc5e04b 100644 --- a/routers/api/v1/packages/pypi/package.go +++ b/routers/api/v1/packages/pypi/package.go @@ -10,7 +10,7 @@ import ( pypi_module "code.gitea.io/gitea/modules/packages/pypi" ) -// Package represents a package with NPM metadata +// Package represents a package with PyPI metadata type Package struct { *models.Package Files []*models.PackageFile diff --git a/templates/repo/packages/content/npm.tmpl b/templates/repo/packages/content/npm.tmpl index 47557feeb7be5..6492d88a48a61 100644 --- a/templates/repo/packages/content/npm.tmpl +++ b/templates/repo/packages/content/npm.tmpl @@ -27,20 +27,24 @@
    {{end}} - {{if .Metadata.Dependencies}} + {{if or .Metadata.Dependencies .Metadata.DevelopmentDependencies .Metadata.PeerDependencies .Metadata.OptionalDependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    - {{range $dependency, $version := .Metadata.Dependencies}} -
    - {{svg "octicon-package-dependencies" 16 ""}} -
    -
    {{$dependency}}
    -
    {{$version}}
    -
    -
    - {{end}} + {{template "repo/packages/content/npm_dependencies" dict "dependencies" .Metadata.Dependencies "title" (.i18n.Tr "repo.packages.npm.dependencies")}} + {{template "repo/packages/content/npm_dependencies" dict "dependencies" .Metadata.DevelopmentDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.development")}} + {{template "repo/packages/content/npm_dependencies" dict "dependencies" .Metadata.PeerDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.peer")}} + {{template "repo/packages/content/npm_dependencies" dict "dependencies" .Metadata.OptionalDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.optional")}}
    {{end}} + + {{if or .Metadata.Keywords}} +

    {{.i18n.Tr "repo.packages.keywords"}}

    +
    + {{range .Metadata.Keywords}} + {{.}} + {{end}} +
    + {{end}} {{end}} diff --git a/templates/repo/packages/content/npm_dependencies.tmpl b/templates/repo/packages/content/npm_dependencies.tmpl new file mode 100644 index 0000000000000..764d1b68504e3 --- /dev/null +++ b/templates/repo/packages/content/npm_dependencies.tmpl @@ -0,0 +1,18 @@ +{{if .dependencies}} +
    +
    +
    {{.title}}
    +
    + {{range $dependency, $version := .dependencies}} +
    + {{svg "octicon-package-dependencies" 16 ""}} +
    +
    {{$dependency}}
    +
    {{$version}}
    +
    +
    + {{end}} +
    +
    +
    +{{end}} diff --git a/templates/repo/packages/content/rubygems.tmpl b/templates/repo/packages/content/rubygems.tmpl index 85a0d307aa737..2648c93e1f726 100644 --- a/templates/repo/packages/content/rubygems.tmpl +++ b/templates/repo/packages/content/rubygems.tmpl @@ -34,42 +34,8 @@ end

    {{.i18n.Tr "repo.packages.dependencies"}}

    - {{if .Metadata.RuntimeDependencies}} -
    -
    -
    Runtime Dependencies
    -
    - {{range .Metadata.RuntimeDependencies}} -
    - {{svg "octicon-package-dependencies" 16 ""}} -
    -
    {{.Name}}
    -
    {{range $i, $v := .Version}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}
    -
    -
    - {{end}} -
    -
    -
    - {{end}} - {{if .Metadata.DevelopmentDependencies}} -
    -
    -
    Development Dependencies
    -
    - {{range .Metadata.DevelopmentDependencies}} -
    - {{svg "octicon-package-dependencies" 16 ""}} -
    -
    {{.Name}}
    -
    {{range $i, $v := .Version}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}
    -
    -
    - {{end}} -
    -
    -
    - {{end}} + {{template "repo/packages/content/rubygems_dependencies" dict "dependencies" .Metadata.RuntimeDependencies "title" (.i18n.Tr "repo.packages.rubygems.dependencies.runtime")}} + {{template "repo/packages/content/rubygems_dependencies" dict "dependencies" .Metadata.DevelopmentDependencies "title" (.i18n.Tr "repo.packages.rubygems.dependencies.development")}}
    {{end}} diff --git a/templates/repo/packages/content/rubygems_dependencies.tmpl b/templates/repo/packages/content/rubygems_dependencies.tmpl new file mode 100644 index 0000000000000..48f9e15ab8c02 --- /dev/null +++ b/templates/repo/packages/content/rubygems_dependencies.tmpl @@ -0,0 +1,18 @@ +{{if .dependencies}} +
    +
    +
    {{.title}}
    +
    + {{range .dependencies}} +
    + {{svg "octicon-package-dependencies" 16 ""}} +
    +
    {{.Name}}
    +
    {{range $i, $v := .Version}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}
    +
    +
    + {{end}} +
    +
    +
    +{{end}} From 08e6f82cc10f8f1c72437a4a76441cdd0f5a49d7 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 7 Sep 2021 19:56:37 +0000 Subject: [PATCH 044/130] Added no-packages information. --- models/package.go | 5 +++++ options/locale/locale_en-US.ini | 3 +++ routers/web/repo/packages.go | 7 +++++++ templates/repo/packages/list.tmpl | 12 +++++++++++- web_src/less/_repository.less | 11 +++++++++++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/models/package.go b/models/package.go index 82cba7dce5590..d48632e5cbcb8 100644 --- a/models/package.go +++ b/models/package.go @@ -256,6 +256,11 @@ func GetPackageByID(packageID int64) (*Package, error) { return p, nil } +// HasRepositoryPackages tests if a repository has packages +func HasRepositoryPackages(repositoryID int64) (bool, error) { + return x.Where("repo_id = ?", repositoryID).Exist(&Package{}) +} + // GetPackagesByRepository returns all packages of a repository func GetPackagesByRepository(repositoryID int64) ([]*Package, error) { packages := make([]*Package, 0, 10) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 68571dac3a8cd..107a0ad858158 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1514,8 +1514,11 @@ milestones.filter_sort.least_issues = Least issues packages = Packages packages.desc = Manage repository packages. +packages.empty = There are no packages yet. +packages.empty.documentation = For more information on the package registry, see the documentation. packages.filter.package_type = Type packages.filter.package_type.all = All +packages.filter.no_result = Your filter produced no results. packages.published_by = Published %[1]s by %[3]s packages.installation = Installation packages.about = About this package diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index f66fff9c25332..41f911b717a42 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -69,6 +69,13 @@ func Packages(ctx *context.Context) { } } + hasPackages, err := models.HasRepositoryPackages(repo.ID) + if err != nil { + ctx.ServerError("HasRepositoryPackages", err) + return + } + + ctx.Data["HasPackages"] = hasPackages ctx.Data["Packages"] = packages ctx.Data["Query"] = query ctx.Data["PackageType"] = packageType diff --git a/templates/repo/packages/list.tmpl b/templates/repo/packages/list.tmpl index 8ea24fd73fa98..209ed391dce10 100644 --- a/templates/repo/packages/list.tmpl +++ b/templates/repo/packages/list.tmpl @@ -33,7 +33,7 @@ -
    +
    {{range .Packages}}
  • @@ -48,6 +48,16 @@
    • + {{else}} + {{if not .HasPackages}} +
    + {{svg "octicon-package" 32}} +

    {{.i18n.Tr "repo.packages.empty"}}

    +

    {{.i18n.Tr "repo.packages.empty.documentation" | Safe}}

    +
    + {{else}} + {{.i18n.Tr "repo.packages.filter.no_result"}} + {{end}} {{end}} {{template "base/paginate" .}}
    diff --git a/web_src/less/_repository.less b/web_src/less/_repository.less index a58fe3698d2ec..89dfc3c89a45b 100644 --- a/web_src/less/_repository.less +++ b/web_src/less/_repository.less @@ -1977,6 +1977,17 @@ } } + &.packages { + .empty { + padding-top: 70px; + padding-bottom: 100px; + + .svg { + height: 48px; + } + } + } + &.wiki { &.start { .ui.segment { From fe02ca2486e2dff8a92dc2939fd56eb5abd05953 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 7 Sep 2021 20:05:04 +0000 Subject: [PATCH 045/130] Display file size. --- templates/repo/packages/view.tmpl | 3 ++- web_src/less/_repository.less | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/templates/repo/packages/view.tmpl b/templates/repo/packages/view.tmpl index 239c183f62918..e9a27b2c99538 100644 --- a/templates/repo/packages/view.tmpl +++ b/templates/repo/packages/view.tmpl @@ -1,5 +1,5 @@ {{template "base/head" .}} -
    +
    {{template "repo/header" .}}
    @@ -40,6 +40,7 @@ {{range .Files}}
    {{.Name}} + {{FileSize .Size}}
    {{end}}
    diff --git a/web_src/less/_repository.less b/web_src/less/_repository.less index 89dfc3c89a45b..e6245e576d6dc 100644 --- a/web_src/less/_repository.less +++ b/web_src/less/_repository.less @@ -1986,6 +1986,10 @@ height: 48px; } } + + .file-size { + white-space: nowrap; + } } &.wiki { From 1cf894b0258de7e4ac3ad8b443b3d82885aa7ae6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 7 Sep 2021 20:08:24 +0000 Subject: [PATCH 046/130] Display asset count. --- options/locale/locale_en-US.ini | 4 ++-- templates/repo/packages/view.tmpl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 107a0ad858158..e150555b2320c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1556,8 +1556,8 @@ packages.pypi.use = To install the package use the following command: packages.pypi.documentation = For more information on the PyPI registry, see the documentation. packages.rubygems.use_1 = Install from the command line packages.rubygems.use_2 = Install via Gemfile -packages.npm.dependencies.runtime = Runtime Dependencies -packages.npm.dependencies.development = Development Dependencies +packages.rubygems.dependencies.runtime = Runtime Dependencies +packages.rubygems.dependencies.development = Development Dependencies packages.rubygems.required.ruby = Requires Ruby version packages.rubygems.required.rubygems = Requires RubyGem version packages.rubygems.documentation = For more information on the RubyGems registry, see the documentation. diff --git a/templates/repo/packages/view.tmpl b/templates/repo/packages/view.tmpl index e9a27b2c99538..f8d37a5c709d4 100644 --- a/templates/repo/packages/view.tmpl +++ b/templates/repo/packages/view.tmpl @@ -35,7 +35,7 @@ {{template "repo/packages/metadata/rubygems" .}}
    - {{.i18n.Tr "repo.packages.assets"}} + {{.i18n.Tr "repo.packages.assets"}} ({{len .Files}})
    {{range .Files}}
    From faf5518b9564826bd86e5ea26bc8fd4ffdb63a16 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 7 Sep 2021 20:34:22 +0000 Subject: [PATCH 047/130] Fixed filter alignment. --- templates/repo/packages/list.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/repo/packages/list.tmpl b/templates/repo/packages/list.tmpl index 209ed391dce10..6c74048b2c78d 100644 --- a/templates/repo/packages/list.tmpl +++ b/templates/repo/packages/list.tmpl @@ -13,8 +13,8 @@
    -
    -
    +
    +
    {{.i18n.Tr "repo.packages.filter.package_type"}} From 4f1ffa6c3a1bd12363ffad41e034a66f4abe5dab Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 9 Sep 2021 16:53:00 +0200 Subject: [PATCH 048/130] Added package icons. --- models/package.go | 19 +++++++++++++++++++ public/img/svg/gitea-maven.svg | 1 + public/img/svg/gitea-npm.svg | 1 + public/img/svg/gitea-nuget.svg | 1 + public/img/svg/gitea-python.svg | 1 + public/img/svg/gitea-rubygems.svg | 1 + templates/repo/packages/list.tmpl | 3 +-- templates/repo/packages/view.tmpl | 2 +- web_src/svg/gitea-maven.svg | 1 + web_src/svg/gitea-npm.svg | 7 +++++++ web_src/svg/gitea-nuget.svg | 15 +++++++++++++++ web_src/svg/gitea-python.svg | 19 +++++++++++++++++++ web_src/svg/gitea-rubygems.svg | 3 +++ 13 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 public/img/svg/gitea-maven.svg create mode 100644 public/img/svg/gitea-npm.svg create mode 100644 public/img/svg/gitea-nuget.svg create mode 100644 public/img/svg/gitea-python.svg create mode 100644 public/img/svg/gitea-rubygems.svg create mode 100644 web_src/svg/gitea-maven.svg create mode 100644 web_src/svg/gitea-npm.svg create mode 100644 web_src/svg/gitea-nuget.svg create mode 100644 web_src/svg/gitea-python.svg create mode 100644 web_src/svg/gitea-rubygems.svg diff --git a/models/package.go b/models/package.go index d48632e5cbcb8..620fd9c00f4ff 100644 --- a/models/package.go +++ b/models/package.go @@ -44,6 +44,25 @@ func (pt PackageType) String() string { return "" } +// SVGName returns the name of the package type svg image +func (pt PackageType) SVGName() string { + switch pt { + case PackageGeneric: + return "octicon-package" + case PackageNuGet: + return "gitea-nuget" + case PackageNpm: + return "gitea-npm" + case PackageMaven: + return "gitea-maven" + case PackagePyPI: + return "gitea-python" + case PackageRubyGems: + return "gitea-rubygems" + } + return "" +} + var ( // ErrDuplicatePackage indicates a duplicated package error ErrDuplicatePackage = errors.New("Package does exist already") diff --git a/public/img/svg/gitea-maven.svg b/public/img/svg/gitea-maven.svg new file mode 100644 index 0000000000000..e83e7282766a8 --- /dev/null +++ b/public/img/svg/gitea-maven.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/svg/gitea-npm.svg b/public/img/svg/gitea-npm.svg new file mode 100644 index 0000000000000..4435e092f2de5 --- /dev/null +++ b/public/img/svg/gitea-npm.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/svg/gitea-nuget.svg b/public/img/svg/gitea-nuget.svg new file mode 100644 index 0000000000000..519b60863807e --- /dev/null +++ b/public/img/svg/gitea-nuget.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/svg/gitea-python.svg b/public/img/svg/gitea-python.svg new file mode 100644 index 0000000000000..07548897e6151 --- /dev/null +++ b/public/img/svg/gitea-python.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/svg/gitea-rubygems.svg b/public/img/svg/gitea-rubygems.svg new file mode 100644 index 0000000000000..5f54dce48dd92 --- /dev/null +++ b/public/img/svg/gitea-rubygems.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/templates/repo/packages/list.tmpl b/templates/repo/packages/list.tmpl index 6c74048b2c78d..09bba18bc7572 100644 --- a/templates/repo/packages/list.tmpl +++ b/templates/repo/packages/list.tmpl @@ -39,8 +39,7 @@
    {{.Name}} - {{.Version}} - {{.Type.String}} + {{svg .Type.SVGName 16}} {{.Type.String}}
    {{ $timeStr := TimeSinceUnix .CreatedUnix $.Lang }} diff --git a/templates/repo/packages/view.tmpl b/templates/repo/packages/view.tmpl index f8d37a5c709d4..e97209ae94109 100644 --- a/templates/repo/packages/view.tmpl +++ b/templates/repo/packages/view.tmpl @@ -9,7 +9,7 @@

    {{.Package.Name}} ({{.Package.Version}})

    {{ $timeStr := TimeSinceUnix .Package.CreatedUnix $.Lang }} -
    {{.i18n.Tr "repo.packages.published_by" $timeStr .Package.Creator.HomeLink (.Package.Creator.GetDisplayName | Escape) | Safe}} {{svg "octicon-package"}} {{.Package.Type.String}}
    +
    {{.i18n.Tr "repo.packages.published_by" $timeStr .Package.Creator.HomeLink (.Package.Creator.GetDisplayName | Escape) | Safe}} {{svg .Package.Type.SVGName}} {{.Package.Type.String}}
    diff --git a/web_src/svg/gitea-maven.svg b/web_src/svg/gitea-maven.svg new file mode 100644 index 0000000000000..8f8502e4a3e38 --- /dev/null +++ b/web_src/svg/gitea-maven.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web_src/svg/gitea-npm.svg b/web_src/svg/gitea-npm.svg new file mode 100644 index 0000000000000..c6d110890e8ee --- /dev/null +++ b/web_src/svg/gitea-npm.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/web_src/svg/gitea-nuget.svg b/web_src/svg/gitea-nuget.svg new file mode 100644 index 0000000000000..f92fb0f114e51 --- /dev/null +++ b/web_src/svg/gitea-nuget.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/web_src/svg/gitea-python.svg b/web_src/svg/gitea-python.svg new file mode 100644 index 0000000000000..b1b19b4fb287d --- /dev/null +++ b/web_src/svg/gitea-python.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/web_src/svg/gitea-rubygems.svg b/web_src/svg/gitea-rubygems.svg new file mode 100644 index 0000000000000..d3eb4f7f00f90 --- /dev/null +++ b/web_src/svg/gitea-rubygems.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file From b1c8554ea711d65562ea9e30069ac21df58f5b41 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 9 Sep 2021 17:57:43 +0000 Subject: [PATCH 049/130] Formatted instructions. --- options/locale/locale_en-US.ini | 24 ++++++++++--------- templates/repo/packages/content/generic.tmpl | 8 ++++--- templates/repo/packages/content/maven.tmpl | 22 ++++++++++------- templates/repo/packages/content/npm.tmpl | 18 +++++++++----- templates/repo/packages/content/nuget.tmpl | 12 ++++++---- templates/repo/packages/content/pypi.tmpl | 8 ++++--- templates/repo/packages/content/rubygems.tmpl | 12 ++++++---- 7 files changed, 62 insertions(+), 42 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e150555b2320c..b41c75236aaf6 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1535,27 +1535,29 @@ packages.versions.on = on packages.delete = Delete package packages.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? packages.delete.success = The package has been deleted. -packages.generic.use = Download the package, for example with curl: +packages.generic.download = Download package from the command line: packages.generic.documentation = For more information on the generic registry, see the documentation. -packages.maven.registry = Setup this registry in your projects pom.xml file: -packages.maven.use = To use the package include the following in the dependencies block in the pom.xml file: -packages.maven.use_2 = Alternatively use this Maven command: +packages.maven.registry = Setup this registry in your project pom.xml file: +packages.maven.install = To use the package include the following in the dependencies block in the pom.xml file: +packages.maven.install2 = Run via command line: +packages.maven.download = To download the dependency, run via command line: packages.maven.documentation = For more information on the Maven registry, see the documentation. -packages.nuget.registry = Setup this registry with the following command: -packages.nuget.use = Install the package with the following command: +packages.nuget.registry = Setup the registry from the command line: +packages.nuget.install = Install from the command line: packages.nuget.documentation = For more information on the NuGet registry, see the documentation. -packages.npm.registry = Setup this registry in your projects .npmrc file: -packages.npm.use = To install the package use the following command: +packages.npm.registry = Setup this registry in your project .npmrc file: +packages.npm.install = Install from the command line: +packages.npm.install2 = Install via package.json: packages.npm.documentation = For more information on the npm registry, see the documentation. packages.npm.dependencies = Dependencies packages.npm.dependencies.development = Development Dependencies packages.npm.dependencies.peer = Peer Dependencies packages.npm.dependencies.optional = Optional Dependencies packages.pypi.requires = Requires Python -packages.pypi.use = To install the package use the following command: +packages.pypi.install = Install from the command line: packages.pypi.documentation = For more information on the PyPI registry, see the documentation. -packages.rubygems.use_1 = Install from the command line -packages.rubygems.use_2 = Install via Gemfile +packages.rubygems.install = Install from the command line: +packages.rubygems.install2 = Install via Gemfile: packages.rubygems.dependencies.runtime = Runtime Dependencies packages.rubygems.dependencies.development = Development Dependencies packages.rubygems.required.ruby = Requires Ruby version diff --git a/templates/repo/packages/content/generic.tmpl b/templates/repo/packages/content/generic.tmpl index c3d398fb90db9..05a0a5fd3fb78 100644 --- a/templates/repo/packages/content/generic.tmpl +++ b/templates/repo/packages/content/generic.tmpl @@ -2,11 +2,13 @@

    {{.i18n.Tr "repo.packages.installation"}}

    - -
    +
    +
    +
    + +
    - {{.i18n.Tr "repo.packages.generic.documentation" | Safe}}
    {{end}} diff --git a/templates/repo/packages/content/maven.tmpl b/templates/repo/packages/content/maven.tmpl index cfe721f9bc70e..5b448e6ac8859 100644 --- a/templates/repo/packages/content/maven.tmpl +++ b/templates/repo/packages/content/maven.tmpl @@ -2,8 +2,8 @@

    {{.i18n.Tr "repo.packages.installation"}}

    - -
    +
    +
    -
    -
    - -
    +
    +
    - -
    +
    + + +
    +
    +
    +
    + +
    - {{.i18n.Tr "repo.packages.maven.documentation" | Safe}}
    {{if .Metadata.Description}} diff --git a/templates/repo/packages/content/npm.tmpl b/templates/repo/packages/content/npm.tmpl index 6492d88a48a61..9075c9461cd66 100644 --- a/templates/repo/packages/content/npm.tmpl +++ b/templates/repo/packages/content/npm.tmpl @@ -2,16 +2,22 @@

    {{.i18n.Tr "repo.packages.installation"}}

    - -
    +
    +
    - -
    - +
    + + +
    +
    + + +
    +
    +
    - {{.i18n.Tr "repo.packages.npm.documentation" | Safe}}
    {{if or .Metadata.Description .Metadata.Readme}} diff --git a/templates/repo/packages/content/nuget.tmpl b/templates/repo/packages/content/nuget.tmpl index 989338b12fcd6..fc0771b65ac00 100644 --- a/templates/repo/packages/content/nuget.tmpl +++ b/templates/repo/packages/content/nuget.tmpl @@ -2,16 +2,18 @@

    {{.i18n.Tr "repo.packages.installation"}}

    - -
    +
    +
    - -
    +
    +
    +
    + +
    - {{.i18n.Tr "repo.packages.nuget.documentation" | Safe}}
    {{if or .Metadata.Description .Metadata.ReleaseNotes}} diff --git a/templates/repo/packages/content/pypi.tmpl b/templates/repo/packages/content/pypi.tmpl index 5c8ed26df9b01..7dbdcd7bffb18 100644 --- a/templates/repo/packages/content/pypi.tmpl +++ b/templates/repo/packages/content/pypi.tmpl @@ -2,12 +2,14 @@

    {{.i18n.Tr "repo.packages.installation"}}

    - -
    +
    +
    +
    + +
    - {{.i18n.Tr "repo.packages.pypi.documentation" | Safe}}
    {{if or .Metadata.Description .Metadata.LongDescription .Metadata.Summary}}

    {{.i18n.Tr "repo.packages.about"}}

    diff --git a/templates/repo/packages/content/rubygems.tmpl b/templates/repo/packages/content/rubygems.tmpl index 2648c93e1f726..430fa421d3e58 100644 --- a/templates/repo/packages/content/rubygems.tmpl +++ b/templates/repo/packages/content/rubygems.tmpl @@ -2,18 +2,20 @@

    {{.i18n.Tr "repo.packages.installation"}}

    - -
    +
    +
    - -
    +
    +
    +
    + +
    - {{.i18n.Tr "repo.packages.rubygems.documentation" | Safe}}
    {{if .Metadata.Description}}

    {{.i18n.Tr "repo.packages.about"}}

    From 3c5c54fbb0c749709c30e5f6c4bcd152b96f3f81 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 9 Sep 2021 19:58:35 +0000 Subject: [PATCH 050/130] Allow anonymous package downloads. --- routers/api/v1/api.go | 38 ++++++++++---------- templates/repo/packages/content/generic.tmpl | 2 +- templates/repo/packages/content/maven.tmpl | 2 +- templates/repo/packages/content/pypi.tmpl | 2 +- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 90c0daa77c206..f890659560f92 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -986,16 +986,17 @@ func Routes() *web.Route { m.Group("/generic", func() { m.Group("/{packagename}/{packageversion}/{filename}", func() { m.Get("", generic.DownloadPackageFile) - m.Put("", reqRepoWriter(models.UnitTypePackages), generic.UploadPackage) - m.Delete("", reqRepoWriter(models.UnitTypePackages), generic.DeletePackage) + m.Group("", func() { + m.Put("", generic.UploadPackage) + m.Delete("", generic.DeletePackage) + }, reqToken(), reqRepoWriter(models.UnitTypePackages)) }) - }, reqToken()) + }) m.Group("/maven", func() { - m.Put("/*", reqRepoWriter(models.UnitTypePackages), maven.UploadPackageFile) + m.Put("/*", reqToken(), reqRepoWriter(models.UnitTypePackages), maven.UploadPackageFile) m.Get("/*", maven.DownloadPackageFile) - }, reqToken()) + }) m.Group("/nuget", func() { - m.Put("/", reqRepoWriter(models.UnitTypePackages), nuget.UploadPackage) m.Get("/index.json", nuget.ServiceIndex) m.Get("/query", nuget.SearchService) m.Group("/registration/{id}", func() { @@ -1004,25 +1005,26 @@ func Routes() *web.Route { }) m.Group("/package/{id}", func() { m.Get("/index.json", nuget.EnumeratePackageVersions) - m.Group("/{version}", func() { - m.Delete("/", reqRepoWriter(models.UnitTypePackages), nuget.DeletePackage) - m.Get("/{filename}", nuget.DownloadPackageFile) - }) + m.Get("/{version}/{filename}", nuget.DownloadPackageFile) }) - m.Put("/symbolpackage", nuget.UploadSymbolPackage) - }, reqBasicAuth()) + m.Group("", func() { + m.Put("/", reqRepoWriter(models.UnitTypePackages), nuget.UploadPackage) + m.Put("/symbolpackage", nuget.UploadSymbolPackage) + m.Delete("/package/{id}/{version}", reqRepoWriter(models.UnitTypePackages), nuget.DeletePackage) + }, reqBasicAuth(), reqRepoWriter(models.UnitTypePackages)) + }) m.Group("/npm", func() { m.Group("/{id}", func() { m.Get("", npm.PackageMetadata) - m.Put("", reqRepoWriter(models.UnitTypePackages), npm.UploadPackage) + m.Put("", reqToken(), reqRepoWriter(models.UnitTypePackages), npm.UploadPackage) m.Get("/-/{version}/{filename}", npm.DownloadPackageFile) }) - }, reqToken()) + }) m.Group("/pypi", func() { - m.Post("/", reqRepoWriter(models.UnitTypePackages), pypi.UploadPackageFile) + m.Post("/", reqBasicAuth(), reqRepoWriter(models.UnitTypePackages), pypi.UploadPackageFile) m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) m.Get("/simple/{id}", pypi.PackageMetadata) - }, reqBasicAuth()) + }) m.Group("/rubygems", func() { m.Get("/specs.4.8.gz", rubygems.EnumeratePackages) m.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) @@ -1032,8 +1034,8 @@ func Routes() *web.Route { m.Group("/api/v1/gems", func() { m.Post("/", rubygems.UploadPackageFile) m.Delete("/yank", rubygems.DeletePackage) - }, reqRepoWriter(models.UnitTypePackages)) - }, reqToken()) + }, reqToken(), reqRepoWriter(models.UnitTypePackages)) + }) m.Group("/{id}", func() { m.Get("/", repo.GetPackage) m.Delete("/", reqRepoWriter(models.UnitTypePackages), repo.DeletePackage) diff --git a/templates/repo/packages/content/generic.tmpl b/templates/repo/packages/content/generic.tmpl index 05a0a5fd3fb78..f443edd17a483 100644 --- a/templates/repo/packages/content/generic.tmpl +++ b/templates/repo/packages/content/generic.tmpl @@ -4,7 +4,7 @@
    - +
    diff --git a/templates/repo/packages/content/maven.tmpl b/templates/repo/packages/content/maven.tmpl index 5b448e6ac8859..fcb794b6e6ab8 100644 --- a/templates/repo/packages/content/maven.tmpl +++ b/templates/repo/packages/content/maven.tmpl @@ -37,7 +37,7 @@
    - +
    diff --git a/templates/repo/packages/content/pypi.tmpl b/templates/repo/packages/content/pypi.tmpl index 7dbdcd7bffb18..0bba746885753 100644 --- a/templates/repo/packages/content/pypi.tmpl +++ b/templates/repo/packages/content/pypi.tmpl @@ -4,7 +4,7 @@
    - +
    From 2b9574e4878d0556a232da418bbb5b9833ca0a0c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 13 Sep 2021 17:06:00 +0200 Subject: [PATCH 051/130] Fixed comments. --- routers/api/v1/packages/nuget/links.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/packages/nuget/links.go b/routers/api/v1/packages/nuget/links.go index 82fa8ec1f94df..f782c7f2cbc1c 100644 --- a/routers/api/v1/packages/nuget/links.go +++ b/routers/api/v1/packages/nuget/links.go @@ -17,12 +17,12 @@ func (l *linkBuilder) GetRegistrationIndexURL(id string) string { return fmt.Sprintf("%s/registration/%s/index.json", l.Base, id) } -// GetRegistrationIndexURL builds the registration leaf url +// GetRegistrationLeafURL builds the registration leaf url func (l *linkBuilder) GetRegistrationLeafURL(id, version string) string { return fmt.Sprintf("%s/registration/%s/%s.json", l.Base, id, version) } -// GetRegistrationIndexURL builds the download url +// GetPackageDownloadURL builds the download url func (l *linkBuilder) GetPackageDownloadURL(id, version string) string { return fmt.Sprintf("%s/package/%s/%s/%s.%s.nupkg", l.Base, id, version, id, version) } From b9e6fb17572f765122ab085cd14d9b48affd9e57 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 13 Sep 2021 17:10:00 +0200 Subject: [PATCH 052/130] Fixed postgres test. --- integrations/api_packages_nuget_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index 9be8315511912..54d39d4655433 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "net/http" + "strings" "testing" "code.gitea.io/gitea/models" @@ -147,8 +148,12 @@ func TestPackageNuGet(t *testing.T) { pfs, err := ps[0].GetFiles() assert.NoError(t, err) assert.Len(t, pfs, 2) - assert.Equal(t, fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion), pfs[1].Name) - assert.Equal(t, int64(368), pfs[1].Size) + i := 0 + if strings.HasSuffix(pfs[1].Name, ".snupkg") { + i = 1 + } + assert.Equal(t, fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion), pfs[i].Name) + assert.Equal(t, int64(368), pfs[i].Size) req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/symbolpackage", url), createPackage(packageName, "SymbolsPackage")) req = AddBasicAuthHeader(req, user.Name) From cb16fe80dc3a7ee032d4c8f738f39a5cfbf7833f Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 28 Sep 2021 14:43:06 +0000 Subject: [PATCH 053/130] Moved file. --- models/{ => packages}/package.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/{ => packages}/package.go (100%) diff --git a/models/package.go b/models/packages/package.go similarity index 100% rename from models/package.go rename to models/packages/package.go From af0ea8ccc97aea68fa1f1caa5e4d7d0986372aaf Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 28 Sep 2021 15:14:47 +0000 Subject: [PATCH 054/130] Moved models to models/packages. --- integrations/api_packages_generic_test.go | 5 +- integrations/api_packages_maven_test.go | 7 +- integrations/api_packages_npm_test.go | 3 +- integrations/api_packages_nuget_test.go | 7 +- integrations/api_packages_pypi_test.go | 5 +- integrations/api_packages_rubygems_test.go | 5 +- models/packages/package.go | 73 ++++++++++---------- modules/convert/package.go | 10 +-- modules/notification/base/notifier.go | 5 +- modules/notification/base/null.go | 5 +- modules/notification/notification.go | 5 +- modules/notification/webhook/webhook.go | 7 +- routers/api/v1/packages/generic/generic.go | 16 ++--- routers/api/v1/packages/maven/maven.go | 22 +++--- routers/api/v1/packages/maven/package.go | 8 +-- routers/api/v1/packages/npm/npm.go | 14 ++-- routers/api/v1/packages/npm/package.go | 10 +-- routers/api/v1/packages/nuget/nuget.go | 32 ++++----- routers/api/v1/packages/nuget/package.go | 8 +-- routers/api/v1/packages/pypi/package.go | 10 +-- routers/api/v1/packages/pypi/pypi.go | 12 ++-- routers/api/v1/packages/rubygems/package.go | 8 +-- routers/api/v1/packages/rubygems/rubygems.go | 30 ++++---- routers/api/v1/repo/package.go | 14 ++-- routers/web/repo/packages.go | 27 ++++---- services/packages/packages.go | 54 ++++++++------- 26 files changed, 208 insertions(+), 194 deletions(-) diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index df06957c1bec9..0d17b11348a79 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" "github.com/stretchr/testify/assert" ) @@ -36,7 +37,7 @@ func TestPackageGeneric(t *testing.T) { req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) MakeRequest(t, req, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageGeneric) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeGeneric) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) @@ -71,7 +72,7 @@ func TestPackageGeneric(t *testing.T) { req := NewRequest(t, "DELETE", url) MakeRequest(t, req, http.StatusOK) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageGeneric) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeGeneric) assert.NoError(t, err) assert.Empty(t, ps) }) diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go index 8dcd5342105ac..80df57acc0245 100644 --- a/integrations/api_packages_maven_test.go +++ b/integrations/api_packages_maven_test.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/maven" @@ -44,7 +45,7 @@ func TestPackageMaven(t *testing.T) { putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusCreated) putFile(t, "/maven-metadata.xml", "test", http.StatusOK) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageMaven) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) @@ -99,7 +100,7 @@ func TestPackageMaven(t *testing.T) { ` + packageDescription + ` ` - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageMaven) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, ps, 1) @@ -110,7 +111,7 @@ func TestPackageMaven(t *testing.T) { putFile(t, fmt.Sprintf("/%s/%s.pom", packageVersion, filename), pomContent, http.StatusCreated) - ps, err = models.GetPackagesByRepositoryAndType(repository.ID, models.PackageMaven) + ps, err = packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, ps, 1) diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index e1d36c33c2306..c002329ae1259 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/setting" @@ -66,7 +67,7 @@ func TestPackageNpm(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNpm) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeNpm) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index e84405a31d78f..28b3342b25e24 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/routers/api/v1/packages/nuget" @@ -92,7 +93,7 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNuGet) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) @@ -142,7 +143,7 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNuGet) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Len(t, ps, 1) @@ -274,7 +275,7 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageNuGet) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Empty(t, ps) }) diff --git a/integrations/api_packages_pypi_test.go b/integrations/api_packages_pypi_test.go index d64e67790a535..c45915f5dff35 100644 --- a/integrations/api_packages_pypi_test.go +++ b/integrations/api_packages_pypi_test.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" "github.com/stretchr/testify/assert" ) @@ -63,7 +64,7 @@ func TestPackagePyPI(t *testing.T) { filename := "test.whl" uploadFile(t, filename, content, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackagePyPI) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypePyPI) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) @@ -82,7 +83,7 @@ func TestPackagePyPI(t *testing.T) { filename := "test.tar.gz" uploadFile(t, filename, content, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackagePyPI) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypePyPI) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) diff --git a/integrations/api_packages_rubygems_test.go b/integrations/api_packages_rubygems_test.go index 0ab73eb60296b..92932ca72f149 100644 --- a/integrations/api_packages_rubygems_test.go +++ b/integrations/api_packages_rubygems_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" "github.com/stretchr/testify/assert" ) @@ -122,7 +123,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) uploadFile(t, http.StatusCreated) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageRubyGems) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Len(t, ps, 1) assert.Equal(t, packageName, ps[0].Name) @@ -198,7 +199,7 @@ gAAAAP//MS06Gw==`) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - ps, err := models.GetPackagesByRepositoryAndType(repository.ID, models.PackageRubyGems) + ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Empty(t, ps) }) diff --git a/models/packages/package.go b/models/packages/package.go index 2b789d83e7cab..329da5e17ab1c 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -2,12 +2,13 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package models +package packages import ( "errors" "strings" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" @@ -19,51 +20,51 @@ func init() { db.RegisterModel(new(PackageFile)) } -// PackageType specifies the different package types -type PackageType int +// Type specifies the different package types +type Type int // Note: new type must append to the end of list to maintain compatibility. const ( - PackageGeneric PackageType = iota - PackageNuGet // 1 - PackageNpm // 2 - PackageMaven // 3 - PackagePyPI // 4 - PackageRubyGems // 5 + TypeGeneric Type = iota + TypeNuGet // 1 + TypeNpm // 2 + TypeMaven // 3 + TypePyPI // 4 + TypeRubyGems // 5 ) -func (pt PackageType) String() string { +func (pt Type) String() string { switch pt { - case PackageGeneric: + case TypeGeneric: return "Generic" - case PackageNuGet: + case TypeNuGet: return "NuGet" - case PackageNpm: + case TypeNpm: return "npm" - case PackageMaven: + case TypeMaven: return "Maven" - case PackagePyPI: + case TypePyPI: return "PyPI" - case PackageRubyGems: + case TypeRubyGems: return "RubyGems" } return "" } // SVGName returns the name of the package type svg image -func (pt PackageType) SVGName() string { +func (pt Type) SVGName() string { switch pt { - case PackageGeneric: + case TypeGeneric: return "octicon-package" - case PackageNuGet: + case TypeNuGet: return "gitea-nuget" - case PackageNpm: + case TypeNpm: return "gitea-npm" - case PackageMaven: + case TypeMaven: return "gitea-maven" - case PackagePyPI: + case TypePyPI: return "gitea-python" - case PackageRubyGems: + case TypeRubyGems: return "gitea-rubygems" } return "" @@ -85,8 +86,8 @@ type Package struct { ID int64 `xorm:"pk autoincr"` RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` CreatorID int64 - Creator *User `xorm:"-"` - Type PackageType `xorm:"UNIQUE(s) INDEX NOT NULL"` + Creator *models.User `xorm:"-"` + Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"` Name string LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` @@ -100,7 +101,7 @@ type Package struct { func (p *Package) LoadCreator() error { if p.Creator == nil { var err error - p.Creator, err = getUserByID(db.GetEngine(db.DefaultContext), p.CreatorID) + p.Creator, err = models.GetUserByID(p.CreatorID) return err } return nil @@ -218,17 +219,17 @@ func (opts *PackageSearchOptions) toConds() builder.Cond { switch opts.Type { case "generic": - cond = cond.And(builder.Eq{"package.type": PackageGeneric}) + cond = cond.And(builder.Eq{"package.type": TypeGeneric}) case "nuget": - cond = cond.And(builder.Eq{"package.type": PackageNuGet}) + cond = cond.And(builder.Eq{"package.type": TypeNuGet}) case "npm": - cond = cond.And(builder.Eq{"package.type": PackageNpm}) + cond = cond.And(builder.Eq{"package.type": TypeNpm}) case "maven": - cond = cond.And(builder.Eq{"package.type": PackageMaven}) + cond = cond.And(builder.Eq{"package.type": TypeMaven}) case "pypi": - cond = cond.And(builder.Eq{"package.type": PackagePyPI}) + cond = cond.And(builder.Eq{"package.type": TypePyPI}) case "rubygems": - cond = cond.And(builder.Eq{"package.type": PackageRubyGems}) + cond = cond.And(builder.Eq{"package.type": TypeRubyGems}) } if opts.Query != "" { @@ -293,19 +294,19 @@ func GetPackagesByRepository(repositoryID int64) ([]*Package, error) { } // GetPackagesByRepositoryAndType returns all packages of a repository with the specific type -func GetPackagesByRepositoryAndType(repositoryID int64, packageType PackageType) ([]*Package, error) { +func GetPackagesByRepositoryAndType(repositoryID int64, packageType Type) ([]*Package, error) { packages := make([]*Package, 0, 10) return packages, db.GetEngine(db.DefaultContext).Where("repo_id = ? AND type = ?", repositoryID, packageType).Find(&packages) } // GetPackagesByName gets all repository packages with the specific name -func GetPackagesByName(repositoryID int64, packageType PackageType, packageName string) ([]*Package, error) { +func GetPackagesByName(repositoryID int64, packageType Type, packageName string) ([]*Package, error) { packages := make([]*Package, 0, 10) return packages, db.GetEngine(db.DefaultContext).Where("repo_id = ? AND type = ? AND lower_name = ?", repositoryID, packageType, strings.ToLower(packageName)).Find(&packages) } // GetPackageByNameAndVersion gets a repository package by name and version -func GetPackageByNameAndVersion(repositoryID int64, packageType PackageType, packageName, packageVersion string) (*Package, error) { +func GetPackageByNameAndVersion(repositoryID int64, packageType Type, packageName, packageVersion string) (*Package, error) { p := &Package{ RepoID: repositoryID, Type: packageType, @@ -322,7 +323,7 @@ func GetPackageByNameAndVersion(repositoryID int64, packageType PackageType, pac } // GetPackagesByFilename gets a repository packages by filename -func GetPackagesByFilename(repositoryID int64, packageType PackageType, packageFilename string) ([]*Package, error) { +func GetPackagesByFilename(repositoryID int64, packageType Type, packageFilename string) ([]*Package, error) { var cond builder.Cond = builder.Eq{ "package.repo_id": repositoryID, "package.type": packageType, diff --git a/modules/convert/package.go b/modules/convert/package.go index 6d0b82ce23d13..e383b0aed28a2 100644 --- a/modules/convert/package.go +++ b/modules/convert/package.go @@ -5,12 +5,12 @@ package convert import ( - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" api "code.gitea.io/gitea/modules/structs" ) -// ToPackage convert a models.Package to api.Package -func ToPackage(p *models.Package) *api.Package { +// ToPackage convert a packages.Package to api.Package +func ToPackage(p *packages.Package) *api.Package { if err := p.LoadCreator(); err != nil { return &api.Package{} } @@ -26,8 +26,8 @@ func ToPackage(p *models.Package) *api.Package { } } -// ToPackageFile converts models.PackageFile to api.PackageFile -func ToPackageFile(pf *models.PackageFile) *api.PackageFile { +// ToPackageFile converts packages.PackageFile to api.PackageFile +func ToPackageFile(pf *packages.PackageFile) *api.PackageFile { return &api.PackageFile{ ID: pf.ID, Size: pf.Size, diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index 6654af7443cbe..8688aaa232fef 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -6,6 +6,7 @@ package base import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/repository" ) @@ -60,6 +61,6 @@ type Notifier interface { NotifyRepoPendingTransfer(doer, newOwner *models.User, repo *models.Repository) - NotifyPackageCreate(repo *models.Repository, p *models.Package) - NotifyPackageDelete(doer *models.User, repo *models.Repository, p *models.Package) + NotifyPackageCreate(repo *models.Repository, p *packages.Package) + NotifyPackageDelete(doer *models.User, repo *models.Repository, p *packages.Package) } diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go index 311dcc2e4a40a..34aadea774ad2 100644 --- a/modules/notification/base/null.go +++ b/modules/notification/base/null.go @@ -6,6 +6,7 @@ package base import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/repository" ) @@ -172,9 +173,9 @@ func (*NullNotifier) NotifyRepoPendingTransfer(doer, newOwner *models.User, repo } // NotifyPackageCreate places a place holder function -func (*NullNotifier) NotifyPackageCreate(repo *models.Repository, p *models.Package) { +func (*NullNotifier) NotifyPackageCreate(repo *models.Repository, p *packages.Package) { } // NotifyPackageDelete places a place holder function -func (*NullNotifier) NotifyPackageDelete(doer *models.User, repo *models.Repository, p *models.Package) { +func (*NullNotifier) NotifyPackageDelete(doer *models.User, repo *models.Repository, p *packages.Package) { } diff --git a/modules/notification/notification.go b/modules/notification/notification.go index a06e332a12c6d..c2c0bdbcf6eac 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -6,6 +6,7 @@ package notification import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/notification/action" "code.gitea.io/gitea/modules/notification/base" "code.gitea.io/gitea/modules/notification/indexer" @@ -299,14 +300,14 @@ func NotifyRepoPendingTransfer(doer, newOwner *models.User, repo *models.Reposit } // NotifyPackageCreate notifies creation of a package to notifiers -func NotifyPackageCreate(repo *models.Repository, p *models.Package) { +func NotifyPackageCreate(repo *models.Repository, p *packages.Package) { for _, notifier := range notifiers { notifier.NotifyPackageCreate(repo, p) } } // NotifyPackageDelete notifies deletion of a package to notifiers -func NotifyPackageDelete(doer *models.User, repo *models.Repository, p *models.Package) { +func NotifyPackageDelete(doer *models.User, repo *models.Repository, p *packages.Package) { for _, notifier := range notifiers { notifier.NotifyPackageDelete(doer, repo, p) } diff --git a/modules/notification/webhook/webhook.go b/modules/notification/webhook/webhook.go index 1c784ec1b1134..9f11e0b98300c 100644 --- a/modules/notification/webhook/webhook.go +++ b/modules/notification/webhook/webhook.go @@ -6,6 +6,7 @@ package webhook import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" @@ -819,7 +820,7 @@ func (m *webhookNotifier) NotifySyncDeleteRef(pusher *models.User, repo *models. m.NotifyDeleteRef(pusher, repo, refType, refFullName) } -func (m *webhookNotifier) NotifyPackageCreate(repo *models.Repository, p *models.Package) { +func (m *webhookNotifier) NotifyPackageCreate(repo *models.Repository, p *packages.Package) { if err := p.LoadCreator(); err != nil { log.Error("LoadCreator: %v", err) } @@ -827,11 +828,11 @@ func (m *webhookNotifier) NotifyPackageCreate(repo *models.Repository, p *models notifyPackage(p.Creator, repo, p, api.HookPackageCreated) } -func (m *webhookNotifier) NotifyPackageDelete(doer *models.User, repo *models.Repository, p *models.Package) { +func (m *webhookNotifier) NotifyPackageDelete(doer *models.User, repo *models.Repository, p *packages.Package) { notifyPackage(doer, repo, p, api.HookPackageDeleted) } -func notifyPackage(sender *models.User, repo *models.Repository, p *models.Package, action api.HookPackageAction) { +func notifyPackage(sender *models.User, repo *models.Repository, p *packages.Package, action api.HookPackageAction) { org := repo.MustOwner() if !org.IsOrganization() { org = nil diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 5d62acf2c6610..a93b0516d9424 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -9,7 +9,7 @@ import ( "net/http" "regexp" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util/filebuffer" @@ -30,9 +30,9 @@ func DownloadPackageFile(ctx *context.APIContext) { return } - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion, filename) + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeGeneric, packageName, packageVersion, filename) if err != nil { - if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { + if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -73,14 +73,14 @@ func UploadPackage(ctx *context.APIContext) { p, err := package_service.CreatePackage( ctx.User, ctx.Repo.Repository, - models.PackageGeneric, + packages.TypeGeneric, packageName, packageVersion, nil, false, ) if err != nil { - if err == models.ErrDuplicatePackage { + if err == packages.ErrDuplicatePackage { ctx.Error(http.StatusBadRequest, "", err) return } @@ -92,7 +92,7 @@ func UploadPackage(ctx *context.APIContext) { _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) if err != nil { log.Error("Error adding file to package: %v", err) - if err := models.DeletePackageByID(p.ID); err != nil { + if err := packages.DeletePackageByID(p.ID); err != nil { log.Error("Error deleting package by id: %v", err) } ctx.Error(http.StatusInternalServerError, "", "") @@ -110,9 +110,9 @@ func DeletePackage(ctx *context.APIContext) { return } - err = package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, models.PackageGeneric, packageName, packageVersion) + err = package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, packages.TypeGeneric, packageName, packageVersion) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) return } diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 019511a476f9a..2569e0538f273 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -19,11 +19,11 @@ import ( "regexp" "strings" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/packages" + packages_module "code.gitea.io/gitea/modules/packages" maven_module "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/util/filebuffer" @@ -62,7 +62,7 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { // /com/foo/project/maven-metadata.xml[.md5/.sha1/.sha256/.sha512] packageName := params.GroupID + "-" + params.ArtifactID - packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageMaven, packageName) + packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeMaven, packageName) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return @@ -112,8 +112,8 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { func servePackageFile(ctx *context.APIContext, params parameters) { packageName := params.GroupID + "-" + params.ArtifactID - p, err := models.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, models.PackageMaven, packageName, params.Version) - if err == models.ErrPackageNotExist { + p, err := packages.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, packages.TypeMaven, packageName, params.Version) + if err == packages.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -127,7 +127,7 @@ func servePackageFile(ctx *context.APIContext, params parameters) { pf, err := p.GetFileByName(filename) if err != nil { - if err == models.ErrPackageFileNotExist { + if err == packages.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", "") return } @@ -152,7 +152,7 @@ func servePackageFile(ctx *context.APIContext, params parameters) { return } - s, err := packages.NewContentStore().Get(p.ID, pf.ID) + s, err := packages_module.NewContentStore().Get(p.ID, pf.ID) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) } @@ -182,7 +182,7 @@ func UploadPackageFile(ctx *context.APIContext) { p, err := package_service.CreatePackage( ctx.User, ctx.Repo.Repository, - models.PackageMaven, + packages.TypeMaven, packageName, params.Version, &maven_module.Metadata{ @@ -209,7 +209,7 @@ func UploadPackageFile(ctx *context.APIContext) { if isChecksumExtension(ext) { pf, err := p.GetFileByName(params.Filename[:len(params.Filename)-len(ext)]) if err != nil { - if err == models.ErrPackageFileNotExist { + if err == packages.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", "") return } @@ -249,7 +249,7 @@ func UploadPackageFile(ctx *context.APIContext) { return } p.MetadataRaw = string(raw) - if err := models.UpdatePackage(p); err != nil { + if err := packages.UpdatePackage(p); err != nil { ctx.Error(http.StatusInternalServerError, "", err) return } @@ -262,7 +262,7 @@ func UploadPackageFile(ctx *context.APIContext) { _, err = package_service.AddFileToPackage(p, params.Filename, buf.Size(), buf) if err != nil { - if err == models.ErrDuplicatePackageFile { + if err == packages.ErrDuplicatePackageFile { ctx.Error(http.StatusBadRequest, "", err) return } diff --git a/routers/api/v1/packages/maven/package.go b/routers/api/v1/packages/maven/package.go index 02dc69e5dffa4..06b2ebcd39b68 100644 --- a/routers/api/v1/packages/maven/package.go +++ b/routers/api/v1/packages/maven/package.go @@ -7,18 +7,18 @@ package maven import ( "sort" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" maven_module "code.gitea.io/gitea/modules/packages/maven" ) // Package represents a package with Maven metadata type Package struct { - *models.Package + *packages.Package Metadata *maven_module.Metadata } -func intializePackages(packages []*models.Package) ([]*Package, error) { +func intializePackages(packages []*packages.Package) ([]*Package, error) { pgs := make([]*Package, 0, len(packages)) for _, p := range packages { np, err := intializePackage(p) @@ -30,7 +30,7 @@ func intializePackages(packages []*models.Package) ([]*Package, error) { return pgs, nil } -func intializePackage(p *models.Package) (*Package, error) { +func intializePackage(p *packages.Package) (*Package, error) { var m *maven_module.Metadata err := json.Unmarshal([]byte(p.MetadataRaw), &m) if err != nil { diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 5f5508fa9df8c..785b85b5e7b8f 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -9,7 +9,7 @@ import ( "net/http" "net/url" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" npm_module "code.gitea.io/gitea/modules/packages/npm" @@ -26,7 +26,7 @@ func PackageMetadata(ctx *context.APIContext) { return } - packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageNpm, packageName) + packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeNpm, packageName) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return @@ -60,9 +60,9 @@ func DownloadPackageFile(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, models.PackageNpm, packageName, packageVersion, filename) + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNpm, packageName, packageVersion, filename) if err != nil { - if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { + if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -85,14 +85,14 @@ func UploadPackage(ctx *context.APIContext) { p, err := package_service.CreatePackage( ctx.User, ctx.Repo.Repository, - models.PackageNpm, + packages.TypeNpm, npmPackage.Name, npmPackage.Version, npmPackage.Metadata, false, ) if err != nil { - if err == models.ErrDuplicatePackage { + if err == packages.ErrDuplicatePackage { ctx.Error(http.StatusBadRequest, "", err) return } @@ -102,7 +102,7 @@ func UploadPackage(ctx *context.APIContext) { _, err = package_service.AddFileToPackage(p, npmPackage.Filename, int64(len(npmPackage.Data)), bytes.NewReader(npmPackage.Data)) if err != nil { - if err := models.DeletePackageByID(p.ID); err != nil { + if err := packages.DeletePackageByID(p.ID); err != nil { log.Error("Error deleting package by id: %v", err) } ctx.Error(http.StatusInternalServerError, "", err) diff --git a/routers/api/v1/packages/npm/package.go b/routers/api/v1/packages/npm/package.go index 90a8b3c251288..5a81a09fafdba 100644 --- a/routers/api/v1/packages/npm/package.go +++ b/routers/api/v1/packages/npm/package.go @@ -7,7 +7,7 @@ package npm import ( "sort" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" npm_module "code.gitea.io/gitea/modules/packages/npm" @@ -16,13 +16,13 @@ import ( // Package represents a package with npm metadata type Package struct { - *models.Package - *models.PackageFile + *packages.Package + *packages.PackageFile SemVer *version.Version Metadata *npm_module.Metadata } -func intializePackages(packages []*models.Package) ([]*Package, error) { +func intializePackages(packages []*packages.Package) ([]*Package, error) { pgs := make([]*Package, 0, len(packages)) for _, p := range packages { np, err := intializePackage(p) @@ -34,7 +34,7 @@ func intializePackages(packages []*models.Package) ([]*Package, error) { return pgs, nil } -func intializePackage(p *models.Package) (*Package, error) { +func intializePackage(p *packages.Package) (*Package, error) { v, err := version.NewSemver(p.Version) if err != nil { return nil, err diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 7d97cd9507631..6169397af3945 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -10,8 +10,8 @@ import ( "net/http" "strings" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" nuget_module "code.gitea.io/gitea/modules/packages/nuget" @@ -32,7 +32,7 @@ func ServiceIndex(ctx *context.APIContext) { // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages func SearchService(ctx *context.APIContext) { - packages, count, err := models.GetPackages(&models.PackageSearchOptions{ + packages, count, err := packages.GetPackages(&packages.PackageSearchOptions{ RepoID: ctx.Repo.Repository.ID, Type: "nuget", Query: ctx.FormTrim("q"), @@ -65,7 +65,7 @@ func SearchService(ctx *context.APIContext) { func RegistrationIndex(ctx *context.APIContext) { packageName := ctx.Params("id") - packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageNuGet, packageName) + packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return @@ -94,9 +94,9 @@ func RegistrationLeaf(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") - p, err := models.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, models.PackageNuGet, packageName, packageVersion) + p, err := packages.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName, packageVersion) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -122,7 +122,7 @@ func RegistrationLeaf(ctx *context.APIContext) { func EnumeratePackageVersions(ctx *context.APIContext) { packageName := ctx.Params("id") - packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackageNuGet, packageName) + packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return @@ -149,9 +149,9 @@ func DownloadPackageFile(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, models.PackageNuGet, packageName, packageVersion, filename) + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNuGet, packageName, packageVersion, filename) if err != nil { - if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { + if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -179,14 +179,14 @@ func UploadPackage(ctx *context.APIContext) { p, err := package_service.CreatePackage( ctx.User, ctx.Repo.Repository, - models.PackageNuGet, + packages.TypeNuGet, meta.ID, meta.Version, meta, false, ) if err != nil { - if err == models.ErrDuplicatePackage { + if err == packages.ErrDuplicatePackage { ctx.Error(http.StatusBadRequest, "", err) return } @@ -197,7 +197,7 @@ func UploadPackage(ctx *context.APIContext) { filename := strings.ToLower(fmt.Sprintf("%s.%s.nupkg", meta.ID, meta.Version)) _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) if err != nil { - if err := models.DeletePackageByID(p.ID); err != nil { + if err := packages.DeletePackageByID(p.ID); err != nil { log.Error("Error deleting package by id: %v", err) } ctx.Error(http.StatusInternalServerError, "", err) @@ -220,9 +220,9 @@ func UploadSymbolPackage(ctx *context.APIContext) { return } - p, err := models.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, models.PackageNuGet, meta.ID, meta.Version) + p, err := packages.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, packages.TypeNuGet, meta.ID, meta.Version) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -233,7 +233,7 @@ func UploadSymbolPackage(ctx *context.APIContext) { filename := strings.ToLower(fmt.Sprintf("%s.%s.snupkg", meta.ID, meta.Version)) _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) if err != nil { - if err == models.ErrDuplicatePackageFile { + if err == packages.ErrDuplicatePackageFile { ctx.Error(http.StatusBadRequest, "", err) return } @@ -286,9 +286,9 @@ func DeletePackage(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") - err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, models.PackageNuGet, packageName, packageVersion) + err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, packages.TypeNuGet, packageName, packageVersion) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) return } diff --git a/routers/api/v1/packages/nuget/package.go b/routers/api/v1/packages/nuget/package.go index 94332fc9fc803..9d3d849f68013 100644 --- a/routers/api/v1/packages/nuget/package.go +++ b/routers/api/v1/packages/nuget/package.go @@ -7,7 +7,7 @@ package nuget import ( "sort" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" nuget_module "code.gitea.io/gitea/modules/packages/nuget" @@ -16,12 +16,12 @@ import ( // Package represents a package with NuGet metadata type Package struct { - *models.Package + *packages.Package SemVer *version.Version Metadata *nuget_module.Metadata } -func intializePackages(packages []*models.Package) ([]*Package, error) { +func intializePackages(packages []*packages.Package) ([]*Package, error) { pgs := make([]*Package, 0, len(packages)) for _, p := range packages { np, err := intializePackage(p) @@ -33,7 +33,7 @@ func intializePackages(packages []*models.Package) ([]*Package, error) { return pgs, nil } -func intializePackage(p *models.Package) (*Package, error) { +func intializePackage(p *packages.Package) (*Package, error) { v, err := version.NewSemver(p.Version) if err != nil { return nil, err diff --git a/routers/api/v1/packages/pypi/package.go b/routers/api/v1/packages/pypi/package.go index 812bbadc5e04b..26ddc4b7a03a9 100644 --- a/routers/api/v1/packages/pypi/package.go +++ b/routers/api/v1/packages/pypi/package.go @@ -5,19 +5,19 @@ package pypi import ( - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" pypi_module "code.gitea.io/gitea/modules/packages/pypi" ) // Package represents a package with PyPI metadata type Package struct { - *models.Package - Files []*models.PackageFile + *packages.Package + Files []*packages.PackageFile Metadata *pypi_module.Metadata } -func intializePackages(packages []*models.Package) ([]*Package, error) { +func intializePackages(packages []*packages.Package) ([]*Package, error) { pgs := make([]*Package, 0, len(packages)) for _, p := range packages { np, err := intializePackage(p) @@ -29,7 +29,7 @@ func intializePackages(packages []*models.Package) ([]*Package, error) { return pgs, nil } -func intializePackage(p *models.Package) (*Package, error) { +func intializePackage(p *packages.Package) (*Package, error) { var m *pypi_module.Metadata err := json.Unmarshal([]byte(p.MetadataRaw), &m) if err != nil { diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index 76888cd7ea665..41b06ca26c892 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -12,7 +12,7 @@ import ( "regexp" "strings" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" pypi_module "code.gitea.io/gitea/modules/packages/pypi" "code.gitea.io/gitea/modules/setting" @@ -33,7 +33,7 @@ var versionMatcher = regexp.MustCompile(`^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1 func PackageMetadata(ctx *context.APIContext) { packageName := normalizer.Replace(ctx.Params("id")) - packages, err := models.GetPackagesByName(ctx.Repo.Repository.ID, models.PackagePyPI, packageName) + packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypePyPI, packageName) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return @@ -62,9 +62,9 @@ func DownloadPackageFile(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, models.PackagePyPI, packageName, packageVersion, filename) + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypePyPI, packageName, packageVersion, filename) if err != nil { - if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { + if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -125,7 +125,7 @@ func UploadPackageFile(ctx *context.APIContext) { p, err := package_service.CreatePackage( ctx.User, ctx.Repo.Repository, - models.PackagePyPI, + packages.TypePyPI, packageName, packageVersion, metadata, @@ -138,7 +138,7 @@ func UploadPackageFile(ctx *context.APIContext) { _, err = package_service.AddFileToPackage(p, fileHeader.Filename, fileHeader.Size, file) if err != nil { - if err == models.ErrDuplicatePackageFile { + if err == packages.ErrDuplicatePackageFile { ctx.Error(http.StatusBadRequest, "", err) return } diff --git a/routers/api/v1/packages/rubygems/package.go b/routers/api/v1/packages/rubygems/package.go index cf978c621a45d..dd60e116855fd 100644 --- a/routers/api/v1/packages/rubygems/package.go +++ b/routers/api/v1/packages/rubygems/package.go @@ -5,18 +5,18 @@ package rubygems import ( - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" ) // Package represents a package with RubyGems metadata type Package struct { - *models.Package + *packages.Package Metadata *rubygems_module.Metadata } -func intializePackages(packages []*models.Package) ([]*Package, error) { +func intializePackages(packages []*packages.Package) ([]*Package, error) { pgs := make([]*Package, 0, len(packages)) for _, p := range packages { np, err := intializePackage(p) @@ -28,7 +28,7 @@ func intializePackages(packages []*models.Package) ([]*Package, error) { return pgs, nil } -func intializePackage(p *models.Package) (*Package, error) { +func intializePackage(p *packages.Package) (*Package, error) { var m *rubygems_module.Metadata if err := json.Unmarshal([]byte(p.MetadataRaw), &m); err != nil { return nil, err diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/v1/packages/rubygems/rubygems.go index 5483dba9fbe0a..66b943353a90e 100644 --- a/routers/api/v1/packages/rubygems/rubygems.go +++ b/routers/api/v1/packages/rubygems/rubygems.go @@ -12,7 +12,7 @@ import ( "net/http" "strings" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" @@ -23,7 +23,7 @@ import ( // EnumeratePackages serves the package list func EnumeratePackages(ctx *context.APIContext) { - packages, err := models.GetPackagesByRepositoryAndType(ctx.Repo.Repository.ID, models.PackageRubyGems) + packages, err := packages.GetPackagesByRepositoryAndType(ctx.Repo.Repository.ID, packages.TypeRubyGems) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return @@ -34,7 +34,7 @@ func EnumeratePackages(ctx *context.APIContext) { // EnumeratePackagesLatest serves the list of the lastest version of every package func EnumeratePackagesLatest(ctx *context.APIContext) { - packages, _, err := models.GetLatestPackagesGrouped(&models.PackageSearchOptions{ + packages, _, err := packages.GetLatestPackagesGrouped(&packages.PackageSearchOptions{ RepoID: ctx.Repo.Repository.ID, Type: "rubygems", }) @@ -48,10 +48,10 @@ func EnumeratePackagesLatest(ctx *context.APIContext) { // EnumeratePackagesPreRelease is not supported and serves an empty list func EnumeratePackagesPreRelease(ctx *context.APIContext) { - enumeratePackages(ctx, "prerelease_specs.4.8", []*models.Package{}) + enumeratePackages(ctx, "prerelease_specs.4.8", []*packages.Package{}) } -func enumeratePackages(ctx *context.APIContext, filename string, packages []*models.Package) { +func enumeratePackages(ctx *context.APIContext, filename string, packages []*packages.Package) { rubygemsPackages, err := intializePackages(packages) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) @@ -91,7 +91,7 @@ func ServePackageSpecification(ctx *context.APIContext) { return } - packages, err := models.GetPackagesByFilename(ctx.Repo.Repository.ID, models.PackageRubyGems, filename[:len(filename)-10]+"gem") + packages, err := packages.GetPackagesByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename[:len(filename)-10]+"gem") if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return @@ -124,20 +124,20 @@ func ServePackageSpecification(ctx *context.APIContext) { func DownloadPackageFile(ctx *context.APIContext) { filename := ctx.Params("filename") - packages, err := models.GetPackagesByFilename(ctx.Repo.Repository.ID, models.PackageRubyGems, filename) + pkgs, err := packages.GetPackagesByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename) if err != nil { ctx.Error(http.StatusInternalServerError, "", err) return } - if len(packages) != 1 { + if len(pkgs) != 1 { ctx.Error(http.StatusNotFound, "", nil) return } - s, pf, err := package_service.GetPackageFileStream(packages[0], filename) + s, pf, err := package_service.GetPackageFileStream(pkgs[0], filename) if err != nil { - if err == models.ErrPackageFileNotExist { + if err == packages.ErrPackageFileNotExist { ctx.Error(http.StatusNotFound, "", err) return } @@ -180,14 +180,14 @@ func UploadPackageFile(ctx *context.APIContext) { p, err := package_service.CreatePackage( ctx.User, ctx.Repo.Repository, - models.PackageRubyGems, + packages.TypeRubyGems, meta.Name, meta.Version, meta, false, ) if err != nil { - if err == models.ErrDuplicatePackage { + if err == packages.ErrDuplicatePackage { ctx.Error(http.StatusBadRequest, "", err) return } @@ -203,7 +203,7 @@ func UploadPackageFile(ctx *context.APIContext) { } _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) if err != nil { - if err := models.DeletePackageByID(p.ID); err != nil { + if err := packages.DeletePackageByID(p.ID); err != nil { log.Error("Error deleting package by id: %v", err) } ctx.Error(http.StatusInternalServerError, "", err) @@ -218,9 +218,9 @@ func DeletePackage(ctx *context.APIContext) { packageName := ctx.FormString("gem_name") packageVersion := ctx.FormString("version") - err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, models.PackageRubyGems, packageName, packageVersion) + err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, packages.TypeRubyGems, packageName, packageVersion) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages.ErrPackageNotExist { ctx.Error(http.StatusNotFound, "", err) return } diff --git a/routers/api/v1/repo/package.go b/routers/api/v1/repo/package.go index e95c553a0255e..0f2b8e037a78e 100644 --- a/routers/api/v1/repo/package.go +++ b/routers/api/v1/repo/package.go @@ -7,7 +7,7 @@ package repo import ( "net/http" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" api "code.gitea.io/gitea/modules/structs" @@ -61,7 +61,7 @@ func ListPackages(ctx *context.APIContext) { repo := ctx.Repo.Repository - packages, count, err := models.GetPackages(&models.PackageSearchOptions{ + packages, count, err := packages.GetPackages(&packages.PackageSearchOptions{ RepoID: repo.ID, Type: packageType, Query: query, @@ -116,9 +116,9 @@ func GetPackage(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - p, err := models.GetPackageByID(ctx.ParamsInt64(":id")) + p, err := packages.GetPackageByID(ctx.ParamsInt64(":id")) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages.ErrPackageNotExist { ctx.NotFound() } else { ctx.Error(http.StatusInternalServerError, "GetPackageByID", err) @@ -159,7 +159,7 @@ func DeletePackage(ctx *context.APIContext) { err := package_service.DeletePackageByID(ctx.User, ctx.Repo.Repository, ctx.ParamsInt64(":id")) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages.ErrPackageNotExist { ctx.NotFound() } else { ctx.Error(http.StatusInternalServerError, "DeletePackageByID", err) @@ -199,9 +199,9 @@ func ListPackageFiles(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - p, err := models.GetPackageByID(ctx.ParamsInt64(":id")) + p, err := packages.GetPackageByID(ctx.ParamsInt64(":id")) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages.ErrPackageNotExist { ctx.NotFound() } else { ctx.Error(http.StatusInternalServerError, "GetPackageByID", err) diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index 7b3ac7574fba9..ab4f9141f27cb 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" @@ -49,7 +50,7 @@ func Packages(ctx *context.Context) { repo := ctx.Repo.Repository - packages, count, err := models.GetLatestPackagesGrouped(&models.PackageSearchOptions{ + pkgs, count, err := packages.GetLatestPackagesGrouped(&packages.PackageSearchOptions{ RepoID: repo.ID, Query: query, Type: packageType, @@ -63,21 +64,21 @@ func Packages(ctx *context.Context) { return } - for _, p := range packages { + for _, p := range pkgs { if err := p.LoadCreator(); err != nil { ctx.ServerError("LoadCreator", err) return } } - hasPackages, err := models.HasRepositoryPackages(repo.ID) + hasPackages, err := packages.HasRepositoryPackages(repo.ID) if err != nil { ctx.ServerError("HasRepositoryPackages", err) return } ctx.Data["HasPackages"] = hasPackages - ctx.Data["Packages"] = packages + ctx.Data["Packages"] = pkgs ctx.Data["Query"] = query ctx.Data["PackageType"] = packageType @@ -91,9 +92,9 @@ func Packages(ctx *context.Context) { // ViewPackage displays a single package func ViewPackage(ctx *context.Context) { - p, err := models.GetPackageByID(ctx.ParamsInt64(":id")) + p, err := packages.GetPackageByID(ctx.ParamsInt64(":id")) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages.ErrPackageNotExist { ctx.NotFound("", nil) } else { ctx.ServerError("GetPackageByID", err) @@ -115,15 +116,15 @@ func ViewPackage(ctx *context.Context) { var metadata interface{} switch p.Type { - case models.PackageNuGet: + case packages.TypeNuGet: metadata = &nuget.Metadata{} - case models.PackageNpm: + case packages.TypeNpm: metadata = &npm.Metadata{} - case models.PackageMaven: + case packages.TypeMaven: metadata = &maven.Metadata{} - case models.PackagePyPI: + case packages.TypePyPI: metadata = &pypi.Metadata{} - case models.PackageRubyGems: + case packages.TypeRubyGems: metadata = &rubygems.Metadata{} } if metadata != nil { @@ -141,7 +142,7 @@ func ViewPackage(ctx *context.Context) { } ctx.Data["Files"] = files - otherVersions, err := models.GetPackagesByName(ctx.Repo.Repository.ID, p.Type, p.LowerName) + otherVersions, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, p.Type, p.LowerName) if err != nil { ctx.ServerError("GetPackagesByName", err) return @@ -174,7 +175,7 @@ func DeletePackagePost(ctx *context.Context) { func DownloadPackageFile(ctx *context.Context) { s, pf, err := package_service.GetFileStreamByPackageID(ctx.Repo.Repository, ctx.ParamsInt64(":id"), ctx.Params(":filename")) if err != nil { - if err == models.ErrPackageNotExist || err == models.ErrPackageFileNotExist { + if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { ctx.NotFound("", err) return } diff --git a/services/packages/packages.go b/services/packages/packages.go index 9abfc5f3d6661..0ab97d41ca280 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -14,6 +14,8 @@ import ( "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" + packages_models "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" @@ -21,7 +23,7 @@ import ( ) // CreatePackage creates a new package -func CreatePackage(creator *models.User, repository *models.Repository, packageType models.PackageType, name, version string, metadata interface{}, allowDuplicate bool) (*models.Package, error) { +func CreatePackage(creator *models.User, repository *models.Repository, packageType packages_models.Type, name, version string, metadata interface{}, allowDuplicate bool) (*packages_models.Package, error) { metadataJSON, err := json.Marshal(metadata) if err != nil { log.Error("Error converting metadata to JSON: %v", err) @@ -30,7 +32,7 @@ func CreatePackage(creator *models.User, repository *models.Repository, packageT log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", creator.ID, repository.ID, packageType, name, version, metadata, allowDuplicate) - p := &models.Package{ + p := &packages_models.Package{ RepoID: repository.ID, CreatorID: creator.ID, Type: packageType, @@ -39,8 +41,8 @@ func CreatePackage(creator *models.User, repository *models.Repository, packageT Version: version, MetadataRaw: string(metadataJSON), } - if p, err = models.TryInsertPackage(p); err != nil { - if err == models.ErrDuplicatePackage { + if p, err = packages_models.TryInsertPackage(p); err != nil { + if err == packages_models.ErrDuplicatePackage { if allowDuplicate { return p, nil } @@ -56,18 +58,18 @@ func CreatePackage(creator *models.User, repository *models.Repository, packageT } // AddFileToPackage adds a new file to package and stores its content -func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reader) (*models.PackageFile, error) { +func AddFileToPackage(p *packages_models.Package, filename string, size int64, r io.Reader) (*packages_models.PackageFile, error) { log.Trace("Creating package file: %v, %v, %s", p.ID, size, filename) - pf := &models.PackageFile{ + pf := &packages_models.PackageFile{ PackageID: p.ID, Size: size, Name: filename, LowerName: strings.ToLower(filename), } var err error - if pf, err = models.TryInsertPackageFile(pf); err != nil { - if err == models.ErrDuplicatePackageFile { + if pf, err = packages_models.TryInsertPackageFile(pf); err != nil { + if err == packages_models.ErrDuplicatePackageFile { return nil, err } log.Error("Error inserting package file: %v", err) @@ -94,7 +96,7 @@ func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reade pf.HashSHA1 = fmt.Sprintf("%x", h1.Sum(nil)) pf.HashSHA256 = fmt.Sprintf("%x", h256.Sum(nil)) pf.HashSHA512 = fmt.Sprintf("%x", h512.Sum(nil)) - if err = models.UpdatePackageFile(pf); err != nil { + if err = packages_models.UpdatePackageFile(pf); err != nil { log.Error("Error updating package file: %v", err) return err } @@ -103,7 +105,7 @@ func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reade if err != nil { _ = contentStore.Delete(p.ID, pf.ID) - if err := models.DeletePackageFileByID(pf.ID); err != nil { + if err := packages_models.DeletePackageFileByID(pf.ID); err != nil { log.Error("Error deleting package file: %v", err) } return nil, err @@ -112,12 +114,12 @@ func AddFileToPackage(p *models.Package, filename string, size int64, r io.Reade } // DeletePackageByNameAndVersion deletes a package and all associated files -func DeletePackageByNameAndVersion(doer *models.User, repository *models.Repository, packageType models.PackageType, name, version string) error { +func DeletePackageByNameAndVersion(doer *models.User, repository *models.Repository, packageType packages_models.Type, name, version string) error { log.Trace("Deleting package: %v, %v, %s, %s", repository.ID, packageType, name, version) - p, err := models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) + p, err := packages_models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages_models.ErrPackageNotExist { return err } log.Error("Error getting package: %v", err) @@ -131,9 +133,9 @@ func DeletePackageByNameAndVersion(doer *models.User, repository *models.Reposit func DeletePackageByID(doer *models.User, repository *models.Repository, packageID int64) error { log.Trace("Deleting package: %v, %v", repository.ID, packageID) - p, err := models.GetPackageByID(packageID) + p, err := packages_models.GetPackageByID(packageID) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages_models.ErrPackageNotExist { return err } log.Error("Error getting package: %v", err) @@ -141,13 +143,13 @@ func DeletePackageByID(doer *models.User, repository *models.Repository, package } if p.RepoID != repository.ID { - return models.ErrPackageNotExist + return packages_models.ErrPackageNotExist } return deletePackage(doer, repository, p) } -func deletePackage(doer *models.User, repository *models.Repository, p *models.Package) error { +func deletePackage(doer *models.User, repository *models.Repository, p *packages_models.Package) error { pfs, err := p.GetFiles() if err != nil { log.Error("Error getting package files: %v", err) @@ -162,7 +164,7 @@ func deletePackage(doer *models.User, repository *models.Repository, p *models.P } } - if err := models.DeletePackageByID(p.ID); err != nil { + if err := packages_models.DeletePackageByID(p.ID); err != nil { log.Error("Error deleting package: %v", err) return err } @@ -173,12 +175,12 @@ func deletePackage(doer *models.User, repository *models.Repository, p *models.P } // GetFileStreamByPackageNameAndVersion returns the content of the specific package file -func GetFileStreamByPackageNameAndVersion(repository *models.Repository, packageType models.PackageType, name, version, filename string) (io.ReadCloser, *models.PackageFile, error) { +func GetFileStreamByPackageNameAndVersion(repository *models.Repository, packageType packages.Type, name, version, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s", repository.ID, packageType, name, version, filename) - p, err := models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) + p, err := packages_models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages_models.ErrPackageNotExist { return nil, nil, err } log.Error("Error getting package: %v", err) @@ -189,12 +191,12 @@ func GetFileStreamByPackageNameAndVersion(repository *models.Repository, package } // GetFileStreamByPackageID returns the content of the specific package file -func GetFileStreamByPackageID(repository *models.Repository, packageID int64, filename string) (io.ReadCloser, *models.PackageFile, error) { +func GetFileStreamByPackageID(repository *models.Repository, packageID int64, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s", repository.ID, packageID, filename) - p, err := models.GetPackageByID(packageID) + p, err := packages_models.GetPackageByID(packageID) if err != nil { - if err == models.ErrPackageNotExist { + if err == packages_models.ErrPackageNotExist { return nil, nil, err } log.Error("Error getting package: %v", err) @@ -202,14 +204,14 @@ func GetFileStreamByPackageID(repository *models.Repository, packageID int64, fi } if p.RepoID != repository.ID { - return nil, nil, models.ErrPackageNotExist + return nil, nil, packages_models.ErrPackageNotExist } return GetPackageFileStream(p, filename) } // GetPackageFileStream returns the cotent of the specific package file -func GetPackageFileStream(p *models.Package, filename string) (io.ReadCloser, *models.PackageFile, error) { +func GetPackageFileStream(p *packages_models.Package, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { pf, err := p.GetFileByName(filename) if err != nil { return nil, nil, err From 775668964d8ef7046260c5ba4a7ab2288c24d2b8 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 30 Sep 2021 14:23:28 +0000 Subject: [PATCH 055/130] Use correct error response format per client. --- routers/api/v1/packages/helper.go | 38 +++++++++++ routers/api/v1/packages/maven/maven.go | 60 +++++++++-------- routers/api/v1/packages/npm/npm.go | 32 +++++---- routers/api/v1/packages/nuget/nuget.go | 69 ++++++++++++-------- routers/api/v1/packages/pypi/pypi.go | 34 ++++++---- routers/api/v1/packages/rubygems/rubygems.go | 48 ++++++++------ 6 files changed, 178 insertions(+), 103 deletions(-) create mode 100644 routers/api/v1/packages/helper.go diff --git a/routers/api/v1/packages/helper.go b/routers/api/v1/packages/helper.go new file mode 100644 index 0000000000000..1b53d2d0f54e5 --- /dev/null +++ b/routers/api/v1/packages/helper.go @@ -0,0 +1,38 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "fmt" + "net/http" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +// LogAndProcessError logs an error and calls a custom callback with the processed error message. +// If the error is an InternalServerError the message is stripped if the user is not an admin. +func LogAndProcessError(ctx *context.APIContext, status int, obj interface{}, cb func(string)) { + var message string + if err, ok := obj.(error); ok { + message = err.Error() + } else { + message = fmt.Sprintf("%s", obj) + } + if status == http.StatusInternalServerError { + log.ErrorWithSkip(1, message) + + if setting.IsProd() && !(ctx.User != nil && ctx.User.IsAdmin) { + message = "" + } + } else { + log.Debug(message) + } + + if cb != nil { + cb(message) + } +} diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 2569e0538f273..7de3d6aeaf16e 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -26,7 +26,7 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" maven_module "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/util/filebuffer" - + package_router "code.gitea.io/gitea/routers/api/v1/packages" package_service "code.gitea.io/gitea/services/packages" ) @@ -39,15 +39,21 @@ const ( ) var ( - errInvalidParameters = errors.New("Request parameters are invalid") + errInvalidParameters = errors.New("request parameters are invalid") illegalCharacters = regexp.MustCompile(`[\\/:"<>|?\*]`) ) +func apiError(ctx *context.APIContext, status int, obj interface{}) { + package_router.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, []byte(message)) + }) +} + // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.APIContext) { params, err := extractPathParameters(ctx) if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } @@ -62,25 +68,25 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { // /com/foo/project/maven-metadata.xml[.md5/.sha1/.sha256/.sha512] packageName := params.GroupID + "-" + params.ArtifactID - packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeMaven, packageName) + pkgs, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeMaven, packageName) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } - if len(packages) == 0 { - ctx.Error(http.StatusNotFound, "", "") + if len(pkgs) == 0 { + apiError(ctx, http.StatusNotFound, packages.ErrPackageNotExist) return } - mavenPackages, err := intializePackages(packages) + mavenPackages, err := intializePackages(pkgs) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } xmlMetadata, err := xml.Marshal(createMetadataResponse(mavenPackages)) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } xmlMetadataWithHeader := append([]byte(xml.Header), xmlMetadata...) @@ -114,7 +120,7 @@ func servePackageFile(ctx *context.APIContext, params parameters) { p, err := packages.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, packages.TypeMaven, packageName, params.Version) if err == packages.ErrPackageNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } @@ -128,11 +134,10 @@ func servePackageFile(ctx *context.APIContext, params parameters) { pf, err := p.GetFileByName(filename) if err != nil { if err == packages.ErrPackageFileNotExist { - ctx.Error(http.StatusNotFound, "", "") + apiError(ctx, http.StatusNotFound, err) return } - log.Error("Error getting file by name: %v", err) - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -154,7 +159,7 @@ func servePackageFile(ctx *context.APIContext, params parameters) { s, err := packages_module.NewContentStore().Get(p.ID, pf.ID) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) } defer s.Close() @@ -165,7 +170,7 @@ func servePackageFile(ctx *context.APIContext, params parameters) { func UploadPackageFile(ctx *context.APIContext) { params, err := extractPathParameters(ctx) if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } @@ -192,13 +197,13 @@ func UploadPackageFile(ctx *context.APIContext) { true, ) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } buf, err := filebuffer.CreateFromReader(ctx.Req.Body, 32*1024*1024) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } defer buf.Close() @@ -210,17 +215,16 @@ func UploadPackageFile(ctx *context.APIContext) { pf, err := p.GetFileByName(params.Filename[:len(params.Filename)-len(ext)]) if err != nil { if err == packages.ErrPackageFileNotExist { - ctx.Error(http.StatusNotFound, "", "") + apiError(ctx, http.StatusNotFound, err) return } - log.Error("GetFileByName: %v", err) - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } hash, err := ioutil.ReadAll(buf) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -228,7 +232,7 @@ func UploadPackageFile(ctx *context.APIContext) { (ext == extensionSHA1 && pf.HashSHA1 != string(hash)) || (ext == extensionSHA256 && pf.HashSHA256 != string(hash)) || (ext == extensionSHA512 && pf.HashSHA512 != string(hash)) { - ctx.Error(http.StatusBadRequest, "", "hash mismatch") + apiError(ctx, http.StatusBadRequest, "hash mismatch") return } @@ -245,17 +249,17 @@ func UploadPackageFile(ctx *context.APIContext) { if metadata != nil { raw, err := json.Marshal(metadata) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } p.MetadataRaw = string(raw) if err := packages.UpdatePackage(p); err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } } if _, err := buf.Seek(0, io.SeekStart); err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } } @@ -263,10 +267,10 @@ func UploadPackageFile(ctx *context.APIContext) { _, err = package_service.AddFileToPackage(p, params.Filename, buf.Size(), buf) if err != nil { if err == packages.ErrDuplicatePackageFile { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 785b85b5e7b8f..23b70ee28720b 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -14,31 +14,39 @@ import ( "code.gitea.io/gitea/modules/log" npm_module "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/setting" - + package_router "code.gitea.io/gitea/routers/api/v1/packages" package_service "code.gitea.io/gitea/services/packages" ) +func apiError(ctx *context.APIContext, status int, obj interface{}) { + package_router.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, map[string]string{ + "error": message, + }) + }) +} + // PackageMetadata returns the metadata for a single package func PackageMetadata(ctx *context.APIContext) { packageName, err := url.QueryUnescape(ctx.Params("id")) if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeNpm, packageName) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } if len(packages) == 0 { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } npmPackages, err := intializePackages(packages) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -54,7 +62,7 @@ func PackageMetadata(ctx *context.APIContext) { func DownloadPackageFile(ctx *context.APIContext) { packageName, err := url.QueryUnescape(ctx.Params("id")) if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } packageVersion := ctx.Params("version") @@ -63,10 +71,10 @@ func DownloadPackageFile(ctx *context.APIContext) { s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNpm, packageName, packageVersion, filename) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } defer s.Close() @@ -78,7 +86,7 @@ func DownloadPackageFile(ctx *context.APIContext) { func UploadPackage(ctx *context.APIContext) { npmPackage, err := npm_module.ParsePackage(ctx.Req.Body) if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } @@ -93,10 +101,10 @@ func UploadPackage(ctx *context.APIContext) { ) if err != nil { if err == packages.ErrDuplicatePackage { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -105,7 +113,7 @@ func UploadPackage(ctx *context.APIContext) { if err := packages.DeletePackageByID(p.ID); err != nil { log.Error("Error deleting package by id: %v", err) } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 6169397af3945..3a643cbaf7a5d 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -5,6 +5,7 @@ package nuget import ( + "errors" "fmt" "io" "net/http" @@ -17,10 +18,18 @@ import ( nuget_module "code.gitea.io/gitea/modules/packages/nuget" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util/filebuffer" - + package_router "code.gitea.io/gitea/routers/api/v1/packages" package_service "code.gitea.io/gitea/services/packages" ) +func apiError(ctx *context.APIContext, status int, obj interface{}) { + package_router.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, map[string]string{ + "Message": message, + }) + }) +} + // ServiceIndex https://docs.microsoft.com/en-us/nuget/api/service-index func ServiceIndex(ctx *context.APIContext) { repoName := ctx.Repo.Repository.FullName() @@ -42,13 +51,13 @@ func SearchService(ctx *context.APIContext) { ), }) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } nugetPackages, err := intializePackages(packages) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -67,17 +76,17 @@ func RegistrationIndex(ctx *context.APIContext) { packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } if len(packages) == 0 { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } nugetPackages, err := intializePackages(packages) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -97,16 +106,16 @@ func RegistrationLeaf(ctx *context.APIContext) { p, err := packages.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName, packageVersion) if err != nil { if err == packages.ErrPackageNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } nugetPackage, err := intializePackage(p) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -124,17 +133,17 @@ func EnumeratePackageVersions(ctx *context.APIContext) { packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } if len(packages) == 0 { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } nugetPackages, err := intializePackages(packages) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -152,10 +161,10 @@ func DownloadPackageFile(ctx *context.APIContext) { s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNuGet, packageName, packageVersion, filename) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } defer s.Close() @@ -187,10 +196,10 @@ func UploadPackage(ctx *context.APIContext) { ) if err != nil { if err == packages.ErrDuplicatePackage { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -200,7 +209,7 @@ func UploadPackage(ctx *context.APIContext) { if err := packages.DeletePackageByID(p.ID); err != nil { log.Error("Error deleting package by id: %v", err) } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -223,10 +232,10 @@ func UploadSymbolPackage(ctx *context.APIContext) { p, err := packages.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, packages.TypeNuGet, meta.ID, meta.Version) if err != nil { if err == packages.ErrPackageNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -234,10 +243,10 @@ func UploadSymbolPackage(ctx *context.APIContext) { _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) if err != nil { if err == packages.ErrDuplicatePackageFile { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -249,7 +258,7 @@ func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.Pack upload, close, err := ctx.UploadStream() if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return nil, nil, closables } @@ -259,22 +268,26 @@ func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.Pack buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return nil, nil, closables } closables = append(closables, buf) meta, err := nuget_module.ParsePackageMetaData(buf, buf.Size()) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + if err == nuget_module.ErrMissingNuspecFile || err == nuget_module.ErrNuspecFileTooLarge || err == nuget_module.ErrNuspecInvalidID || err == nuget_module.ErrNuspecInvalidVersion { + apiError(ctx, http.StatusBadRequest, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } return nil, nil, closables } if meta.PackageType != expectedType { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, errors.New("unexpected package type")) return nil, nil, closables } if _, err := buf.Seek(0, io.SeekStart); err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return nil, nil, closables } return meta, buf, closables @@ -289,9 +302,9 @@ func DeletePackage(ctx *context.APIContext) { err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, packages.TypeNuGet, packageName, packageVersion) if err != nil { if err == packages.ErrPackageNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", "") + apiError(ctx, http.StatusInternalServerError, err) } } diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index 41b06ca26c892..15ac61cad74d6 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -18,7 +18,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/validation" - + package_router "code.gitea.io/gitea/routers/api/v1/packages" package_service "code.gitea.io/gitea/services/packages" ) @@ -29,23 +29,29 @@ var nameMatcher = regexp.MustCompile(`\A[a-z0-9\.\-_]+\z`) // https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions var versionMatcher = regexp.MustCompile(`^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$`) +func apiError(ctx *context.APIContext, status int, obj interface{}) { + package_router.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, []byte(message)) + }) +} + // PackageMetadata returns the metadata for a single package func PackageMetadata(ctx *context.APIContext) { packageName := normalizer.Replace(ctx.Params("id")) packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypePyPI, packageName) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } if len(packages) == 0 { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } pypiPackages, err := intializePackages(packages) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -65,10 +71,10 @@ func DownloadPackageFile(ctx *context.APIContext) { s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypePyPI, packageName, packageVersion, filename) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } defer s.Close() @@ -80,30 +86,30 @@ func DownloadPackageFile(ctx *context.APIContext) { func UploadPackageFile(ctx *context.APIContext) { file, fileHeader, err := ctx.Req.FormFile("content") if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } defer file.Close() h256 := sha256.New() if _, err := io.Copy(h256, file); err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } if !strings.EqualFold(ctx.Req.FormValue("sha256_digest"), fmt.Sprintf("%x", h256.Sum(nil))) { - ctx.Error(http.StatusBadRequest, "", "hash mismatch") + apiError(ctx, http.StatusBadRequest, "hash mismatch") return } if _, err := file.Seek(0, io.SeekStart); err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } packageName := normalizer.Replace(ctx.Req.FormValue("name")) packageVersion := ctx.Req.FormValue("version") if !nameMatcher.MatchString(packageName) || !versionMatcher.MatchString(packageVersion) { - ctx.Error(http.StatusBadRequest, "", "invalid name or version") + apiError(ctx, http.StatusBadRequest, "invalid name or version") return } @@ -132,17 +138,17 @@ func UploadPackageFile(ctx *context.APIContext) { true, ) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } _, err = package_service.AddFileToPackage(p, fileHeader.Filename, fileHeader.Size, file) if err != nil { if err == packages.ErrDuplicatePackageFile { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/v1/packages/rubygems/rubygems.go index 66b943353a90e..25b5da576b590 100644 --- a/routers/api/v1/packages/rubygems/rubygems.go +++ b/routers/api/v1/packages/rubygems/rubygems.go @@ -17,15 +17,21 @@ import ( "code.gitea.io/gitea/modules/log" rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" "code.gitea.io/gitea/modules/util/filebuffer" - + package_router "code.gitea.io/gitea/routers/api/v1/packages" package_service "code.gitea.io/gitea/services/packages" ) +func apiError(ctx *context.APIContext, status int, obj interface{}) { + package_router.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, []byte(message)) + }) +} + // EnumeratePackages serves the package list func EnumeratePackages(ctx *context.APIContext) { packages, err := packages.GetPackagesByRepositoryAndType(ctx.Repo.Repository.ID, packages.TypeRubyGems) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -39,7 +45,7 @@ func EnumeratePackagesLatest(ctx *context.APIContext) { Type: "rubygems", }) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -54,7 +60,7 @@ func EnumeratePackagesPreRelease(ctx *context.APIContext) { func enumeratePackages(ctx *context.APIContext, filename string, packages []*packages.Package) { rubygemsPackages, err := intializePackages(packages) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -87,24 +93,24 @@ func ServePackageSpecification(ctx *context.APIContext) { filename := ctx.Params("filename") if !strings.HasSuffix(filename, ".gemspec.rz") { - ctx.Error(http.StatusBadRequest, "", nil) + apiError(ctx, http.StatusBadRequest, nil) return } packages, err := packages.GetPackagesByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename[:len(filename)-10]+"gem") if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } if len(packages) != 1 { - ctx.Error(http.StatusNotFound, "", nil) + apiError(ctx, http.StatusNotFound, nil) return } p, err := intializePackage(packages[0]) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -126,22 +132,22 @@ func DownloadPackageFile(ctx *context.APIContext) { pkgs, err := packages.GetPackagesByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } if len(pkgs) != 1 { - ctx.Error(http.StatusNotFound, "", nil) + apiError(ctx, http.StatusNotFound, nil) return } s, pf, err := package_service.GetPackageFileStream(pkgs[0], filename) if err != nil { if err == packages.ErrPackageFileNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } defer s.Close() @@ -153,7 +159,7 @@ func DownloadPackageFile(ctx *context.APIContext) { func UploadPackageFile(ctx *context.APIContext) { upload, close, err := ctx.UploadStream() if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } if close { @@ -162,18 +168,18 @@ func UploadPackageFile(ctx *context.APIContext) { buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } defer buf.Close() meta, err := rubygems_module.ParsePackageMetaData(buf) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } if _, err := buf.Seek(0, io.SeekStart); err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -188,10 +194,10 @@ func UploadPackageFile(ctx *context.APIContext) { ) if err != nil { if err == packages.ErrDuplicatePackage { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -206,7 +212,7 @@ func UploadPackageFile(ctx *context.APIContext) { if err := packages.DeletePackageByID(p.ID); err != nil { log.Error("Error deleting package by id: %v", err) } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -221,9 +227,9 @@ func DeletePackage(ctx *context.APIContext) { err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, packages.TypeRubyGems, packageName, packageVersion) if err != nil { if err == packages.ErrPackageNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", "") + apiError(ctx, http.StatusInternalServerError, err) } } From 88aa66f7114b271237a94b1d1bb676fbda08ddfb Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 2 Oct 2021 11:56:09 +0000 Subject: [PATCH 056/130] Use simpler search form. --- templates/repo/packages/list.tmpl | 44 +++++++++++-------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/templates/repo/packages/list.tmpl b/templates/repo/packages/list.tmpl index 09bba18bc7572..295f82af07213 100644 --- a/templates/repo/packages/list.tmpl +++ b/templates/repo/packages/list.tmpl @@ -3,36 +3,22 @@ {{template "repo/header" .}}
    {{template "base/alert" .}} -
    -
    -
    -
    - - - -
    -
    +
    +
    + + +
    -
    -
    -
    - - {{.i18n.Tr "repo.packages.filter.package_type"}} - {{svg "octicon-triangle-down" 14 "dropdown icon"}} - -
    - {{.i18n.Tr "repo.packages.filter.package_type.all"}} - Generic - NuGet - npm - Maven - PyPI - RubyGems -
    -
    -
    -
    -
    +
    {{range .Packages}}
  • From c444d4bc459a1794cc2b700ac4c71ecb0504590d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 30 Oct 2021 13:00:21 +0000 Subject: [PATCH 057/130] Fixed IsProd. --- routers/api/v1/packages/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/packages/helper.go b/routers/api/v1/packages/helper.go index 1b53d2d0f54e5..3a0c5c522146f 100644 --- a/routers/api/v1/packages/helper.go +++ b/routers/api/v1/packages/helper.go @@ -25,7 +25,7 @@ func LogAndProcessError(ctx *context.APIContext, status int, obj interface{}, cb if status == http.StatusInternalServerError { log.ErrorWithSkip(1, message) - if setting.IsProd() && !(ctx.User != nil && ctx.User.IsAdmin) { + if setting.IsProd && !(ctx.User != nil && ctx.User.IsAdmin) { message = "" } } else { From f3b0d770ff341c9dd4042361786471ed3ab05b35 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 7 Nov 2021 21:10:00 +0200 Subject: [PATCH 058/130] Restructured data model. --- integrations/api_packages_generic_test.go | 30 +- integrations/api_packages_maven_test.go | 78 +++- integrations/api_packages_npm_test.go | 26 +- integrations/api_packages_nuget_test.go | 57 ++- integrations/api_packages_pypi_test.go | 53 ++- integrations/api_packages_rubygems_test.go | 36 +- integrations/api_packages_test.go | 21 +- models/migrations/v202.go | 72 +++- models/packages/descriptor.go | 129 ++++++ models/packages/package.go | 342 +++------------- models/packages/package_blob.go | 78 ++++ models/packages/package_file.go | 85 ++++ models/packages/package_version.go | 219 ++++++++++ models/packages/package_version_property.go | 60 +++ modules/convert/package.go | 35 +- modules/notification/base/notifier.go | 4 +- modules/notification/base/null.go | 4 +- modules/notification/notification.go | 8 +- modules/notification/webhook/webhook.go | 22 +- modules/packages/content_store.go | 24 +- modules/packages/hashed_buffer.go | 80 ++++ modules/packages/nuget/metadata.go | 26 +- modules/packages/nuget/metadata_test.go | 66 +-- modules/packages/rubygems/metadata.go | 21 +- modules/packages/rubygems/metadata_test.go | 60 +-- modules/structs/package.go | 6 - routers/api/v1/api.go | 2 +- routers/api/v1/packages/generic/generic.go | 84 ++-- routers/api/v1/packages/maven/api.go | 35 +- routers/api/v1/packages/maven/maven.go | 141 ++++--- routers/api/v1/packages/maven/package.go | 59 --- routers/api/v1/packages/npm/api.go | 56 +-- routers/api/v1/packages/npm/npm.go | 55 +-- routers/api/v1/packages/npm/package.go | 74 ---- routers/api/v1/packages/nuget/api.go | 124 +++--- routers/api/v1/packages/nuget/nuget.go | 132 +++--- routers/api/v1/packages/nuget/package.go | 66 --- routers/api/v1/packages/pypi/package.go | 52 --- routers/api/v1/packages/pypi/pypi.go | 77 ++-- routers/api/v1/packages/rubygems/package.go | 75 ---- routers/api/v1/packages/rubygems/rubygems.go | 141 ++++--- routers/api/v1/repo/package.go | 56 ++- routers/web/repo/packages.go | 78 +--- services/packages/packages.go | 383 +++++++++++++----- templates/api/packages/pypi/simple.tmpl | 8 +- templates/repo/packages/content/maven.tmpl | 18 +- templates/repo/packages/content/npm.tmpl | 32 +- templates/repo/packages/content/nuget.tmpl | 16 +- templates/repo/packages/content/pypi.tmpl | 20 +- templates/repo/packages/content/rubygems.tmpl | 26 +- templates/repo/packages/list.tmpl | 14 +- templates/repo/packages/metadata/maven.tmpl | 8 +- templates/repo/packages/metadata/npm.tmpl | 8 +- templates/repo/packages/metadata/nuget.tmpl | 6 +- templates/repo/packages/metadata/pypi.tmpl | 8 +- .../repo/packages/metadata/rubygems.tmpl | 8 +- templates/repo/packages/view.tmpl | 23 +- 57 files changed, 2031 insertions(+), 1496 deletions(-) create mode 100644 models/packages/descriptor.go create mode 100644 models/packages/package_blob.go create mode 100644 models/packages/package_file.go create mode 100644 models/packages/package_version.go create mode 100644 models/packages/package_version_property.go create mode 100644 modules/packages/hashed_buffer.go delete mode 100644 routers/api/v1/packages/maven/package.go delete mode 100644 routers/api/v1/packages/npm/package.go delete mode 100644 routers/api/v1/packages/nuget/package.go delete mode 100644 routers/api/v1/packages/pypi/package.go delete mode 100644 routers/api/v1/packages/rubygems/package.go diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index 0d17b11348a79..9fc6346e2c6eb 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -37,17 +37,26 @@ func TestPackageGeneric(t *testing.T) { req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) MakeRequest(t, req, http.StatusCreated) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeGeneric) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeGeneric) assert.NoError(t, err) - assert.Len(t, ps, 1) - assert.Equal(t, packageName, ps[0].Name) - assert.Equal(t, packageVersion, ps[0].Version) + assert.Len(t, pvs, 1) - pfs, err := ps[0].GetFiles() + pd, err := packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.Nil(t, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) assert.Len(t, pfs, 1) assert.Equal(t, filename, pfs[0].Name) - assert.Equal(t, int64(len(content)), pfs[0].Size) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), pb.Size) }) t.Run("UploadExists", func(t *testing.T) { @@ -64,6 +73,11 @@ func TestPackageGeneric(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) assert.Equal(t, content, resp.Body.Bytes()) + + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeGeneric) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(1), pvs[0].DownloadCount) }) t.Run("Delete", func(t *testing.T) { @@ -72,9 +86,9 @@ func TestPackageGeneric(t *testing.T) { req := NewRequest(t, "DELETE", url) MakeRequest(t, req, http.StatusOK) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeGeneric) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeGeneric) assert.NoError(t, err) - assert.Empty(t, ps) + assert.Empty(t, pvs) }) t.Run("DownloadNotExists", func(t *testing.T) { diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go index 80df57acc0245..da526e23cc75d 100644 --- a/integrations/api_packages_maven_test.go +++ b/integrations/api_packages_maven_test.go @@ -13,7 +13,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/maven" "github.com/stretchr/testify/assert" @@ -45,17 +44,26 @@ func TestPackageMaven(t *testing.T) { putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusCreated) putFile(t, "/maven-metadata.xml", "test", http.StatusOK) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeMaven) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) assert.NoError(t, err) - assert.Len(t, ps, 1) - assert.Equal(t, packageName, ps[0].Name) - assert.Equal(t, packageVersion, ps[0].Version) + assert.Len(t, pvs, 1) - pfs, err := ps[0].GetFiles() + pd, err := packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.Nil(t, pd.SemVer) + assert.Nil(t, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) assert.Len(t, pfs, 1) assert.Equal(t, filename, pfs[0].Name) - assert.Equal(t, int64(4), pfs[0].Size) + assert.False(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(4), pb.Size) }) t.Run("UploadExists", func(t *testing.T) { @@ -72,6 +80,11 @@ func TestPackageMaven(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) assert.Equal(t, []byte("test"), resp.Body.Bytes()) + + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(0), pvs[0].DownloadCount) }) t.Run("UploadVerifySHA1", func(t *testing.T) { @@ -89,10 +102,7 @@ func TestPackageMaven(t *testing.T) { }) }) - t.Run("UploadPOM", func(t *testing.T) { - defer PrintCurrentTest(t)() - - pomContent := ` + pomContent := ` ` + groupID + ` ` + artifactID + ` @@ -100,23 +110,51 @@ func TestPackageMaven(t *testing.T) { ` + packageDescription + ` ` - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeMaven) + t.Run("UploadPOM", func(t *testing.T) { + defer PrintCurrentTest(t)() + + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) assert.NoError(t, err) - assert.Len(t, ps, 1) + assert.Len(t, pvs, 1) - var m *maven.Metadata - err = json.Unmarshal([]byte(ps[0].MetadataRaw), &m) + pd, err := packages.GetPackageDescriptor(pvs[0]) assert.NoError(t, err) - assert.Empty(t, m.Description) + assert.Nil(t, pd.Metadata) putFile(t, fmt.Sprintf("/%s/%s.pom", packageVersion, filename), pomContent, http.StatusCreated) - ps, err = packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeMaven) + pvs, err = packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) assert.NoError(t, err) - assert.Len(t, ps, 1) + assert.Len(t, pvs, 1) + + pd, err = packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.IsType(t, &maven.Metadata{}, pd.Metadata) + assert.Equal(t, packageDescription, pd.Metadata.(*maven.Metadata).Description) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 2) + i := 0 + if strings.HasSuffix(pfs[1].Name, ".pom") { + i = 1 + } + assert.Equal(t, filename+".pom", pfs[i].Name) + assert.True(t, pfs[i].IsLead) + }) + + t.Run("DownloadPOM", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, []byte(pomContent), resp.Body.Bytes()) - err = json.Unmarshal([]byte(ps[0].MetadataRaw), &m) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) assert.NoError(t, err) - assert.Equal(t, packageDescription, m.Description) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(1), pvs[0].DownloadCount) }) } diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index c002329ae1259..893c7c5f2206b 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -67,17 +67,26 @@ func TestPackageNpm(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeNpm) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNpm) assert.NoError(t, err) - assert.Len(t, ps, 1) - assert.Equal(t, packageName, ps[0].Name) - assert.Equal(t, packageVersion, ps[0].Version) + assert.Len(t, pvs, 1) - pfs, err := ps[0].GetFiles() + pd, err := packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &npm.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) assert.Len(t, pfs, 1) assert.Equal(t, filename, pfs[0].Name) - assert.Equal(t, int64(192), pfs[0].Size) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(192), pb.Size) }) t.Run("UploadExists", func(t *testing.T) { @@ -97,6 +106,11 @@ func TestPackageNpm(t *testing.T) { b, _ := base64.StdEncoding.DecodeString(data) assert.Equal(t, b, resp.Body.Bytes()) + + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNpm) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(1), pvs[0].DownloadCount) }) t.Run("PackageMetadata", func(t *testing.T) { diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index 28b3342b25e24..5af16127a9a99 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" + nuget_module "code.gitea.io/gitea/modules/packages/nuget" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/routers/api/v1/packages/nuget" @@ -93,17 +94,26 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) assert.NoError(t, err) - assert.Len(t, ps, 1) - assert.Equal(t, packageName, ps[0].Name) - assert.Equal(t, packageVersion, ps[0].Version) + assert.Len(t, pvs, 1) - pfs, err := ps[0].GetFiles() + pd, err := packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &nuget_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) assert.Len(t, pfs, 1) assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name) - assert.Equal(t, int64(len(content)), pfs[0].Size) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), pb.Size) req = NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) req = AddBasicAuthHeader(req, user.Name) @@ -143,11 +153,18 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) assert.NoError(t, err) - assert.Len(t, ps, 1) + assert.Len(t, pvs, 1) - pfs, err := ps[0].GetFiles() + pd, err := packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &nuget_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) assert.Len(t, pfs, 2) i := 0 @@ -155,7 +172,11 @@ func TestPackageNuGet(t *testing.T) { i = 1 } assert.Equal(t, fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion), pfs[i].Name) - assert.Equal(t, int64(368), pfs[i].Size) + assert.False(t, pfs[i].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[i].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(368), pb.Size) req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/symbolpackage", url), createPackage(packageName, "SymbolsPackage")) req = AddBasicAuthHeader(req, user.Name) @@ -172,9 +193,19 @@ func TestPackageNuGet(t *testing.T) { assert.Equal(t, content, resp.Body.Bytes()) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(1), pvs[0].DownloadCount) + req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.snupkg", url, packageName, packageVersion, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) + + pvs, err = packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(1), pvs[0].DownloadCount) }) t.Run("SearchService", func(t *testing.T) { @@ -271,13 +302,13 @@ func TestPackageNuGet(t *testing.T) { t.Run("Delete", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "DELETE", fmt.Sprintf("%s/package/%s/%s", url, packageName, packageVersion)) + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/%s/%s", url, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) assert.NoError(t, err) - assert.Empty(t, ps) + assert.Empty(t, pvs) }) t.Run("DownloadNotExists", func(t *testing.T) { diff --git a/integrations/api_packages_pypi_test.go b/integrations/api_packages_pypi_test.go index c45915f5dff35..e0fdce67f8ed5 100644 --- a/integrations/api_packages_pypi_test.go +++ b/integrations/api_packages_pypi_test.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/packages/pypi" "github.com/stretchr/testify/assert" ) @@ -64,17 +65,26 @@ func TestPackagePyPI(t *testing.T) { filename := "test.whl" uploadFile(t, filename, content, http.StatusCreated) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypePyPI) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypePyPI) assert.NoError(t, err) - assert.Len(t, ps, 1) - assert.Equal(t, packageName, ps[0].Name) - assert.Equal(t, packageVersion, ps[0].Version) + assert.Len(t, pvs, 1) - pfs, err := ps[0].GetFiles() + pd, err := packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &pypi.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) assert.Len(t, pfs, 1) assert.Equal(t, filename, pfs[0].Name) - assert.Equal(t, int64(4), pfs[0].Size) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(4), pb.Size) }) t.Run("UploadAddFile", func(t *testing.T) { @@ -83,21 +93,29 @@ func TestPackagePyPI(t *testing.T) { filename := "test.tar.gz" uploadFile(t, filename, content, http.StatusCreated) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypePyPI) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypePyPI) assert.NoError(t, err) - assert.Len(t, ps, 1) - assert.Equal(t, packageName, ps[0].Name) - assert.Equal(t, packageVersion, ps[0].Version) + assert.Len(t, pvs, 1) - pf, err := ps[0].GetFileByName(filename) + pd, err := packages.GetPackageDescriptor(pvs[0]) assert.NoError(t, err) - assert.NotNil(t, pf) - assert.Equal(t, filename, pf.Name) - assert.Equal(t, int64(4), pf.Size) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &pypi.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) - pfs, err := ps[0].GetFiles() + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) assert.Len(t, pfs, 2) + + pf, err := packages.GetFileForVersionByName(db.DefaultContext, pvs[0].ID, filename) + assert.NoError(t, err) + assert.Equal(t, filename, pf.Name) + assert.True(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(4), pb.Size) }) t.Run("UploadHashMismatch", func(t *testing.T) { @@ -127,6 +145,11 @@ func TestPackagePyPI(t *testing.T) { downloadFile("test.whl") downloadFile("test.tar.gz") + + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypePyPI) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(2), pvs[0].DownloadCount) }) t.Run("PackageMetadata", func(t *testing.T) { diff --git a/integrations/api_packages_rubygems_test.go b/integrations/api_packages_rubygems_test.go index 92932ca72f149..80f310045332c 100644 --- a/integrations/api_packages_rubygems_test.go +++ b/integrations/api_packages_rubygems_test.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/packages/rubygems" "github.com/stretchr/testify/assert" ) @@ -123,17 +124,26 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) uploadFile(t, http.StatusCreated) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeRubyGems) assert.NoError(t, err) - assert.Len(t, ps, 1) - assert.Equal(t, packageName, ps[0].Name) - assert.Equal(t, packageVersion, ps[0].Version) + assert.Len(t, pvs, 1) - pfs, err := ps[0].GetFiles() + pd, err := packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &rubygems.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) assert.Len(t, pfs, 1) assert.Equal(t, packageFilename, pfs[0].Name) - assert.Equal(t, int64(4608), pfs[0].Size) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(4608), pb.Size) }) t.Run("UploadExists", func(t *testing.T) { @@ -150,6 +160,11 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) resp := MakeRequest(t, req, http.StatusOK) assert.Equal(t, gemContent, resp.Body.Bytes()) + + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeRubyGems) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(1), pvs[0].DownloadCount) }) t.Run("DownloadGemspec", func(t *testing.T) { @@ -164,6 +179,11 @@ EBJqJQjWFZZaVJyZnxfN5qnEZahnoGcKkjTwVBJyB6lUKEhMzk5MTwULGngqcRaVJlWCONEMBp5K DGAWSKc7zFhPJamg0qRK99TcYphehZLU4hKInFhGSUlBsZW+PtgZepn5+iDxECRzDUDGcfh6hoA4 gAAAAP//MS06Gw==`) assert.Equal(t, b, resp.Body.Bytes()) + + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeRubyGems) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(1), pvs[0].DownloadCount) }) t.Run("EnumeratePackages", func(t *testing.T) { @@ -199,8 +219,8 @@ gAAAAP//MS06Gw==`) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - ps, err := packages.GetPackagesByRepositoryAndType(repository.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeRubyGems) assert.NoError(t, err) - assert.Empty(t, ps) + assert.Empty(t, pvs) }) } diff --git a/integrations/api_packages_test.go b/integrations/api_packages_test.go index 8eceb43a8261f..0db6cf5786c37 100644 --- a/integrations/api_packages_test.go +++ b/integrations/api_packages_test.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" api "code.gitea.io/gitea/modules/structs" "github.com/stretchr/testify/assert" @@ -40,17 +41,17 @@ func TestPackageAPI(t *testing.T) { req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages?token=%s", user.Name, repository.Name, token)) resp := MakeRequest(t, req, http.StatusOK) - var packages []*api.Package - DecodeJSON(t, resp, &packages) + var apiPackages []*api.Package + DecodeJSON(t, resp, &apiPackages) - assert.Len(t, packages, 1) - assert.Equal(t, "Generic", packages[0].Type) - assert.Equal(t, packageName, packages[0].Name) - assert.Equal(t, packageVersion, packages[0].Version) - assert.NotNil(t, packages[0].Creator) - assert.Equal(t, user.Name, packages[0].Creator.UserName) + assert.Len(t, apiPackages, 1) + assert.Equal(t, string(packages.TypeGeneric), apiPackages[0].Type) + assert.Equal(t, packageName, apiPackages[0].Name) + assert.Equal(t, packageVersion, apiPackages[0].Version) + assert.NotNil(t, apiPackages[0].Creator) + assert.Equal(t, user.Name, apiPackages[0].Creator.UserName) - packageID = packages[0].ID + packageID = apiPackages[0].ID }) t.Run("GetPackage", func(t *testing.T) { @@ -65,7 +66,7 @@ func TestPackageAPI(t *testing.T) { var p *api.Package DecodeJSON(t, resp, &p) - assert.Equal(t, "Generic", p.Type) + assert.Equal(t, string(packages.TypeGeneric), p.Type) assert.Equal(t, packageName, p.Name) assert.Equal(t, packageVersion, p.Version) assert.NotNil(t, p.Creator) diff --git a/models/migrations/v202.go b/models/migrations/v202.go index 56ee43c6942f5..77e677a5a652d 100644 --- a/models/migrations/v202.go +++ b/models/migrations/v202.go @@ -12,37 +12,65 @@ import ( func addPackageTables(x *xorm.Engine) error { type Package struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - CreatorID int64 - Type int `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` - MetadataRaw string `xorm:"TEXT"` - - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + SemverCompatible bool } if err := x.Sync2(new(Package)); err != nil { return err } + type PackageVersion struct { + ID int64 `xorm:"pk autoincr"` + PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatorID int64 + Version string + LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + MetadataJSON string `xorm:"TEXT metadata_json"` + DownloadCount int64 + } + + if err := x.Sync2(new(PackageVersion)); err != nil { + return err + } + + type PackageVersionProperty struct { + ID int64 `xorm:"pk autoincr"` + VersionID int64 `xorm:"INDEX NOT NULL"` + Name string `xorm:"INDEX NOT NULL"` + Value string + } + + if err := x.Sync2(new(PackageVersionProperty)); err != nil { + return err + } + type PackageFile struct { + ID int64 `xorm:"pk autoincr"` + VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + BlobID int64 `xorm:"INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + IsLead bool + } + + if err := x.Sync2(new(PackageFile)); err != nil { + return err + } + + type PackageBlob struct { ID int64 `xorm:"pk autoincr"` - PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` Size int64 - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - HashMD5 string `xorm:"hash_md5"` - HashSHA1 string `xorm:"hash_sha1"` - HashSHA256 string `xorm:"hash_sha256"` - HashSHA512 string `xorm:"hash_sha512"` - - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + HashMD5 string `xorm:"hash_md5 char(32) UNIQUE(md5) INDEX NOT NULL"` + HashSHA1 string `xorm:"hash_sha1 char(40) UNIQUE(sha1) INDEX NOT NULL"` + HashSHA256 string `xorm:"hash_sha256 char(64) UNIQUE(sha256) INDEX NOT NULL"` + HashSHA512 string `xorm:"hash_sha512 char(128) UNIQUE(sha512) INDEX NOT NULL"` } - return x.Sync2(new(PackageFile)) + return x.Sync2(new(PackageBlob)) } diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go new file mode 100644 index 0000000000000..b4ea0161afa2d --- /dev/null +++ b/models/packages/descriptor.go @@ -0,0 +1,129 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/maven" + "code.gitea.io/gitea/modules/packages/npm" + "code.gitea.io/gitea/modules/packages/nuget" + "code.gitea.io/gitea/modules/packages/pypi" + "code.gitea.io/gitea/modules/packages/rubygems" + + "github.com/hashicorp/go-version" +) + +// PackageDescriptor describes a package +type PackageDescriptor struct { + Package *Package + Repository *models.Repository + Version *PackageVersion + SemVer *version.Version + Creator *models.User + Properties []*PackageVersionProperty + Metadata interface{} + Files []PackageFileDescriptor +} + +// PackageFileDescriptor describes a package file +type PackageFileDescriptor struct { + File *PackageFile + Blob *PackageBlob +} + +// GetPackageDescriptor gets the package description for a version +func GetPackageDescriptor(pv *PackageVersion) (*PackageDescriptor, error) { + return GetPackageDescriptorCtx(db.DefaultContext, pv) +} + +// GetPackageDescriptorCtx gets the package description for a version +func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageDescriptor, error) { + p, err := GetPackageByID(ctx, pv.PackageID) + if err != nil { + return nil, err + } + repository, err := models.GetRepositoryByIDCtx(ctx, p.RepoID) + if err != nil { + return nil, err + } + creator, err := models.GetUserByID(pv.CreatorID) + if err != nil { + return nil, err + } + var semVer *version.Version + if p.SemverCompatible { + semVer, err = version.NewVersion(pv.Version) + if err != nil { + return nil, err + } + } + pvps, err := GetVersionProperties(ctx, pv.ID) + if err != nil { + return nil, err + } + pfs, err := GetFilesByVersionID(ctx, pv.ID) + if err != nil { + return nil, err + } + + pfds := make([]PackageFileDescriptor, 0, len(pfs)) + for _, pf := range pfs { + pb, err := GetBlobByID(ctx, pf.BlobID) + if err != nil { + return nil, err + } + pfds = append(pfds, PackageFileDescriptor{ + pf, + pb, + }) + } + + var metadata interface{} + switch p.Type { + case TypeNuGet: + metadata = &nuget.Metadata{} + case TypeNpm: + metadata = &npm.Metadata{} + case TypeMaven: + metadata = &maven.Metadata{} + case TypePyPI: + metadata = &pypi.Metadata{} + case TypeRubyGems: + metadata = &rubygems.Metadata{} + } + if metadata != nil { + if err := json.Unmarshal([]byte(pv.MetadataJSON), &metadata); err != nil { + return nil, err + } + } + + return &PackageDescriptor{ + Package: p, + Repository: repository, + Version: pv, + SemVer: semVer, + Creator: creator, + Properties: pvps, + Metadata: metadata, + Files: pfds, + }, nil +} + +// GetPackageDescriptors gets the package descriptions for the versions +func GetPackageDescriptors(pvs []*PackageVersion) ([]*PackageDescriptor, error) { + pds := make([]*PackageDescriptor, 0, len(pvs)) + for _, pv := range pvs { + pd, err := GetPackageDescriptor(pv) + if err != nil { + return nil, err + } + pds = append(pds, pd) + } + return pds, nil +} diff --git a/models/packages/package.go b/models/packages/package.go index 329da5e17ab1c..7fcbcc13835da 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -5,35 +5,38 @@ package packages import ( + "context" "errors" - "strings" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/modules/timeutil" - - "xorm.io/builder" ) func init() { db.RegisterModel(new(Package)) - db.RegisterModel(new(PackageFile)) } -// Type specifies the different package types -type Type int +var ( + // ErrDuplicatePackage indicates a duplicated package error + ErrDuplicatePackage = errors.New("Package does exist already") + // ErrPackageNotExist indicates a package not exist error + ErrPackageNotExist = errors.New("Package does not exist") +) + +// Type of a package +type Type string -// Note: new type must append to the end of list to maintain compatibility. +// List of supported packages const ( - TypeGeneric Type = iota - TypeNuGet // 1 - TypeNpm // 2 - TypeMaven // 3 - TypePyPI // 4 - TypeRubyGems // 5 + TypeGeneric Type = "generic" + TypeNuGet Type = "nuget" + TypeNpm Type = "npm" + TypeMaven Type = "maven" + TypePyPI Type = "pypi" + TypeRubyGems Type = "rubygems" ) -func (pt Type) String() string { +// Name gets the name of the package type +func (pt Type) Name() string { switch pt { case TypeGeneric: return "Generic" @@ -51,7 +54,7 @@ func (pt Type) String() string { return "" } -// SVGName returns the name of the package type svg image +// SVGName gets the name of the package type svg image func (pt Type) SVGName() string { switch pt { case TypeGeneric: @@ -70,314 +73,71 @@ func (pt Type) SVGName() string { return "" } -var ( - // ErrDuplicatePackage indicates a duplicated package error - ErrDuplicatePackage = errors.New("Package does exist already") - // ErrPackageNotExist indicates a package not exist error - ErrPackageNotExist = errors.New("Package does not exist") - // ErrDuplicatePackageFile indicates a duplicated package file error - ErrDuplicatePackageFile = errors.New("Package file does exist already") - // ErrPackageFileNotExist indicates a package file not exist error - ErrPackageFileNotExist = errors.New("Package file does not exist") -) - // Package represents a package type Package struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - CreatorID int64 - Creator *models.User `xorm:"-"` - Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Version string `xorm:"UNIQUE(s) INDEX NOT NULL"` - MetadataRaw string `xorm:"TEXT"` - - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` -} - -// LoadCreator loads the creator -func (p *Package) LoadCreator() error { - if p.Creator == nil { - var err error - p.Creator, err = models.GetUserByID(p.CreatorID) - return err - } - return nil -} - -// PackageFile represents files associated with a package -type PackageFile struct { - ID int64 `xorm:"pk autoincr"` - PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - Size int64 - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - HashMD5 string `xorm:"hash_md5"` - HashSHA1 string `xorm:"hash_sha1"` - HashSHA256 string `xorm:"hash_sha256"` - HashSHA512 string `xorm:"hash_sha512"` - - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` -} - -// GetFiles loads all files associated with the package -func (p *Package) GetFiles() ([]*PackageFile, error) { - packageFiles := make([]*PackageFile, 0, 10) - return packageFiles, db.GetEngine(db.DefaultContext).Where("package_id = ?", p.ID).Find(&packageFiles) -} - -// GetFileByName gets the specific package file by name -func (p *Package) GetFileByName(name string) (*PackageFile, error) { - pf := &PackageFile{ - PackageID: p.ID, - LowerName: strings.ToLower(name), - } - has, err := db.GetEngine(db.DefaultContext).Get(pf) - if err != nil { - return nil, err - } - if !has { - return nil, ErrPackageFileNotExist - } - return pf, nil + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + SemverCompatible bool } -// TryInsertPackage inserts a package -// If a package already exists ErrDuplicatePackage is returned -func TryInsertPackage(p *Package) (*Package, error) { - sess := db.NewSession(db.DefaultContext) - defer sess.Close() - if err := sess.Begin(); err != nil { - return nil, err - } +// TryInsertPackage inserts a package. If a package exists already, ErrDuplicatePackage is returned +func TryInsertPackage(ctx context.Context, p *Package) (*Package, error) { + e := db.GetEngine(ctx) key := &Package{ RepoID: p.RepoID, Type: p.Type, LowerName: p.LowerName, - Version: p.Version, } - has, err := sess.Get(key) + has, err := e.Get(key) if err != nil { return nil, err } if has { return key, ErrDuplicatePackage } - if _, err = sess.Insert(p); err != nil { + if _, err = e.Insert(p); err != nil { return nil, err } - return p, sess.Commit() -} - -// UpdatePackage updates a package -func UpdatePackage(p *Package) error { - _, err := db.GetEngine(db.DefaultContext).ID(p.ID).Update(p) - return err -} - -// DeletePackageByID deletes a package and its files by ID -func DeletePackageByID(packageID int64) error { - if err := DeletePackageFilesByPackageID(packageID); err != nil { - return err - } - - _, err := db.GetEngine(db.DefaultContext).ID(packageID).Delete(&Package{}) - return err -} - -// DeletePackagesByRepositoryID deletes all packages of a repository -func DeletePackagesByRepositoryID(repositoryID int64) error { - packages, err := GetPackagesByRepository(repositoryID) - if err != nil { - return err - } - - for _, p := range packages { - if err := DeletePackageByID(p.ID); err != nil { - return err - } - } - - return nil -} - -// PackageSearchOptions are options for GetLatestPackagesGrouped -type PackageSearchOptions struct { - RepoID int64 - Type string - Query string - db.Paginator -} - -func (opts *PackageSearchOptions) toConds() builder.Cond { - var cond builder.Cond = builder.Eq{"package.repo_id": opts.RepoID} - - switch opts.Type { - case "generic": - cond = cond.And(builder.Eq{"package.type": TypeGeneric}) - case "nuget": - cond = cond.And(builder.Eq{"package.type": TypeNuGet}) - case "npm": - cond = cond.And(builder.Eq{"package.type": TypeNpm}) - case "maven": - cond = cond.And(builder.Eq{"package.type": TypeMaven}) - case "pypi": - cond = cond.And(builder.Eq{"package.type": TypePyPI}) - case "rubygems": - cond = cond.And(builder.Eq{"package.type": TypeRubyGems}) - } - - if opts.Query != "" { - cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.Query)}) - } - - return cond -} - -// GetPackages returns a list of all packages of the repository -func GetPackages(opts *PackageSearchOptions) ([]*Package, int64, error) { - sess := db.GetEngine(db.DefaultContext).Where(opts.toConds()) - - if opts.Paginator != nil { - sess = db.SetSessionPagination(sess, opts) - } - - packages := make([]*Package, 0, 10) - count, err := sess.FindAndCount(&packages) - return packages, count, err -} - -// GetLatestPackagesGrouped returns a list of all packages in their latest version of the repository -func GetLatestPackagesGrouped(opts *PackageSearchOptions) ([]*Package, int64, error) { - cond := opts.toConds(). - And(builder.Expr("p2.id IS NULL")) - - sess := db.GetEngine(db.DefaultContext).Where(cond). - Table("package"). - Join("left", "package p2", "package.repo_id = p2.repo_id AND package.type = p2.type AND package.lower_name = p2.lower_name AND package.version < p2.version") - - if opts.Paginator != nil { - sess = db.SetSessionPagination(sess, opts) - } - - packages := make([]*Package, 0, 10) - count, err := sess.FindAndCount(&packages) - return packages, count, err -} - -// GetPackageByID returns the package with the specific id -func GetPackageByID(packageID int64) (*Package, error) { - p := &Package{} - has, err := db.GetEngine(db.DefaultContext).ID(packageID).Get(p) - if err != nil { - return nil, err - } else if !has { - return nil, ErrPackageNotExist - } return p, nil } -// HasRepositoryPackages tests if a repository has packages -func HasRepositoryPackages(repositoryID int64) (bool, error) { - return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repositoryID).Exist(&Package{}) -} - -// GetPackagesByRepository returns all packages of a repository -func GetPackagesByRepository(repositoryID int64) ([]*Package, error) { - packages := make([]*Package, 0, 10) - return packages, db.GetEngine(db.DefaultContext).Where("repo_id = ?", repositoryID).Find(&packages) -} - -// GetPackagesByRepositoryAndType returns all packages of a repository with the specific type -func GetPackagesByRepositoryAndType(repositoryID int64, packageType Type) ([]*Package, error) { - packages := make([]*Package, 0, 10) - return packages, db.GetEngine(db.DefaultContext).Where("repo_id = ? AND type = ?", repositoryID, packageType).Find(&packages) -} - -// GetPackagesByName gets all repository packages with the specific name -func GetPackagesByName(repositoryID int64, packageType Type, packageName string) ([]*Package, error) { - packages := make([]*Package, 0, 10) - return packages, db.GetEngine(db.DefaultContext).Where("repo_id = ? AND type = ? AND lower_name = ?", repositoryID, packageType, strings.ToLower(packageName)).Find(&packages) -} +// GetPackageByID gets a package by id +func GetPackageByID(ctx context.Context, packageID int64) (*Package, error) { + p := &Package{} -// GetPackageByNameAndVersion gets a repository package by name and version -func GetPackageByNameAndVersion(repositoryID int64, packageType Type, packageName, packageVersion string) (*Package, error) { - p := &Package{ - RepoID: repositoryID, - Type: packageType, - LowerName: strings.ToLower(packageName), - Version: packageVersion, - } - has, err := db.GetEngine(db.DefaultContext).Get(p) + has, err := db.GetEngine(ctx).ID(packageID).Get(p) if err != nil { return nil, err - } else if !has { + } + if !has { return nil, ErrPackageNotExist } return p, nil } -// GetPackagesByFilename gets a repository packages by filename -func GetPackagesByFilename(repositoryID int64, packageType Type, packageFilename string) ([]*Package, error) { - var cond builder.Cond = builder.Eq{ - "package.repo_id": repositoryID, - "package.type": packageType, - "package_file.lower_name": strings.ToLower(packageFilename), - } - - packages := make([]*Package, 0, 10) - return packages, db.GetEngine(db.DefaultContext). - Table("package"). - Where(cond). - Join("INNER", "package_file", "package.id = package_file.package_id"). - Find(&packages) -} - -// TryInsertPackageFile inserts a package file -func TryInsertPackageFile(pf *PackageFile) (*PackageFile, error) { - sess := db.NewSession(db.DefaultContext) - defer sess.Close() - if err := sess.Begin(); err != nil { - return nil, err - } - - key := &PackageFile{ - PackageID: pf.PackageID, - LowerName: pf.LowerName, - } +// DeletePackageByIDIfUnreferenced deletes a package if there are no associated versions +func DeletePackageByIDIfUnreferenced(ctx context.Context, packageID int64) error { + e := db.GetEngine(ctx) - has, err := sess.Get(key) + count, err := e.Count(&PackageVersion{ + PackageID: packageID, + }) if err != nil { - return nil, err - } - if has { - return key, ErrDuplicatePackageFile + return err } - if _, err = sess.Insert(pf); err != nil { - return nil, err + if count == 0 { + _, err := e.ID(packageID).Delete(&Package{}) + return err } - return pf, sess.Commit() -} - -// UpdatePackageFile updates a package file -func UpdatePackageFile(pf *PackageFile) error { - _, err := db.GetEngine(db.DefaultContext).ID(pf.ID).Update(pf) - return err -} - -// DeletePackageFileByID deletes a package file -func DeletePackageFileByID(fileID int64) error { - _, err := db.GetEngine(db.DefaultContext).ID(fileID).Delete(&PackageFile{}) - return err + return nil } -// DeletePackageFilesByPackageID deletes all files associated with the package -func DeletePackageFilesByPackageID(packageID int64) error { - _, err := db.GetEngine(db.DefaultContext).Where("package_id = ?", packageID).Delete(&PackageFile{}) - return err +// HasRepositoryPackages tests if a repository has packages +func HasRepositoryPackages(repositoryID int64) (bool, error) { + return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repositoryID).Exist(&Package{}) } diff --git a/models/packages/package_blob.go b/models/packages/package_blob.go new file mode 100644 index 0000000000000..4f4414bdad4fc --- /dev/null +++ b/models/packages/package_blob.go @@ -0,0 +1,78 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + "errors" + + "code.gitea.io/gitea/models/db" +) + +var ( + // ErrPackageBlobNotExist indicates a package blob not exist error + ErrPackageBlobNotExist = errors.New("Package blob does not exist") +) + +func init() { + db.RegisterModel(new(PackageBlob)) +} + +// PackageBlob represents a package blob +type PackageBlob struct { + ID int64 `xorm:"pk autoincr"` + Size int64 + HashMD5 string `xorm:"hash_md5 char(32) UNIQUE(md5) INDEX NOT NULL"` + HashSHA1 string `xorm:"hash_sha1 char(40) UNIQUE(sha1) INDEX NOT NULL"` + HashSHA256 string `xorm:"hash_sha256 char(64) UNIQUE(sha256) INDEX NOT NULL"` + HashSHA512 string `xorm:"hash_sha512 char(128) UNIQUE(sha512) INDEX NOT NULL"` +} + +// GetOrInsertBlob inserts a blob. If the blob exists already the existing blob is returned +func GetOrInsertBlob(ctx context.Context, pb *PackageBlob) (*PackageBlob, bool, error) { + e := db.GetEngine(ctx) + + has, err := e.Get(pb) + if err != nil { + return nil, false, err + } + if has { + return pb, true, nil + } + if _, err = e.Insert(pb); err != nil { + return nil, false, err + } + return pb, false, nil +} + +// GetBlobByID gets a blob by id +func GetBlobByID(ctx context.Context, blobID int64) (*PackageBlob, error) { + pb := &PackageBlob{} + + has, err := db.GetEngine(ctx).ID(blobID).Get(pb) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageBlobNotExist + } + return pb, nil +} + +// GetUnreferencedBlobs gets all blobs without associated files +func GetUnreferencedBlobs(ctx context.Context) ([]*PackageBlob, error) { + pbs := make([]*PackageBlob, 0, 10) + return pbs, db.GetEngine(ctx). + Table("package_blob"). + Join("LEFT OUTER", "package_file", "package_file.blob_id = package_blob.id"). + Where("package_file.id IS NULL"). + Find(&pbs) +} + +// DeleteBlobByID deletes a blob by id +func DeleteBlobByID(ctx context.Context, blobID int64) error { + _, err := db.GetEngine(ctx).ID(blobID).Delete(&PackageBlob{}) + return err +} diff --git a/models/packages/package_file.go b/models/packages/package_file.go new file mode 100644 index 0000000000000..524ab72b25ce6 --- /dev/null +++ b/models/packages/package_file.go @@ -0,0 +1,85 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + "errors" + "strings" + + "code.gitea.io/gitea/models/db" +) + +func init() { + db.RegisterModel(new(PackageFile)) +} + +var ( + // ErrDuplicatePackageFile indicates a duplicated package file error + ErrDuplicatePackageFile = errors.New("Package file does exist already") + // ErrPackageFileNotExist indicates a package file not exist error + ErrPackageFileNotExist = errors.New("Package file does not exist") +) + +// PackageFile represents a package file +type PackageFile struct { + ID int64 `xorm:"pk autoincr"` + VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + BlobID int64 `xorm:"INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + IsLead bool +} + +// TryInsertFile inserts a file. If the file exists already ErrDuplicatePackageFile is returned +func TryInsertFile(ctx context.Context, pf *PackageFile) (*PackageFile, error) { + e := db.GetEngine(ctx) + + key := &PackageFile{ + VersionID: pf.VersionID, + LowerName: pf.LowerName, + } + + has, err := e.Get(key) + if err != nil { + return nil, err + } + if has { + return pf, ErrDuplicatePackageFile + } + if _, err = e.Insert(pf); err != nil { + return nil, err + } + return pf, nil +} + +// GetFilesByVersionID gets all files of a version +func GetFilesByVersionID(ctx context.Context, versionID int64) ([]*PackageFile, error) { + pfs := make([]*PackageFile, 0, 10) + return pfs, db.GetEngine(ctx).Where("version_id = ?", versionID).Find(&pfs) +} + +// GetFileForVersionByName gets a file of a version by name +func GetFileForVersionByName(ctx context.Context, versionID int64, name string) (*PackageFile, error) { + pf := &PackageFile{ + VersionID: versionID, + LowerName: strings.ToLower(name), + } + + has, err := db.GetEngine(ctx).Get(pf) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageFileNotExist + } + return pf, nil +} + +// DeleteFilesByVersionID deletes all files of a version +func DeleteFilesByVersionID(ctx context.Context, versionID int64) error { + _, err := db.GetEngine(ctx).Where("version_id = ?", versionID).Delete(&PackageFile{}) + return err +} diff --git a/models/packages/package_version.go b/models/packages/package_version.go new file mode 100644 index 0000000000000..67256e1311c23 --- /dev/null +++ b/models/packages/package_version.go @@ -0,0 +1,219 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + "errors" + "strings" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" +) + +var ( + // ErrDuplicatePackageVersion indicates a duplicated package version error + ErrDuplicatePackageVersion = errors.New("Package version does exist already") + // ErrPackageVersionNotExist indicates a package version not exist error + ErrPackageVersionNotExist = errors.New("Package version does not exist") +) + +func init() { + db.RegisterModel(new(PackageVersion)) +} + +// PackageVersion represents a package version +type PackageVersion struct { + ID int64 `xorm:"pk autoincr"` + PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatorID int64 + Version string + LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + MetadataJSON string `xorm:"TEXT metadata_json"` + DownloadCount int64 +} + +// GetOrInsertVersion inserts a version. If the same version exist already ErrDuplicatePackageVersion is returned +func GetOrInsertVersion(ctx context.Context, pv *PackageVersion) (*PackageVersion, error) { + e := db.GetEngine(ctx) + + key := &PackageVersion{ + PackageID: pv.PackageID, + LowerVersion: pv.LowerVersion, + } + + has, err := e.Get(key) + if err != nil { + return nil, err + } + if has { + return key, ErrDuplicatePackageVersion + } + if _, err = e.Insert(pv); err != nil { + return nil, err + } + return pv, nil +} + +// UpdateVersion updates a version +func UpdateVersion(pv *PackageVersion) error { + _, err := db.GetEngine(db.DefaultContext).ID(pv.ID).Update(pv) + return err +} + +// IncrementDownloadCounter increments the download counter of a version +func IncrementDownloadCounter(versionID int64) error { + _, err := db.GetEngine(db.DefaultContext).Exec("UPDATE `package_version` SET `download_count` = `download_count` + 1 WHERE `id` = ?", versionID) + return err +} + +// GetVersionByID gets a version by id +func GetVersionByID(ctx context.Context, versionID int64) (*PackageVersion, error) { + pv := &PackageVersion{} + + has, err := db.GetEngine(ctx).ID(versionID).Get(pv) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageNotExist + } + return pv, nil +} + +// GetVersionByNameAndVersion gets a version by name and version number +func GetVersionByNameAndVersion(ctx context.Context, repositoryID int64, packageType Type, name, version string) (*PackageVersion, error) { + var cond builder.Cond = builder.Eq{ + "package.repo_id": repositoryID, + "package.type": packageType, + "package.lower_name": strings.ToLower(name), + } + pv := &PackageVersion{ + LowerVersion: strings.ToLower(version), + } + has, err := db.GetEngine(ctx). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond). + Get(pv) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageNotExist + } + + return pv, nil +} + +// GetVersionsByPackageType gets all versions of a specific type +func GetVersionsByPackageType(repositoryID int64, packageType Type) ([]*PackageVersion, error) { + var cond builder.Cond = builder.Eq{ + "package.repo_id": repositoryID, + "package.type": packageType, + } + + pvs := make([]*PackageVersion, 0, 10) + return pvs, db.GetEngine(db.DefaultContext). + Where(cond). + Join("INNER", "package", "package.id = package_version.package_id"). + Find(&pvs) +} + +// GetVersionsByPackageName gets all versions of a specific package +func GetVersionsByPackageName(repositoryID int64, packageType Type, name string) ([]*PackageVersion, error) { + var cond builder.Cond = builder.Eq{ + "package.repo_id": repositoryID, + "package.type": packageType, + "package.lower_name": strings.ToLower(name), + } + + pvs := make([]*PackageVersion, 0, 10) + return pvs, db.GetEngine(db.DefaultContext). + Where(cond). + Join("INNER", "package", "package.id = package_version.package_id"). + Find(&pvs) +} + +// GetVersionsByFilename gets all versions which are linked to a filename +func GetVersionsByFilename(repositoryID int64, packageType Type, filename string) ([]*PackageVersion, error) { + var cond builder.Cond = builder.Eq{ + "package.repo_id": repositoryID, + "package.type": packageType, + "package_file.lower_name": strings.ToLower(filename), + } + + pvs := make([]*PackageVersion, 0, 10) + return pvs, db.GetEngine(db.DefaultContext). + Where(cond). + Join("INNER", "package_file", "package_file.version_id = package_version.id"). + Join("INNER", "package", "package.id = package_version.package_id"). + Find(&pvs) +} + +// DeleteVersionByID deletes a version by id +func DeleteVersionByID(ctx context.Context, versionID int64) error { + _, err := db.GetEngine(ctx).ID(versionID).Delete(&PackageVersion{}) + return err +} + +// PackageSearchOptions are options for SearchXXX methods +type PackageSearchOptions struct { + RepoID int64 + Type string + Query string + db.Paginator +} + +func (opts *PackageSearchOptions) toConds() builder.Cond { + var cond builder.Cond = builder.Eq{"package.repo_id": opts.RepoID} + + if opts.Type != "" { + cond = cond.And(builder.Eq{"package.type": opts.Type}) + } + + if opts.Query != "" { + cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.Query)}) + } + + return cond +} + +// SearchVersions gets all versions of packages matching the search options +func SearchVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, error) { + sess := db.GetEngine(db.DefaultContext).Where(opts.toConds()). + Table("package_version"). + Join("INNER", "package", "package.id = package_version.package_id") + + if opts.Paginator != nil { + sess = db.SetSessionPagination(sess, opts) + } + + pvs := make([]*PackageVersion, 0, 10) + count, err := sess.FindAndCount(&pvs) + return pvs, count, err +} + +// SearchLatestVersions gets the latest version of every package matching the search options +func SearchLatestVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, error) { + cond := opts.toConds(). + And(builder.Expr("pv2.id IS NULL")) + + sess := db.GetEngine(db.DefaultContext). + Table("package_version"). + Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND package_version.version < pv2.version"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond) + + if opts.Paginator != nil { + sess = db.SetSessionPagination(sess, opts) + } + + pvs := make([]*PackageVersion, 0, 10) + count, err := sess.FindAndCount(&pvs) + return pvs, count, err +} diff --git a/models/packages/package_version_property.go b/models/packages/package_version_property.go new file mode 100644 index 0000000000000..3590c1a168884 --- /dev/null +++ b/models/packages/package_version_property.go @@ -0,0 +1,60 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + + "code.gitea.io/gitea/models/db" +) + +func init() { + db.RegisterModel(new(PackageVersionProperty)) +} + +// PackageVersionProperty represents a property of a package version +type PackageVersionProperty struct { + ID int64 `xorm:"pk autoincr"` + VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Value string `xorm:"INDEX NOT NULL"` +} + +// SetVersionProperties sets the properties of the version +func SetVersionProperties(ctx context.Context, versionID int64, properties map[string]string) error { + e := db.GetEngine(ctx) + + for name, value := range properties { + pvp := &PackageVersionProperty{ + VersionID: versionID, + Name: name, + Value: value, + } + + count, err := e.Where("version_id = ? AND name = ?", versionID, pvp.Name).Update(pvp) + if err != nil { + return err + } + if count != 1 { + if _, err = e.Insert(pvp); err != nil { + return err + } + } + } + + return nil +} + +// GetVersionProperties gets all properties of the version +func GetVersionProperties(ctx context.Context, versionID int64) ([]*PackageVersionProperty, error) { + pvps := make([]*PackageVersionProperty, 0, 10) + return pvps, db.GetEngine(ctx).Where("version_id = ?", versionID).Find(&pvps) +} + +// DeleteVersionPropertiesByVersionID deletes all properties of the version +func DeleteVersionPropertiesByVersionID(ctx context.Context, versionID int64) error { + _, err := db.GetEngine(ctx).Where("version_id = ?", versionID).Delete(&PackageVersionProperty{}) + return err +} diff --git a/modules/convert/package.go b/modules/convert/package.go index e383b0aed28a2..942576d0bdb2a 100644 --- a/modules/convert/package.go +++ b/modules/convert/package.go @@ -9,34 +9,27 @@ import ( api "code.gitea.io/gitea/modules/structs" ) -// ToPackage convert a packages.Package to api.Package -func ToPackage(p *packages.Package) *api.Package { - if err := p.LoadCreator(); err != nil { - return &api.Package{} - } - +// ToPackage convert a packages.PackageDescriptor to api.Package +func ToPackage(pd *packages.PackageDescriptor) *api.Package { return &api.Package{ - ID: p.ID, - Creator: ToUser(p.Creator, nil), - Type: p.Type.String(), - Name: p.Name, - Version: p.Version, - CreatedAt: p.CreatedUnix.AsTime(), - UpdatedAt: p.CreatedUnix.AsTime(), + ID: pd.Version.ID, + Creator: ToUser(pd.Creator, nil), + Type: string(pd.Package.Type), + Name: pd.Package.Name, + Version: pd.Version.Version, + CreatedAt: pd.Version.CreatedUnix.AsTime(), } } // ToPackageFile converts packages.PackageFile to api.PackageFile -func ToPackageFile(pf *packages.PackageFile) *api.PackageFile { +func ToPackageFile(pf *packages.PackageFile, pb *packages.PackageBlob) *api.PackageFile { return &api.PackageFile{ ID: pf.ID, - Size: pf.Size, + Size: pb.Size, Name: pf.Name, - HashMD5: pf.HashMD5, - HashSHA1: pf.HashSHA1, - HashSHA256: pf.HashSHA256, - HashSHA512: pf.HashSHA512, - CreatedAt: pf.CreatedUnix.AsTime(), - UpdatedAt: pf.UpdatedUnix.AsTime(), + HashMD5: pb.HashMD5, + HashSHA1: pb.HashSHA1, + HashSHA256: pb.HashSHA256, + HashSHA512: pb.HashSHA512, } } diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index 8688aaa232fef..4fe05b18c673e 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -61,6 +61,6 @@ type Notifier interface { NotifyRepoPendingTransfer(doer, newOwner *models.User, repo *models.Repository) - NotifyPackageCreate(repo *models.Repository, p *packages.Package) - NotifyPackageDelete(doer *models.User, repo *models.Repository, p *packages.Package) + NotifyPackageCreate(doer *models.User, pd *packages.PackageDescriptor) + NotifyPackageDelete(doer *models.User, pd *packages.PackageDescriptor) } diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go index 34aadea774ad2..a62f97b39cf74 100644 --- a/modules/notification/base/null.go +++ b/modules/notification/base/null.go @@ -173,9 +173,9 @@ func (*NullNotifier) NotifyRepoPendingTransfer(doer, newOwner *models.User, repo } // NotifyPackageCreate places a place holder function -func (*NullNotifier) NotifyPackageCreate(repo *models.Repository, p *packages.Package) { +func (*NullNotifier) NotifyPackageCreate(doer *models.User, pd *packages.PackageDescriptor) { } // NotifyPackageDelete places a place holder function -func (*NullNotifier) NotifyPackageDelete(doer *models.User, repo *models.Repository, p *packages.Package) { +func (*NullNotifier) NotifyPackageDelete(doer *models.User, pd *packages.PackageDescriptor) { } diff --git a/modules/notification/notification.go b/modules/notification/notification.go index c2c0bdbcf6eac..6b227329566cc 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -300,15 +300,15 @@ func NotifyRepoPendingTransfer(doer, newOwner *models.User, repo *models.Reposit } // NotifyPackageCreate notifies creation of a package to notifiers -func NotifyPackageCreate(repo *models.Repository, p *packages.Package) { +func NotifyPackageCreate(doer *models.User, pd *packages.PackageDescriptor) { for _, notifier := range notifiers { - notifier.NotifyPackageCreate(repo, p) + notifier.NotifyPackageCreate(doer, pd) } } // NotifyPackageDelete notifies deletion of a package to notifiers -func NotifyPackageDelete(doer *models.User, repo *models.Repository, p *packages.Package) { +func NotifyPackageDelete(doer *models.User, pd *packages.PackageDescriptor) { for _, notifier := range notifiers { - notifier.NotifyPackageDelete(doer, repo, p) + notifier.NotifyPackageDelete(doer, pd) } } diff --git a/modules/notification/webhook/webhook.go b/modules/notification/webhook/webhook.go index 8ee53299c7313..5751eedb82f11 100644 --- a/modules/notification/webhook/webhook.go +++ b/modules/notification/webhook/webhook.go @@ -820,28 +820,24 @@ func (m *webhookNotifier) NotifySyncDeleteRef(pusher *models.User, repo *models. m.NotifyDeleteRef(pusher, repo, refType, refFullName) } -func (m *webhookNotifier) NotifyPackageCreate(repo *models.Repository, p *packages.Package) { - if err := p.LoadCreator(); err != nil { - log.Error("LoadCreator: %v", err) - } - - notifyPackage(p.Creator, repo, p, api.HookPackageCreated) +func (m *webhookNotifier) NotifyPackageCreate(doer *models.User, pd *packages.PackageDescriptor) { + notifyPackage(doer, pd, api.HookPackageCreated) } -func (m *webhookNotifier) NotifyPackageDelete(doer *models.User, repo *models.Repository, p *packages.Package) { - notifyPackage(doer, repo, p, api.HookPackageDeleted) +func (m *webhookNotifier) NotifyPackageDelete(doer *models.User, pd *packages.PackageDescriptor) { + notifyPackage(doer, pd, api.HookPackageDeleted) } -func notifyPackage(sender *models.User, repo *models.Repository, p *packages.Package, action api.HookPackageAction) { - org := repo.MustOwner() +func notifyPackage(sender *models.User, pd *packages.PackageDescriptor, action api.HookPackageAction) { + org := pd.Repository.MustOwner() if !org.IsOrganization() { org = nil } - if err := webhook_services.PrepareWebhooks(repo, models.HookEventPackage, &api.PackagePayload{ + if err := webhook_services.PrepareWebhooks(pd.Repository, models.HookEventPackage, &api.PackagePayload{ Action: action, - Repository: convert.ToRepo(repo, models.AccessModeNone), - Package: convert.ToPackage(p), + Repository: convert.ToRepo(pd.Repository, models.AccessModeNone), + Package: convert.ToPackage(pd), Organization: convert.ToUser(org, nil), Sender: convert.ToUser(sender, nil), }); err != nil { diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index c1bfb99f36d69..26471fdd38cba 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -5,8 +5,8 @@ package packages import ( - "fmt" "io" + "strconv" "code.gitea.io/gitea/modules/storage" ) @@ -22,22 +22,18 @@ func NewContentStore() *ContentStore { return contentStore } -// Get get the package file content -func (s *ContentStore) Get(packageID, packageFileID int64) (storage.Object, error) { - return s.store.Open(toRelativePath(packageID, packageFileID)) +// Get gets a package blob +func (s *ContentStore) Get(packageBlobID int64) (storage.Object, error) { + return s.store.Open(strconv.FormatInt(packageBlobID, 10)) } -// Save stores the package file content -func (s *ContentStore) Save(packageID, packageFileID int64, r io.Reader, size int64) error { - _, err := s.store.Save(toRelativePath(packageID, packageFileID), r, size) +// Save stores a package blob +func (s *ContentStore) Save(packageBlobID int64, r io.Reader, size int64) error { + _, err := s.store.Save(strconv.FormatInt(packageBlobID, 10), r, size) return err } -// Delete deletes the package file content -func (s *ContentStore) Delete(packageID, packageFileID int64) error { - return s.store.Delete(toRelativePath(packageID, packageFileID)) -} - -func toRelativePath(packageID, packageFileID int64) string { - return fmt.Sprintf("%d/%d", packageID, packageFileID) +// Delete deletes a package blob +func (s *ContentStore) Delete(packageBlobID int64) error { + return s.store.Delete(strconv.FormatInt(packageBlobID, 10)) } diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go new file mode 100644 index 0000000000000..5f2d7132c5ed0 --- /dev/null +++ b/modules/packages/hashed_buffer.go @@ -0,0 +1,80 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "hash" + "io" + + "code.gitea.io/gitea/modules/util/filebuffer" +) + +// HashedBuffer is buffer which calculates multiple checksums +type HashedBuffer struct { + *filebuffer.FileBackedBuffer + + md5 hash.Hash + sha1 hash.Hash + sha256 hash.Hash + sha512 hash.Hash + + combinedWriter io.Writer +} + +// NewHashedBuffer creates a hashed buffer with a specific maximum memory size +func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) { + b, err := filebuffer.New(maxMemorySize) + if err != nil { + return nil, err + } + md5 := md5.New() + sha1 := sha1.New() + sha256 := sha256.New() + sha512 := sha512.New() + + combinedWriter := io.MultiWriter(b, md5, sha1, sha256, sha512) + + return &HashedBuffer{ + b, + md5, + sha1, + sha256, + sha512, + combinedWriter, + }, nil +} + +// CreateHashedBufferFromReader creates a hashed buffer and copies the provided reader data into it. +func CreateHashedBufferFromReader(r io.Reader, maxMemorySize int) (*HashedBuffer, error) { + b, err := NewHashedBuffer(maxMemorySize) + if err != nil { + return nil, err + } + + _, err = io.Copy(b, r) + if err != nil { + return nil, err + } + + return b, nil +} + +// Write implements io.Writer +func (b *HashedBuffer) Write(p []byte) (int, error) { + return b.combinedWriter.Write(p) +} + +// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data +func (b *HashedBuffer) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) { + hashMD5 = b.md5.Sum(nil) + hashSHA1 = b.sha1.Sum(nil) + hashSHA256 = b.sha256.Sum(nil) + hashSHA512 = b.sha512.Sum(nil) + return +} diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 2ac99e3fda121..6f6e3fa18d64d 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -43,11 +43,16 @@ var idmatch = regexp.MustCompile(`\A\w+(?:[.-]\w+)*\z`) const maxNuspecFileSize = 3 * 1024 * 1024 +// Package represents a Nuget package +type Package struct { + PackageType PackageType + ID string + Version string + Metadata *Metadata +} + // Metadata represents the metadata of a Nuget package type Metadata struct { - PackageType PackageType `json:"-"` - ID string `json:"-"` - Version string `json:"-"` Description string `json:"description"` ReleaseNotes string `json:"release_notes"` Authors string `json:"authors"` @@ -93,7 +98,7 @@ type nuspecPackage struct { } // ParsePackageMetaData parses the metadata of a Nuget package file -func ParsePackageMetaData(r io.ReaderAt, size int64) (*Metadata, error) { +func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) { archive, err := zip.NewReader(r, size) if err != nil { return nil, err @@ -120,7 +125,7 @@ func ParsePackageMetaData(r io.ReaderAt, size int64) (*Metadata, error) { } // ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package -func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { +func ParseNuspecMetaData(r io.Reader) (*Package, error) { var p nuspecPackage if err := xml.NewDecoder(r).Decode(&p); err != nil { return nil, err @@ -148,9 +153,6 @@ func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { } m := &Metadata{ - PackageType: packageType, - ID: p.Metadata.ID, - Version: v.String(), Description: p.Metadata.Description, ReleaseNotes: p.Metadata.ReleaseNotes, Authors: p.Metadata.Authors, @@ -158,6 +160,7 @@ func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { RepositoryURL: p.Metadata.Repository.URL, Dependencies: make(map[string][]Dependency), } + for _, group := range p.Metadata.Dependencies.Group { deps := make([]Dependency, 0, len(group.Dependency)) for _, dep := range group.Dependency { @@ -173,5 +176,10 @@ func ParseNuspecMetaData(r io.Reader) (*Metadata, error) { m.Dependencies[group.TargetFramework] = deps } } - return m, nil + return &Package{ + PackageType: packageType, + ID: p.Metadata.ID, + Version: v.String(), + Metadata: m, + }, nil } diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go index e4418c3cd2e84..e8c7773e97069 100644 --- a/modules/packages/nuget/metadata_test.go +++ b/modules/packages/nuget/metadata_test.go @@ -73,24 +73,24 @@ func TestParsePackageMetaData(t *testing.T) { t.Run("MissingNuspecFile", func(t *testing.T) { data := createArchive("dummy.txt", "") - m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) - assert.Nil(t, m) + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) assert.ErrorIs(t, err, ErrMissingNuspecFile) }) t.Run("MissingNuspecFileInRoot", func(t *testing.T) { data := createArchive("sub/package.nuspec", "") - m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) - assert.Nil(t, m) + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) assert.ErrorIs(t, err, ErrMissingNuspecFile) }) t.Run("InvalidNuspecFile", func(t *testing.T) { data := createArchive("package.nuspec", "") - m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) - assert.Nil(t, m) + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) assert.Error(t, err) }) @@ -100,8 +100,8 @@ func TestParsePackageMetaData(t *testing.T) { `) - m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) - assert.Nil(t, m) + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) assert.ErrorIs(t, err, ErrNuspecInvalidID) }) @@ -113,51 +113,51 @@ func TestParsePackageMetaData(t *testing.T) { `) - m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) - assert.Nil(t, m) + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) assert.ErrorIs(t, err, ErrNuspecInvalidVersion) }) t.Run("Valid", func(t *testing.T) { data := createArchive("package.nuspec", nuspecContent) - m, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) assert.NoError(t, err) - assert.NotNil(t, m) + assert.NotNil(t, np) }) } func TestParseNuspecMetaData(t *testing.T) { t.Run("Dependency Package", func(t *testing.T) { - m, err := ParseNuspecMetaData(strings.NewReader(nuspecContent)) + np, err := ParseNuspecMetaData(strings.NewReader(nuspecContent)) assert.NoError(t, err) - assert.NotNil(t, m) - assert.Equal(t, DependencyPackage, m.PackageType) - - assert.Equal(t, id, m.ID) - assert.Equal(t, semver, m.Version) - assert.Equal(t, authors, m.Authors) - assert.Equal(t, projectURL, m.ProjectURL) - assert.Equal(t, description, m.Description) - assert.Equal(t, releaseNotes, m.ReleaseNotes) - assert.Equal(t, repositoryURL, m.RepositoryURL) - assert.Len(t, m.Dependencies, 1) - assert.Contains(t, m.Dependencies, targetFramework) - deps := m.Dependencies[targetFramework] + assert.NotNil(t, np) + assert.Equal(t, DependencyPackage, np.PackageType) + + assert.Equal(t, id, np.ID) + assert.Equal(t, semver, np.Version) + assert.Equal(t, authors, np.Metadata.Authors) + assert.Equal(t, projectURL, np.Metadata.ProjectURL) + assert.Equal(t, description, np.Metadata.Description) + assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) + assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL) + assert.Len(t, np.Metadata.Dependencies, 1) + assert.Contains(t, np.Metadata.Dependencies, targetFramework) + deps := np.Metadata.Dependencies[targetFramework] assert.Len(t, deps, 1) assert.Equal(t, dependencyID, deps[0].ID) assert.Equal(t, dependencyVersion, deps[0].Version) }) t.Run("Symbols Package", func(t *testing.T) { - m, err := ParseNuspecMetaData(strings.NewReader(symbolsNuspecContent)) + np, err := ParseNuspecMetaData(strings.NewReader(symbolsNuspecContent)) assert.NoError(t, err) - assert.NotNil(t, m) - assert.Equal(t, SymbolsPackage, m.PackageType) + assert.NotNil(t, np) + assert.Equal(t, SymbolsPackage, np.PackageType) - assert.Equal(t, id, m.ID) - assert.Equal(t, semver, m.Version) - assert.Equal(t, description, m.Description) - assert.Empty(t, m.Dependencies) + assert.Equal(t, id, np.ID) + assert.Equal(t, semver, np.Version) + assert.Equal(t, description, np.Metadata.Description) + assert.Empty(t, np.Metadata.Dependencies) }) } diff --git a/modules/packages/rubygems/metadata.go b/modules/packages/rubygems/metadata.go index 4fe90850bf2f7..f337ba3530f2d 100644 --- a/modules/packages/rubygems/metadata.go +++ b/modules/packages/rubygems/metadata.go @@ -28,10 +28,15 @@ var ( var versionMatcher = regexp.MustCompile(`\A[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z`) +// Package represents a RubyGems package +type Package struct { + Name string + Version string + Metadata *Metadata +} + // Metadata represents the metadata of a RubyGems package type Metadata struct { - Name string `json:"-"` - Version string `json:"-"` Platform string `json:"platform"` Description string `json:"description"` Summary string `json:"summary"` @@ -138,7 +143,7 @@ func (r requirement) AsVersionRequirement() []VersionRequirement { } // ParsePackageMetaData parses the metadata of a Gem package file -func ParsePackageMetaData(r io.Reader) (*Metadata, error) { +func ParsePackageMetaData(r io.Reader) (*Package, error) { archive := tar.NewReader(r) for { hdr, err := archive.Next() @@ -157,7 +162,7 @@ func ParsePackageMetaData(r io.Reader) (*Metadata, error) { return nil, ErrMissingMetadataFile } -func parseMetadataFile(r io.Reader) (*Metadata, error) { +func parseMetadataFile(r io.Reader) (*Package, error) { zr, err := gzip.NewReader(r) if err != nil { return nil, err @@ -185,8 +190,6 @@ func parseMetadataFile(r io.Reader) (*Metadata, error) { } m := &Metadata{ - Name: spec.Name, - Version: spec.Version.Version, Platform: spec.Platform, Description: spec.Description, Summary: spec.Summary, @@ -211,5 +214,9 @@ func parseMetadataFile(r io.Reader) (*Metadata, error) { } } - return m, nil + return &Package{ + Name: spec.Name, + Version: spec.Version.Version, + Metadata: m, + }, nil } diff --git a/modules/packages/rubygems/metadata_test.go b/modules/packages/rubygems/metadata_test.go index cf674f1306e4c..f3f248021e5c8 100644 --- a/modules/packages/rubygems/metadata_test.go +++ b/modules/packages/rubygems/metadata_test.go @@ -32,18 +32,18 @@ func TestParsePackageMetaData(t *testing.T) { t.Run("MissingMetadataFile", func(t *testing.T) { data := createArchive("dummy.txt", []byte{0}) - m, err := ParsePackageMetaData(data) + rp, err := ParsePackageMetaData(data) assert.ErrorIs(t, err, ErrMissingMetadataFile) - assert.Nil(t, m) + assert.Nil(t, rp) }) t.Run("Valid", func(t *testing.T) { content, _ := base64.StdEncoding.DecodeString("H4sICHC/I2EEAG1ldGFkYXRhAAEeAOH/bmFtZTogZwp2ZXJzaW9uOgogIHZlcnNpb246IDEKWw35Tx4AAAA=") data := createArchive("metadata.gz", content) - m, err := ParsePackageMetaData(data) + rp, err := ParsePackageMetaData(data) assert.NoError(t, err) - assert.NotNil(t, m) + assert.NotNil(t, rp) }) } @@ -58,32 +58,32 @@ uv9Q9Aau/6ngP84elvNpXh5oRVlB5/yW7BH0+qu0G4gqaI/JdEHBFBS5l+pKtsARIjIwUnfj8Le0 dVoR6hj07u0HZgAl3SRS8G/fmXcRK20jyq6rDMSYQFgidamqkXbbuspLXE/0k7GphtKqe67GuRC/ yjAbmt9LsOMp8xMamFkSQ38fP5EFjdz8LA4do2C69VvqWXAJgrPbKZb58/xZXrKoW6ttW13Bhvzi 4ftn7/yUxd4YGcglvTmmY8aGY3ZwRn4CqcWcidUGAAA=`) - m, err := parseMetadataFile(bytes.NewReader(content)) + rp, err := parseMetadataFile(bytes.NewReader(content)) assert.NoError(t, err) - assert.NotNil(t, m) + assert.NotNil(t, rp) - assert.Equal(t, "gitea", m.Name) - assert.Equal(t, "1.0.5", m.Version) - assert.Equal(t, "ruby", m.Platform) - assert.Equal(t, "Gitea package", m.Summary) - assert.Equal(t, "RubyGems package test", m.Description) - assert.Equal(t, []string{"Gitea"}, m.Authors) - assert.Equal(t, "https://gitea.io/", m.ProjectURL) - assert.Equal(t, []string{"MIT"}, m.Licenses) - assert.Empty(t, m.RequiredRubygemsVersion) - assert.Len(t, m.RequiredRubyVersion, 1) - assert.Equal(t, ">=", m.RequiredRubyVersion[0].Restriction) - assert.Equal(t, "2.3.0", m.RequiredRubyVersion[0].Version) - assert.Len(t, m.RuntimeDependencies, 1) - assert.Equal(t, "runtime-dep", m.RuntimeDependencies[0].Name) - assert.Len(t, m.RuntimeDependencies[0].Version, 2) - assert.Equal(t, ">=", m.RuntimeDependencies[0].Version[0].Restriction) - assert.Equal(t, "1.2.0", m.RuntimeDependencies[0].Version[0].Version) - assert.Equal(t, "<", m.RuntimeDependencies[0].Version[1].Restriction) - assert.Equal(t, "2.0", m.RuntimeDependencies[0].Version[1].Version) - assert.Len(t, m.DevelopmentDependencies, 1) - assert.Equal(t, "dev-dep", m.DevelopmentDependencies[0].Name) - assert.Len(t, m.DevelopmentDependencies[0].Version, 1) - assert.Equal(t, "~>", m.DevelopmentDependencies[0].Version[0].Restriction) - assert.Equal(t, "5.2", m.DevelopmentDependencies[0].Version[0].Version) + assert.Equal(t, "gitea", rp.Name) + assert.Equal(t, "1.0.5", rp.Version) + assert.Equal(t, "ruby", rp.Metadata.Platform) + assert.Equal(t, "Gitea package", rp.Metadata.Summary) + assert.Equal(t, "RubyGems package test", rp.Metadata.Description) + assert.Equal(t, []string{"Gitea"}, rp.Metadata.Authors) + assert.Equal(t, "https://gitea.io/", rp.Metadata.ProjectURL) + assert.Equal(t, []string{"MIT"}, rp.Metadata.Licenses) + assert.Empty(t, rp.Metadata.RequiredRubygemsVersion) + assert.Len(t, rp.Metadata.RequiredRubyVersion, 1) + assert.Equal(t, ">=", rp.Metadata.RequiredRubyVersion[0].Restriction) + assert.Equal(t, "2.3.0", rp.Metadata.RequiredRubyVersion[0].Version) + assert.Len(t, rp.Metadata.RuntimeDependencies, 1) + assert.Equal(t, "runtime-dep", rp.Metadata.RuntimeDependencies[0].Name) + assert.Len(t, rp.Metadata.RuntimeDependencies[0].Version, 2) + assert.Equal(t, ">=", rp.Metadata.RuntimeDependencies[0].Version[0].Restriction) + assert.Equal(t, "1.2.0", rp.Metadata.RuntimeDependencies[0].Version[0].Version) + assert.Equal(t, "<", rp.Metadata.RuntimeDependencies[0].Version[1].Restriction) + assert.Equal(t, "2.0", rp.Metadata.RuntimeDependencies[0].Version[1].Version) + assert.Len(t, rp.Metadata.DevelopmentDependencies, 1) + assert.Equal(t, "dev-dep", rp.Metadata.DevelopmentDependencies[0].Name) + assert.Len(t, rp.Metadata.DevelopmentDependencies[0].Version, 1) + assert.Equal(t, "~>", rp.Metadata.DevelopmentDependencies[0].Version[0].Restriction) + assert.Equal(t, "5.2", rp.Metadata.DevelopmentDependencies[0].Version[0].Version) } diff --git a/modules/structs/package.go b/modules/structs/package.go index 0ed7232c3f750..cf00334de9b73 100644 --- a/modules/structs/package.go +++ b/modules/structs/package.go @@ -17,8 +17,6 @@ type Package struct { Version string `json:"version"` // swagger:strfmt date-time CreatedAt time.Time `json:"created_at"` - // swagger:strfmt date-time - UpdatedAt time.Time `json:"updated_at"` } // PackageFile represents a package file @@ -30,8 +28,4 @@ type PackageFile struct { HashSHA1 string `json:"sha1"` HashSHA256 string `json:"sha256"` HashSHA512 string `json:"sha512"` - // swagger:strfmt date-time - CreatedAt time.Time `json:"created_at"` - // swagger:strfmt date-time - UpdatedAt time.Time `json:"updated_at"` } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index f7c85b8cae631..6f873379482a3 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1019,7 +1019,7 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { m.Group("", func() { m.Put("/", reqRepoWriter(models.UnitTypePackages), nuget.UploadPackage) m.Put("/symbolpackage", nuget.UploadSymbolPackage) - m.Delete("/package/{id}/{version}", reqRepoWriter(models.UnitTypePackages), nuget.DeletePackage) + m.Delete("/{id}/{version}", reqRepoWriter(models.UnitTypePackages), nuget.DeletePackage) }, reqBasicAuth(), reqRepoWriter(models.UnitTypePackages)) }) m.Group("/npm", func() { diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index a93b0516d9424..f56221ac1f302 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -12,8 +12,8 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util/filebuffer" - + package_module "code.gitea.io/gitea/modules/packages" + package_router "code.gitea.io/gitea/routers/api/v1/packages" package_service "code.gitea.io/gitea/services/packages" "github.com/hashicorp/go-version" @@ -22,21 +22,27 @@ import ( var packageNameRegex = regexp.MustCompile(`\A[A-Za-z0-9\.\_\-\+]+\z`) var filenameRegex = packageNameRegex +func apiError(ctx *context.APIContext, status int, obj interface{}) { + package_router.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, []byte(message)) + }) +} + // DownloadPackageFile serves the specific generic package. func DownloadPackageFile(ctx *context.APIContext) { packageName, packageVersion, filename, err := sanitizeParameters(ctx) if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeGeneric, packageName, packageVersion, filename) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } defer s.Close() @@ -49,53 +55,50 @@ func DownloadPackageFile(ctx *context.APIContext) { func UploadPackage(ctx *context.APIContext) { packageName, packageVersion, filename, err := sanitizeParameters(ctx) if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } upload, close, err := ctx.UploadStream() if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } if close { defer upload.Close() } - buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) + buf, err := package_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { - log.Error("Error creating file buffer: %v", err) - ctx.Error(http.StatusInternalServerError, "", "") + log.Error("Error creating hashed buffer: %v", err) + apiError(ctx, http.StatusInternalServerError, err) return } defer buf.Close() - p, err := package_service.CreatePackage( - ctx.User, - ctx.Repo.Repository, - packages.TypeGeneric, - packageName, - packageVersion, - nil, - false, + _, _, err = package_service.CreatePackageAndAddFile( + &package_service.PackageCreationInfo{ + PackageInfo: package_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypeGeneric, + Name: packageName, + Version: packageVersion, + }, + SemverCompatible: true, + Creator: ctx.User, + }, + &package_service.PackageFileInfo{ + Filename: filename, + Data: buf, + IsLead: true, + }, ) if err != nil { - if err == packages.ErrDuplicatePackage { - ctx.Error(http.StatusBadRequest, "", err) + if err == packages.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) return } - log.Error("Error creating package: %v", err) - ctx.Error(http.StatusInternalServerError, "", "") - return - } - - _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) - if err != nil { - log.Error("Error adding file to package: %v", err) - if err := packages.DeletePackageByID(p.ID); err != nil { - log.Error("Error deleting package by id: %v", err) - } - ctx.Error(http.StatusInternalServerError, "", "") + apiError(ctx, http.StatusInternalServerError, err) return } @@ -106,20 +109,27 @@ func UploadPackage(ctx *context.APIContext) { func DeletePackage(ctx *context.APIContext) { packageName, packageVersion, _, err := sanitizeParameters(ctx) if err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } - err = package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, packages.TypeGeneric, packageName, packageVersion) + err = package_service.DeletePackageVersionByNameAndVersion( + ctx.User, + &package_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypeGeneric, + Name: packageName, + Version: packageVersion, + }, + ) if err != nil { if err == packages.ErrPackageNotExist { - ctx.Error(http.StatusNotFound, "", err) + apiError(ctx, http.StatusNotFound, err) return } - ctx.Error(http.StatusInternalServerError, "", "") + apiError(ctx, http.StatusInternalServerError, err) return } - ctx.PlainText(http.StatusOK, nil) } func sanitizeParameters(ctx *context.APIContext) (string, string, string, error) { diff --git a/routers/api/v1/packages/maven/api.go b/routers/api/v1/packages/maven/api.go index 60eeaff3f5372..8cc5711aa4d6d 100644 --- a/routers/api/v1/packages/maven/api.go +++ b/routers/api/v1/packages/maven/api.go @@ -6,7 +6,11 @@ package maven import ( "encoding/xml" + "sort" "strings" + + packages_models "code.gitea.io/gitea/models/packages" + maven_module "code.gitea.io/gitea/modules/packages/maven" ) // MetadataResponse https://maven.apache.org/ref/3.2.5/maven-repository-metadata/repository-metadata.html @@ -19,29 +23,34 @@ type MetadataResponse struct { Version []string `xml:"versioning>versions>version"` } -func createMetadataResponse(packages []*Package) *MetadataResponse { - sortedPackages := sortPackagesByCreationASC(packages) +func createMetadataResponse(pds []*packages_models.PackageDescriptor) *MetadataResponse { + sort.Slice(pds, func(i, j int) bool { + // Maven and Gradle order packages by their creation timestamp and not by their version string + return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix + }) - var release *Package + var release *packages_models.PackageDescriptor - versions := make([]string, 0, len(sortedPackages)) - for _, p := range sortedPackages { - if !strings.HasSuffix(p.Version, "-SNAPSHOT") { - release = p + versions := make([]string, 0, len(pds)) + for _, pd := range pds { + if !strings.HasSuffix(pd.Version.Version, "-SNAPSHOT") { + release = pd } - versions = append(versions, p.Version) + versions = append(versions, pd.Version.Version) } - latest := sortedPackages[len(sortedPackages)-1] + latest := pds[len(pds)-1] + + metadata := latest.Metadata.(*maven_module.Metadata) resp := &MetadataResponse{ - GroupID: latest.Metadata.GroupID, - ArtifactID: latest.Metadata.ArtifactID, - Latest: latest.Version, + GroupID: metadata.GroupID, + ArtifactID: metadata.ArtifactID, + Latest: latest.Version.Version, Version: versions, } if release != nil { - resp.Release = release.Version + resp.Release = release.Version.Version } return resp } diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 7de3d6aeaf16e..7cc0fef974104 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -19,15 +19,15 @@ import ( "regexp" "strings" + "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" maven_module "code.gitea.io/gitea/modules/packages/maven" - "code.gitea.io/gitea/modules/util/filebuffer" package_router "code.gitea.io/gitea/routers/api/v1/packages" - package_service "code.gitea.io/gitea/services/packages" + packages_service "code.gitea.io/gitea/services/packages" ) const ( @@ -68,23 +68,23 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { // /com/foo/project/maven-metadata.xml[.md5/.sha1/.sha256/.sha512] packageName := params.GroupID + "-" + params.ArtifactID - pkgs, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeMaven, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypeMaven, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - if len(pkgs) == 0 { + if len(pvs) == 0 { apiError(ctx, http.StatusNotFound, packages.ErrPackageNotExist) return } - mavenPackages, err := intializePackages(pkgs) + pds, err := packages.GetPackageDescriptors(pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - xmlMetadata, err := xml.Marshal(createMetadataResponse(mavenPackages)) + xmlMetadata, err := xml.Marshal(createMetadataResponse(pds)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -118,9 +118,13 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { func servePackageFile(ctx *context.APIContext, params parameters) { packageName := params.GroupID + "-" + params.ArtifactID - p, err := packages.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, packages.TypeMaven, packageName, params.Version) - if err == packages.ErrPackageNotExist { - apiError(ctx, http.StatusNotFound, err) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Repo.Repository.ID, packages.TypeMaven, packageName, params.Version) + if err != nil { + if err == packages.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } return } @@ -131,38 +135,50 @@ func servePackageFile(ctx *context.APIContext, params parameters) { filename = filename[:len(filename)-len(ext)] } - pf, err := p.GetFileByName(filename) + pf, err := packages.GetFileForVersionByName(db.DefaultContext, pv.ID, filename) if err != nil { if err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) - return + } else { + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } if isChecksumExtension(ext) { + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + var hash string switch ext { case extensionMD5: - hash = pf.HashMD5 + hash = pb.HashMD5 case extensionSHA1: - hash = pf.HashSHA1 + hash = pb.HashSHA1 case extensionSHA256: - hash = pf.HashSHA256 + hash = pb.HashSHA256 case extensionSHA512: - hash = pf.HashSHA512 + hash = pb.HashSHA512 } ctx.PlainText(http.StatusOK, []byte(hash)) return } - s, err := packages_module.NewContentStore().Get(p.ID, pf.ID) + s, err := packages_module.NewContentStore().Get(pf.BlobID) if err != nil { apiError(ctx, http.StatusInternalServerError, err) } defer s.Close() + if pf.IsLead { + if err := packages.IncrementDownloadCounter(pv.ID); err != nil { + log.Error("Error incrementing download counter: %v", err) + } + } + ctx.ServeStream(s, pf.Name) } @@ -184,35 +200,38 @@ func UploadPackageFile(ctx *context.APIContext) { packageName := params.GroupID + "-" + params.ArtifactID - p, err := package_service.CreatePackage( - ctx.User, - ctx.Repo.Repository, - packages.TypeMaven, - packageName, - params.Version, - &maven_module.Metadata{ - GroupID: params.GroupID, - ArtifactID: params.ArtifactID, - }, - true, - ) + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } + defer buf.Close() - buf, err := filebuffer.CreateFromReader(ctx.Req.Body, 32*1024*1024) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return + pvci := &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypeMaven, + Name: packageName, + Version: params.Version, + }, + SemverCompatible: false, + Creator: ctx.User, } - defer buf.Close() ext := filepath.Ext(params.Filename) // Do not upload checksum files but compare the hashes. if isChecksumExtension(ext) { - pf, err := p.GetFileByName(params.Filename[:len(params.Filename)-len(ext)]) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Repository.ID, pvci.PackageType, pvci.Name, pvci.Version) + if err != nil { + if err == packages.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + pf, err := packages.GetFileForVersionByName(db.DefaultContext, pv.ID, params.Filename[:len(params.Filename)-len(ext)]) if err != nil { if err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -221,6 +240,11 @@ func UploadPackageFile(ctx *context.APIContext) { apiError(ctx, http.StatusInternalServerError, err) return } + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } hash, err := ioutil.ReadAll(buf) if err != nil { @@ -228,10 +252,10 @@ func UploadPackageFile(ctx *context.APIContext) { return } - if (ext == extensionMD5 && pf.HashMD5 != string(hash)) || - (ext == extensionSHA1 && pf.HashSHA1 != string(hash)) || - (ext == extensionSHA256 && pf.HashSHA256 != string(hash)) || - (ext == extensionSHA512 && pf.HashSHA512 != string(hash)) { + if (ext == extensionMD5 && pb.HashMD5 != string(hash)) || + (ext == extensionSHA1 && pb.HashSHA1 != string(hash)) || + (ext == extensionSHA256 && pb.HashSHA256 != string(hash)) || + (ext == extensionSHA512 && pb.HashSHA512 != string(hash)) { apiError(ctx, http.StatusBadRequest, "hash mismatch") return } @@ -240,31 +264,52 @@ func UploadPackageFile(ctx *context.APIContext) { return } + pfi := &packages_service.PackageFileInfo{ + Filename: params.Filename, + Data: buf, + IsLead: false, + } + // If it's the package pom file extract the metadata if ext == ".pom" { - metadata, err := maven_module.ParsePackageMetaData(buf) + pfi.IsLead = true + + var err error + pvci.Metadata, err = maven_module.ParsePackageMetaData(buf) if err != nil { log.Error("Error parsing package metadata: %v", err) } - if metadata != nil { - raw, err := json.Marshal(metadata) - if err != nil { + + if pvci.Metadata != nil { + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Repository.ID, pvci.PackageType, pvci.Name, pvci.Version) + if err != nil && err != packages.ErrPackageNotExist { apiError(ctx, http.StatusInternalServerError, err) return } - p.MetadataRaw = string(raw) - if err := packages.UpdatePackage(p); err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return + if pv != nil { + raw, err := json.Marshal(pvci.Metadata) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + pv.MetadataJSON = string(raw) + if err := packages.UpdateVersion(pv); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } } } + if _, err := buf.Seek(0, io.SeekStart); err != nil { apiError(ctx, http.StatusInternalServerError, err) return } } - _, err = package_service.AddFileToPackage(p, params.Filename, buf.Size(), buf) + _, _, err = packages_service.CreatePackageOrAddFileToExisting( + pvci, + pfi, + ) if err != nil { if err == packages.ErrDuplicatePackageFile { apiError(ctx, http.StatusBadRequest, err) diff --git a/routers/api/v1/packages/maven/package.go b/routers/api/v1/packages/maven/package.go deleted file mode 100644 index 06b2ebcd39b68..0000000000000 --- a/routers/api/v1/packages/maven/package.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package maven - -import ( - "sort" - - "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/modules/json" - maven_module "code.gitea.io/gitea/modules/packages/maven" -) - -// Package represents a package with Maven metadata -type Package struct { - *packages.Package - Metadata *maven_module.Metadata -} - -func intializePackages(packages []*packages.Package) ([]*Package, error) { - pgs := make([]*Package, 0, len(packages)) - for _, p := range packages { - np, err := intializePackage(p) - if err != nil { - return nil, err - } - pgs = append(pgs, np) - } - return pgs, nil -} - -func intializePackage(p *packages.Package) (*Package, error) { - var m *maven_module.Metadata - err := json.Unmarshal([]byte(p.MetadataRaw), &m) - if err != nil { - return nil, err - } - if m == nil { - m = &maven_module.Metadata{} - } - - return &Package{ - Package: p, - Metadata: m, - }, nil -} - -// Maven and Gradle order packages by their creation timestamp and not by their version string -func sortPackagesByCreationASC(packages []*Package) []*Package { - sortedPackages := make([]*Package, len(packages)) - copy(sortedPackages, packages) - - sort.Slice(sortedPackages, func(i, j int) bool { - return sortedPackages[i].CreatedUnix < sortedPackages[j].CreatedUnix - }) - - return sortedPackages -} diff --git a/routers/api/v1/packages/npm/api.go b/routers/api/v1/packages/npm/api.go index d4e9592b48510..fea3827ab4e89 100644 --- a/routers/api/v1/packages/npm/api.go +++ b/routers/api/v1/packages/npm/api.go @@ -9,53 +9,61 @@ import ( "encoding/hex" "fmt" "net/url" + "sort" + packages_models "code.gitea.io/gitea/models/packages" npm_module "code.gitea.io/gitea/modules/packages/npm" ) -func createPackageMetadataResponse(registryURL string, packages []*Package) *npm_module.PackageMetadata { - sortedPackages := sortPackagesByVersionASC(packages) +func createPackageMetadataResponse(registryURL string, pds []*packages_models.PackageDescriptor) *npm_module.PackageMetadata { + sort.Slice(pds, func(i, j int) bool { + return pds[i].SemVer.LessThan(pds[j].SemVer) + }) versions := make(map[string]*npm_module.PackageMetadataVersion) - for _, p := range sortedPackages { - versions[p.SemVer.String()] = createPackageMetadataVersion(registryURL, p) + for _, pd := range pds { + versions[pd.SemVer.String()] = createPackageMetadataVersion(registryURL, pd) } - latest := sortedPackages[len(sortedPackages)-1] + latest := pds[len(pds)-1] distTags := make(map[string]string) - distTags["latest"] = latest.Version + distTags["latest"] = latest.Version.Version + + metadata := latest.Metadata.(*npm_module.Metadata) return &npm_module.PackageMetadata{ ID: latest.Package.Name, Name: latest.Package.Name, DistTags: distTags, - Description: latest.Metadata.Description, - Readme: latest.Metadata.Readme, - Homepage: latest.Metadata.ProjectURL, - Author: npm_module.User{Name: latest.Metadata.Author}, - License: latest.Metadata.License, + Description: metadata.Description, + Readme: metadata.Readme, + Homepage: metadata.ProjectURL, + Author: npm_module.User{Name: metadata.Author}, + License: metadata.License, Versions: versions, } } -func createPackageMetadataVersion(registryURL string, p *Package) *npm_module.PackageMetadataVersion { - hashBytes, _ := hex.DecodeString(p.PackageFile.HashSHA512) +func createPackageMetadataVersion(registryURL string, pd *packages_models.PackageDescriptor) *npm_module.PackageMetadataVersion { + hashBytes, _ := hex.DecodeString(pd.Files[0].Blob.HashSHA512) + + metadata := pd.Metadata.(*npm_module.Metadata) return &npm_module.PackageMetadataVersion{ - ID: fmt.Sprintf("%s@%s", p.Package.Name, p.Package.Version), - Name: p.Package.Name, - Version: p.Package.Version, - Description: p.Metadata.Description, - Author: npm_module.User{Name: p.Metadata.Author}, - Homepage: p.Metadata.ProjectURL, - License: p.Metadata.License, - Dependencies: p.Metadata.Dependencies, - Readme: p.Metadata.Readme, + ID: fmt.Sprintf("%s@%s", pd.Package.Name, pd.Version.Version), + Name: pd.Package.Name, + Version: pd.Version.Version, + Description: metadata.Description, + Author: npm_module.User{Name: metadata.Author}, + Homepage: metadata.ProjectURL, + License: metadata.License, + Dependencies: metadata.Dependencies, + Readme: metadata.Readme, Dist: npm_module.PackageDistribution{ - Shasum: p.PackageFile.HashSHA1, + Shasum: pd.Files[0].Blob.HashSHA1, Integrity: "sha512-" + base64.StdEncoding.EncodeToString(hashBytes), - Tarball: fmt.Sprintf("%s/%s/-/%s/%s", registryURL, url.QueryEscape(p.Package.Name), p.Package.Version, p.PackageFile.LowerName), + Tarball: fmt.Sprintf("%s/%s/-/%s/%s", registryURL, url.QueryEscape(pd.Package.Name), pd.Version.Version, pd.Files[0].File.LowerName), }, } } diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 23b70ee28720b..b651cfc65630d 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -11,11 +11,11 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/log" + package_module "code.gitea.io/gitea/modules/packages" npm_module "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/setting" package_router "code.gitea.io/gitea/routers/api/v1/packages" - package_service "code.gitea.io/gitea/services/packages" + packages_service "code.gitea.io/gitea/services/packages" ) func apiError(ctx *context.APIContext, status int, obj interface{}) { @@ -34,17 +34,17 @@ func PackageMetadata(ctx *context.APIContext) { return } - packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeNpm, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypeNpm, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - if len(packages) == 0 { + if len(pvs) == 0 { apiError(ctx, http.StatusNotFound, err) return } - npmPackages, err := intializePackages(packages) + pds, err := packages.GetPackageDescriptors(pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -52,7 +52,7 @@ func PackageMetadata(ctx *context.APIContext) { resp := createPackageMetadataResponse( setting.AppURL+"api/v1/repos/"+ctx.Repo.Repository.FullName()+"/packages/npm", - npmPackages, + pds, ) ctx.JSON(http.StatusOK, resp) @@ -68,7 +68,7 @@ func DownloadPackageFile(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNpm, packageName, packageVersion, filename) + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNpm, packageName, packageVersion, filename) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -90,28 +90,35 @@ func UploadPackage(ctx *context.APIContext) { return } - p, err := package_service.CreatePackage( - ctx.User, - ctx.Repo.Repository, - packages.TypeNpm, - npmPackage.Name, - npmPackage.Version, - npmPackage.Metadata, - false, - ) + buf, err := package_module.CreateHashedBufferFromReader(bytes.NewReader(npmPackage.Data), 32*1024*1024) if err != nil { - if err == packages.ErrDuplicatePackage { - apiError(ctx, http.StatusBadRequest, err) - return - } apiError(ctx, http.StatusInternalServerError, err) return } - - _, err = package_service.AddFileToPackage(p, npmPackage.Filename, int64(len(npmPackage.Data)), bytes.NewReader(npmPackage.Data)) + defer buf.Close() + + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypeNpm, + Name: npmPackage.Name, + Version: npmPackage.Version, + }, + SemverCompatible: true, + Creator: ctx.User, + Metadata: npmPackage.Metadata, + }, + &packages_service.PackageFileInfo{ + Filename: npmPackage.Filename, + Data: buf, + IsLead: true, + }, + ) if err != nil { - if err := packages.DeletePackageByID(p.ID); err != nil { - log.Error("Error deleting package by id: %v", err) + if err == packages.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) + return } apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/npm/package.go b/routers/api/v1/packages/npm/package.go deleted file mode 100644 index 5a81a09fafdba..0000000000000 --- a/routers/api/v1/packages/npm/package.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package npm - -import ( - "sort" - - "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/modules/json" - npm_module "code.gitea.io/gitea/modules/packages/npm" - - "github.com/hashicorp/go-version" -) - -// Package represents a package with npm metadata -type Package struct { - *packages.Package - *packages.PackageFile - SemVer *version.Version - Metadata *npm_module.Metadata -} - -func intializePackages(packages []*packages.Package) ([]*Package, error) { - pgs := make([]*Package, 0, len(packages)) - for _, p := range packages { - np, err := intializePackage(p) - if err != nil { - return nil, err - } - pgs = append(pgs, np) - } - return pgs, nil -} - -func intializePackage(p *packages.Package) (*Package, error) { - v, err := version.NewSemver(p.Version) - if err != nil { - return nil, err - } - - var m *npm_module.Metadata - err = json.Unmarshal([]byte(p.MetadataRaw), &m) - if err != nil { - return nil, err - } - if m == nil { - m = &npm_module.Metadata{} - } - - pfs, err := p.GetFiles() - if err != nil { - return nil, err - } - - return &Package{ - Package: p, - PackageFile: pfs[0], - SemVer: v, - Metadata: m, - }, nil -} - -func sortPackagesByVersionASC(packages []*Package) []*Package { - sortedPackages := make([]*Package, len(packages)) - copy(sortedPackages, packages) - - sort.Slice(sortedPackages, func(i, j int) bool { - return sortedPackages[i].SemVer.LessThan(sortedPackages[j].SemVer) - }) - - return sortedPackages -} diff --git a/routers/api/v1/packages/nuget/api.go b/routers/api/v1/packages/nuget/api.go index 18515283ffb79..c6db443056ca6 100644 --- a/routers/api/v1/packages/nuget/api.go +++ b/routers/api/v1/packages/nuget/api.go @@ -7,8 +7,12 @@ package nuget import ( "bytes" "fmt" + "sort" "time" + packages_models "code.gitea.io/gitea/models/packages" + nuget_module "code.gitea.io/gitea/modules/packages/nuget" + "github.com/hashicorp/go-version" ) @@ -91,51 +95,57 @@ type PackageDependency struct { Range string `json:"range"` } -func createRegistrationIndexResponse(l *linkBuilder, packages []*Package) *RegistrationIndexResponse { - sortedPackages := sortPackagesByVersionASC(packages) +func createRegistrationIndexResponse(l *linkBuilder, pds []*packages_models.PackageDescriptor) *RegistrationIndexResponse { + sort.Slice(pds, func(i, j int) bool { + return pds[i].SemVer.LessThan(pds[j].SemVer) + }) - items := make([]*RegistrationIndexPageItem, 0, len(packages)) - for _, p := range sortedPackages { + items := make([]*RegistrationIndexPageItem, 0, len(pds)) + for _, p := range pds { items = append(items, createRegistrationIndexPageItem(l, p)) } return &RegistrationIndexResponse{ - RegistrationIndexURL: l.GetRegistrationIndexURL(packages[0].Name), + RegistrationIndexURL: l.GetRegistrationIndexURL(pds[0].Package.Name), Type: []string{"catalog:CatalogRoot", "PackageRegistration", "catalog:Permalink"}, Count: 1, Pages: []*RegistrationIndexPage{ { - RegistrationPageURL: l.GetRegistrationIndexURL(packages[0].Name), - Count: len(packages), - Lower: normalizeVersion(packages[0].SemVer), - Upper: normalizeVersion(packages[len(packages)-1].SemVer), + RegistrationPageURL: l.GetRegistrationIndexURL(pds[0].Package.Name), + Count: len(pds), + Lower: normalizeVersion(pds[0].SemVer), + Upper: normalizeVersion(pds[len(pds)-1].SemVer), Items: items, }, }, } } -func createRegistrationIndexPageItem(l *linkBuilder, p *Package) *RegistrationIndexPageItem { +func createRegistrationIndexPageItem(l *linkBuilder, pd *packages_models.PackageDescriptor) *RegistrationIndexPageItem { + metadata := pd.Metadata.(*nuget_module.Metadata) + return &RegistrationIndexPageItem{ - RegistrationLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), - PackageContentURL: l.GetPackageDownloadURL(p.Name, p.Version), + RegistrationLeafURL: l.GetRegistrationLeafURL(pd.Package.Name, pd.Version.Version), + PackageContentURL: l.GetPackageDownloadURL(pd.Package.Name, pd.Version.Version), CatalogEntry: &CatalogEntry{ - CatalogLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), - PackageContentURL: l.GetPackageDownloadURL(p.Name, p.Version), - ID: p.Name, - Version: p.Version, - Description: p.Metadata.Description, - ReleaseNotes: p.Metadata.ReleaseNotes, - Authors: p.Metadata.Authors, - ProjectURL: p.Metadata.ProjectURL, - DependencyGroups: createDependencyGroups(p), + CatalogLeafURL: l.GetRegistrationLeafURL(pd.Package.Name, pd.Version.Version), + PackageContentURL: l.GetPackageDownloadURL(pd.Package.Name, pd.Version.Version), + ID: pd.Package.Name, + Version: pd.Version.Version, + Description: metadata.Description, + ReleaseNotes: metadata.ReleaseNotes, + Authors: metadata.Authors, + ProjectURL: metadata.ProjectURL, + DependencyGroups: createDependencyGroups(pd), }, } } -func createDependencyGroups(p *Package) []*PackageDependencyGroup { - dependencyGroups := make([]*PackageDependencyGroup, 0, len(p.Metadata.Dependencies)) - for k, v := range p.Metadata.Dependencies { +func createDependencyGroups(pd *packages_models.PackageDescriptor) []*PackageDependencyGroup { + metadata := pd.Metadata.(*nuget_module.Metadata) + + dependencyGroups := make([]*PackageDependencyGroup, 0, len(metadata.Dependencies)) + for k, v := range metadata.Dependencies { dependencies := make([]*PackageDependency, 0, len(v)) for _, dep := range v { dependencies = append(dependencies, &PackageDependency{ @@ -162,14 +172,14 @@ type RegistrationLeafResponse struct { RegistrationIndexURL string `json:"registration"` } -func createRegistrationLeafResponse(l *linkBuilder, p *Package) *RegistrationLeafResponse { +func createRegistrationLeafResponse(l *linkBuilder, pd *packages_models.PackageDescriptor) *RegistrationLeafResponse { return &RegistrationLeafResponse{ Type: []string{"Package", "http://schema.nuget.org/catalog#Permalink"}, Listed: true, - Published: time.Unix(int64(p.CreatedUnix), 0), - RegistrationLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), - PackageContentURL: l.GetPackageDownloadURL(p.Name, p.Version), - RegistrationIndexURL: l.GetRegistrationIndexURL(p.Name), + Published: time.Unix(int64(pd.Version.CreatedUnix), 0), + RegistrationLeafURL: l.GetRegistrationLeafURL(pd.Package.Name, pd.Version.Version), + PackageContentURL: l.GetPackageDownloadURL(pd.Package.Name, pd.Version.Version), + RegistrationIndexURL: l.GetRegistrationIndexURL(pd.Package.Name), } } @@ -178,10 +188,10 @@ type PackageVersionsResponse struct { Versions []string `json:"versions"` } -func createPackageVersionsResponse(packages []*Package) *PackageVersionsResponse { - versions := make([]string, 0, len(packages)) - for _, p := range packages { - versions = append(versions, normalizeVersion(p.SemVer)) +func createPackageVersionsResponse(pds []*packages_models.PackageDescriptor) *PackageVersionsResponse { + versions := make([]string, 0, len(pds)) + for _, pd := range pds { + versions = append(versions, normalizeVersion(pd.SemVer)) } return &PackageVersionsResponse{ @@ -213,20 +223,20 @@ type SearchResultVersion struct { Downloads int64 `json:"downloads"` } -func createSearchResultResponse(l *linkBuilder, totalHits int64, packages []*Package) *SearchResultResponse { - data := make([]*SearchResult, 0, len(packages)) +func createSearchResultResponse(l *linkBuilder, totalHits int64, pds []*packages_models.PackageDescriptor) *SearchResultResponse { + data := make([]*SearchResult, 0, len(pds)) - if len(packages) > 0 { - groupID := packages[0].Name - group := make([]*Package, 0, 10) + if len(pds) > 0 { + groupID := pds[0].Package.Name + group := make([]*packages_models.PackageDescriptor, 0, 10) - for i := 0; i < len(packages); i++ { - if groupID != packages[i].Name { + for i := 0; i < len(pds); i++ { + if groupID != pds[i].Package.Name { data = append(data, createSearchResult(l, group)) - groupID = packages[i].Name + groupID = pds[i].Package.Name group = group[:0] } - group = append(group, packages[i]) + group = append(group, pds[i]) } data = append(data, createSearchResult(l, group)) } @@ -237,28 +247,30 @@ func createSearchResultResponse(l *linkBuilder, totalHits int64, packages []*Pac } } -func createSearchResult(l *linkBuilder, packages []*Package) *SearchResult { - latest := packages[0] - versions := make([]*SearchResultVersion, 0, len(packages)) - for _, p := range packages { - if latest.SemVer.LessThan(p.SemVer) { - latest = p +func createSearchResult(l *linkBuilder, pds []*packages_models.PackageDescriptor) *SearchResult { + latest := pds[0] + versions := make([]*SearchResultVersion, 0, len(pds)) + for _, pd := range pds { + if latest.SemVer.LessThan(pd.SemVer) { + latest = pd } versions = append(versions, &SearchResultVersion{ - RegistrationLeafURL: l.GetRegistrationLeafURL(p.Name, p.Version), - Version: p.Version, + RegistrationLeafURL: l.GetRegistrationLeafURL(pd.Package.Name, pd.Version.Version), + Version: pd.Version.Version, }) } + metadata := latest.Metadata.(*nuget_module.Metadata) + return &SearchResult{ - ID: latest.Name, - Version: latest.Version, + ID: latest.Package.Name, + Version: latest.Version.Version, Versions: versions, - Description: latest.Metadata.Description, - Authors: latest.Metadata.Authors, - ProjectURL: latest.Metadata.ProjectURL, - RegistrationIndexURL: l.GetRegistrationIndexURL(latest.Name), + Description: metadata.Description, + Authors: metadata.Authors, + ProjectURL: metadata.ProjectURL, + RegistrationIndexURL: l.GetRegistrationIndexURL(latest.Package.Name), } } diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 3a643cbaf7a5d..658fb8755ab2b 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -14,12 +14,11 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/log" + package_module "code.gitea.io/gitea/modules/packages" nuget_module "code.gitea.io/gitea/modules/packages/nuget" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util/filebuffer" package_router "code.gitea.io/gitea/routers/api/v1/packages" - package_service "code.gitea.io/gitea/services/packages" + packages_service "code.gitea.io/gitea/services/packages" ) func apiError(ctx *context.APIContext, status int, obj interface{}) { @@ -41,7 +40,7 @@ func ServiceIndex(ctx *context.APIContext) { // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages func SearchService(ctx *context.APIContext) { - packages, count, err := packages.GetPackages(&packages.PackageSearchOptions{ + pvs, count, err := packages.SearchVersions(&packages.PackageSearchOptions{ RepoID: ctx.Repo.Repository.ID, Type: "nuget", Query: ctx.FormTrim("q"), @@ -55,7 +54,7 @@ func SearchService(ctx *context.APIContext) { return } - nugetPackages, err := intializePackages(packages) + pds, err := packages.GetPackageDescriptors(pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -64,7 +63,7 @@ func SearchService(ctx *context.APIContext) { resp := createSearchResultResponse( &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, count, - nugetPackages, + pds, ) ctx.JSON(http.StatusOK, resp) @@ -74,17 +73,17 @@ func SearchService(ctx *context.APIContext) { func RegistrationIndex(ctx *context.APIContext) { packageName := ctx.Params("id") - packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - if len(packages) == 0 { + if len(pvs) == 0 { apiError(ctx, http.StatusNotFound, err) return } - nugetPackages, err := intializePackages(packages) + pds, err := packages.GetPackageDescriptors(pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -92,7 +91,7 @@ func RegistrationIndex(ctx *context.APIContext) { resp := createRegistrationIndexResponse( &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, - nugetPackages, + pds, ) ctx.JSON(http.StatusOK, resp) @@ -103,7 +102,7 @@ func RegistrationLeaf(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") - p, err := packages.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName, packageVersion) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Repo.Repository.ID, packages.TypeNuGet, packageName, packageVersion) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -113,7 +112,7 @@ func RegistrationLeaf(ctx *context.APIContext) { return } - nugetPackage, err := intializePackage(p) + pd, err := packages.GetPackageDescriptor(pv) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -121,7 +120,7 @@ func RegistrationLeaf(ctx *context.APIContext) { resp := createRegistrationLeafResponse( &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, - nugetPackage, + pd, ) ctx.JSON(http.StatusOK, resp) @@ -131,23 +130,23 @@ func RegistrationLeaf(ctx *context.APIContext) { func EnumeratePackageVersions(ctx *context.APIContext) { packageName := ctx.Params("id") - packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - if len(packages) == 0 { + if len(pvs) == 0 { apiError(ctx, http.StatusNotFound, err) return } - nugetPackages, err := intializePackages(packages) + pds, err := packages.GetPackageDescriptors(pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - resp := createPackageVersionsResponse(nugetPackages) + resp := createPackageVersionsResponse(pds) ctx.JSON(http.StatusOK, resp) } @@ -158,7 +157,7 @@ func DownloadPackageFile(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNuGet, packageName, packageVersion, filename) + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNuGet, packageName, packageVersion, filename) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -175,27 +174,36 @@ func DownloadPackageFile(ctx *context.APIContext) { // UploadPackage creates a new package with the metadata contained in the uploaded nupgk file // https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#push-a-package func UploadPackage(ctx *context.APIContext) { - meta, buf, closables := processUploadedFile(ctx, nuget_module.DependencyPackage) + np, buf, closables := processUploadedFile(ctx, nuget_module.DependencyPackage) defer func() { for _, c := range closables { c.Close() } }() - if meta == nil { + if np == nil { return } - p, err := package_service.CreatePackage( - ctx.User, - ctx.Repo.Repository, - packages.TypeNuGet, - meta.ID, - meta.Version, - meta, - false, + _, _, err := packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypeNuGet, + Name: np.ID, + Version: np.Version, + }, + SemverCompatible: true, + Creator: ctx.User, + Metadata: np.Metadata, + }, + &packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.nupkg", np.ID, np.Version)), + Data: buf, + IsLead: true, + }, ) if err != nil { - if err == packages.ErrDuplicatePackage { + if err == packages.ErrDuplicatePackageVersion { apiError(ctx, http.StatusBadRequest, err) return } @@ -203,57 +211,51 @@ func UploadPackage(ctx *context.APIContext) { return } - filename := strings.ToLower(fmt.Sprintf("%s.%s.nupkg", meta.ID, meta.Version)) - _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) - if err != nil { - if err := packages.DeletePackageByID(p.ID); err != nil { - log.Error("Error deleting package by id: %v", err) - } - apiError(ctx, http.StatusInternalServerError, err) - return - } - ctx.PlainText(http.StatusCreated, nil) } // UploadSymbolPackage adds a symbol package to an existing package // https://docs.microsoft.com/en-us/nuget/api/symbol-package-publish-resource func UploadSymbolPackage(ctx *context.APIContext) { - meta, buf, closables := processUploadedFile(ctx, nuget_module.SymbolsPackage) + np, buf, closables := processUploadedFile(ctx, nuget_module.SymbolsPackage) defer func() { for _, c := range closables { c.Close() } }() - if meta == nil { + if np == nil { return } - p, err := packages.GetPackageByNameAndVersion(ctx.Repo.Repository.ID, packages.TypeNuGet, meta.ID, meta.Version) + _, _, err := packages_service.AddFileToExistingPackage( + &packages_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypeNuGet, + Name: np.ID, + Version: np.Version, + }, + &packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.snupkg", np.ID, np.Version)), + Data: buf, + IsLead: false, + }, + ) if err != nil { - if err == packages.ErrPackageNotExist { + switch err { + case packages.ErrPackageNotExist: apiError(ctx, http.StatusNotFound, err) - return - } - apiError(ctx, http.StatusInternalServerError, err) - return - } - - filename := strings.ToLower(fmt.Sprintf("%s.%s.snupkg", meta.ID, meta.Version)) - _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) - if err != nil { - if err == packages.ErrDuplicatePackageFile { + case packages.ErrDuplicatePackageFile: apiError(ctx, http.StatusBadRequest, err) - return + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } ctx.PlainText(http.StatusCreated, nil) } -func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.PackageType) (*nuget_module.Metadata, *filebuffer.FileBackedBuffer, []io.Closer) { +func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.PackageType) (*nuget_module.Package, *package_module.HashedBuffer, []io.Closer) { closables := make([]io.Closer, 0, 2) upload, close, err := ctx.UploadStream() @@ -266,14 +268,14 @@ func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.Pack closables = append(closables, upload) } - buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) + buf, err := package_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return nil, nil, closables } closables = append(closables, buf) - meta, err := nuget_module.ParsePackageMetaData(buf, buf.Size()) + np, err := nuget_module.ParsePackageMetaData(buf, buf.Size()) if err != nil { if err == nuget_module.ErrMissingNuspecFile || err == nuget_module.ErrNuspecFileTooLarge || err == nuget_module.ErrNuspecInvalidID || err == nuget_module.ErrNuspecInvalidVersion { apiError(ctx, http.StatusBadRequest, err) @@ -282,7 +284,7 @@ func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.Pack } return nil, nil, closables } - if meta.PackageType != expectedType { + if np.PackageType != expectedType { apiError(ctx, http.StatusBadRequest, errors.New("unexpected package type")) return nil, nil, closables } @@ -290,7 +292,7 @@ func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.Pack apiError(ctx, http.StatusInternalServerError, err) return nil, nil, closables } - return meta, buf, closables + return np, buf, closables } // DeletePackage hard deletes the package @@ -299,7 +301,15 @@ func DeletePackage(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") - err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, packages.TypeNuGet, packageName, packageVersion) + err := packages_service.DeletePackageVersionByNameAndVersion( + ctx.User, + &packages_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypeNuGet, + Name: packageName, + Version: packageVersion, + }, + ) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/v1/packages/nuget/package.go b/routers/api/v1/packages/nuget/package.go deleted file mode 100644 index 9d3d849f68013..0000000000000 --- a/routers/api/v1/packages/nuget/package.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package nuget - -import ( - "sort" - - "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/modules/json" - nuget_module "code.gitea.io/gitea/modules/packages/nuget" - - "github.com/hashicorp/go-version" -) - -// Package represents a package with NuGet metadata -type Package struct { - *packages.Package - SemVer *version.Version - Metadata *nuget_module.Metadata -} - -func intializePackages(packages []*packages.Package) ([]*Package, error) { - pgs := make([]*Package, 0, len(packages)) - for _, p := range packages { - np, err := intializePackage(p) - if err != nil { - return nil, err - } - pgs = append(pgs, np) - } - return pgs, nil -} - -func intializePackage(p *packages.Package) (*Package, error) { - v, err := version.NewSemver(p.Version) - if err != nil { - return nil, err - } - - var m *nuget_module.Metadata - err = json.Unmarshal([]byte(p.MetadataRaw), &m) - if err != nil { - return nil, err - } - if m == nil { - m = &nuget_module.Metadata{} - } - return &Package{ - Package: p, - SemVer: v, - Metadata: m, - }, nil -} - -func sortPackagesByVersionASC(packages []*Package) []*Package { - sortedPackages := make([]*Package, len(packages)) - copy(sortedPackages, packages) - - sort.Slice(sortedPackages, func(i, j int) bool { - return sortedPackages[i].SemVer.LessThan(sortedPackages[j].SemVer) - }) - - return sortedPackages -} diff --git a/routers/api/v1/packages/pypi/package.go b/routers/api/v1/packages/pypi/package.go deleted file mode 100644 index 26ddc4b7a03a9..0000000000000 --- a/routers/api/v1/packages/pypi/package.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package pypi - -import ( - "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/modules/json" - pypi_module "code.gitea.io/gitea/modules/packages/pypi" -) - -// Package represents a package with PyPI metadata -type Package struct { - *packages.Package - Files []*packages.PackageFile - Metadata *pypi_module.Metadata -} - -func intializePackages(packages []*packages.Package) ([]*Package, error) { - pgs := make([]*Package, 0, len(packages)) - for _, p := range packages { - np, err := intializePackage(p) - if err != nil { - return nil, err - } - pgs = append(pgs, np) - } - return pgs, nil -} - -func intializePackage(p *packages.Package) (*Package, error) { - var m *pypi_module.Metadata - err := json.Unmarshal([]byte(p.MetadataRaw), &m) - if err != nil { - return nil, err - } - if m == nil { - m = &pypi_module.Metadata{} - } - - pfs, err := p.GetFiles() - if err != nil { - return nil, err - } - - return &Package{ - Package: p, - Files: pfs, - Metadata: m, - }, nil -} diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index 15ac61cad74d6..021805619abd7 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -5,7 +5,6 @@ package pypi import ( - "crypto/sha256" "fmt" "io" "net/http" @@ -14,12 +13,13 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" + package_module "code.gitea.io/gitea/modules/packages" pypi_module "code.gitea.io/gitea/modules/packages/pypi" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/validation" package_router "code.gitea.io/gitea/routers/api/v1/packages" - package_service "code.gitea.io/gitea/services/packages" + packages_service "code.gitea.io/gitea/services/packages" ) // https://www.python.org/dev/peps/pep-0503/#normalized-names @@ -39,25 +39,25 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { func PackageMetadata(ctx *context.APIContext) { packageName := normalizer.Replace(ctx.Params("id")) - packages, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, packages.TypePyPI, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypePyPI, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - if len(packages) == 0 { + if len(pvs) == 0 { apiError(ctx, http.StatusNotFound, err) return } - pypiPackages, err := intializePackages(packages) + pds, err := packages.GetPackageDescriptors(pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } ctx.Data["RegistryURL"] = setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/pypi" - ctx.Data["Package"] = pypiPackages[0] - ctx.Data["Packages"] = pypiPackages + ctx.Data["PackageDescriptor"] = pds[0] + ctx.Data["PackageDescriptors"] = pds ctx.Render = templates.HTMLRenderer() ctx.HTML(http.StatusOK, "api/packages/pypi/simple") } @@ -68,7 +68,7 @@ func DownloadPackageFile(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypePyPI, packageName, packageVersion, filename) + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypePyPI, packageName, packageVersion, filename) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -91,17 +91,21 @@ func UploadPackageFile(ctx *context.APIContext) { } defer file.Close() - h256 := sha256.New() - if _, err := io.Copy(h256, file); err != nil { + buf, err := package_module.CreateHashedBufferFromReader(file, 32*1024*1024) + if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - if !strings.EqualFold(ctx.Req.FormValue("sha256_digest"), fmt.Sprintf("%x", h256.Sum(nil))) { + defer buf.Close() + + _, _, hashSHA256, _ := buf.Sums() + + if !strings.EqualFold(ctx.Req.FormValue("sha256_digest"), fmt.Sprintf("%x", hashSHA256)) { apiError(ctx, http.StatusBadRequest, "hash mismatch") return } - if _, err := file.Seek(0, io.SeekStart); err != nil { + if _, err := buf.Seek(0, io.SeekStart); err != nil { apiError(ctx, http.StatusInternalServerError, err) return } @@ -118,31 +122,32 @@ func UploadPackageFile(ctx *context.APIContext) { projectURL = "" } - metadata := &pypi_module.Metadata{ - Author: ctx.Req.FormValue("author"), - Description: ctx.Req.FormValue("description"), - LongDescription: ctx.Req.FormValue("long_description"), - Summary: ctx.Req.FormValue("summary"), - ProjectURL: projectURL, - License: ctx.Req.FormValue("license"), - RequiresPython: ctx.Req.FormValue("requires_python"), - } - - p, err := package_service.CreatePackage( - ctx.User, - ctx.Repo.Repository, - packages.TypePyPI, - packageName, - packageVersion, - metadata, - true, + _, _, err = packages_service.CreatePackageOrAddFileToExisting( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypePyPI, + Name: packageName, + Version: packageVersion, + }, + SemverCompatible: true, + Creator: ctx.User, + Metadata: &pypi_module.Metadata{ + Author: ctx.Req.FormValue("author"), + Description: ctx.Req.FormValue("description"), + LongDescription: ctx.Req.FormValue("long_description"), + Summary: ctx.Req.FormValue("summary"), + ProjectURL: projectURL, + License: ctx.Req.FormValue("license"), + RequiresPython: ctx.Req.FormValue("requires_python"), + }, + }, + &packages_service.PackageFileInfo{ + Filename: fileHeader.Filename, + Data: buf, + IsLead: true, + }, ) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - - _, err = package_service.AddFileToPackage(p, fileHeader.Filename, fileHeader.Size, file) if err != nil { if err == packages.ErrDuplicatePackageFile { apiError(ctx, http.StatusBadRequest, err) diff --git a/routers/api/v1/packages/rubygems/package.go b/routers/api/v1/packages/rubygems/package.go deleted file mode 100644 index dd60e116855fd..0000000000000 --- a/routers/api/v1/packages/rubygems/package.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package rubygems - -import ( - "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/modules/json" - rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" -) - -// Package represents a package with RubyGems metadata -type Package struct { - *packages.Package - Metadata *rubygems_module.Metadata -} - -func intializePackages(packages []*packages.Package) ([]*Package, error) { - pgs := make([]*Package, 0, len(packages)) - for _, p := range packages { - np, err := intializePackage(p) - if err != nil { - return nil, err - } - pgs = append(pgs, np) - } - return pgs, nil -} - -func intializePackage(p *packages.Package) (*Package, error) { - var m *rubygems_module.Metadata - if err := json.Unmarshal([]byte(p.MetadataRaw), &m); err != nil { - return nil, err - } - if m == nil { - m = &rubygems_module.Metadata{} - } - - return &Package{ - Package: p, - Metadata: m, - }, nil -} - -// AsSpecification creates a Ruby Gem::Specification object used by ServePackageSpecification -func (p *Package) AsSpecification() *rubygems_module.RubyUserDef { - return &rubygems_module.RubyUserDef{ - Name: "Gem::Specification", - Value: []interface{}{ - "3.2.3", // @rubygems_version - 4, // @specification_version, - p.Name, - &rubygems_module.RubyUserMarshal{ - Name: "Gem::Version", - Value: []string{p.Version}, - }, - nil, // date - p.Metadata.Summary, // @summary - nil, // @required_ruby_version - nil, // @required_rubygems_version - p.Metadata.Platform, // @original_platform - []interface{}{}, // @dependencies - nil, // rubyforge_project - "", // @email - p.Metadata.Authors, - p.Metadata.Description, - p.Metadata.ProjectURL, - true, // has_rdoc - p.Metadata.Platform, // @new_platform - nil, - p.Metadata.Licenses, - }, - } -} diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/v1/packages/rubygems/rubygems.go index 25b5da576b590..288b627627f64 100644 --- a/routers/api/v1/packages/rubygems/rubygems.go +++ b/routers/api/v1/packages/rubygems/rubygems.go @@ -14,11 +14,11 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/log" + //"code.gitea.io/gitea/modules/log" + package_module "code.gitea.io/gitea/modules/packages" rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" - "code.gitea.io/gitea/modules/util/filebuffer" package_router "code.gitea.io/gitea/routers/api/v1/packages" - package_service "code.gitea.io/gitea/services/packages" + packages_service "code.gitea.io/gitea/services/packages" ) func apiError(ctx *context.APIContext, status int, obj interface{}) { @@ -29,7 +29,7 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { // EnumeratePackages serves the package list func EnumeratePackages(ctx *context.APIContext) { - packages, err := packages.GetPackagesByRepositoryAndType(ctx.Repo.Repository.ID, packages.TypeRubyGems) + packages, err := packages.GetVersionsByPackageType(ctx.Repo.Repository.ID, packages.TypeRubyGems) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -40,7 +40,7 @@ func EnumeratePackages(ctx *context.APIContext) { // EnumeratePackagesLatest serves the list of the lastest version of every package func EnumeratePackagesLatest(ctx *context.APIContext) { - packages, _, err := packages.GetLatestPackagesGrouped(&packages.PackageSearchOptions{ + pvs, _, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ RepoID: ctx.Repo.Repository.ID, Type: "rubygems", }) @@ -49,30 +49,30 @@ func EnumeratePackagesLatest(ctx *context.APIContext) { return } - enumeratePackages(ctx, "latest_specs.4.8", packages) + enumeratePackages(ctx, "latest_specs.4.8", pvs) } // EnumeratePackagesPreRelease is not supported and serves an empty list func EnumeratePackagesPreRelease(ctx *context.APIContext) { - enumeratePackages(ctx, "prerelease_specs.4.8", []*packages.Package{}) + enumeratePackages(ctx, "prerelease_specs.4.8", []*packages.PackageVersion{}) } -func enumeratePackages(ctx *context.APIContext, filename string, packages []*packages.Package) { - rubygemsPackages, err := intializePackages(packages) +func enumeratePackages(ctx *context.APIContext, filename string, pvs []*packages.PackageVersion) { + pds, err := packages.GetPackageDescriptors(pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - specs := make([]interface{}, 0, len(rubygemsPackages)) - for _, p := range rubygemsPackages { + specs := make([]interface{}, 0, len(pds)) + for _, p := range pds { specs = append(specs, []interface{}{ - p.Name, + p.Package.Name, &rubygems_module.RubyUserMarshal{ Name: "Gem::Version", - Value: []string{p.Version}, + Value: []string{p.Version.Version}, }, - p.Metadata.Platform, + p.Metadata.(*rubygems_module.Metadata).Platform, }) } @@ -93,22 +93,22 @@ func ServePackageSpecification(ctx *context.APIContext) { filename := ctx.Params("filename") if !strings.HasSuffix(filename, ".gemspec.rz") { - apiError(ctx, http.StatusBadRequest, nil) + apiError(ctx, http.StatusNotImplemented, nil) return } - packages, err := packages.GetPackagesByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename[:len(filename)-10]+"gem") + pvs, err := packages.GetVersionsByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename[:len(filename)-10]+"gem") if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - if len(packages) != 1 { + if len(pvs) != 1 { apiError(ctx, http.StatusNotFound, nil) return } - p, err := intializePackage(packages[0]) + pd, err := packages.GetPackageDescriptor(pvs[0]) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -119,7 +119,36 @@ func ServePackageSpecification(ctx *context.APIContext) { zw := zlib.NewWriter(ctx.Resp) defer zw.Close() - spec := p.AsSpecification() + metadata := pd.Metadata.(*rubygems_module.Metadata) + + // create a Ruby Gem::Specification object + spec := &rubygems_module.RubyUserDef{ + Name: "Gem::Specification", + Value: []interface{}{ + "3.2.3", // @rubygems_version + 4, // @specification_version, + pd.Package.Name, + &rubygems_module.RubyUserMarshal{ + Name: "Gem::Version", + Value: []string{pd.Version.Version}, + }, + nil, // date + metadata.Summary, // @summary + nil, // @required_ruby_version + nil, // @required_rubygems_version + metadata.Platform, // @original_platform + []interface{}{}, // @dependencies + nil, // rubyforge_project + "", // @email + metadata.Authors, + metadata.Description, + metadata.ProjectURL, + true, // has_rdoc + metadata.Platform, // @new_platform + nil, + metadata.Licenses, + }, + } if err := rubygems_module.NewMarshalEncoder(zw).Encode(spec); err != nil { ctx.ServerError("Download file failed", err) @@ -130,18 +159,18 @@ func ServePackageSpecification(ctx *context.APIContext) { func DownloadPackageFile(ctx *context.APIContext) { filename := ctx.Params("filename") - pkgs, err := packages.GetPackagesByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename) + pvs, err := packages.GetVersionsByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - if len(pkgs) != 1 { + if len(pvs) != 1 { apiError(ctx, http.StatusNotFound, nil) return } - s, pf, err := package_service.GetPackageFileStream(pkgs[0], filename) + s, pf, err := packages_service.GetPackageFileStream(pvs[0], filename) if err != nil { if err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -166,14 +195,14 @@ func UploadPackageFile(ctx *context.APIContext) { defer upload.Close() } - buf, err := filebuffer.CreateFromReader(upload, 32*1024*1024) + buf, err := package_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } defer buf.Close() - meta, err := rubygems_module.ParsePackageMetaData(buf) + rp, err := rubygems_module.ParsePackageMetaData(buf) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -183,34 +212,35 @@ func UploadPackageFile(ctx *context.APIContext) { return } - p, err := package_service.CreatePackage( - ctx.User, - ctx.Repo.Repository, - packages.TypeRubyGems, - meta.Name, - meta.Version, - meta, - false, - ) - if err != nil { - if err == packages.ErrDuplicatePackage { - apiError(ctx, http.StatusBadRequest, err) - return - } - apiError(ctx, http.StatusInternalServerError, err) - return - } - var filename string - if len(meta.Platform) == 0 || meta.Platform == "ruby" { - filename = strings.ToLower(fmt.Sprintf("%s-%s.gem", meta.Name, meta.Version)) + if rp.Metadata.Platform == "" || rp.Metadata.Platform == "ruby" { + filename = strings.ToLower(fmt.Sprintf("%s-%s.gem", rp.Name, rp.Version)) } else { - filename = strings.ToLower(fmt.Sprintf("%s-%s-%s.gem", meta.Name, meta.Version, meta.Platform)) + filename = strings.ToLower(fmt.Sprintf("%s-%s-%s.gem", rp.Name, rp.Version, rp.Metadata.Platform)) } - _, err = package_service.AddFileToPackage(p, filename, buf.Size(), buf) + + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypeRubyGems, + Name: rp.Name, + Version: rp.Version, + }, + SemverCompatible: true, + Creator: ctx.User, + Metadata: rp.Metadata, + }, + &packages_service.PackageFileInfo{ + Filename: filename, + Data: buf, + IsLead: true, + }, + ) if err != nil { - if err := packages.DeletePackageByID(p.ID); err != nil { - log.Error("Error deleting package by id: %v", err) + if err == packages.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) + return } apiError(ctx, http.StatusInternalServerError, err) return @@ -221,10 +251,23 @@ func UploadPackageFile(ctx *context.APIContext) { // DeletePackage deletes a package func DeletePackage(ctx *context.APIContext) { + // Go populates the form only for POST, PUT and PATCH requests + if err := ctx.Req.ParseMultipartForm(32 << 20); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } packageName := ctx.FormString("gem_name") packageVersion := ctx.FormString("version") - err := package_service.DeletePackageByNameAndVersion(ctx.User, ctx.Repo.Repository, packages.TypeRubyGems, packageName, packageVersion) + err := packages_service.DeletePackageVersionByNameAndVersion( + ctx.User, + &packages_service.PackageInfo{ + Repository: ctx.Repo.Repository, + PackageType: packages.TypeRubyGems, + Name: packageName, + Version: packageVersion, + }, + ) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/v1/repo/package.go b/routers/api/v1/repo/package.go index 0f2b8e037a78e..9395626e2158f 100644 --- a/routers/api/v1/repo/package.go +++ b/routers/api/v1/repo/package.go @@ -7,6 +7,7 @@ package repo import ( "net/http" + "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" @@ -61,24 +62,26 @@ func ListPackages(ctx *context.APIContext) { repo := ctx.Repo.Repository - packages, count, err := packages.GetPackages(&packages.PackageSearchOptions{ + pvs, count, err := packages.SearchVersions(&packages.PackageSearchOptions{ RepoID: repo.ID, Type: packageType, Query: query, Paginator: &listOptions, }) if err != nil { - ctx.Error(http.StatusInternalServerError, "GetPackages", err) + ctx.Error(http.StatusInternalServerError, "SearchVersions", err) return } - apiPackages := make([]*api.Package, 0, len(packages)) - for _, p := range packages { - if err := p.LoadCreator(); err != nil { - ctx.Error(http.StatusInternalServerError, "LoadCreator", err) - return - } - apiPackages = append(apiPackages, convert.ToPackage(p)) + pds, err := packages.GetPackageDescriptors(pvs) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetPackageDescriptors", err) + return + } + + apiPackages := make([]*api.Package, 0, len(pds)) + for _, pd := range pds { + apiPackages = append(apiPackages, convert.ToPackage(pd)) } ctx.SetLinkHeader(int(count), listOptions.PageSize) @@ -116,17 +119,23 @@ func GetPackage(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - p, err := packages.GetPackageByID(ctx.ParamsInt64(":id")) + pv, err := packages.GetVersionByID(db.DefaultContext, ctx.ParamsInt64(":id")) if err != nil { if err == packages.ErrPackageNotExist { ctx.NotFound() } else { - ctx.Error(http.StatusInternalServerError, "GetPackageByID", err) + ctx.Error(http.StatusInternalServerError, "GetVersionByID", err) } return } - ctx.JSON(http.StatusOK, convert.ToPackage(p)) + pd, err := packages.GetPackageDescriptor(pv) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetPackageDescriptor", err) + return + } + + ctx.JSON(http.StatusOK, convert.ToPackage(pd)) } // DeletePackage delete a package from a repository @@ -157,12 +166,12 @@ func DeletePackage(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - err := package_service.DeletePackageByID(ctx.User, ctx.Repo.Repository, ctx.ParamsInt64(":id")) + err := package_service.DeleteVersionByID(ctx.User, ctx.Repo.Repository, ctx.ParamsInt64(":id")) if err != nil { if err == packages.ErrPackageNotExist { ctx.NotFound() } else { - ctx.Error(http.StatusInternalServerError, "DeletePackageByID", err) + ctx.Error(http.StatusInternalServerError, "DeleteVersionByID", err) } return } @@ -199,25 +208,30 @@ func ListPackageFiles(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - p, err := packages.GetPackageByID(ctx.ParamsInt64(":id")) + pv, err := packages.GetVersionByID(db.DefaultContext, ctx.ParamsInt64(":id")) if err != nil { if err == packages.ErrPackageNotExist { ctx.NotFound() } else { - ctx.Error(http.StatusInternalServerError, "GetPackageByID", err) + ctx.Error(http.StatusInternalServerError, "GetVersionByID", err) } return } - files, err := p.GetFiles() + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pv.ID) if err != nil { - ctx.Error(http.StatusInternalServerError, "GetFiles", err) + ctx.Error(http.StatusInternalServerError, "GetFilesByVersionID", err) return } - apiPackageFiles := make([]*api.PackageFile, 0, len(files)) - for _, pf := range files { - apiPackageFiles = append(apiPackageFiles, convert.ToPackageFile(pf)) + apiPackageFiles := make([]*api.PackageFile, 0, len(pfs)) + for _, pf := range pfs { + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetBlobByID", err) + return + } + apiPackageFiles = append(apiPackageFiles, convert.ToPackageFile(pf, pb)) } ctx.JSON(http.StatusOK, apiPackageFiles) diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index ab4f9141f27cb..4de3c7e671b6e 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -13,15 +13,9 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/packages/maven" - "code.gitea.io/gitea/modules/packages/npm" - "code.gitea.io/gitea/modules/packages/nuget" - "code.gitea.io/gitea/modules/packages/pypi" - "code.gitea.io/gitea/modules/packages/rubygems" "code.gitea.io/gitea/modules/setting" - package_service "code.gitea.io/gitea/services/packages" + packages_service "code.gitea.io/gitea/services/packages" ) const ( @@ -50,7 +44,7 @@ func Packages(ctx *context.Context) { repo := ctx.Repo.Repository - pkgs, count, err := packages.GetLatestPackagesGrouped(&packages.PackageSearchOptions{ + pvs, count, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ RepoID: repo.ID, Query: query, Type: packageType, @@ -60,15 +54,14 @@ func Packages(ctx *context.Context) { }, }) if err != nil { - ctx.ServerError("GetLatestPackagesGrouped", err) + ctx.ServerError("SearchLatestVersions", err) return } - for _, p := range pkgs { - if err := p.LoadCreator(); err != nil { - ctx.ServerError("LoadCreator", err) - return - } + pds, err := packages.GetPackageDescriptors(pvs) + if err != nil { + ctx.ServerError("GetPackageDescriptors", err) + return } hasPackages, err := packages.HasRepositoryPackages(repo.ID) @@ -78,7 +71,7 @@ func Packages(ctx *context.Context) { } ctx.Data["HasPackages"] = hasPackages - ctx.Data["Packages"] = pkgs + ctx.Data["PackageDescriptors"] = pds ctx.Data["Query"] = query ctx.Data["PackageType"] = packageType @@ -92,59 +85,32 @@ func Packages(ctx *context.Context) { // ViewPackage displays a single package func ViewPackage(ctx *context.Context) { - p, err := packages.GetPackageByID(ctx.ParamsInt64(":id")) + pv, err := packages.GetVersionByID(db.DefaultContext, ctx.ParamsInt64(":id")) if err != nil { if err == packages.ErrPackageNotExist { ctx.NotFound("", nil) } else { - ctx.ServerError("GetPackageByID", err) + ctx.ServerError("GetVersionByID", err) } return } - if p.RepoID != ctx.Repo.Repository.ID { - ctx.NotFound("", nil) + pd, err := packages.GetPackageDescriptor(pv) + if err != nil { + ctx.ServerError("GetPackageDescriptor", err) return } - if err := p.LoadCreator(); err != nil { - ctx.ServerError("LoadCreator", err) + if pd.Repository.ID != ctx.Repo.Repository.ID { + ctx.NotFound("", nil) return } - ctx.Data["Title"] = p.Name + ctx.Data["Title"] = pd.Package.Name ctx.Data["IsPackagesPage"] = true - ctx.Data["Package"] = p - - var metadata interface{} - switch p.Type { - case packages.TypeNuGet: - metadata = &nuget.Metadata{} - case packages.TypeNpm: - metadata = &npm.Metadata{} - case packages.TypeMaven: - metadata = &maven.Metadata{} - case packages.TypePyPI: - metadata = &pypi.Metadata{} - case packages.TypeRubyGems: - metadata = &rubygems.Metadata{} - } - if metadata != nil { - if err := json.Unmarshal([]byte(p.MetadataRaw), &metadata); err != nil { - ctx.ServerError("Unmarshal", err) - return - } - } - ctx.Data["Metadata"] = metadata - - files, err := p.GetFiles() - if err != nil { - ctx.ServerError("GetFiles", err) - return - } - ctx.Data["Files"] = files + ctx.Data["PackageDescriptor"] = pd - otherVersions, err := packages.GetPackagesByName(ctx.Repo.Repository.ID, p.Type, p.LowerName) + otherVersions, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, pd.Package.Type, pd.Package.LowerName) if err != nil { - ctx.ServerError("GetPackagesByName", err) + ctx.ServerError("GetVersionsByPackageName", err) return } sort.Slice(otherVersions, func(i, j int) bool { @@ -161,7 +127,7 @@ func ViewPackage(ctx *context.Context) { // DeletePackagePost deletes a package func DeletePackagePost(ctx *context.Context) { - err := package_service.DeletePackageByID(ctx.User, ctx.Repo.Repository, ctx.ParamsInt64(":id")) + err := packages_service.DeleteVersionByID(ctx.User, ctx.Repo.Repository, ctx.ParamsInt64(":id")) if err != nil { ctx.Flash.Error(err.Error()) } else { @@ -173,13 +139,13 @@ func DeletePackagePost(ctx *context.Context) { // DownloadPackageFile serves the content of a package file func DownloadPackageFile(ctx *context.Context) { - s, pf, err := package_service.GetFileStreamByPackageID(ctx.Repo.Repository, ctx.ParamsInt64(":id"), ctx.Params(":filename")) + s, pf, err := packages_service.GetFileStreamByPackageVersionID(ctx.Repo.Repository, ctx.ParamsInt64(":id"), ctx.Params(":filename")) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { ctx.NotFound("", err) return } - ctx.ServerError("GetFileStreamByPackageID", err) + ctx.ServerError("GetFileStreamByPackageVersionID", err) return } defer s.Close() diff --git a/services/packages/packages.go b/services/packages/packages.go index 0ab97d41ca280..e77eb48935c38 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -5,16 +5,13 @@ package packages import ( - "crypto/md5" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" + "context" "fmt" "io" "strings" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/db" packages_models "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" @@ -22,163 +19,316 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" ) -// CreatePackage creates a new package -func CreatePackage(creator *models.User, repository *models.Repository, packageType packages_models.Type, name, version string, metadata interface{}, allowDuplicate bool) (*packages_models.Package, error) { - metadataJSON, err := json.Marshal(metadata) +// PackageInfo describes a package +type PackageInfo struct { + Repository *models.Repository + PackageType packages_models.Type + Name string + Version string +} + +// PackageCreationInfo describes a package to create +type PackageCreationInfo struct { + PackageInfo + SemverCompatible bool + Creator *models.User + Metadata interface{} + Properties map[string]string +} + +// PackageFileInfo describes a package file +type PackageFileInfo struct { + Filename string + Data *packages_module.HashedBuffer + IsLead bool +} + +// CreatePackageAndAddFile creates a package with a file. If the same package exists already, ErrDuplicatePackageVersion is returned +func CreatePackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo) (*packages_models.PackageVersion, *packages_models.PackageFile, error) { + return createPackageAndAddFile(pvci, pfi, false) +} + +// CreatePackageOrAddFileToExisting creates a package with a file or adds the file if the package exists already +func CreatePackageOrAddFileToExisting(pvci *PackageCreationInfo, pfi *PackageFileInfo) (*packages_models.PackageVersion, *packages_models.PackageFile, error) { + return createPackageAndAddFile(pvci, pfi, true) +} + +func createPackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo, allowDuplicate bool) (*packages_models.PackageVersion, *packages_models.PackageFile, error) { + ctx, committer, err := db.TxContext() if err != nil { - log.Error("Error converting metadata to JSON: %v", err) - return nil, err + return nil, nil, err } + defer committer.Close() - log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", creator.ID, repository.ID, packageType, name, version, metadata, allowDuplicate) + pv, created, err := createPackageAndVersion(ctx, pvci, allowDuplicate) + if err != nil { + return nil, nil, err + } - p := &packages_models.Package{ - RepoID: repository.ID, - CreatorID: creator.ID, - Type: packageType, - Name: name, - LowerName: strings.ToLower(name), - Version: version, - MetadataRaw: string(metadataJSON), - } - if p, err = packages_models.TryInsertPackage(p); err != nil { - if err == packages_models.ErrDuplicatePackage { - if allowDuplicate { - return p, nil + pf, blobCreated, err := addFileToPackageVersion(ctx, pv, pfi) + removeBlob := false + defer func() { + if blobCreated && removeBlob { + contentStore := packages_module.NewContentStore() + if err := contentStore.Delete(pf.BlobID); err != nil { + log.Error("Error deleting package blob from content store: %v", err) } - return nil, err } - log.Error("Error inserting package: %v", err) - return nil, err + }() + if err != nil { + removeBlob = true + return nil, nil, err } - notification.NotifyPackageCreate(repository, p) + if err := committer.Commit(); err != nil { + removeBlob = true + return nil, nil, err + } + + if created { + pd, err := packages_models.GetPackageDescriptor(pv) + if err != nil { + return nil, nil, err + } + + notification.NotifyPackageCreate(pvci.Creator, pd) + } - return p, nil + return pv, pf, nil } -// AddFileToPackage adds a new file to package and stores its content -func AddFileToPackage(p *packages_models.Package, filename string, size int64, r io.Reader) (*packages_models.PackageFile, error) { - log.Trace("Creating package file: %v, %v, %s", p.ID, size, filename) +func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, allowDuplicate bool) (*packages_models.PackageVersion, bool, error) { + log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", pvci.Creator.ID, pvci.Repository.ID, pvci.PackageType, pvci.Name, pvci.Version, pvci.Properties, allowDuplicate) - pf := &packages_models.PackageFile{ - PackageID: p.ID, - Size: size, - Name: filename, - LowerName: strings.ToLower(filename), + p := &packages_models.Package{ + RepoID: pvci.Repository.ID, + Type: pvci.PackageType, + Name: pvci.Name, + LowerName: strings.ToLower(pvci.Name), + SemverCompatible: pvci.SemverCompatible, } var err error - if pf, err = packages_models.TryInsertPackageFile(pf); err != nil { - if err == packages_models.ErrDuplicatePackageFile { - return nil, err + if p, err = packages_models.TryInsertPackage(ctx, p); err != nil { + if err != packages_models.ErrDuplicatePackage { + log.Error("Error inserting package: %v", err) + return nil, false, err } - log.Error("Error inserting package file: %v", err) - return nil, err } - md5 := md5.New() - h1 := sha1.New() - h256 := sha256.New() - h512 := sha512.New() + metadataJSON, err := json.Marshal(pvci.Metadata) + if err != nil { + return nil, false, err + } - r = io.TeeReader(r, io.MultiWriter(md5, h1, h256, h512)) + created := true + pv := &packages_models.PackageVersion{ + PackageID: p.ID, + CreatorID: pvci.Creator.ID, + Version: pvci.Version, + LowerVersion: strings.ToLower(pvci.Version), + MetadataJSON: string(metadataJSON), + } + if pv, err = packages_models.GetOrInsertVersion(ctx, pv); err != nil { + if err == packages_models.ErrDuplicatePackageVersion { + created = false + } + if err != packages_models.ErrDuplicatePackageVersion || !allowDuplicate { + log.Error("Error inserting package: %v", err) + return nil, false, err + } + } - contentStore := packages_module.NewContentStore() + if err := packages_models.SetVersionProperties(ctx, pv.ID, pvci.Properties); err != nil { + log.Error("Error setting package version properties: %v", err) + return nil, false, err + } - err = func() error { - err := contentStore.Save(p.ID, pf.ID, r, size) - if err != nil { - log.Error("Error saving package file in content store: %v", err) - return err - } + return pv, created, nil +} - pf.HashMD5 = fmt.Sprintf("%x", md5.Sum(nil)) - pf.HashSHA1 = fmt.Sprintf("%x", h1.Sum(nil)) - pf.HashSHA256 = fmt.Sprintf("%x", h256.Sum(nil)) - pf.HashSHA512 = fmt.Sprintf("%x", h512.Sum(nil)) - if err = packages_models.UpdatePackageFile(pf); err != nil { - log.Error("Error updating package file: %v", err) - return err +// AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned +func AddFileToExistingPackage(pvi *PackageInfo, pfi *PackageFileInfo) (*packages_models.PackageVersion, *packages_models.PackageFile, error) { + ctx, committer, err := db.TxContext() + if err != nil { + return nil, nil, err + } + defer committer.Close() + + pv, err := packages_models.GetVersionByNameAndVersion(ctx, pvi.Repository.ID, pvi.PackageType, pvi.Name, pvi.Version) + if err != nil { + return nil, nil, err + } + + pf, blobCreated, err := addFileToPackageVersion(ctx, pv, pfi) + removeBlob := false + defer func() { + if blobCreated && removeBlob { + contentStore := packages_module.NewContentStore() + if err := contentStore.Delete(pf.BlobID); err != nil { + log.Error("Error deleting package blob from content store: %v", err) + } } - return nil }() if err != nil { - _ = contentStore.Delete(p.ID, pf.ID) + removeBlob = true + return nil, nil, err + } - if err := packages_models.DeletePackageFileByID(pf.ID); err != nil { - log.Error("Error deleting package file: %v", err) - } - return nil, err + if err := committer.Commit(); err != nil { + return nil, nil, err } - return pf, nil + + return pv, pf, nil } -// DeletePackageByNameAndVersion deletes a package and all associated files -func DeletePackageByNameAndVersion(doer *models.User, repository *models.Repository, packageType packages_models.Type, name, version string) error { - log.Trace("Deleting package: %v, %v, %s, %s", repository.ID, packageType, name, version) +func addFileToPackageVersion(ctx context.Context, pv *packages_models.PackageVersion, pfi *PackageFileInfo) (*packages_models.PackageFile, bool, error) { + log.Trace("Adding package file: %v, %s", pv.ID, pfi.Filename) - p, err := packages_models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) + hashMD5, hashSHA1, hashSHA256, hashSHA512 := pfi.Data.Sums() + pb := &packages_models.PackageBlob{ + Size: pfi.Data.Size(), + HashMD5: fmt.Sprintf("%x", hashMD5), + HashSHA1: fmt.Sprintf("%x", hashSHA1), + HashSHA256: fmt.Sprintf("%x", hashSHA256), + HashSHA512: fmt.Sprintf("%x", hashSHA512), + } + pb, exists, err := packages_models.GetOrInsertBlob(ctx, pb) if err != nil { - if err == packages_models.ErrPackageNotExist { - return err + log.Error("Error inserting package blob: %v", err) + return nil, false, err + } + if !exists { + contentStore := packages_module.NewContentStore() + if err := contentStore.Save(pb.ID, pfi.Data, pfi.Data.Size()); err != nil { + log.Error("Error saving package blob in content store: %v", err) + return nil, false, err } - log.Error("Error getting package: %v", err) - return err } - return deletePackage(doer, repository, p) + pf := &packages_models.PackageFile{ + VersionID: pv.ID, + BlobID: pb.ID, + Name: pfi.Filename, + LowerName: strings.ToLower(pfi.Filename), + IsLead: pfi.IsLead, + } + if pf, err = packages_models.TryInsertFile(ctx, pf); err != nil { + if err != packages_models.ErrDuplicatePackageFile { + log.Error("Error inserting package file: %v", err) + } + return nil, !exists, err + } + + return pf, !exists, nil } -// DeletePackageByID deletes a package and all associated files -func DeletePackageByID(doer *models.User, repository *models.Repository, packageID int64) error { - log.Trace("Deleting package: %v, %v", repository.ID, packageID) +// DeletePackageVersionByNameAndVersion deletes a package version and all associated files +func DeletePackageVersionByNameAndVersion(doer *models.User, pvi *PackageInfo) error { + return deletePackageVersion( + doer, + pvi.Repository, + func(ctx context.Context) (*packages_models.PackageVersion, error) { + return packages_models.GetVersionByNameAndVersion(ctx, pvi.Repository.ID, pvi.PackageType, pvi.Name, pvi.Version) + }, + ) +} - p, err := packages_models.GetPackageByID(packageID) +// DeleteVersionByID deletes a package version and all associated files +func DeleteVersionByID(doer *models.User, repository *models.Repository, versionID int64) error { + return deletePackageVersion( + doer, + repository, + func(ctx context.Context) (*packages_models.PackageVersion, error) { + return packages_models.GetVersionByID(ctx, versionID) + }, + ) +} + +func deletePackageVersion(doer *models.User, repository *models.Repository, cb func(context.Context) (*packages_models.PackageVersion, error)) error { + ctx, committer, err := db.TxContext() if err != nil { - if err == packages_models.ErrPackageNotExist { - return err + return err + } + defer committer.Close() + + pv, err := cb(ctx) + if err != nil { + if err != packages_models.ErrPackageNotExist { + log.Error("Error getting package: %v", err) } - log.Error("Error getting package: %v", err) return err } - if p.RepoID != repository.ID { - return packages_models.ErrPackageNotExist + pd, err := packages_models.GetPackageDescriptorCtx(ctx, pv) + if err != nil { + return err + } + + log.Trace("Deleting package: %v, %v", repository.ID, pv.ID) + + if err := packages_models.DeleteVersionPropertiesByVersionID(ctx, pv.ID); err != nil { + return err + } + + if err := packages_models.DeleteFilesByVersionID(ctx, pv.ID); err != nil { + return err + } + + if err := packages_models.DeleteVersionByID(ctx, pv.ID); err != nil { + return err + } + + if err := packages_models.DeletePackageByIDIfUnreferenced(ctx, pv.PackageID); err != nil { + return err } - return deletePackage(doer, repository, p) + if err := committer.Commit(); err != nil { + return err + } + + notification.NotifyPackageDelete(doer, pd) + + return DeleteUnreferencedBlobs() } -func deletePackage(doer *models.User, repository *models.Repository, p *packages_models.Package) error { - pfs, err := p.GetFiles() +// DeleteUnreferencedBlobs deletes all unreferenced package blobs +func DeleteUnreferencedBlobs() error { + ctx, committer, err := db.TxContext() if err != nil { - log.Error("Error getting package files: %v", err) return err } + defer committer.Close() - contentStore := packages_module.NewContentStore() - for _, pf := range pfs { - if err := contentStore.Delete(p.ID, pf.ID); err != nil { - log.Error("Error deleting package file [%s]: %v", pf.Name, err) + pbs, err := packages_models.GetUnreferencedBlobs(ctx) + if err != nil { + return err + } + + for _, pb := range pbs { + if err := packages_models.DeleteBlobByID(ctx, pb.ID); err != nil { return err } } - if err := packages_models.DeletePackageByID(p.ID); err != nil { - log.Error("Error deleting package: %v", err) + if err := committer.Commit(); err != nil { return err } - notification.NotifyPackageDelete(doer, repository, p) + contentStore := packages_module.NewContentStore() + for _, pb := range pbs { + if err := contentStore.Delete(pb.ID); err != nil { + log.Error("Error deleting package blob [%v]: %v", pb.ID, err) + } + } return nil } // GetFileStreamByPackageNameAndVersion returns the content of the specific package file -func GetFileStreamByPackageNameAndVersion(repository *models.Repository, packageType packages.Type, name, version, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { +func GetFileStreamByPackageNameAndVersion(repository *models.Repository, packageType packages_models.Type, name, version, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s", repository.ID, packageType, name, version, filename) - p, err := packages_models.GetPackageByNameAndVersion(repository.ID, packageType, name, version) + pv, err := packages_models.GetVersionByNameAndVersion(db.DefaultContext, repository.ID, packageType, name, version) if err != nil { if err == packages_models.ErrPackageNotExist { return nil, nil, err @@ -187,18 +337,24 @@ func GetFileStreamByPackageNameAndVersion(repository *models.Repository, package return nil, nil, err } - return GetPackageFileStream(p, filename) + return GetPackageFileStream(pv, filename) } -// GetFileStreamByPackageID returns the content of the specific package file -func GetFileStreamByPackageID(repository *models.Repository, packageID int64, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { - log.Trace("Getting package file stream: %v, %v, %s", repository.ID, packageID, filename) +// GetFileStreamByPackageVersionID returns the content of the specific package file +func GetFileStreamByPackageVersionID(repository *models.Repository, versionID int64, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { + log.Trace("Getting package file stream: %v, %v, %s", repository.ID, versionID, filename) - p, err := packages_models.GetPackageByID(packageID) + pv, err := packages_models.GetVersionByID(db.DefaultContext, versionID) if err != nil { - if err == packages_models.ErrPackageNotExist { - return nil, nil, err + if err == packages_models.ErrPackageVersionNotExist { + return nil, nil, packages_models.ErrPackageNotExist } + log.Error("Error getting package version: %v", err) + return nil, nil, err + } + + p, err := packages_models.GetPackageByID(db.DefaultContext, pv.PackageID) + if err != nil { log.Error("Error getting package: %v", err) return nil, nil, err } @@ -207,16 +363,23 @@ func GetFileStreamByPackageID(repository *models.Repository, packageID int64, fi return nil, nil, packages_models.ErrPackageNotExist } - return GetPackageFileStream(p, filename) + return GetPackageFileStream(pv, filename) } // GetPackageFileStream returns the cotent of the specific package file -func GetPackageFileStream(p *packages_models.Package, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { - pf, err := p.GetFileByName(filename) +func GetPackageFileStream(pv *packages_models.PackageVersion, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { + pf, err := packages_models.GetFileForVersionByName(db.DefaultContext, pv.ID, filename) if err != nil { return nil, nil, err } - s, err := packages_module.NewContentStore().Get(p.ID, pf.ID) + s, err := packages_module.NewContentStore().Get(pf.BlobID) + if err == nil { + if pf.IsLead { + if err := packages_models.IncrementDownloadCounter(pv.ID); err != nil { + log.Error("Error incrementing download counter: %v", err) + } + } + } return s, pf, err } diff --git a/templates/api/packages/pypi/simple.tmpl b/templates/api/packages/pypi/simple.tmpl index 3815b56e6f33b..d8e480d9c675d 100644 --- a/templates/api/packages/pypi/simple.tmpl +++ b/templates/api/packages/pypi/simple.tmpl @@ -1,14 +1,14 @@ - Links for {{.Package.Name}} + Links for {{.PackageDescriptor.Package.Name}} -

    Links for {{.Package.Name}}

    - {{range .Packages}} +

    Links for {{.PackageDescriptor.Package.Name}}

    + {{range .PackageDescriptors}} {{$p := .}} {{range .Files}} - {{.Name}}
    + {{.File.Name}}
    {{end}} {{end}} diff --git a/templates/repo/packages/content/maven.tmpl b/templates/repo/packages/content/maven.tmpl index fcb794b6e6ab8..3f9e703358b7e 100644 --- a/templates/repo/packages/content/maven.tmpl +++ b/templates/repo/packages/content/maven.tmpl @@ -1,4 +1,4 @@ -{{if eq .Package.Type 3}} +{{if eq .PackageDescriptor.Package.Type "maven"}}

    {{.i18n.Tr "repo.packages.installation"}}

    @@ -26,9 +26,9 @@
    @@ -37,7 +37,7 @@
    - +
    @@ -45,18 +45,18 @@
    - {{if .Metadata.Description}} + {{if .PackageDescriptor.Metadata.Description}}

    {{.i18n.Tr "repo.packages.about"}}

    - {{.Metadata.Description}} + {{.PackageDescriptor.Metadata.Description}}
    {{end}} - {{if .Metadata.Dependencies}} + {{if .PackageDescriptor.Metadata.Dependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    - {{range .Metadata.Dependencies}} + {{range .PackageDescriptor.Metadata.Dependencies}}
    {{svg "octicon-package-dependencies" 16 ""}}
    diff --git a/templates/repo/packages/content/npm.tmpl b/templates/repo/packages/content/npm.tmpl index 9075c9461cd66..5d862a4a50fe7 100644 --- a/templates/repo/packages/content/npm.tmpl +++ b/templates/repo/packages/content/npm.tmpl @@ -1,18 +1,18 @@ -{{if eq .Package.Type 2}} +{{if eq .PackageDescriptor.Package.Type "npm"}}

    {{.i18n.Tr "repo.packages.installation"}}

    - +
    - +
    - +
    @@ -20,35 +20,35 @@
    - {{if or .Metadata.Description .Metadata.Readme}} + {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.Readme}}

    {{.i18n.Tr "repo.packages.about"}}

    - {{if .Metadata.Readme}} + {{if .PackageDescriptor.Metadata.Readme}}
    - {{RenderMarkdownToHtml .Metadata.Readme}} + {{RenderMarkdownToHtml .PackageDescriptor.Metadata.Readme}}
    - {{else if .Metadata.Description}} - {{.Metadata.Description}} + {{else if .PackageDescriptor.Metadata.Description}} + {{.PackageDescriptor.Metadata.Description}} {{end}}
    {{end}} - {{if or .Metadata.Dependencies .Metadata.DevelopmentDependencies .Metadata.PeerDependencies .Metadata.OptionalDependencies}} + {{if or .PackageDescriptor.Metadata.Dependencies .PackageDescriptor.Metadata.DevelopmentDependencies .PackageDescriptor.Metadata.PeerDependencies .PackageDescriptor.Metadata.OptionalDependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    - {{template "repo/packages/content/npm_dependencies" dict "dependencies" .Metadata.Dependencies "title" (.i18n.Tr "repo.packages.npm.dependencies")}} - {{template "repo/packages/content/npm_dependencies" dict "dependencies" .Metadata.DevelopmentDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.development")}} - {{template "repo/packages/content/npm_dependencies" dict "dependencies" .Metadata.PeerDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.peer")}} - {{template "repo/packages/content/npm_dependencies" dict "dependencies" .Metadata.OptionalDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.optional")}} + {{template "repo/packages/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.Dependencies "title" (.i18n.Tr "repo.packages.npm.dependencies")}} + {{template "repo/packages/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.development")}} + {{template "repo/packages/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.PeerDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.peer")}} + {{template "repo/packages/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.OptionalDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.optional")}}
    {{end}} - {{if or .Metadata.Keywords}} + {{if or .PackageDescriptor.Metadata.Keywords}}

    {{.i18n.Tr "repo.packages.keywords"}}

    - {{range .Metadata.Keywords}} + {{range .PackageDescriptor.Metadata.Keywords}} {{.}} {{end}}
    diff --git a/templates/repo/packages/content/nuget.tmpl b/templates/repo/packages/content/nuget.tmpl index fc0771b65ac00..b0ae1bdd2e27a 100644 --- a/templates/repo/packages/content/nuget.tmpl +++ b/templates/repo/packages/content/nuget.tmpl @@ -1,14 +1,14 @@ -{{if eq .Package.Type 1}} +{{if eq .PackageDescriptor.Package.Type "nuget"}}

    {{.i18n.Tr "repo.packages.installation"}}

    - +
    - +
    @@ -16,19 +16,19 @@
    - {{if or .Metadata.Description .Metadata.ReleaseNotes}} + {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.ReleaseNotes}}

    {{.i18n.Tr "repo.packages.about"}}

    - {{if .Metadata.Description}}{{.Metadata.Description}}{{end}} - {{if .Metadata.ReleaseNotes}}{{Str2html .Metadata.ReleaseNotes}}{{end}} + {{if .PackageDescriptor.Metadata.Description}}{{.PackageDescriptor.Metadata.Description}}{{end}} + {{if .PackageDescriptor.Metadata.ReleaseNotes}}{{Str2html .PackageDescriptor.Metadata.ReleaseNotes}}{{end}}
    {{end}} - {{if .Metadata.Dependencies}} + {{if .PackageDescriptor.Metadata.Dependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    - {{range $framework, $dependencies := .Metadata.Dependencies}} + {{range $framework, $dependencies := .PackageDescriptor.Metadata.Dependencies}}
    {{$framework}}
    diff --git a/templates/repo/packages/content/pypi.tmpl b/templates/repo/packages/content/pypi.tmpl index 0bba746885753..d2e11f3ca0b4c 100644 --- a/templates/repo/packages/content/pypi.tmpl +++ b/templates/repo/packages/content/pypi.tmpl @@ -1,31 +1,31 @@ -{{if eq .Package.Type 4}} +{{if eq .PackageDescriptor.Package.Type "pypi"}}

    {{.i18n.Tr "repo.packages.installation"}}

    - +
    - {{if or .Metadata.Description .Metadata.LongDescription .Metadata.Summary}} + {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.LongDescription .PackageDescriptor.Metadata.Summary}}

    {{.i18n.Tr "repo.packages.about"}}

    -

    {{if .Metadata.Summary}}{{.Metadata.Summary}}{{end}}

    - {{if .Metadata.LongDescription}} - {{RenderMarkdownToHtml .Metadata.LongDescription}} - {{else if .Metadata.Description}} - {{RenderMarkdownToHtml .Metadata.Description}} +

    {{if .PackageDescriptor.Metadata.Summary}}{{.PackageDescriptor.Metadata.Summary}}{{end}}

    + {{if .PackageDescriptor.Metadata.LongDescription}} + {{RenderMarkdownToHtml .PackageDescriptor.Metadata.LongDescription}} + {{else if .PackageDescriptor.Metadata.Description}} + {{RenderMarkdownToHtml .PackageDescriptor.Metadata.Description}} {{end}}
    {{end}} - {{if .Metadata.RequiresPython}} + {{if .PackageDescriptor.Metadata.RequiresPython}}

    {{.i18n.Tr "repo.packages.requirements"}}

    - {{.i18n.Tr "repo.packages.pypi.requires"}}: {{.Metadata.RequiresPython}} + {{.i18n.Tr "repo.packages.pypi.requires"}}: {{.PackageDescriptor.Metadata.RequiresPython}}
    {{end}} {{end}} diff --git a/templates/repo/packages/content/rubygems.tmpl b/templates/repo/packages/content/rubygems.tmpl index 430fa421d3e58..214714a8457c6 100644 --- a/templates/repo/packages/content/rubygems.tmpl +++ b/templates/repo/packages/content/rubygems.tmpl @@ -1,15 +1,15 @@ -{{if eq .Package.Type 5}} +{{if eq .PackageDescriptor.Package.Type "rubygems"}}

    {{.i18n.Tr "repo.packages.installation"}}

    - +
    @@ -17,27 +17,23 @@ end
    - {{if .Metadata.Description}} + {{if .PackageDescriptor.Metadata.Description}}

    {{.i18n.Tr "repo.packages.about"}}

    -
    - {{if .Metadata.Description}} - {{.Metadata.Description}} - {{end}} -
    +
    {{.PackageDescriptor.Metadata.Description}}
    {{end}} - {{if or .Metadata.RequiredRubyVersion .Metadata.RequiredRubygemsVersion}} + {{if or .PackageDescriptor.Metadata.RequiredRubyVersion .PackageDescriptor.Metadata.RequiredRubygemsVersion}}

    {{.i18n.Tr "repo.packages.requirements"}}

    - {{if .Metadata.RequiredRubyVersion}}

    {{.i18n.Tr "repo.packages.rubygems.required.ruby"}}: {{range $i, $v := .Metadata.RequiredRubyVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}} - {{if .Metadata.RequiredRubygemsVersion}}

    {{.i18n.Tr "repo.packages.rubygems.required.rubygems"}}: {{range $i, $v := .Metadata.RequiredRubygemsVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}} + {{if .PackageDescriptor.Metadata.RequiredRubyVersion}}

    {{.i18n.Tr "repo.packages.rubygems.required.ruby"}}: {{range $i, $v := .PackageDescriptor.Metadata.RequiredRubyVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}} + {{if .PackageDescriptor.Metadata.RequiredRubygemsVersion}}

    {{.i18n.Tr "repo.packages.rubygems.required.rubygems"}}: {{range $i, $v := .PackageDescriptor.Metadata.RequiredRubygemsVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}}
    {{end}} - {{if or .Metadata.RuntimeDependencies .Metadata.DevelopmentDependencies}} + {{if or .PackageDescriptor.Metadata.RuntimeDependencies .PackageDescriptor.Metadata.DevelopmentDependencies}}

    {{.i18n.Tr "repo.packages.dependencies"}}

    - {{template "repo/packages/content/rubygems_dependencies" dict "dependencies" .Metadata.RuntimeDependencies "title" (.i18n.Tr "repo.packages.rubygems.dependencies.runtime")}} - {{template "repo/packages/content/rubygems_dependencies" dict "dependencies" .Metadata.DevelopmentDependencies "title" (.i18n.Tr "repo.packages.rubygems.dependencies.development")}} + {{template "repo/packages/content/rubygems_dependencies" dict "dependencies" .PackageDescriptor.Metadata.RuntimeDependencies "title" (.i18n.Tr "repo.packages.rubygems.dependencies.runtime")}} + {{template "repo/packages/content/rubygems_dependencies" dict "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "repo.packages.rubygems.dependencies.development")}}
    {{end}} diff --git a/templates/repo/packages/list.tmpl b/templates/repo/packages/list.tmpl index 295f82af07213..bac0d4afae318 100644 --- a/templates/repo/packages/list.tmpl +++ b/templates/repo/packages/list.tmpl @@ -15,20 +15,20 @@ - +
    -
    - {{range .Packages}} +
    + {{range .PackageDescriptors}}
  • - {{.Name}} - {{svg .Type.SVGName 16}} {{.Type.String}} + {{.Package.Name}} + {{svg .Package.Type.SVGName 16}} {{.Package.Type.Name}}
    - {{ $timeStr := TimeSinceUnix .CreatedUnix $.Lang }} + {{ $timeStr := TimeSinceUnix .Version.CreatedUnix $.Lang }} {{$.i18n.Tr "repo.packages.published_by" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) | Safe}}
    @@ -41,7 +41,7 @@

    {{.i18n.Tr "repo.packages.empty.documentation" | Safe}}

    • {{else}} - {{.i18n.Tr "repo.packages.filter.no_result"}} +

    {{.i18n.Tr "repo.packages.filter.no_result"}}

    {{end}} {{end}} {{template "base/paginate" .}} diff --git a/templates/repo/packages/metadata/maven.tmpl b/templates/repo/packages/metadata/maven.tmpl index c9566b398f5cc..cf52b21c2e5c4 100644 --- a/templates/repo/packages/metadata/maven.tmpl +++ b/templates/repo/packages/metadata/maven.tmpl @@ -1,5 +1,5 @@ -{{if eq .Package.Type 3}} - {{if .Metadata.Name}}
    {{svg "octicon-note" 16 "mr-3"}} {{.Metadata.Name}}
    {{end}} - {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{range .Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} +{{if eq .PackageDescriptor.Package.Type "maven"}} + {{if .PackageDescriptor.Metadata.Name}}
    {{svg "octicon-note" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Name}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{range .PackageDescriptor.Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/npm.tmpl b/templates/repo/packages/metadata/npm.tmpl index a05a84356f69a..7a17d81ea72f8 100644 --- a/templates/repo/packages/metadata/npm.tmpl +++ b/templates/repo/packages/metadata/npm.tmpl @@ -1,5 +1,5 @@ -{{if eq .Package.Type 2}} - {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.Metadata.Author}}
    {{end}} - {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.Metadata.License}}
    {{end}} +{{if eq .PackageDescriptor.Package.Type "npm"}} + {{if .PackageDescriptor.Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .PackageDescriptor.Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/nuget.tmpl b/templates/repo/packages/metadata/nuget.tmpl index d9cb79ec388c3..3357e68467340 100644 --- a/templates/repo/packages/metadata/nuget.tmpl +++ b/templates/repo/packages/metadata/nuget.tmpl @@ -1,4 +1,4 @@ -{{if eq .Package.Type 1}} - {{if .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.Metadata.Authors}}
    {{end}} - {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} +{{if eq .PackageDescriptor.Package.Type "nuget"}} + {{if .PackageDescriptor.Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Authors}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/pypi.tmpl b/templates/repo/packages/metadata/pypi.tmpl index c8215bffe73b0..33f940a79b5d4 100644 --- a/templates/repo/packages/metadata/pypi.tmpl +++ b/templates/repo/packages/metadata/pypi.tmpl @@ -1,5 +1,5 @@ -{{if eq .Package.Type 4}} - {{if .Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.Metadata.Author}}
    {{end}} - {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{if .Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.Metadata.License}}
    {{end}} +{{if eq .PackageDescriptor.Package.Type "pypi"}} + {{if .PackageDescriptor.Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .PackageDescriptor.Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/rubygems.tmpl b/templates/repo/packages/metadata/rubygems.tmpl index f3937d7324ebf..c2fa7b8d82438 100644 --- a/templates/repo/packages/metadata/rubygems.tmpl +++ b/templates/repo/packages/metadata/rubygems.tmpl @@ -1,5 +1,5 @@ -{{if eq .Package.Type 5}} - {{range .Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.}}
    {{end}} - {{if .Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{range .Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} +{{if eq .PackageDescriptor.Package.Type "rubygems"}} + {{range .PackageDescriptor.Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{range .PackageDescriptor.Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} {{end}} diff --git a/templates/repo/packages/view.tmpl b/templates/repo/packages/view.tmpl index e97209ae94109..28621913edbe4 100644 --- a/templates/repo/packages/view.tmpl +++ b/templates/repo/packages/view.tmpl @@ -6,10 +6,10 @@
    -

    {{.Package.Name}} ({{.Package.Version}})

    +

    {{.PackageDescriptor.Package.Name}} ({{.PackageDescriptor.Version.Version}})

    - {{ $timeStr := TimeSinceUnix .Package.CreatedUnix $.Lang }} -
    {{.i18n.Tr "repo.packages.published_by" $timeStr .Package.Creator.HomeLink (.Package.Creator.GetDisplayName | Escape) | Safe}} {{svg .Package.Type.SVGName}} {{.Package.Type.String}}
    + {{ $timeStr := TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.Lang }} +
    {{.i18n.Tr "repo.packages.published_by" $timeStr .PackageDescriptor.Creator.HomeLink (.PackageDescriptor.Creator.GetDisplayName | Escape) | Safe}} {{svg .PackageDescriptor.Package.Type.SVGName}} {{.PackageDescriptor.Package.Type.Name}}
    @@ -24,9 +24,8 @@
    {{.i18n.Tr "repo.packages.details"}}
    -
    - {{svg "octicon-calendar" 16 "mr-3"}} {{.Package.CreatedUnix.FormatDate}} -
    +
    {{svg "octicon-calendar" 16 "mr-3"}} {{.PackageDescriptor.Version.CreatedUnix.FormatDate}}
    +
    {{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
    {{template "repo/packages/metadata/generic" .}} {{template "repo/packages/metadata/nuget" .}} {{template "repo/packages/metadata/npm" .}} @@ -35,12 +34,12 @@ {{template "repo/packages/metadata/rubygems" .}}
    - {{.i18n.Tr "repo.packages.assets"}} ({{len .Files}}) + {{.i18n.Tr "repo.packages.assets"}} ({{len .PackageDescriptor.Files}})
    - {{range .Files}} + {{range .PackageDescriptor.Files}}
    - {{.Name}} - {{FileSize .Size}} + {{.File.Name}} + {{FileSize .Blob.Size}}
    {{end}}
    @@ -50,7 +49,7 @@
    {{range .OtherVersions}}
    - {{.Version}} + {{.Name}} {{$.i18n.Tr "repo.packages.versions.on"}} {{.CreatedUnix.FormatDate}}
    {{end}} @@ -67,7 +66,7 @@
    - {{.i18n.Tr "repo.packages.delete.notice" .Package.Name .Package.Version}} + {{.i18n.Tr "repo.packages.delete.notice" .PackageDescriptor.Package.Name .PackageDescriptor.Version.Version}}
    {{.CsrfTokenHtml}} From 1cef19a315723bcb297d0bec97bab1ae0c79b92c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 7 Nov 2021 21:11:00 +0200 Subject: [PATCH 059/130] Prevent empty filename. --- models/packages/package_file.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 524ab72b25ce6..20c0593aa3c4d 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -63,6 +63,10 @@ func GetFilesByVersionID(ctx context.Context, versionID int64) ([]*PackageFile, // GetFileForVersionByName gets a file of a version by name func GetFileForVersionByName(ctx context.Context, versionID int64, name string) (*PackageFile, error) { + if name == "" { + return nil, ErrPackageFileNotExist + } + pf := &PackageFile{ VersionID: versionID, LowerName: strings.ToLower(name), From 4162676d471c6d3defde68f72a93ed4166c34881 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 9 Nov 2021 22:22:47 +0000 Subject: [PATCH 060/130] Fix swagger. --- templates/swagger/v1_json.tmpl | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 0492ac55026fb..a705e55684ae9 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -16486,11 +16486,6 @@ "type": "string", "x-go-name": "Type" }, - "updated_at": { - "type": "string", - "format": "date-time", - "x-go-name": "UpdatedAt" - }, "version": { "type": "string", "x-go-name": "Version" @@ -16506,11 +16501,6 @@ "type": "integer", "format": "int64" }, - "created_at": { - "type": "string", - "format": "date-time", - "x-go-name": "CreatedAt" - }, "id": { "type": "integer", "format": "int64", @@ -16535,11 +16525,6 @@ "sha512": { "type": "string", "x-go-name": "HashSHA512" - }, - "updated_at": { - "type": "string", - "format": "date-time", - "x-go-name": "UpdatedAt" } }, "x-go-package": "code.gitea.io/gitea/modules/structs" From 86af6114d624ac9f590e28129d3c44fcd0e8b095 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 9 Nov 2021 22:31:54 +0000 Subject: [PATCH 061/130] Implemented user/org registry. --- docs/content/doc/packages/generic.en-us.md | 16 +- docs/content/doc/packages/maven.en-us.md | 11 +- docs/content/doc/packages/npm.en-us.md | 13 +- docs/content/doc/packages/nuget.en-us.md | 9 +- docs/content/doc/packages/pypi.en-us.md | 14 +- docs/content/doc/packages/rubygems.en-us.md | 25 ++- integrations/api_packages_generic_test.go | 11 +- integrations/api_packages_maven_test.go | 15 +- integrations/api_packages_npm_test.go | 9 +- integrations/api_packages_nuget_test.go | 15 +- integrations/api_packages_pypi_test.go | 11 +- integrations/api_packages_rubygems_test.go | 13 +- integrations/api_packages_test.go | 19 +- models/migrations/v202.go | 3 +- models/packages/descriptor.go | 8 +- models/packages/package.go | 10 +- models/packages/package_version.go | 31 ++-- modules/context/api.go | 3 +- modules/context/api_package.go | 15 ++ modules/convert/package.go | 19 +- modules/notification/webhook/webhook.go | 7 +- modules/setting/setting.go | 4 +- modules/structs/package.go | 1 + routers/api/v1/api.go | 178 ++++++++++++------- routers/api/v1/packages/generic/generic.go | 14 +- routers/api/v1/packages/maven/maven.go | 10 +- routers/api/v1/packages/npm/npm.go | 16 +- routers/api/v1/packages/nuget/nuget.go | 38 ++-- routers/api/v1/packages/pypi/pypi.go | 16 +- routers/api/v1/packages/rubygems/rubygems.go | 14 +- routers/api/v1/repo/package.go | 31 ++-- routers/web/repo/packages.go | 4 +- services/packages/packages.go | 32 ++-- 33 files changed, 372 insertions(+), 263 deletions(-) create mode 100644 modules/context/api_package.go diff --git a/docs/content/doc/packages/generic.en-us.md b/docs/content/doc/packages/generic.en-us.md index cb40c5e5b00e3..72652cd8b3562 100644 --- a/docs/content/doc/packages/generic.en-us.md +++ b/docs/content/doc/packages/generic.en-us.md @@ -14,7 +14,7 @@ menu: # Generic Packages Repository -Publish generic files, like release binaries or other output, in your project’s Package Registry. +Publish generic files, like release binaries or other output, for your user or organization. **Table of Contents** @@ -30,13 +30,12 @@ To publish a generic package perform a HTTP PUT operation with the package conte You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. ``` -PUT https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/generic/{package_name}/{package_version}/{file_name} +PUT https://gitea.example.com/api/v1/packages/{owner}/generic/{package_name}/{package_version}/{file_name} ``` | Parameter | Description | | ----------------- | ----------- | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | | `package_name` | The package name. It can contain only lowercase letters (`a-z`), uppercase letter (`A-Z`), numbers (`0-9`), dots (`.`), hyphens (`-`), or underscores (`_`). | | `package_version` | The package version as described in the [SemVer](https://semver.org/) spec. | | `file_name` | The filename. It can contain only lowercase letters (`a-z`), uppercase letter (`A-Z`), numbers (`0-9`), dots (`.`), hyphens (`-`), or underscores (`_`). | @@ -46,7 +45,7 @@ Example request using HTTP Basic authentication: ```shell curl --user your_username:your_password_or_token \ --upload-file path/to/file.bin \ - "https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/generic/test_package/1.0.0/file.bin" + "https://gitea.example.com/api/v1/packages/testuser/generic/test_package/1.0.0/file.bin" ``` The server reponds with the following HTTP Status codes. @@ -61,13 +60,12 @@ The server reponds with the following HTTP Status codes. To download a generic package perform a HTTP GET operation. ``` -GET https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/generic/{package_name}/{package_version}/{file_name} +GET https://gitea.example.com/api/v1/packages/{owner}/generic/{package_name}/{package_version}/{file_name} ``` | Parameter | Description | | ----------------- | ----------- | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | | `package_name` | The package name. | | `package_version` | The package version. | | `file_name` | The filename. | @@ -78,5 +76,5 @@ Example request using HTTP Basic authentication: ```shell curl --user your_username:your_token_or_password \ - "https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/generic/test_package/1.0.0/file.bin" + "https://gitea.example.com/api/v1/packages/testuser/generic/test_package/1.0.0/file.bin" ``` diff --git a/docs/content/doc/packages/maven.en-us.md b/docs/content/doc/packages/maven.en-us.md index 173acb16194a9..c8f627528d49f 100644 --- a/docs/content/doc/packages/maven.en-us.md +++ b/docs/content/doc/packages/maven.en-us.md @@ -14,7 +14,7 @@ menu: # Maven Packages Repository -Publish [Maven](https://maven.apache.org) packages in your project’s Package Registry. +Publish [Maven](https://maven.apache.org) packages for your user or organization. **Table of Contents** @@ -53,17 +53,17 @@ Afterwards add the following sections to your project `pom.xml` file: gitea - https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/maven + https://gitea.example.com/api/v1/packages/{owner}/maven gitea - https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/maven + https://gitea.example.com/api/v1/packages/{owner}/maven gitea - https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/maven + https://gitea.example.com/api/v1/packages/{owner}/maven ``` @@ -71,8 +71,7 @@ Afterwards add the following sections to your project `pom.xml` file: | Parameter | Description | | -------------- | ----------- | | `access_token` | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | ## Publish a package diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index c676ad23cba74..729e82bb5a54d 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -14,7 +14,7 @@ menu: # npm Packages Repository -Publish [npm](https://www.npmjs.com/) packages in your project’s Package Registry. +Publish [npm](https://www.npmjs.com/) packages for your user or organization. **Table of Contents** @@ -33,22 +33,21 @@ The following examples use the `npm` tool with the scope `@test`. To register the project’s package registry you need to configure a new package source. ```shell -npm config set {scope}:registry https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/npm/ -npm config set -- '//gitea.example.com/api/v1/repos/{owner}/{repository}/packages/npm/:_authToken' "{token}" +npm config set {scope}:registry https://gitea.example.com/api/v1/packages/{owner}/npm/ +npm config set -- '//gitea.example.com/api/v1/packages/{owner}/npm/:_authToken' "{token}" ``` | Parameter | Description | | ------------ | ----------- | | `scope` | The scope of the packages. | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | | `token` | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). | For example: ```shell -npm config set @test:registry https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/npm/ -npm config set -- '//gitea.example.com/api/v1/repos/testuser/test-repository/packages/npm/:_authToken' "personal_access_token" +npm config set @test:registry https://gitea.example.com/api/v1/packages/testuser/npm/ +npm config set -- '//gitea.example.com/api/v1/packages/testuser/npm/:_authToken' "personal_access_token" ``` ## Publish a package diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md index 8aaef7d661908..d68384758d287 100644 --- a/docs/content/doc/packages/nuget.en-us.md +++ b/docs/content/doc/packages/nuget.en-us.md @@ -14,7 +14,7 @@ menu: # NuGet Packages Repository -Publish [NuGet](https://www.nuget.org/) packages in your project’s Package Registry. +Publish [NuGet](https://www.nuget.org/) packages for your user or organization. **Table of Contents** @@ -31,7 +31,7 @@ The following examples use the `dotnet nuget` tool. To register the project’s package registry you need to configure a new NuGet feed source: ```shell -dotnet nuget add source --name {source_name} --username {username} --password {password} https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/nuget/index.json +dotnet nuget add source --name {source_name} --username {username} --password {password} https://gitea.example.com/api/v1/packages/{owner}/nuget/index.json ``` | Parameter | Description | @@ -39,13 +39,12 @@ dotnet nuget add source --name {source_name} --username {username} --password {p | `source_name` | The desired source name. | | `username` | Your Gitea username. | | `password` | Your Gitea password or a personal access token. | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | For example: ```shell -dotnet nuget add source --name gitea --username testuser --password password123 https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/nuget/index.json +dotnet nuget add source --name gitea --username testuser --password password123 https://gitea.example.com/api/v1/packages/testuser/nuget/index.json ``` ## Publish a package diff --git a/docs/content/doc/packages/pypi.en-us.md b/docs/content/doc/packages/pypi.en-us.md index 33372ee553c56..34058562b57a9 100644 --- a/docs/content/doc/packages/pypi.en-us.md +++ b/docs/content/doc/packages/pypi.en-us.md @@ -14,7 +14,7 @@ menu: # PyPI Packages Repository -Publish [PyPI](https://pypi.org/) packages in your project’s Package Registry. +Publish [PyPI](https://pypi.org/) packages for your user or organization. **Table of Contents** @@ -33,15 +33,14 @@ To register the project’s package registry you need to edit your local `~/.pyp index-servers = gitea [gitea] -repository = https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/pypi +repository = https://gitea.example.com/api/v1/packages/{owner}/pypi username = {username} password = {password} ``` | Placeholder | Description | | ------------ | ----------- | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | | `username` | Your Gitea username. | | `password` | Your Gitea password or a [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}). | @@ -62,19 +61,18 @@ You cannot publish a package if a package of the same name and version already e To install a PyPI package from the package registry, execute the following command: ```shell -pip install --index-url https://{username}:{password}@gitea.example.com/api/v1/repos/{owner}/{repository}/packages/pypi/simple --no-deps {package_name} +pip install --index-url https://{username}:{password}@gitea.example.com/api/v1/packages/{owner}/pypi/simple --no-deps {package_name} ``` | Parameter | Description | | ----------------- | ----------- | | `username` | Your Gitea username. | | `password` | Your Gitea password or a personal access token. | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | | `package_name` | The package name. | For example: ```shell -pip install --index-url https://testuser:password123@gitea.example.com/api/v1/repos/testuser/test-repository/packages/pypi/simple --no-deps test_package +pip install --index-url https://testuser:password123@gitea.example.com/api/v1/packages/testuser/pypi/simple --no-deps test_package ``` diff --git a/docs/content/doc/packages/rubygems.en-us.md b/docs/content/doc/packages/rubygems.en-us.md index 16e4125b9dac0..c02e367f771ed 100644 --- a/docs/content/doc/packages/rubygems.en-us.md +++ b/docs/content/doc/packages/rubygems.en-us.md @@ -14,7 +14,7 @@ menu: # RubyGems Packages Repository -Publish [RubyGems](https://guides.rubygems.org/) packages in your project’s Package Registry. +Publish [RubyGems](https://guides.rubygems.org/) packages for your user or organization. **Table of Contents** @@ -30,20 +30,19 @@ To register the project’s package registry edit the `~/.gem/credentials` file ```ini --- -https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/rubygems: Bearer {token} +https://gitea.example.com/api/v1/packages/{owner}/rubygems: Bearer {token} ``` | Parameter | Description | | ------------- | ----------- | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | | `token` | Your personal access token. | For example: ```ini --- -https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/rubygems: Bearer 3bd626f84b01cd26b873931eace1e430a5773cc4 +https://gitea.example.com/api/v1/packages/testuser/rubygems: Bearer 3bd626f84b01cd26b873931eace1e430a5773cc4 ``` ## Publish a package @@ -62,7 +61,7 @@ gem push --host {host} {package_file} For example: ```shell -gem push --host https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/rubygems test_package-1.0.0.gem +gem push --host https://gitea.example.com/api/v1/packages/testuser/rubygems test_package-1.0.0.gem ``` You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. @@ -76,21 +75,20 @@ To install a package from the package registry you can use [Bundler](https://bun Add a new `source` block to your `Gemfile`: ``` -source "https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/rubygems" do +source "https://gitea.example.com/api/v1/packages/{owner}/rubygems" do gem "{package_name}" end ``` | Parameter | Description | | ----------------- | ----------- | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | | `package_name` | The package name. | For example: ``` -source "https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/rubygems" do +source "https://gitea.example.com/api/v1/packages/testuser/rubygems" do gem "test_package" end ``` @@ -106,17 +104,16 @@ bundle install Execute the following command: ```shell -gem install --host https://gitea.example.com/api/v1/repos/{owner}/{repository}/packages/rubygems {package_name} +gem install --host https://gitea.example.com/api/v1/packages/{owner}/rubygems {package_name} ``` | Parameter | Description | | ----------------- | ----------- | -| `owner` | The owner of the repository. | -| `repository` | The name of the repository. | +| `owner` | The owner of the package. | | `package_name` | The package name. | For example: ```shell -gem install --host https://gitea.example.com/api/v1/repos/testuser/test-repository/packages/rubygems test_package +gem install --host https://gitea.example.com/api/v1/packages/testuser/rubygems test_package ``` \ No newline at end of file diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index 9fc6346e2c6eb..3a5faae302997 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -19,8 +19,7 @@ import ( func TestPackageGeneric(t *testing.T) { defer prepareTestEnv(t)() - repository := db.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) - user := db.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + user := db.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) session := loginUser(t, user.Name) token := getTokenForLoggedInUser(t, session) @@ -29,7 +28,7 @@ func TestPackageGeneric(t *testing.T) { filename := "fi-le_na.me" content := []byte{1, 2, 3} - url := fmt.Sprintf("/api/v1/repos/%s/%s/packages/generic/%s/%s/%s?token=%s", user.Name, repository.Name, packageName, packageVersion, filename, token) + url := fmt.Sprintf("/api/v1/packages/%s/generic/%s/%s/%s?token=%s", user.Name, packageName, packageVersion, filename, token) t.Run("Upload", func(t *testing.T) { defer PrintCurrentTest(t)() @@ -37,7 +36,7 @@ func TestPackageGeneric(t *testing.T) { req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) MakeRequest(t, req, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeGeneric) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeGeneric) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -74,7 +73,7 @@ func TestPackageGeneric(t *testing.T) { assert.Equal(t, content, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeGeneric) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeGeneric) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) @@ -86,7 +85,7 @@ func TestPackageGeneric(t *testing.T) { req := NewRequest(t, "DELETE", url) MakeRequest(t, req, http.StatusOK) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeGeneric) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeGeneric) assert.NoError(t, err) assert.Empty(t, pvs) }) diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go index da526e23cc75d..d35a7ecfb3c15 100644 --- a/integrations/api_packages_maven_test.go +++ b/integrations/api_packages_maven_test.go @@ -20,8 +20,7 @@ import ( func TestPackageMaven(t *testing.T) { defer prepareTestEnv(t)() - repository := db.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) - user := db.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + user := db.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) groupID := "com.gitea" artifactID := "test-project" @@ -29,7 +28,7 @@ func TestPackageMaven(t *testing.T) { packageVersion := "1.0.1" packageDescription := "Test Description" - root := fmt.Sprintf("/api/v1/repos/%s/%s/packages/maven/%s/%s", user.Name, repository.Name, strings.ReplaceAll(groupID, ".", "/"), artifactID) + root := fmt.Sprintf("/api/v1/packages/%s/maven/%s/%s", user.Name, strings.ReplaceAll(groupID, ".", "/"), artifactID) filename := fmt.Sprintf("%s-%s.jar", packageName, packageVersion) putFile := func(t *testing.T, path, content string, expectedStatus int) { @@ -44,7 +43,7 @@ func TestPackageMaven(t *testing.T) { putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusCreated) putFile(t, "/maven-metadata.xml", "test", http.StatusOK) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -81,7 +80,7 @@ func TestPackageMaven(t *testing.T) { assert.Equal(t, []byte("test"), resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(0), pvs[0].DownloadCount) @@ -113,7 +112,7 @@ func TestPackageMaven(t *testing.T) { t.Run("UploadPOM", func(t *testing.T) { defer PrintCurrentTest(t)() - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -123,7 +122,7 @@ func TestPackageMaven(t *testing.T) { putFile(t, fmt.Sprintf("/%s/%s.pom", packageVersion, filename), pomContent, http.StatusCreated) - pvs, err = packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) + pvs, err = packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -152,7 +151,7 @@ func TestPackageMaven(t *testing.T) { assert.Equal(t, []byte(pomContent), resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeMaven) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index 893c7c5f2206b..0dd473e89fbfb 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -23,8 +23,7 @@ import ( func TestPackageNpm(t *testing.T) { defer prepareTestEnv(t)() - repository := db.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) - user := db.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + user := db.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) packageName := "@scope/test-package" packageVersion := "1.0.1-pre" @@ -57,7 +56,7 @@ func TestPackageNpm(t *testing.T) { } }` - root := fmt.Sprintf("/api/v1/repos/%s/%s/packages/npm/%s", user.Name, repository.Name, url.QueryEscape(packageName)) + root := fmt.Sprintf("/api/v1/packages/%s/npm/%s", user.Name, url.QueryEscape(packageName)) filename := fmt.Sprintf("%s-%s.tgz", strings.Split(packageName, "/")[1], packageVersion) t.Run("Upload", func(t *testing.T) { @@ -67,7 +66,7 @@ func TestPackageNpm(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNpm) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNpm) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -107,7 +106,7 @@ func TestPackageNpm(t *testing.T) { b, _ := base64.StdEncoding.DecodeString(data) assert.Equal(t, b, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNpm) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNpm) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index 5af16127a9a99..b7d8fb1afcd57 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -25,8 +25,7 @@ import ( func TestPackageNuGet(t *testing.T) { defer prepareTestEnv(t)() - repository := db.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) - user := db.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + user := db.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) packageName := "test.package" packageVersion := "1.0.3" @@ -48,7 +47,7 @@ func TestPackageNuGet(t *testing.T) { archive.Close() content := buf.Bytes() - url := fmt.Sprintf("/api/v1/repos/%s/%s/packages/nuget", user.Name, repository.Name) + url := fmt.Sprintf("/api/v1/packages/%s/nuget", user.Name) t.Run("ServiceIndex", func(t *testing.T) { defer PrintCurrentTest(t)() @@ -94,7 +93,7 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -153,7 +152,7 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -193,7 +192,7 @@ func TestPackageNuGet(t *testing.T) { assert.Equal(t, content, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) @@ -202,7 +201,7 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - pvs, err = packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) + pvs, err = packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) @@ -306,7 +305,7 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Empty(t, pvs) }) diff --git a/integrations/api_packages_pypi_test.go b/integrations/api_packages_pypi_test.go index e0fdce67f8ed5..dcc4d6628dbc4 100644 --- a/integrations/api_packages_pypi_test.go +++ b/integrations/api_packages_pypi_test.go @@ -24,8 +24,7 @@ import ( func TestPackagePyPI(t *testing.T) { defer prepareTestEnv(t)() - repository := db.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) - user := db.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + user := db.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) packageName := "test-package" packageVersion := "1.0.1" @@ -35,7 +34,7 @@ func TestPackagePyPI(t *testing.T) { content := "test" hashSHA256 := "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" - root := fmt.Sprintf("/api/v1/repos/%s/%s/packages/pypi", user.Name, repository.Name) + root := fmt.Sprintf("/api/v1/packages/%s/pypi", user.Name) uploadFile := func(t *testing.T, filename, content string, expectedStatus int) { body := &bytes.Buffer{} @@ -65,7 +64,7 @@ func TestPackagePyPI(t *testing.T) { filename := "test.whl" uploadFile(t, filename, content, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypePyPI) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypePyPI) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -93,7 +92,7 @@ func TestPackagePyPI(t *testing.T) { filename := "test.tar.gz" uploadFile(t, filename, content, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypePyPI) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypePyPI) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -146,7 +145,7 @@ func TestPackagePyPI(t *testing.T) { downloadFile("test.whl") downloadFile("test.tar.gz") - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypePyPI) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypePyPI) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(2), pvs[0].DownloadCount) diff --git a/integrations/api_packages_rubygems_test.go b/integrations/api_packages_rubygems_test.go index 80f310045332c..7392e0220513d 100644 --- a/integrations/api_packages_rubygems_test.go +++ b/integrations/api_packages_rubygems_test.go @@ -22,8 +22,7 @@ import ( func TestPackageRubyGems(t *testing.T) { defer prepareTestEnv(t)() - repository := db.AssertExistsAndLoadBean(t, &models.Repository{ID: 2}).(*models.Repository) - user := db.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + user := db.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) packageName := "gitea" packageVersion := "1.0.5" @@ -111,7 +110,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) - root := fmt.Sprintf("/api/v1/repos/%s/%s/packages/rubygems", user.Name, repository.Name) + root := fmt.Sprintf("/api/v1/packages/%s/rubygems", user.Name) uploadFile := func(t *testing.T, expectedStatus int) { req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/api/v1/gems", root), bytes.NewReader(gemContent)) @@ -124,7 +123,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) uploadFile(t, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Len(t, pvs, 1) @@ -161,7 +160,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) assert.Equal(t, gemContent, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) @@ -180,7 +179,7 @@ DGAWSKc7zFhPJamg0qRK99TcYphehZLU4hKInFhGSUlBsZW+PtgZepn5+iDxECRzDUDGcfh6hoA4 gAAAAP//MS06Gw==`) assert.Equal(t, b, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) @@ -219,7 +218,7 @@ gAAAAP//MS06Gw==`) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - pvs, err := packages.GetVersionsByPackageType(repository.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Empty(t, pvs) }) diff --git a/integrations/api_packages_test.go b/integrations/api_packages_test.go index 0db6cf5786c37..9ed426b4d75ff 100644 --- a/integrations/api_packages_test.go +++ b/integrations/api_packages_test.go @@ -20,8 +20,7 @@ import ( func TestPackageAPI(t *testing.T) { defer prepareTestEnv(t)() - repository := db.AssertExistsAndLoadBean(t, &models.Repository{ID: 15}).(*models.Repository) - user := db.AssertExistsAndLoadBean(t, &models.User{ID: repository.OwnerID}).(*models.User) + user := db.AssertExistsAndLoadBean(t, &models.User{ID: 4}).(*models.User) session := loginUser(t, user.Name) token := getTokenForLoggedInUser(t, session) @@ -29,7 +28,7 @@ func TestPackageAPI(t *testing.T) { packageVersion := "1.0.3" filename := "file.bin" - url := fmt.Sprintf("/api/v1/repos/%s/%s/packages/generic/%s/%s/%s?token=%s", user.Name, repository.Name, packageName, packageVersion, filename, token) + url := fmt.Sprintf("/api/v1/packages/%s/generic/%s/%s/%s?token=%s", user.Name, packageName, packageVersion, filename, token) req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{})) MakeRequest(t, req, http.StatusCreated) @@ -38,7 +37,7 @@ func TestPackageAPI(t *testing.T) { t.Run("ListPackages", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages?token=%s", user.Name, repository.Name, token)) + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s?token=%s", user.Name, token)) resp := MakeRequest(t, req, http.StatusOK) var apiPackages []*api.Package @@ -57,10 +56,10 @@ func TestPackageAPI(t *testing.T) { t.Run("GetPackage", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d?token=%s", user.Name, repository.Name, 123456, token)) + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/%d?token=%s", user.Name, 123456, token)) MakeRequest(t, req, http.StatusNotFound) - req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d?token=%s", user.Name, repository.Name, packageID, token)) + req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/%d?token=%s", user.Name, packageID, token)) resp := MakeRequest(t, req, http.StatusOK) var p *api.Package @@ -76,10 +75,10 @@ func TestPackageAPI(t *testing.T) { t.Run("ListPackageFiles", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d/files?token=%s", user.Name, repository.Name, 123456, token)) + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/%d/files?token=%s", user.Name, 123456, token)) MakeRequest(t, req, http.StatusNotFound) - req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d/files?token=%s", user.Name, repository.Name, packageID, token)) + req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/%d/files?token=%s", user.Name, packageID, token)) resp := MakeRequest(t, req, http.StatusOK) var files []*api.PackageFile @@ -97,10 +96,10 @@ func TestPackageAPI(t *testing.T) { t.Run("DeletePackage", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d?token=%s", user.Name, repository.Name, 123456, token)) + req := NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/packages/%s/%d?token=%s", user.Name, 123456, token)) MakeRequest(t, req, http.StatusNotFound) - req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/repos/%s/%s/packages/%d?token=%s", user.Name, repository.Name, packageID, token)) + req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/packages/%s/%d?token=%s", user.Name, packageID, token)) MakeRequest(t, req, http.StatusNoContent) }) } diff --git a/models/migrations/v202.go b/models/migrations/v202.go index 77e677a5a652d..68b66500bc5e0 100644 --- a/models/migrations/v202.go +++ b/models/migrations/v202.go @@ -13,7 +13,8 @@ import ( func addPackageTables(x *xorm.Engine) error { type Package struct { ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + RepoID int64 `xorm:"INDEX"` Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` Name string LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index b4ea0161afa2d..4fbde61fdb704 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -22,6 +22,7 @@ import ( // PackageDescriptor describes a package type PackageDescriptor struct { Package *Package + Owner *models.User Repository *models.Repository Version *PackageVersion SemVer *version.Version @@ -48,10 +49,14 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD if err != nil { return nil, err } - repository, err := models.GetRepositoryByIDCtx(ctx, p.RepoID) + o, err := models.GetUserByID(p.OwnerID) if err != nil { return nil, err } + repository, err := models.GetRepositoryByIDCtx(ctx, p.RepoID) + if err != nil && !models.IsErrRepoNotExist(err) { + return nil, err + } creator, err := models.GetUserByID(pv.CreatorID) if err != nil { return nil, err @@ -105,6 +110,7 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD return &PackageDescriptor{ Package: p, + Owner: o, Repository: repository, Version: pv, SemVer: semVer, diff --git a/models/packages/package.go b/models/packages/package.go index 7fcbcc13835da..d309016b878e7 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -76,7 +76,8 @@ func (pt Type) SVGName() string { // Package represents a package type Package struct { ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + RepoID int64 `xorm:"INDEX"` Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"` Name string LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` @@ -88,7 +89,7 @@ func TryInsertPackage(ctx context.Context, p *Package) (*Package, error) { e := db.GetEngine(ctx) key := &Package{ - RepoID: p.RepoID, + OwnerID: p.OwnerID, Type: p.Type, LowerName: p.LowerName, } @@ -137,6 +138,11 @@ func DeletePackageByIDIfUnreferenced(ctx context.Context, packageID int64) error return nil } +// HasOwnerPackages tests if a user/org has packages +func HasOwnerPackages(ownerID int64) (bool, error) { + return db.GetEngine(db.DefaultContext).Where("owner_id = ?", ownerID).Exist(&Package{}) +} + // HasRepositoryPackages tests if a repository has packages func HasRepositoryPackages(repositoryID int64) (bool, error) { return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repositoryID).Exist(&Package{}) diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 67256e1311c23..826bbbe9bf2de 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -87,9 +87,9 @@ func GetVersionByID(ctx context.Context, versionID int64) (*PackageVersion, erro } // GetVersionByNameAndVersion gets a version by name and version number -func GetVersionByNameAndVersion(ctx context.Context, repositoryID int64, packageType Type, name, version string) (*PackageVersion, error) { +func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType Type, name, version string) (*PackageVersion, error) { var cond builder.Cond = builder.Eq{ - "package.repo_id": repositoryID, + "package.owner_id": ownerID, "package.type": packageType, "package.lower_name": strings.ToLower(name), } @@ -111,10 +111,10 @@ func GetVersionByNameAndVersion(ctx context.Context, repositoryID int64, package } // GetVersionsByPackageType gets all versions of a specific type -func GetVersionsByPackageType(repositoryID int64, packageType Type) ([]*PackageVersion, error) { +func GetVersionsByPackageType(ownerID int64, packageType Type) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ - "package.repo_id": repositoryID, - "package.type": packageType, + "package.owner_id": ownerID, + "package.type": packageType, } pvs := make([]*PackageVersion, 0, 10) @@ -125,9 +125,9 @@ func GetVersionsByPackageType(repositoryID int64, packageType Type) ([]*PackageV } // GetVersionsByPackageName gets all versions of a specific package -func GetVersionsByPackageName(repositoryID int64, packageType Type, name string) ([]*PackageVersion, error) { +func GetVersionsByPackageName(ownerID int64, packageType Type, name string) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ - "package.repo_id": repositoryID, + "package.owner_id": ownerID, "package.type": packageType, "package.lower_name": strings.ToLower(name), } @@ -140,9 +140,9 @@ func GetVersionsByPackageName(repositoryID int64, packageType Type, name string) } // GetVersionsByFilename gets all versions which are linked to a filename -func GetVersionsByFilename(repositoryID int64, packageType Type, filename string) ([]*PackageVersion, error) { +func GetVersionsByFilename(ownerID int64, packageType Type, filename string) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ - "package.repo_id": repositoryID, + "package.owner_id": ownerID, "package.type": packageType, "package_file.lower_name": strings.ToLower(filename), } @@ -163,19 +163,22 @@ func DeleteVersionByID(ctx context.Context, versionID int64) error { // PackageSearchOptions are options for SearchXXX methods type PackageSearchOptions struct { - RepoID int64 - Type string - Query string + OwnerID int64 + RepoID int64 + Type string + Query string db.Paginator } func (opts *PackageSearchOptions) toConds() builder.Cond { - var cond builder.Cond = builder.Eq{"package.repo_id": opts.RepoID} + var cond builder.Cond = builder.Eq{"package.owner_id": opts.OwnerID} + if opts.RepoID != 0 { + cond = cond.And(builder.Eq{"package.repo_id": opts.RepoID}) + } if opts.Type != "" { cond = cond.And(builder.Eq{"package.type": opts.Type}) } - if opts.Query != "" { cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.Query)}) } diff --git a/modules/context/api.go b/modules/context/api.go index c978835af8749..d82b0d7a708d7 100644 --- a/modules/context/api.go +++ b/modules/context/api.go @@ -27,7 +27,8 @@ import ( // APIContext is a specific context for API service type APIContext struct { *Context - Org *APIOrganization + Org *APIOrganization + Package *APIPackage } // APIError is error format response diff --git a/modules/context/api_package.go b/modules/context/api_package.go new file mode 100644 index 0000000000000..6e9ab4913c3c1 --- /dev/null +++ b/modules/context/api_package.go @@ -0,0 +1,15 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package context + +import ( + "code.gitea.io/gitea/models" +) + +// APIPackage contains organization and team +type APIPackage struct { + Owner *models.User + AccessMode models.AccessMode +} diff --git a/modules/convert/package.go b/modules/convert/package.go index 942576d0bdb2a..ca8533a34e3d9 100644 --- a/modules/convert/package.go +++ b/modules/convert/package.go @@ -13,6 +13,7 @@ import ( func ToPackage(pd *packages.PackageDescriptor) *api.Package { return &api.Package{ ID: pd.Version.ID, + Owner: ToUser(pd.Owner, nil), Creator: ToUser(pd.Creator, nil), Type: string(pd.Package.Type), Name: pd.Package.Name, @@ -21,15 +22,15 @@ func ToPackage(pd *packages.PackageDescriptor) *api.Package { } } -// ToPackageFile converts packages.PackageFile to api.PackageFile -func ToPackageFile(pf *packages.PackageFile, pb *packages.PackageBlob) *api.PackageFile { +// ToPackageFile converts packages.PackageFileDescriptor to api.PackageFile +func ToPackageFile(pfd *packages.PackageFileDescriptor) *api.PackageFile { return &api.PackageFile{ - ID: pf.ID, - Size: pb.Size, - Name: pf.Name, - HashMD5: pb.HashMD5, - HashSHA1: pb.HashSHA1, - HashSHA256: pb.HashSHA256, - HashSHA512: pb.HashSHA512, + ID: pfd.File.ID, + Size: pfd.Blob.Size, + Name: pfd.File.Name, + HashMD5: pfd.Blob.HashMD5, + HashSHA1: pfd.Blob.HashSHA1, + HashSHA256: pfd.Blob.HashSHA256, + HashSHA512: pfd.Blob.HashSHA512, } } diff --git a/modules/notification/webhook/webhook.go b/modules/notification/webhook/webhook.go index fa2507aa3cb64..2bdd5fa01ce3f 100644 --- a/modules/notification/webhook/webhook.go +++ b/modules/notification/webhook/webhook.go @@ -830,7 +830,12 @@ func (m *webhookNotifier) NotifyPackageDelete(doer *models.User, pd *packages.Pa } func notifyPackage(sender *models.User, pd *packages.PackageDescriptor, action api.HookPackageAction) { - org := pd.Repository.MustOwner() + if pd.Repository == nil { + //TODO + return + } + + org := pd.Owner if !org.IsOrganization() { org = nil } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 866857f1a1ebb..54f166f7ab538 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -929,9 +929,9 @@ func NewContext() { if os.Getuid() == 0 { if !unsafeAllowRunAsRoot { // Special thanks to VLC which inspired the wording of this messaging. - log.Fatal("Gitea is not supposed to be run as root. Sorry. If you need to use privileged TCP ports please instead use setcap and the `cap_net_bind_service` permission") + //log.Fatal("Gitea is not supposed to be run as root. Sorry. If you need to use privileged TCP ports please instead use setcap and the `cap_net_bind_service` permission") } - log.Critical("You are running Gitea using the root user, and have purposely chosen to skip built-in protections around this. You have been warned against this.") + //log.Critical("You are running Gitea using the root user, and have purposely chosen to skip built-in protections around this. You have been warned against this.") } SSH.BuiltinServerUser = Cfg.Section("server").Key("BUILTIN_SSH_SERVER_USER").MustString(RunUser) diff --git a/modules/structs/package.go b/modules/structs/package.go index cf00334de9b73..9424878b1315d 100644 --- a/modules/structs/package.go +++ b/modules/structs/package.go @@ -11,6 +11,7 @@ import ( // Package represents a package type Package struct { ID int64 `json:"id"` + Owner *User `json:"owner"` Creator *User `json:"creator"` Type string `json:"type"` Name string `json:"name"` diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 6c9188f9926ab..891c9c8882c4d 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -193,6 +193,59 @@ func repoAssignment() func(ctx *context.APIContext) { } } +func packageOwnerAssignment() func(ctx *context.APIContext) { + return func(ctx *context.APIContext) { + ownerName := ctx.Params("owner") + + ctx.Package = &context.APIPackage{} + + if ctx.IsSigned && ctx.User.LowerName == strings.ToLower(ownerName) { + ctx.Package.Owner = ctx.User + ctx.Package.AccessMode = models.AccessModeOwner + } else { + owner, err := models.GetUserByName(ownerName) + if err != nil { + if models.IsErrUserNotExist(err) { + if redirectUserID, err := models.LookupUserRedirect(ownerName); err == nil { + context.RedirectToUser(ctx.Context, ownerName, redirectUserID) + } else if models.IsErrUserRedirectNotExist(err) { + ctx.NotFound("GetUserByName", err) + } else { + ctx.Error(http.StatusInternalServerError, "LookupUserRedirect", err) + } + } else { + ctx.Error(http.StatusInternalServerError, "GetUserByName", err) + } + return + } + ctx.Package.Owner = owner + + if owner.IsOrganization() { + if ctx.User != nil && models.HasOrgOrUserVisible(owner, ctx.User) { + var err error + ctx.Package.AccessMode, err = owner.GetOrgUserMaxAuthorizeLevel(ctx.User.ID) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetOrgUserMaxAuthorizeLevel", err) + return + } + } + } else { + ctx.Package.AccessMode = models.AccessModeRead + } + } + } +} + +func reqPackageAccess(accessMode models.AccessMode) func(ctx *context.APIContext) { + return func(ctx *context.APIContext) { + if ctx.Package.AccessMode < accessMode && !ctx.IsUserSiteAdmin() { + ctx.Resp.Header().Set("WWW-Authenticate", `Basic realm="Gitea Package API"`) + ctx.Error(http.StatusUnauthorized, "reqPackageAccess", "user should have specific permission or be a site admin") + return + } + } +} + // Contexter middleware already checks token for user sign in process. func reqToken() func(ctx *context.APIContext) { return func(ctx *context.APIContext) { @@ -222,7 +275,7 @@ func reqExploreSignIn() func(ctx *context.APIContext) { func reqBasicAuth() func(ctx *context.APIContext) { return func(ctx *context.APIContext) { if !ctx.Context.IsBasicAuth { - ctx.Resp.Header().Set("WWW-Authenticate", `Basic realm="API"`) + ctx.Resp.Header().Set("WWW-Authenticate", `Basic realm="Gitea API"`) ctx.Error(http.StatusUnauthorized, "reqBasicAuth", "basic auth required") return } @@ -992,70 +1045,71 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { }, reqAnyRepoReader()) m.Get("/issue_templates", context.ReferencesGitRepo(false), repo.GetIssueTemplates) m.Get("/languages", reqRepoReader(unit.TypeCode), repo.GetLanguages) - m.Group("/packages", func() { - m.Group("/generic", func() { - m.Group("/{packagename}/{packageversion}/{filename}", func() { - m.Get("", generic.DownloadPackageFile) - m.Group("", func() { - m.Put("", generic.UploadPackage) - m.Delete("", generic.DeletePackage) - }, reqToken(), reqRepoWriter(unit.TypePackages)) - }) - }) - m.Group("/maven", func() { - m.Put("/*", reqToken(), reqRepoWriter(unit.TypePackages), maven.UploadPackageFile) - m.Get("/*", maven.DownloadPackageFile) - }) - m.Group("/nuget", func() { - m.Get("/index.json", nuget.ServiceIndex) - m.Get("/query", nuget.SearchService) - m.Group("/registration/{id}", func() { - m.Get("/index.json", nuget.RegistrationIndex) - m.Get("/{version}", nuget.RegistrationLeaf) - }) - m.Group("/package/{id}", func() { - m.Get("/index.json", nuget.EnumeratePackageVersions) - m.Get("/{version}/{filename}", nuget.DownloadPackageFile) - }) - m.Group("", func() { - m.Put("/", reqRepoWriter(unit.TypePackages), nuget.UploadPackage) - m.Put("/symbolpackage", nuget.UploadSymbolPackage) - m.Delete("/{id}/{version}", reqRepoWriter(unit.TypePackages), nuget.DeletePackage) - }, reqBasicAuth(), reqRepoWriter(unit.TypePackages)) - }) - m.Group("/npm", func() { - m.Group("/{id}", func() { - m.Get("", npm.PackageMetadata) - m.Put("", reqToken(), reqRepoWriter(unit.TypePackages), npm.UploadPackage) - m.Get("/-/{version}/{filename}", npm.DownloadPackageFile) - }) - }) - m.Group("/pypi", func() { - m.Post("/", reqBasicAuth(), reqRepoWriter(unit.TypePackages), pypi.UploadPackageFile) - m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) - m.Get("/simple/{id}", pypi.PackageMetadata) - }) - m.Group("/rubygems", func() { - m.Get("/specs.4.8.gz", rubygems.EnumeratePackages) - m.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) - m.Get("/prerelease_specs.4.8.gz", rubygems.EnumeratePackagesPreRelease) - m.Get("/quick/Marshal.4.8/{filename}", rubygems.ServePackageSpecification) - m.Get("/gems/{filename}", rubygems.DownloadPackageFile) - m.Group("/api/v1/gems", func() { - m.Post("/", rubygems.UploadPackageFile) - m.Delete("/yank", rubygems.DeletePackage) - }, reqToken(), reqRepoWriter(unit.TypePackages)) - }) - m.Group("/{id}", func() { - m.Get("/", repo.GetPackage) - m.Delete("/", reqRepoWriter(unit.TypePackages), repo.DeletePackage) - m.Get("/files", repo.ListPackageFiles) - }) - m.Get("/", repo.ListPackages) - }, reqRepoReader(unit.TypePackages)) }, repoAssignment()) }) + m.Group("/packages/{owner}", func() { + m.Group("/generic", func() { + m.Group("/{packagename}/{packageversion}/{filename}", func() { + m.Get("", generic.DownloadPackageFile) + m.Group("", func() { + m.Put("", generic.UploadPackage) + m.Delete("", generic.DeletePackage) + }, reqToken(), reqPackageAccess(models.AccessModeWrite)) + }) + }) + m.Group("/maven", func() { + m.Put("/*", reqToken(), reqPackageAccess(models.AccessModeWrite), maven.UploadPackageFile) + m.Get("/*", maven.DownloadPackageFile) + }) + m.Group("/nuget", func() { + m.Get("/index.json", nuget.ServiceIndex) + m.Get("/query", nuget.SearchService) + m.Group("/registration/{id}", func() { + m.Get("/index.json", nuget.RegistrationIndex) + m.Get("/{version}", nuget.RegistrationLeaf) + }) + m.Group("/package/{id}", func() { + m.Get("/index.json", nuget.EnumeratePackageVersions) + m.Get("/{version}/{filename}", nuget.DownloadPackageFile) + }) + m.Group("", func() { + m.Put("/", nuget.UploadPackage) + m.Put("/symbolpackage", nuget.UploadSymbolPackage) + m.Delete("/{id}/{version}", nuget.DeletePackage) + }, reqBasicAuth(), reqPackageAccess(models.AccessModeWrite)) + }) + m.Group("/npm", func() { + m.Group("/{id}", func() { + m.Get("", npm.PackageMetadata) + m.Put("", reqToken(), reqPackageAccess(models.AccessModeWrite), npm.UploadPackage) + m.Get("/-/{version}/{filename}", npm.DownloadPackageFile) + }) + }) + m.Group("/pypi", func() { + m.Post("/", reqBasicAuth(), reqPackageAccess(models.AccessModeWrite), pypi.UploadPackageFile) + m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) + m.Get("/simple/{id}", pypi.PackageMetadata) + }) + m.Group("/rubygems", func() { + m.Get("/specs.4.8.gz", rubygems.EnumeratePackages) + m.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) + m.Get("/prerelease_specs.4.8.gz", rubygems.EnumeratePackagesPreRelease) + m.Get("/quick/Marshal.4.8/{filename}", rubygems.ServePackageSpecification) + m.Get("/gems/{filename}", rubygems.DownloadPackageFile) + m.Group("/api/v1/gems", func() { + m.Post("/", rubygems.UploadPackageFile) + m.Delete("/yank", rubygems.DeletePackage) + }, reqToken(), reqPackageAccess(models.AccessModeWrite)) + }) + m.Group("/{id}", func() { + m.Get("/", repo.GetPackage) + m.Delete("/", reqPackageAccess(models.AccessModeWrite), repo.DeletePackage) + m.Get("/files", repo.ListPackageFiles) + }) + m.Get("/", repo.ListPackages) + }, packageOwnerAssignment(), reqPackageAccess(models.AccessModeRead)) + // Organizations m.Get("/user/orgs", reqToken(), org.ListMyOrgs) m.Group("/users/{username}/orgs", func() { diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index f56221ac1f302..28d3b461ee2f8 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -36,7 +36,15 @@ func DownloadPackageFile(ctx *context.APIContext) { return } - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeGeneric, packageName, packageVersion, filename) + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion( + &package_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages.TypeGeneric, + Name: packageName, + Version: packageVersion, + }, + filename, + ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -79,7 +87,7 @@ func UploadPackage(ctx *context.APIContext) { _, _, err = package_service.CreatePackageAndAddFile( &package_service.PackageCreationInfo{ PackageInfo: package_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypeGeneric, Name: packageName, Version: packageVersion, @@ -116,7 +124,7 @@ func DeletePackage(ctx *context.APIContext) { err = package_service.DeletePackageVersionByNameAndVersion( ctx.User, &package_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypeGeneric, Name: packageName, Version: packageVersion, diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 7cc0fef974104..784213881420e 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -68,7 +68,7 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { // /com/foo/project/maven-metadata.xml[.md5/.sha1/.sha256/.sha512] packageName := params.GroupID + "-" + params.ArtifactID - pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypeMaven, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeMaven, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -118,7 +118,7 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { func servePackageFile(ctx *context.APIContext, params parameters) { packageName := params.GroupID + "-" + params.ArtifactID - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Repo.Repository.ID, packages.TypeMaven, packageName, params.Version) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeMaven, packageName, params.Version) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -209,7 +209,7 @@ func UploadPackageFile(ctx *context.APIContext) { pvci := &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypeMaven, Name: packageName, Version: params.Version, @@ -222,7 +222,7 @@ func UploadPackageFile(ctx *context.APIContext) { // Do not upload checksum files but compare the hashes. if isChecksumExtension(ext) { - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Repository.ID, pvci.PackageType, pvci.Name, pvci.Version) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -281,7 +281,7 @@ func UploadPackageFile(ctx *context.APIContext) { } if pvci.Metadata != nil { - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Repository.ID, pvci.PackageType, pvci.Name, pvci.Version) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) if err != nil && err != packages.ErrPackageNotExist { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index b651cfc65630d..1f5da23a3bc16 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -34,7 +34,7 @@ func PackageMetadata(ctx *context.APIContext) { return } - pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypeNpm, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNpm, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -51,7 +51,7 @@ func PackageMetadata(ctx *context.APIContext) { } resp := createPackageMetadataResponse( - setting.AppURL+"api/v1/repos/"+ctx.Repo.Repository.FullName()+"/packages/npm", + setting.AppURL+"api/v1/packages/"+ctx.Package.Owner.Name+"/npm", pds, ) @@ -68,7 +68,15 @@ func DownloadPackageFile(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNpm, packageName, packageVersion, filename) + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages.TypeNpm, + Name: packageName, + Version: packageVersion, + }, + filename, + ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -100,7 +108,7 @@ func UploadPackage(ctx *context.APIContext) { _, _, err = packages_service.CreatePackageAndAddFile( &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypeNpm, Name: npmPackage.Name, Version: npmPackage.Version, diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 658fb8755ab2b..75370d91c0e12 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -31,9 +31,7 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { // ServiceIndex https://docs.microsoft.com/en-us/nuget/api/service-index func ServiceIndex(ctx *context.APIContext) { - repoName := ctx.Repo.Repository.FullName() - - resp := createServiceIndexResponse(setting.AppURL + "api/v1/repos/" + repoName + "/packages/nuget") + resp := createServiceIndexResponse(setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/nuget") ctx.JSON(http.StatusOK, resp) } @@ -41,9 +39,9 @@ func ServiceIndex(ctx *context.APIContext) { // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages func SearchService(ctx *context.APIContext) { pvs, count, err := packages.SearchVersions(&packages.PackageSearchOptions{ - RepoID: ctx.Repo.Repository.ID, - Type: "nuget", - Query: ctx.FormTrim("q"), + OwnerID: ctx.Package.Owner.ID, + Type: string(packages.TypeNuGet), + Query: ctx.FormTrim("q"), Paginator: db.NewAbsoluteListOptions( ctx.FormInt("skip"), ctx.FormInt("take"), @@ -61,7 +59,7 @@ func SearchService(ctx *context.APIContext) { } resp := createSearchResultResponse( - &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, + &linkBuilder{setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/nuget"}, count, pds, ) @@ -73,7 +71,7 @@ func SearchService(ctx *context.APIContext) { func RegistrationIndex(ctx *context.APIContext) { packageName := ctx.Params("id") - pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNuGet, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -90,7 +88,7 @@ func RegistrationIndex(ctx *context.APIContext) { } resp := createRegistrationIndexResponse( - &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, + &linkBuilder{setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/nuget"}, pds, ) @@ -102,7 +100,7 @@ func RegistrationLeaf(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Repo.Repository.ID, packages.TypeNuGet, packageName, packageVersion) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeNuGet, packageName, packageVersion) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -119,7 +117,7 @@ func RegistrationLeaf(ctx *context.APIContext) { } resp := createRegistrationLeafResponse( - &linkBuilder{setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/nuget"}, + &linkBuilder{setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/nuget"}, pd, ) @@ -130,7 +128,7 @@ func RegistrationLeaf(ctx *context.APIContext) { func EnumeratePackageVersions(ctx *context.APIContext) { packageName := ctx.Params("id") - pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypeNuGet, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNuGet, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -157,7 +155,15 @@ func DownloadPackageFile(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypeNuGet, packageName, packageVersion, filename) + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages.TypeNuGet, + Name: packageName, + Version: packageVersion, + }, + filename, + ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -187,7 +193,7 @@ func UploadPackage(ctx *context.APIContext) { _, _, err := packages_service.CreatePackageAndAddFile( &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypeNuGet, Name: np.ID, Version: np.Version, @@ -229,7 +235,7 @@ func UploadSymbolPackage(ctx *context.APIContext) { _, _, err := packages_service.AddFileToExistingPackage( &packages_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypeNuGet, Name: np.ID, Version: np.Version, @@ -304,7 +310,7 @@ func DeletePackage(ctx *context.APIContext) { err := packages_service.DeletePackageVersionByNameAndVersion( ctx.User, &packages_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypeNuGet, Name: packageName, Version: packageVersion, diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index 021805619abd7..ac01939521f49 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -39,7 +39,7 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { func PackageMetadata(ctx *context.APIContext) { packageName := normalizer.Replace(ctx.Params("id")) - pvs, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, packages.TypePyPI, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypePyPI, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -55,7 +55,7 @@ func PackageMetadata(ctx *context.APIContext) { return } - ctx.Data["RegistryURL"] = setting.AppURL + "api/v1/repos/" + ctx.Repo.Repository.FullName() + "/packages/pypi" + ctx.Data["RegistryURL"] = setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/pypi" ctx.Data["PackageDescriptor"] = pds[0] ctx.Data["PackageDescriptors"] = pds ctx.Render = templates.HTMLRenderer() @@ -68,7 +68,15 @@ func DownloadPackageFile(ctx *context.APIContext) { packageVersion := ctx.Params("version") filename := ctx.Params("filename") - s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion(ctx.Repo.Repository, packages.TypePyPI, packageName, packageVersion, filename) + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages.TypePyPI, + Name: packageName, + Version: packageVersion, + }, + filename, + ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -125,7 +133,7 @@ func UploadPackageFile(ctx *context.APIContext) { _, _, err = packages_service.CreatePackageOrAddFileToExisting( &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypePyPI, Name: packageName, Version: packageVersion, diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/v1/packages/rubygems/rubygems.go index 288b627627f64..d77aa3695462f 100644 --- a/routers/api/v1/packages/rubygems/rubygems.go +++ b/routers/api/v1/packages/rubygems/rubygems.go @@ -29,7 +29,7 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { // EnumeratePackages serves the package list func EnumeratePackages(ctx *context.APIContext) { - packages, err := packages.GetVersionsByPackageType(ctx.Repo.Repository.ID, packages.TypeRubyGems) + packages, err := packages.GetVersionsByPackageType(ctx.Package.Owner.ID, packages.TypeRubyGems) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -41,8 +41,8 @@ func EnumeratePackages(ctx *context.APIContext) { // EnumeratePackagesLatest serves the list of the lastest version of every package func EnumeratePackagesLatest(ctx *context.APIContext) { pvs, _, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ - RepoID: ctx.Repo.Repository.ID, - Type: "rubygems", + OwnerID: ctx.Package.Owner.ID, + Type: string(packages.TypeRubyGems), }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -97,7 +97,7 @@ func ServePackageSpecification(ctx *context.APIContext) { return } - pvs, err := packages.GetVersionsByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename[:len(filename)-10]+"gem") + pvs, err := packages.GetVersionsByFilename(ctx.Package.Owner.ID, packages.TypeRubyGems, filename[:len(filename)-10]+"gem") if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -159,7 +159,7 @@ func ServePackageSpecification(ctx *context.APIContext) { func DownloadPackageFile(ctx *context.APIContext) { filename := ctx.Params("filename") - pvs, err := packages.GetVersionsByFilename(ctx.Repo.Repository.ID, packages.TypeRubyGems, filename) + pvs, err := packages.GetVersionsByFilename(ctx.Package.Owner.ID, packages.TypeRubyGems, filename) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -222,7 +222,7 @@ func UploadPackageFile(ctx *context.APIContext) { _, _, err = packages_service.CreatePackageAndAddFile( &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypeRubyGems, Name: rp.Name, Version: rp.Version, @@ -262,7 +262,7 @@ func DeletePackage(ctx *context.APIContext) { err := packages_service.DeletePackageVersionByNameAndVersion( ctx.User, &packages_service.PackageInfo{ - Repository: ctx.Repo.Repository, + Owner: ctx.Package.Owner, PackageType: packages.TypeRubyGems, Name: packageName, Version: packageVersion, diff --git a/routers/api/v1/repo/package.go b/routers/api/v1/repo/package.go index 9395626e2158f..8c051c33d4a78 100644 --- a/routers/api/v1/repo/package.go +++ b/routers/api/v1/repo/package.go @@ -60,10 +60,8 @@ func ListPackages(ctx *context.APIContext) { packageType := ctx.FormTrim("package_type") query := ctx.FormTrim("q") - repo := ctx.Repo.Repository - pvs, count, err := packages.SearchVersions(&packages.PackageSearchOptions{ - RepoID: repo.ID, + OwnerID: ctx.Package.Owner.ID, Type: packageType, Query: query, Paginator: &listOptions, @@ -135,6 +133,11 @@ func GetPackage(ctx *context.APIContext) { return } + if ctx.Package.Owner.ID != pd.Owner.ID { + ctx.NotFound() + return + } + ctx.JSON(http.StatusOK, convert.ToPackage(pd)) } @@ -166,7 +169,7 @@ func DeletePackage(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - err := package_service.DeleteVersionByID(ctx.User, ctx.Repo.Repository, ctx.ParamsInt64(":id")) + err := package_service.DeleteVersionByID(ctx.User, ctx.Package.Owner, ctx.ParamsInt64(":id")) if err != nil { if err == packages.ErrPackageNotExist { ctx.NotFound() @@ -218,20 +221,20 @@ func ListPackageFiles(ctx *context.APIContext) { return } - pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pv.ID) + pd, err := packages.GetPackageDescriptor(pv) if err != nil { - ctx.Error(http.StatusInternalServerError, "GetFilesByVersionID", err) + ctx.Error(http.StatusInternalServerError, "GetPackageDescriptor", err) return } - apiPackageFiles := make([]*api.PackageFile, 0, len(pfs)) - for _, pf := range pfs { - pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) - if err != nil { - ctx.Error(http.StatusInternalServerError, "GetBlobByID", err) - return - } - apiPackageFiles = append(apiPackageFiles, convert.ToPackageFile(pf, pb)) + if ctx.Package.Owner.ID != pd.Owner.ID { + ctx.NotFound() + return + } + + apiPackageFiles := make([]*api.PackageFile, 0, len(pd.Files)) + for _, pfd := range pd.Files { + apiPackageFiles = append(apiPackageFiles, convert.ToPackageFile(&pfd)) } ctx.JSON(http.StatusOK, apiPackageFiles) diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index 86dd9f81ff431..53ac5865eeac5 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -127,7 +127,7 @@ func ViewPackage(ctx *context.Context) { // DeletePackagePost deletes a package func DeletePackagePost(ctx *context.Context) { - err := packages_service.DeleteVersionByID(ctx.User, ctx.Repo.Repository, ctx.ParamsInt64(":id")) + err := packages_service.DeleteVersionByID(ctx.User, nil, ctx.ParamsInt64(":id")) // TODO if err != nil { ctx.Flash.Error(err.Error()) } else { @@ -139,7 +139,7 @@ func DeletePackagePost(ctx *context.Context) { // DownloadPackageFile serves the content of a package file func DownloadPackageFile(ctx *context.Context) { - s, pf, err := packages_service.GetFileStreamByPackageVersionID(ctx.Repo.Repository, ctx.ParamsInt64(":id"), ctx.Params(":filename")) + s, pf, err := packages_service.GetFileStreamByPackageVersionID(nil, ctx.ParamsInt64(":id"), ctx.Params(":filename")) // TODO if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { ctx.NotFound("", err) diff --git a/services/packages/packages.go b/services/packages/packages.go index e77eb48935c38..96ec58da87a97 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -21,7 +21,7 @@ import ( // PackageInfo describes a package type PackageInfo struct { - Repository *models.Repository + Owner *models.User PackageType packages_models.Type Name string Version string @@ -98,10 +98,10 @@ func createPackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo, al } func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, allowDuplicate bool) (*packages_models.PackageVersion, bool, error) { - log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", pvci.Creator.ID, pvci.Repository.ID, pvci.PackageType, pvci.Name, pvci.Version, pvci.Properties, allowDuplicate) + log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", pvci.Creator.ID, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, pvci.Properties, allowDuplicate) p := &packages_models.Package{ - RepoID: pvci.Repository.ID, + OwnerID: pvci.Owner.ID, Type: pvci.PackageType, Name: pvci.Name, LowerName: strings.ToLower(pvci.Name), @@ -154,7 +154,7 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfi *PackageFileInfo) (*packages } defer committer.Close() - pv, err := packages_models.GetVersionByNameAndVersion(ctx, pvi.Repository.ID, pvi.PackageType, pvi.Name, pvi.Version) + pv, err := packages_models.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { return nil, nil, err } @@ -226,25 +226,25 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_models.PackageVer func DeletePackageVersionByNameAndVersion(doer *models.User, pvi *PackageInfo) error { return deletePackageVersion( doer, - pvi.Repository, + pvi.Owner, func(ctx context.Context) (*packages_models.PackageVersion, error) { - return packages_models.GetVersionByNameAndVersion(ctx, pvi.Repository.ID, pvi.PackageType, pvi.Name, pvi.Version) + return packages_models.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) }, ) } // DeleteVersionByID deletes a package version and all associated files -func DeleteVersionByID(doer *models.User, repository *models.Repository, versionID int64) error { +func DeleteVersionByID(doer *models.User, owner *models.User, versionID int64) error { return deletePackageVersion( doer, - repository, + owner, func(ctx context.Context) (*packages_models.PackageVersion, error) { return packages_models.GetVersionByID(ctx, versionID) }, ) } -func deletePackageVersion(doer *models.User, repository *models.Repository, cb func(context.Context) (*packages_models.PackageVersion, error)) error { +func deletePackageVersion(doer *models.User, owner *models.User, cb func(context.Context) (*packages_models.PackageVersion, error)) error { ctx, committer, err := db.TxContext() if err != nil { return err @@ -264,7 +264,7 @@ func deletePackageVersion(doer *models.User, repository *models.Repository, cb f return err } - log.Trace("Deleting package: %v, %v", repository.ID, pv.ID) + log.Trace("Deleting package: %v, %v", owner.ID, pv.ID) if err := packages_models.DeleteVersionPropertiesByVersionID(ctx, pv.ID); err != nil { return err @@ -325,10 +325,10 @@ func DeleteUnreferencedBlobs() error { } // GetFileStreamByPackageNameAndVersion returns the content of the specific package file -func GetFileStreamByPackageNameAndVersion(repository *models.Repository, packageType packages_models.Type, name, version, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { - log.Trace("Getting package file stream: %v, %v, %s, %s, %s", repository.ID, packageType, name, version, filename) +func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { + log.Trace("Getting package file stream: %v, %v, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, filename) - pv, err := packages_models.GetVersionByNameAndVersion(db.DefaultContext, repository.ID, packageType, name, version) + pv, err := packages_models.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { if err == packages_models.ErrPackageNotExist { return nil, nil, err @@ -341,8 +341,8 @@ func GetFileStreamByPackageNameAndVersion(repository *models.Repository, package } // GetFileStreamByPackageVersionID returns the content of the specific package file -func GetFileStreamByPackageVersionID(repository *models.Repository, versionID int64, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { - log.Trace("Getting package file stream: %v, %v, %s", repository.ID, versionID, filename) +func GetFileStreamByPackageVersionID(owner *models.User, versionID int64, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { + log.Trace("Getting package file stream: %v, %v, %s", owner.ID, versionID, filename) pv, err := packages_models.GetVersionByID(db.DefaultContext, versionID) if err != nil { @@ -359,7 +359,7 @@ func GetFileStreamByPackageVersionID(repository *models.Repository, versionID in return nil, nil, err } - if p.RepoID != repository.ID { + if p.OwnerID != owner.ID { return nil, nil, packages_models.ErrPackageNotExist } From df1ceb5f596f666d89dbe1c653a32297816fb574 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 12 Nov 2021 23:54:56 +0000 Subject: [PATCH 062/130] Implemented UI. --- models/packages/descriptor.go | 6 + models/packages/package.go | 6 + models/repo.go | 8 +- models/repo_unit.go | 2 +- models/unit/unit.go | 14 - modules/context/api.go | 3 +- modules/context/api_package.go | 15 - modules/context/context.go | 7 +- modules/context/package.go | 98 +++++ modules/context/repo.go | 2 +- modules/context/user.go | 61 +++ modules/convert/package.go | 21 +- modules/setting/setting.go | 4 +- modules/structs/package.go | 13 +- options/locale/locale_en-US.ini | 114 +++--- routers/api/v1/api.go | 54 +-- routers/api/v1/repo/package.go | 117 ++---- routers/web/repo/packages.go | 120 +----- routers/web/repo/setting.go | 9 - routers/web/user/package.go | 200 ++++++++++ routers/web/web.go | 34 +- services/forms/repo_form.go | 1 - services/forms/user_form.go | 12 + services/packages/packages.go | 35 +- templates/package/content/generic.tmpl | 14 + .../packages => package}/content/maven.tmpl | 24 +- .../packages => package}/content/npm.tmpl | 26 +- .../content/npm_dependencies.tmpl | 0 .../packages => package}/content/nuget.tmpl | 14 +- .../packages => package}/content/pypi.tmpl | 14 +- templates/package/content/rubygems.tmpl | 40 ++ .../content/rubygems_dependencies.tmpl | 0 .../metadata/generic.tmpl | 0 .../packages => package}/metadata/maven.tmpl | 4 +- .../packages => package}/metadata/npm.tmpl | 6 +- .../packages => package}/metadata/nuget.tmpl | 4 +- .../packages => package}/metadata/pypi.tmpl | 6 +- .../metadata/rubygems.tmpl | 6 +- templates/package/settings.tmpl | 72 ++++ templates/package/shared/list.tmpl | 46 +++ templates/package/view.tmpl | 85 +++++ templates/repo/header.tmpl | 8 +- templates/repo/packages.tmpl | 6 + templates/repo/packages/content/generic.tmpl | 14 - templates/repo/packages/content/rubygems.tmpl | 40 -- templates/repo/packages/list.tmpl | 51 --- templates/repo/packages/view.tmpl | 87 ----- templates/repo/settings/options.tmpl | 13 - templates/swagger/v1_json.tmpl | 360 ++++++++---------- templates/user/overview/header.tmpl | 25 ++ templates/user/overview/packages.tmpl | 6 + templates/user/profile.tmpl | 3 + 52 files changed, 1084 insertions(+), 846 deletions(-) delete mode 100644 modules/context/api_package.go create mode 100644 modules/context/package.go create mode 100644 modules/context/user.go create mode 100644 routers/web/user/package.go create mode 100644 templates/package/content/generic.tmpl rename templates/{repo/packages => package}/content/maven.tmpl (64%) rename templates/{repo/packages => package}/content/npm.tmpl (57%) rename templates/{repo/packages => package}/content/npm_dependencies.tmpl (100%) rename templates/{repo/packages => package}/content/nuget.tmpl (76%) rename templates/{repo/packages => package}/content/pypi.tmpl (61%) create mode 100644 templates/package/content/rubygems.tmpl rename templates/{repo/packages => package}/content/rubygems_dependencies.tmpl (100%) rename templates/{repo/packages => package}/metadata/generic.tmpl (100%) rename templates/{repo/packages => package}/metadata/maven.tmpl (71%) rename templates/{repo/packages => package}/metadata/npm.tmpl (51%) rename templates/{repo/packages => package}/metadata/nuget.tmpl (59%) rename templates/{repo/packages => package}/metadata/pypi.tmpl (51%) rename templates/{repo/packages => package}/metadata/rubygems.tmpl (56%) create mode 100644 templates/package/settings.tmpl create mode 100644 templates/package/shared/list.tmpl create mode 100644 templates/package/view.tmpl create mode 100644 templates/repo/packages.tmpl delete mode 100644 templates/repo/packages/content/generic.tmpl delete mode 100644 templates/repo/packages/content/rubygems.tmpl delete mode 100644 templates/repo/packages/list.tmpl delete mode 100644 templates/repo/packages/view.tmpl create mode 100644 templates/user/overview/header.tmpl create mode 100644 templates/user/overview/packages.tmpl diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 4fbde61fdb704..e44d7c12173ad 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -6,6 +6,7 @@ package packages import ( "context" + "strconv" "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" @@ -38,6 +39,11 @@ type PackageFileDescriptor struct { Blob *PackageBlob } +// WebLink returns the package's web link +func (pd *PackageDescriptor) WebLink() string { + return pd.Owner.HTMLURL() + "/-/packages/" + strconv.FormatInt(pd.Version.ID, 10) +} + // GetPackageDescriptor gets the package description for a version func GetPackageDescriptor(pv *PackageVersion) (*PackageDescriptor, error) { return GetPackageDescriptorCtx(db.DefaultContext, pv) diff --git a/models/packages/package.go b/models/packages/package.go index d309016b878e7..d28ec63fc2aae 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -107,6 +107,12 @@ func TryInsertPackage(ctx context.Context, p *Package) (*Package, error) { return p, nil } +// SetRepositoryLink sets the linked repository +func SetRepositoryLink(packageID, repoID int64) error { + _, err := db.GetEngine(db.DefaultContext).ID(packageID).Cols("repo_id").Update(&Package{RepoID: repoID}) + return err +} + // GetPackageByID gets a package by id func GetPackageByID(ctx context.Context, packageID int64) (*Package, error) { p := &Package{} diff --git a/models/repo.go b/models/repo.go index 10dc8d9b09388..14d3f1f2095fa 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1029,7 +1029,7 @@ func GetRepoInitFile(tp, name string) ([]byte, error) { } var ( - reservedRepoNames = []string{".", ".."} + reservedRepoNames = []string{".", "..", "-"} reservedRepoPatterns = []string{"*.git", "*.wiki", "*.rss", "*.atom"} ) @@ -1773,6 +1773,10 @@ func GetUserRepositories(opts *SearchRepoOptions) ([]*Repository, int64, error) sess := db.NewSession(db.DefaultContext) defer sess.Close() + if opts.Page != 0 { + sess = db.SetSessionPagination(sess, opts) + } + count, err := sess.Where(cond).Count(new(Repository)) if err != nil { return nil, 0, fmt.Errorf("Count: %v", err) @@ -1780,7 +1784,7 @@ func GetUserRepositories(opts *SearchRepoOptions) ([]*Repository, int64, error) sess.Where(cond).OrderBy(opts.OrderBy.String()) repos := make([]*Repository, 0, opts.PageSize) - return repos, count, db.SetSessionPagination(sess, opts).Find(&repos) + return repos, count, sess.Find(&repos) } // GetUserMirrorRepositories returns a list of mirror repositories of given user. diff --git a/models/repo_unit.go b/models/repo_unit.go index 1de3340dde33e..4dac15366b95d 100644 --- a/models/repo_unit.go +++ b/models/repo_unit.go @@ -156,7 +156,7 @@ func (r *RepoUnit) BeforeSet(colName string, val xorm.Cell) { switch colName { case "type": switch unit.Type(login.Cell2Int64(val)) { - case unit.TypeCode, unit.TypeReleases, unit.TypeWiki, unit.TypeProjects, unit.TypePackages: + case unit.TypeCode, unit.TypeReleases, unit.TypeWiki, unit.TypeProjects: r.Config = new(UnitConfig) case unit.TypeExternalWiki: r.Config = new(ExternalWikiConfig) diff --git a/models/unit/unit.go b/models/unit/unit.go index 4fe9fa5265696..0af4640b7a4b4 100644 --- a/models/unit/unit.go +++ b/models/unit/unit.go @@ -25,7 +25,6 @@ const ( TypeExternalWiki // 6 ExternalWiki TypeExternalTracker // 7 ExternalTracker TypeProjects // 8 Kanban board - TypePackages // 9 Packages ) // Value returns integer value for unit type @@ -51,8 +50,6 @@ func (u Type) String() string { return "TypeExternalTracker" case TypeProjects: return "TypeProjects" - case TypePackages: - return "TypePackages" } return fmt.Sprintf("Unknown Type %d", u) } @@ -75,7 +72,6 @@ var ( TypeExternalWiki, TypeExternalTracker, TypeProjects, - TypePackages, } // DefaultRepoUnits contains the default unit types @@ -86,7 +82,6 @@ var ( TypeReleases, TypeWiki, TypeProjects, - TypePackages, } // NotAllowedDefaultRepoUnits contains units that can't be default @@ -261,14 +256,6 @@ var ( 5, } - UnitPackages = Unit{ - TypePackages, - "repo.packages", - "/packages", - "repo.packages.desc", - 6, - } - // Units contains all the units Units = map[Type]Unit{ TypeCode: UnitCode, @@ -279,7 +266,6 @@ var ( TypeWiki: UnitWiki, TypeExternalWiki: UnitExternalWiki, TypeProjects: UnitProjects, - TypePackages: UnitPackages, } ) diff --git a/modules/context/api.go b/modules/context/api.go index d82b0d7a708d7..c978835af8749 100644 --- a/modules/context/api.go +++ b/modules/context/api.go @@ -27,8 +27,7 @@ import ( // APIContext is a specific context for API service type APIContext struct { *Context - Org *APIOrganization - Package *APIPackage + Org *APIOrganization } // APIError is error format response diff --git a/modules/context/api_package.go b/modules/context/api_package.go deleted file mode 100644 index 6e9ab4913c3c1..0000000000000 --- a/modules/context/api_package.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package context - -import ( - "code.gitea.io/gitea/models" -) - -// APIPackage contains organization and team -type APIPackage struct { - Owner *models.User - AccessMode models.AccessMode -} diff --git a/modules/context/context.go b/modules/context/context.go index ce558c57b0624..a19f6ba9ee078 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -66,8 +66,10 @@ type Context struct { IsSigned bool IsBasicAuth bool - Repo *Repository - Org *Organization + ContextUser *models.User + Repo *Repository + Org *Organization + Package *Package } // GetData returns the data @@ -753,7 +755,6 @@ func Contexter() func(next http.Handler) http.Handler { ctx.Data["UnitIssuesGlobalDisabled"] = unit.TypeIssues.UnitGlobalDisabled() ctx.Data["UnitPullsGlobalDisabled"] = unit.TypePullRequests.UnitGlobalDisabled() ctx.Data["UnitProjectsGlobalDisabled"] = unit.TypeProjects.UnitGlobalDisabled() - ctx.Data["UnitPackagesGlobalDisabled"] = unit.TypePackages.UnitGlobalDisabled() ctx.Data["i18n"] = locale ctx.Data["Tr"] = i18n.Tr diff --git a/modules/context/package.go b/modules/context/package.go new file mode 100644 index 0000000000000..b0a6c9be6b20e --- /dev/null +++ b/modules/context/package.go @@ -0,0 +1,98 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package context + +import ( + "fmt" + "net/http" + "strconv" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/packages" +) + +// Package contains owner, access mode and optional the package +type Package struct { + Owner *models.User + AccessMode models.AccessMode + Descriptor *packages.PackageDescriptor +} + +// PackageAssignment returns a middleware to handle Context.Package assignment +func PackageAssignment() func(ctx *Context) { + return func(ctx *Context) { + packageAssignment(ctx, func(status int, title string, obj interface{}) { + err, ok := obj.(error) + if !ok { + err = fmt.Errorf("%s", obj) + } + if status == http.StatusNotFound { + ctx.NotFound(title, err) + } else { + ctx.ServerError(title, err) + } + }) + } +} + +// PackageAssignmentAPI returns a middleware to handle Context.Package assignment +func PackageAssignmentAPI() func(ctx *APIContext) { + return func(ctx *APIContext) { + packageAssignment(ctx.Context, ctx.Error) + } +} + +func packageAssignment(ctx *Context, errCb func(int, string, interface{})) { + ctx.Package = &Package{ + Owner: ctx.ContextUser, + } + + if ctx.IsSigned && ctx.User.ID == ctx.ContextUser.ID { + ctx.Package.AccessMode = models.AccessModeOwner + } else { + if ctx.Package.Owner.IsOrganization() { + if ctx.User != nil && models.HasOrgOrUserVisible(ctx.Package.Owner, ctx.User) { + var err error + ctx.Package.AccessMode, err = ctx.Package.Owner.GetOrgUserMaxAuthorizeLevel(ctx.User.ID) + if err != nil { + errCb(http.StatusInternalServerError, "GetOrgUserMaxAuthorizeLevel", err) + return + } + } + } else { + ctx.Package.AccessMode = models.AccessModeRead + } + } + + versionID := ctx.Params(":versionid") + if versionID != "" { + id, err := strconv.ParseInt(versionID, 10, 64) + if err != nil { + errCb(http.StatusInternalServerError, "ParseInt", err) + return + } + + pv, err := packages.GetVersionByID(ctx, id) + if err != nil { + if err == packages.ErrPackageNotExist { + errCb(http.StatusNotFound, "GetVersionByID", err) + } else { + errCb(http.StatusInternalServerError, "GetVersionByID", err) + } + return + } + + ctx.Package.Descriptor, err = packages.GetPackageDescriptorCtx(ctx, pv) + if err != nil { + errCb(http.StatusInternalServerError, "GetPackageDescriptorCtx", err) + return + } + + if ctx.Package.Descriptor.Owner.ID != ctx.Package.Owner.ID { + errCb(http.StatusNotFound, "Package owner does not match", "Package owner does not match") + return + } + } +} diff --git a/modules/context/repo.go b/modules/context/repo.go index 879876e3994cc..af8ac4af2df2c 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -429,6 +429,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { } ctx.Repo.Owner = owner ctx.Data["Username"] = ctx.Repo.Owner.Name + ctx.ContextUser = owner // Get repository. repo, err := models.GetRepositoryByName(owner.ID, repoName) @@ -906,7 +907,6 @@ func UnitTypes() func(ctx *Context) { ctx.Data["UnitTypeExternalWiki"] = unit_model.TypeExternalWiki ctx.Data["UnitTypeExternalTracker"] = unit_model.TypeExternalTracker ctx.Data["UnitTypeProjects"] = unit_model.TypeProjects - ctx.Data["UnitTypePackages"] = unit_model.TypePackages } } diff --git a/modules/context/user.go b/modules/context/user.go new file mode 100644 index 0000000000000..90662ae0be966 --- /dev/null +++ b/modules/context/user.go @@ -0,0 +1,61 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package context + +import ( + "fmt" + "net/http" + "strings" + + "code.gitea.io/gitea/models" +) + +// UserAssignment returns a middleware to handle context-user assignment +func UserAssignment() func(ctx *Context) { + return func(ctx *Context) { + userAssignment(ctx, func(status int, title string, obj interface{}) { + err, ok := obj.(error) + if !ok { + err = fmt.Errorf("%s", obj) + } + if status == http.StatusNotFound { + ctx.NotFound(title, err) + } else { + ctx.ServerError(title, err) + } + }) + } +} + +// UserAssignmentAPI returns a middleware to handle context-user assignment +func UserAssignmentAPI() func(ctx *APIContext) { + return func(ctx *APIContext) { + userAssignment(ctx.Context, ctx.Error) + } +} + +func userAssignment(ctx *Context, errCb func(int, string, interface{})) { + username := ctx.Params(":username") + + if ctx.IsSigned && ctx.User.LowerName == strings.ToLower(username) { + ctx.ContextUser = ctx.User + } else { + var err error + ctx.ContextUser, err = models.GetUserByName(username) + if err != nil { + if models.IsErrUserNotExist(err) { + if redirectUserID, err := models.LookupUserRedirect(username); err == nil { + RedirectToUser(ctx, username, redirectUserID) + } else if models.IsErrUserRedirectNotExist(err) { + errCb(http.StatusNotFound, "GetUserByName", err) + } else { + errCb(http.StatusInternalServerError, "LookupUserRedirect", err) + } + } else { + errCb(http.StatusInternalServerError, "GetUserByName", err) + } + } + } +} diff --git a/modules/convert/package.go b/modules/convert/package.go index ca8533a34e3d9..a51062766ac08 100644 --- a/modules/convert/package.go +++ b/modules/convert/package.go @@ -5,20 +5,27 @@ package convert import ( + "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/packages" api "code.gitea.io/gitea/modules/structs" ) // ToPackage convert a packages.PackageDescriptor to api.Package func ToPackage(pd *packages.PackageDescriptor) *api.Package { + var repo *api.Repository + if pd.Repository != nil { + repo = ToRepo(pd.Repository, models.AccessModeNone) + } + return &api.Package{ - ID: pd.Version.ID, - Owner: ToUser(pd.Owner, nil), - Creator: ToUser(pd.Creator, nil), - Type: string(pd.Package.Type), - Name: pd.Package.Name, - Version: pd.Version.Version, - CreatedAt: pd.Version.CreatedUnix.AsTime(), + ID: pd.Version.ID, + Owner: ToUser(pd.Owner, nil), + Repository: repo, + Creator: ToUser(pd.Creator, nil), + Type: string(pd.Package.Type), + Name: pd.Package.Name, + Version: pd.Version.Version, + CreatedAt: pd.Version.CreatedUnix.AsTime(), } } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 54f166f7ab538..866857f1a1ebb 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -929,9 +929,9 @@ func NewContext() { if os.Getuid() == 0 { if !unsafeAllowRunAsRoot { // Special thanks to VLC which inspired the wording of this messaging. - //log.Fatal("Gitea is not supposed to be run as root. Sorry. If you need to use privileged TCP ports please instead use setcap and the `cap_net_bind_service` permission") + log.Fatal("Gitea is not supposed to be run as root. Sorry. If you need to use privileged TCP ports please instead use setcap and the `cap_net_bind_service` permission") } - //log.Critical("You are running Gitea using the root user, and have purposely chosen to skip built-in protections around this. You have been warned against this.") + log.Critical("You are running Gitea using the root user, and have purposely chosen to skip built-in protections around this. You have been warned against this.") } SSH.BuiltinServerUser = Cfg.Section("server").Key("BUILTIN_SSH_SERVER_USER").MustString(RunUser) diff --git a/modules/structs/package.go b/modules/structs/package.go index 9424878b1315d..fbdd6c90aa862 100644 --- a/modules/structs/package.go +++ b/modules/structs/package.go @@ -10,12 +10,13 @@ import ( // Package represents a package type Package struct { - ID int64 `json:"id"` - Owner *User `json:"owner"` - Creator *User `json:"creator"` - Type string `json:"type"` - Name string `json:"name"` - Version string `json:"version"` + ID int64 `json:"id"` + Owner *User `json:"owner"` + Repository *Repository `json:"repository"` + Creator *User `json:"creator"` + Type string `json:"type"` + Name string `json:"name"` + Version string `json:"version"` // swagger:strfmt date-time CreatedAt time.Time `json:"created_at"` } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index c63fdff8f3ca7..461ae62c41534 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1529,58 +1529,6 @@ milestones.filter_sort.most_complete = Most complete milestones.filter_sort.most_issues = Most issues milestones.filter_sort.least_issues = Least issues -packages = Packages -packages.desc = Manage repository packages. -packages.empty = There are no packages yet. -packages.empty.documentation = For more information on the package registry, see the documentation. -packages.filter.package_type = Type -packages.filter.package_type.all = All -packages.filter.no_result = Your filter produced no results. -packages.published_by = Published %[1]s by %[3]s -packages.installation = Installation -packages.about = About this package -packages.requirements = Requirements -packages.dependencies = Dependencies -packages.keywords = Keywords -packages.details = Details -packages.details.author = Author -packages.details.project_site = Project Site -packages.details.license = License -packages.assets = Assets -packages.versions = Versions -packages.versions.on = on -packages.delete = Delete package -packages.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? -packages.delete.success = The package has been deleted. -packages.generic.download = Download package from the command line: -packages.generic.documentation = For more information on the generic registry, see the documentation. -packages.maven.registry = Setup this registry in your project pom.xml file: -packages.maven.install = To use the package include the following in the dependencies block in the pom.xml file: -packages.maven.install2 = Run via command line: -packages.maven.download = To download the dependency, run via command line: -packages.maven.documentation = For more information on the Maven registry, see the documentation. -packages.nuget.registry = Setup the registry from the command line: -packages.nuget.install = Install from the command line: -packages.nuget.documentation = For more information on the NuGet registry, see the documentation. -packages.npm.registry = Setup this registry in your project .npmrc file: -packages.npm.install = Install from the command line: -packages.npm.install2 = Install via package.json: -packages.npm.documentation = For more information on the npm registry, see the documentation. -packages.npm.dependencies = Dependencies -packages.npm.dependencies.development = Development Dependencies -packages.npm.dependencies.peer = Peer Dependencies -packages.npm.dependencies.optional = Optional Dependencies -packages.pypi.requires = Requires Python -packages.pypi.install = Install from the command line: -packages.pypi.documentation = For more information on the PyPI registry, see the documentation. -packages.rubygems.install = Install from the command line: -packages.rubygems.install2 = Install via Gemfile: -packages.rubygems.dependencies.runtime = Runtime Dependencies -packages.rubygems.dependencies.development = Development Dependencies -packages.rubygems.required.ruby = Requires Ruby version -packages.rubygems.required.rubygems = Requires RubyGem version -packages.rubygems.documentation = For more information on the RubyGems registry, see the documentation. - signing.will_sign = This commit will be signed with key '%s' signing.wont_sign.error = There was an error whilst checking if the commit could be signed signing.wont_sign.nokey = There is no key available to sign this commit @@ -2896,3 +2844,65 @@ error.probable_bad_default_signature = "WARNING! Although the default key has th [units] error.no_unit_allowed_repo = You are not allowed to access any section of this repository. error.unit_not_allowed = You are not allowed to access this repository section. + +[packages] +title = Packages +desc = Manage repository packages. +empty = There are no packages yet. +empty.documentation = For more information on the package registry, see the documentation. +filter.type = Type +filter.type.all = All +filter.no_result = Your filter produced no results. +published_by = Published %[1]s by %[3]s +published_by_in = Published %[1]s by %[3]s in %[5]s +installation = Installation +about = About this package +requirements = Requirements +dependencies = Dependencies +keywords = Keywords +details = Details +details.author = Author +details.project_site = Project Site +details.license = License +assets = Assets +versions = Versions +versions.on = on +generic.download = Download package from the command line: +generic.documentation = For more information on the generic registry, see the documentation. +maven.registry = Setup this registry in your project pom.xml file: +maven.install = To use the package include the following in the dependencies block in the pom.xml file: +maven.install2 = Run via command line: +maven.download = To download the dependency, run via command line: +maven.documentation = For more information on the Maven registry, see the documentation. +nuget.registry = Setup the registry from the command line: +nuget.install = Install from the command line: +nuget.documentation = For more information on the NuGet registry, see the documentation. +npm.registry = Setup this registry in your project .npmrc file: +npm.install = Install from the command line: +npm.install2 = Install via package.json: +npm.documentation = For more information on the npm registry, see the documentation. +npm.dependencies = Dependencies +npm.dependencies.development = Development Dependencies +npm.dependencies.peer = Peer Dependencies +npm.dependencies.optional = Optional Dependencies +pypi.requires = Requires Python +pypi.install = Install from the command line: +pypi.documentation = For more information on the PyPI registry, see the documentation. +rubygems.install = Install from the command line: +rubygems.install2 = Install via Gemfile: +rubygems.dependencies.runtime = Runtime Dependencies +rubygems.dependencies.development = Development Dependencies +rubygems.required.ruby = Requires Ruby version +rubygems.required.rubygems = Requires RubyGem version +rubygems.documentation = For more information on the RubyGems registry, see the documentation. +settings.link = Link this package to a repository +settings.link.description = If you link a package with a repository, the package is listed in the repository's package list. +settings.link.select = Select Repository +settings.link.button = Update Repository Link +settings.link.success = Repository link was successfully updated. +settings.link.error = Failed to update repository link. +settings.delete = Delete package +settings.delete.description = Deleting a package is permanent and cannot be undone. +settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? +settings.delete.success = The package has been deleted. +settings.delete.error = Failed to delete the package. \ No newline at end of file diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 891c9c8882c4d..bb35dd202a9f1 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -158,6 +158,7 @@ func repoAssignment() func(ctx *context.APIContext) { } } ctx.Repo.Owner = owner + ctx.ContextUser = owner // Get repository. repo, err := models.GetRepositoryByName(owner.ID, repoName) @@ -193,49 +194,6 @@ func repoAssignment() func(ctx *context.APIContext) { } } -func packageOwnerAssignment() func(ctx *context.APIContext) { - return func(ctx *context.APIContext) { - ownerName := ctx.Params("owner") - - ctx.Package = &context.APIPackage{} - - if ctx.IsSigned && ctx.User.LowerName == strings.ToLower(ownerName) { - ctx.Package.Owner = ctx.User - ctx.Package.AccessMode = models.AccessModeOwner - } else { - owner, err := models.GetUserByName(ownerName) - if err != nil { - if models.IsErrUserNotExist(err) { - if redirectUserID, err := models.LookupUserRedirect(ownerName); err == nil { - context.RedirectToUser(ctx.Context, ownerName, redirectUserID) - } else if models.IsErrUserRedirectNotExist(err) { - ctx.NotFound("GetUserByName", err) - } else { - ctx.Error(http.StatusInternalServerError, "LookupUserRedirect", err) - } - } else { - ctx.Error(http.StatusInternalServerError, "GetUserByName", err) - } - return - } - ctx.Package.Owner = owner - - if owner.IsOrganization() { - if ctx.User != nil && models.HasOrgOrUserVisible(owner, ctx.User) { - var err error - ctx.Package.AccessMode, err = owner.GetOrgUserMaxAuthorizeLevel(ctx.User.ID) - if err != nil { - ctx.Error(http.StatusInternalServerError, "GetOrgUserMaxAuthorizeLevel", err) - return - } - } - } else { - ctx.Package.AccessMode = models.AccessModeRead - } - } - } -} - func reqPackageAccess(accessMode models.AccessMode) func(ctx *context.APIContext) { return func(ctx *context.APIContext) { if ctx.Package.AccessMode < accessMode && !ctx.IsUserSiteAdmin() { @@ -1048,7 +1006,7 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { }, repoAssignment()) }) - m.Group("/packages/{owner}", func() { + m.Group("/packages/{username}", func() { m.Group("/generic", func() { m.Group("/{packagename}/{packageversion}/{filename}", func() { m.Get("", generic.DownloadPackageFile) @@ -1102,13 +1060,13 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { m.Delete("/yank", rubygems.DeletePackage) }, reqToken(), reqPackageAccess(models.AccessModeWrite)) }) - m.Group("/{id}", func() { - m.Get("/", repo.GetPackage) - m.Delete("/", reqPackageAccess(models.AccessModeWrite), repo.DeletePackage) + m.Group("/{versionid}", func() { + m.Get("", repo.GetPackage) + m.Delete("", reqPackageAccess(models.AccessModeWrite), repo.DeletePackage) m.Get("/files", repo.ListPackageFiles) }) m.Get("/", repo.ListPackages) - }, packageOwnerAssignment(), reqPackageAccess(models.AccessModeRead)) + }, context.UserAssignmentAPI(), context.PackageAssignmentAPI(), reqPackageAccess(models.AccessModeRead)) // Organizations m.Get("/user/orgs", reqToken(), org.ListMyOrgs) diff --git a/routers/api/v1/repo/package.go b/routers/api/v1/repo/package.go index 8c051c33d4a78..2c8382ccfc251 100644 --- a/routers/api/v1/repo/package.go +++ b/routers/api/v1/repo/package.go @@ -7,7 +7,6 @@ package repo import ( "net/http" - "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" @@ -16,22 +15,17 @@ import ( package_service "code.gitea.io/gitea/services/packages" ) -// ListPackages gets all packages of a repository +// ListPackages gets all packages of an owner func ListPackages(ctx *context.APIContext) { - // swagger:operation GET /repos/{owner}/{repo}/packages repository repoListPackages + // swagger:operation GET /packages/{owner} package listPackages // --- - // summary: Gets all packages of a repository + // summary: Gets all packages of an owner // produces: // - application/json // parameters: // - name: owner // in: path - // description: owner of the repo - // type: string - // required: true - // - name: repo - // in: path - // description: name of the repo + // description: owner of the packages // type: string // required: true // - name: page @@ -42,7 +36,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: page size of results // type: integer - // - name: package_type + // - name: type // in: query // description: package type filter // type: string @@ -57,7 +51,7 @@ func ListPackages(ctx *context.APIContext) { listOptions := utils.GetListOptions(ctx) - packageType := ctx.FormTrim("package_type") + packageType := ctx.FormTrim("type") query := ctx.FormTrim("q") pvs, count, err := packages.SearchVersions(&packages.PackageSearchOptions{ @@ -89,7 +83,7 @@ func ListPackages(ctx *context.APIContext) { // GetPackage gets a package func GetPackage(ctx *context.APIContext) { - // swagger:operation GET /repos/{owner}/{repo}/packages/{id} repository repoGetPackage + // swagger:operation GET /packages/{owner}/{versionid} package getPackage // --- // summary: Gets a package // produces: @@ -97,17 +91,12 @@ func GetPackage(ctx *context.APIContext) { // parameters: // - name: owner // in: path - // description: owner of the repo + // description: owner of the package // type: string // required: true - // - name: repo + // - name: versionid // in: path - // description: name of the repo - // type: string - // required: true - // - name: id - // in: path - // description: id of the package + // description: id of the package version // type: integer // format: int64 // required: true @@ -117,49 +106,23 @@ func GetPackage(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - pv, err := packages.GetVersionByID(db.DefaultContext, ctx.ParamsInt64(":id")) - if err != nil { - if err == packages.ErrPackageNotExist { - ctx.NotFound() - } else { - ctx.Error(http.StatusInternalServerError, "GetVersionByID", err) - } - return - } - - pd, err := packages.GetPackageDescriptor(pv) - if err != nil { - ctx.Error(http.StatusInternalServerError, "GetPackageDescriptor", err) - return - } - - if ctx.Package.Owner.ID != pd.Owner.ID { - ctx.NotFound() - return - } - - ctx.JSON(http.StatusOK, convert.ToPackage(pd)) + ctx.JSON(http.StatusOK, convert.ToPackage(ctx.Package.Descriptor)) } -// DeletePackage delete a package from a repository +// DeletePackage deletes a package func DeletePackage(ctx *context.APIContext) { - // swagger:operation DELETE /repos/{owner}/{repo}/packages/{id} repository repoDeletePackage + // swagger:operation DELETE /packages/{owner}/{versionid} package deletePackage // --- - // summary: Delete a package from a repository + // summary: Delete a package // parameters: // - name: owner // in: path - // description: owner of the repo + // description: owner of the package // type: string // required: true - // - name: repo + // - name: versionid // in: path - // description: name of the repo - // type: string - // required: true - // - name: id - // in: path - // description: id of the package + // description: id of the package version // type: integer // format: int64 // required: true @@ -169,13 +132,9 @@ func DeletePackage(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - err := package_service.DeleteVersionByID(ctx.User, ctx.Package.Owner, ctx.ParamsInt64(":id")) + err := package_service.DeletePackageVersion(ctx.User, ctx.Package.Descriptor.Version) if err != nil { - if err == packages.ErrPackageNotExist { - ctx.NotFound() - } else { - ctx.Error(http.StatusInternalServerError, "DeleteVersionByID", err) - } + ctx.Error(http.StatusInternalServerError, "DeletePackageVersion", err) return } ctx.Status(http.StatusNoContent) @@ -183,7 +142,7 @@ func DeletePackage(ctx *context.APIContext) { // ListPackageFiles gets all files of a package func ListPackageFiles(ctx *context.APIContext) { - // swagger:operation GET /repos/{owner}/{repo}/packages/{id}/files repository repoListPackageFiles + // swagger:operation GET /packages/{owner}/{versionid}/files package listPackageFiles // --- // summary: Gets all files of a package // produces: @@ -191,17 +150,12 @@ func ListPackageFiles(ctx *context.APIContext) { // parameters: // - name: owner // in: path - // description: owner of the repo + // description: owner of the package // type: string // required: true - // - name: repo + // - name: versionid // in: path - // description: name of the repo - // type: string - // required: true - // - name: id - // in: path - // description: id of the package + // description: id of the package version // type: integer // format: int64 // required: true @@ -211,29 +165,8 @@ func ListPackageFiles(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - pv, err := packages.GetVersionByID(db.DefaultContext, ctx.ParamsInt64(":id")) - if err != nil { - if err == packages.ErrPackageNotExist { - ctx.NotFound() - } else { - ctx.Error(http.StatusInternalServerError, "GetVersionByID", err) - } - return - } - - pd, err := packages.GetPackageDescriptor(pv) - if err != nil { - ctx.Error(http.StatusInternalServerError, "GetPackageDescriptor", err) - return - } - - if ctx.Package.Owner.ID != pd.Owner.ID { - ctx.NotFound() - return - } - - apiPackageFiles := make([]*api.PackageFile, 0, len(pd.Files)) - for _, pfd := range pd.Files { + apiPackageFiles := make([]*api.PackageFile, 0, len(ctx.Package.Descriptor.Files)) + for _, pfd := range ctx.Package.Descriptor.Files { apiPackageFiles = append(apiPackageFiles, convert.ToPackageFile(&pfd)) } diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index 53ac5865eeac5..20abd827bfdb9 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -6,52 +6,36 @@ package repo import ( "net/http" - "sort" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" - "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" - - packages_service "code.gitea.io/gitea/services/packages" ) const ( - tplPackages base.TplName = "repo/packages/list" - tplPackagesView base.TplName = "repo/packages/view" + tplPackagesList base.TplName = "repo/packages" ) -// MustEnablePackages checks if packages are enabled -func MustEnablePackages(ctx *context.Context) { - if unit.TypePackages.UnitGlobalDisabled() || !ctx.Repo.CanRead(unit.TypePackages) { - ctx.NotFound("MustEnablePackages", nil) - } -} - // Packages displays a list of all packages in the repository func Packages(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("repo.packages") - ctx.Data["IsPackagesPage"] = true - - query := ctx.FormTrim("q") - packageType := ctx.FormTrim("package_type") page := ctx.FormInt("page") if page <= 1 { page = 1 } + query := ctx.FormTrim("q") + packageType := ctx.FormTrim("type") - repo := ctx.Repo.Repository - - pvs, count, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ - RepoID: repo.ID, - Query: query, - Type: packageType, + pvs, total, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ Paginator: &db.ListOptions{ - Page: page, PageSize: setting.UI.PackagesPagingNum, + Page: page, }, + OwnerID: ctx.ContextUser.ID, + RepoID: ctx.Repo.Repository.ID, + Query: query, + Type: packageType, }) if err != nil { ctx.ServerError("SearchLatestVersions", err) @@ -64,91 +48,25 @@ func Packages(ctx *context.Context) { return } - hasPackages, err := packages.HasRepositoryPackages(repo.ID) + hasPackages, err := packages.HasRepositoryPackages(ctx.Repo.Repository.ID) if err != nil { ctx.ServerError("HasRepositoryPackages", err) return } - ctx.Data["HasPackages"] = hasPackages - ctx.Data["PackageDescriptors"] = pds + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["IsPackagesPage"] = true + ctx.Data["ContextUser"] = ctx.ContextUser ctx.Data["Query"] = query ctx.Data["PackageType"] = packageType + ctx.Data["HasPackages"] = hasPackages + ctx.Data["PackageDescriptors"] = pds + ctx.Data["Total"] = total - pager := context.NewPagination(int(count), setting.UI.PackagesPagingNum, page, 5) + pager := context.NewPagination(int(total), setting.UI.PackagesPagingNum, page, 5) pager.AddParam(ctx, "q", "Query") - pager.AddParam(ctx, "package_type", "PackageType") + pager.AddParam(ctx, "type", "PackageType") ctx.Data["Page"] = pager - ctx.HTML(http.StatusOK, tplPackages) -} - -// ViewPackage displays a single package -func ViewPackage(ctx *context.Context) { - pv, err := packages.GetVersionByID(db.DefaultContext, ctx.ParamsInt64(":id")) - if err != nil { - if err == packages.ErrPackageNotExist { - ctx.NotFound("", nil) - } else { - ctx.ServerError("GetVersionByID", err) - } - return - } - pd, err := packages.GetPackageDescriptor(pv) - if err != nil { - ctx.ServerError("GetPackageDescriptor", err) - return - } - if pd.Repository.ID != ctx.Repo.Repository.ID { - ctx.NotFound("", nil) - return - } - - ctx.Data["Title"] = pd.Package.Name - ctx.Data["IsPackagesPage"] = true - ctx.Data["PackageDescriptor"] = pd - - otherVersions, err := packages.GetVersionsByPackageName(ctx.Repo.Repository.ID, pd.Package.Type, pd.Package.LowerName) - if err != nil { - ctx.ServerError("GetVersionsByPackageName", err) - return - } - sort.Slice(otherVersions, func(i, j int) bool { - return otherVersions[i].Version > otherVersions[j].Version - }) - ctx.Data["OtherVersions"] = otherVersions - - ctx.Data["Repo"] = ctx.Repo.Repository - ctx.Data["CanWritePackages"] = ctx.Repo.Permission.CanWrite(unit.TypePackages) - ctx.Data["PageIsPackages"] = true - - ctx.HTML(http.StatusOK, tplPackagesView) -} - -// DeletePackagePost deletes a package -func DeletePackagePost(ctx *context.Context) { - err := packages_service.DeleteVersionByID(ctx.User, nil, ctx.ParamsInt64(":id")) // TODO - if err != nil { - ctx.Flash.Error(err.Error()) - } else { - ctx.Flash.Success(ctx.Tr("repo.packages.delete.success")) - } - - ctx.Redirect(ctx.Repo.RepoLink + "/packages") -} - -// DownloadPackageFile serves the content of a package file -func DownloadPackageFile(ctx *context.Context) { - s, pf, err := packages_service.GetFileStreamByPackageVersionID(nil, ctx.ParamsInt64(":id"), ctx.Params(":filename")) // TODO - if err != nil { - if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { - ctx.NotFound("", err) - return - } - ctx.ServerError("GetFileStreamByPackageVersionID", err) - return - } - defer s.Close() - - ctx.ServeStream(s, pf.Name) + ctx.HTML(http.StatusOK, tplPackagesList) } diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 9657075e4726c..b104ede005221 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -421,15 +421,6 @@ func SettingsPost(ctx *context.Context) { } } - if form.EnablePackages && !unit_model.TypePackages.UnitGlobalDisabled() { - units = append(units, models.RepoUnit{ - RepoID: repo.ID, - Type: unit_model.TypePackages, - }) - } else if !unit_model.TypePackages.UnitGlobalDisabled() { - deleteUnitTypes = append(deleteUnitTypes, unit_model.TypePackages) - } - if form.EnableProjects && !unit_model.TypeProjects.UnitGlobalDisabled() { units = append(units, models.RepoUnit{ RepoID: repo.ID, diff --git a/routers/web/user/package.go b/routers/web/user/package.go new file mode 100644 index 0000000000000..de079cfd45598 --- /dev/null +++ b/routers/web/user/package.go @@ -0,0 +1,200 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package user + +import ( + "net/http" + "sort" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/services/forms" + packages_service "code.gitea.io/gitea/services/packages" +) + +const ( + tplPackagesList base.TplName = "user/overview/packages" + tplPackagesView base.TplName = "package/view" + tplPackagesSettings base.TplName = "package/settings" +) + +// Packages displays a list of all packages of the context user +func Packages(ctx *context.Context) { + page := ctx.FormInt("page") + if page <= 1 { + page = 1 + } + query := ctx.FormTrim("q") + packageType := ctx.FormTrim("type") + + pvs, total, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ + Paginator: &db.ListOptions{ + PageSize: setting.UI.PackagesPagingNum, + Page: page, + }, + OwnerID: ctx.ContextUser.ID, + Query: query, + Type: packageType, + }) + if err != nil { + ctx.ServerError("SearchLatestVersions", err) + return + } + + pds, err := packages.GetPackageDescriptors(pvs) + if err != nil { + ctx.ServerError("GetPackageDescriptors", err) + return + } + + hasPackages, err := packages.HasOwnerPackages(ctx.ContextUser.ID) + if err != nil { + ctx.ServerError("HasOwnerPackages", err) + return + } + + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["IsPackagesPage"] = true + ctx.Data["ContextUser"] = ctx.ContextUser + ctx.Data["Query"] = query + ctx.Data["PackageType"] = packageType + ctx.Data["HasPackages"] = hasPackages + ctx.Data["PackageDescriptors"] = pds + ctx.Data["Total"] = total + + pager := context.NewPagination(int(total), setting.UI.PackagesPagingNum, page, 5) + pager.AddParam(ctx, "q", "Query") + pager.AddParam(ctx, "type", "PackageType") + ctx.Data["Page"] = pager + + ctx.HTML(http.StatusOK, tplPackagesList) +} + +// ViewPackage displays a single package +func ViewPackage(ctx *context.Context) { + pd := ctx.Package.Descriptor + + if pd == nil { + ctx.NotFound("Package does not exist", nil) + return + } + + ctx.Data["Title"] = pd.Package.Name + ctx.Data["IsPackagesPage"] = true + ctx.Data["ContextUser"] = ctx.ContextUser + ctx.Data["PackageDescriptor"] = pd + + otherVersions, err := packages.GetVersionsByPackageName(pd.Owner.ID, pd.Package.Type, pd.Package.LowerName) + if err != nil { + ctx.ServerError("GetVersionsByPackageName", err) + return + } + sort.Slice(otherVersions, func(i, j int) bool { + return otherVersions[i].Version > otherVersions[j].Version + }) + ctx.Data["OtherVersions"] = otherVersions + + ctx.Data["CanWritePackages"] = ctx.Package.AccessMode >= models.AccessModeWrite || ctx.IsUserSiteAdmin() + + ctx.HTML(http.StatusOK, tplPackagesView) +} + +// PackageSettings displays the package settings page +func PackageSettings(ctx *context.Context) { + pd := ctx.Package.Descriptor + + if pd == nil { + ctx.NotFound("Package does not exist", nil) + return + } + + ctx.Data["Title"] = pd.Package.Name + ctx.Data["IsPackagesPage"] = true + ctx.Data["ContextUser"] = ctx.ContextUser + ctx.Data["PackageDescriptor"] = pd + + repos, _, _ := models.GetUserRepositories(&models.SearchRepoOptions{ + Actor: pd.Owner, + }) + ctx.Data["Repos"] = repos + ctx.Data["CanWritePackages"] = ctx.Package.AccessMode >= models.AccessModeWrite || ctx.IsUserSiteAdmin() + + ctx.HTML(http.StatusOK, tplPackagesSettings) +} + +// PackageSettingsPost updates the package settings +func PackageSettingsPost(ctx *context.Context) { + pd := ctx.Package.Descriptor + + if pd == nil { + ctx.NotFound("Package does not exist", nil) + return + } + + form := web.GetForm(ctx).(*forms.PackageSettingForm) + switch form.Action { + case "link": + success := func() bool { + repo, err := models.GetRepositoryByID(form.RepoID) + if err != nil { + log.Error("Error getting repository: %v", err) + return false + } + + if repo.OwnerID != pd.Owner.ID { + return false + } + + if err = packages.SetRepositoryLink(pd.Package.ID, repo.ID); err != nil { + log.Error("Error updating package: %v", err) + return false + } + + return true + }() + + if success { + ctx.Flash.Success(ctx.Tr("packages.settings.link.success")) + } else { + ctx.Flash.Error(ctx.Tr("packages.settings.link.error")) + } + + ctx.Redirect(ctx.Link) + return + case "delete": + err := packages_service.DeletePackageVersion(ctx.User, ctx.Package.Descriptor.Version) + if err != nil { + log.Error("Error deleting package: %v", err) + ctx.Flash.Error(ctx.Tr("packages.settings.delete.error")) + } else { + ctx.Flash.Success(ctx.Tr("packages.settings.delete.success")) + } + + ctx.Redirect(ctx.Package.Owner.HTMLURL() + "/-/packages") + return + } +} + +// DownloadPackageFile serves the content of a package file +func DownloadPackageFile(ctx *context.Context) { + s, pf, err := packages_service.GetFileStreamByPackageVersionID(ctx.ContextUser, ctx.ParamsInt64(":versionid"), ctx.Params(":filename")) + if err != nil { + if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + ctx.NotFound("", err) + return + } + ctx.ServerError("GetFileStreamByPackageVersionID", err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} diff --git a/routers/web/web.go b/routers/web/web.go index e48d2b0075fb9..caae8c8012ea5 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -10,6 +10,7 @@ import ( "os" "path" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" @@ -498,8 +499,14 @@ func RegisterRoutes(m *web.Route) { reqRepoIssuesOrPullsReader := context.RequireRepoReaderOr(unit.TypeIssues, unit.TypePullRequests) reqRepoProjectsReader := context.RequireRepoReader(unit.TypeProjects) reqRepoProjectsWriter := context.RequireRepoWriter(unit.TypeProjects) - reqRepoPackagesReader := context.RequireRepoReader(unit.TypePackages) - reqRepoPackagesWriter := context.RequireRepoWriter(unit.TypePackages) + + reqPackageAccess := func(accessMode models.AccessMode) func(ctx *context.Context) { + return func(ctx *context.Context) { + if ctx.Package.AccessMode < accessMode && !ctx.IsUserSiteAdmin() { + ctx.Error(http.StatusUnauthorized, "reqPackageAccess", "user should have specific permission or be a site admin") + } + } + } // ***** START: Organization ***** m.Group("/org", func() { @@ -593,6 +600,20 @@ func RegisterRoutes(m *web.Route) { }, context.RepoIDAssignment(), context.UnitTypes(), reqRepoCodeReader) }, reqSignIn) + m.Group("/{username}/-", func() { + m.Group("/packages", func() { + m.Get("", user.Packages) + m.Group("/{versionid}", func() { + m.Get("", user.ViewPackage) + m.Get("/files/{filename}", user.DownloadPackageFile) + m.Group("/settings", func() { + m.Get("", user.PackageSettings) + m.Post("", bindIgnErr(forms.PackageSettingForm{}), user.PackageSettingsPost) + }, reqPackageAccess(models.AccessModeWrite)) + }) + }, context.PackageAssignment(), reqPackageAccess(models.AccessModeRead)) + }, context.UserAssignment()) + // ***** Release Attachment Download without Signin m.Get("/{username}/{reponame}/releases/download/{vTag}/{fileName}", ignSignIn, context.RepoAssignment, repo.MustBeNotEmpty, repo.RedirectDownload) @@ -871,14 +892,7 @@ func RegisterRoutes(m *web.Route) { m.Get("/milestones", reqRepoIssuesOrPullsReader, repo.Milestones) }, context.RepoRef()) - m.Group("/packages", func() { - m.Get("", repo.Packages) - m.Group("/{id}", func() { - m.Get("/", repo.ViewPackage) - m.Post("/", reqRepoPackagesWriter, repo.DeletePackagePost) - m.Get("/files/{filename}", repo.DownloadPackageFile) - }) - }, reqRepoPackagesReader, repo.MustEnablePackages) + m.Get("/packages", repo.Packages) m.Group("/projects", func() { m.Get("", repo.Projects) diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 0b56b930a0a04..ee96260385d58 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -141,7 +141,6 @@ type RepoSettingForm struct { TrackerURLFormat string TrackerIssueStyle string EnableCloseIssuesViaCommitInAnyBranch bool - EnablePackages bool EnableProjects bool EnablePulls bool PullsIgnoreWhitespace bool diff --git a/services/forms/user_form.go b/services/forms/user_form.go index 9f86bf61661f6..cb485f8749569 100644 --- a/services/forms/user_form.go +++ b/services/forms/user_form.go @@ -424,3 +424,15 @@ func (f *U2FDeleteForm) Validate(req *http.Request, errs binding.Errors) binding ctx := context.GetContext(req) return middleware.Validate(errs, ctx.Data, f, ctx.Locale) } + +// PackageSettingForm form for package settings +type PackageSettingForm struct { + Action string + RepoID int64 `form:"repo_id"` +} + +// Validate validates the fields +func (f *PackageSettingForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/services/packages/packages.go b/services/packages/packages.go index 96ec58da87a97..61da546ae3eca 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -224,47 +224,28 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_models.PackageVer // DeletePackageVersionByNameAndVersion deletes a package version and all associated files func DeletePackageVersionByNameAndVersion(doer *models.User, pvi *PackageInfo) error { - return deletePackageVersion( - doer, - pvi.Owner, - func(ctx context.Context) (*packages_models.PackageVersion, error) { - return packages_models.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) - }, - ) -} + pv, err := packages_models.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + if err != nil { + return err + } -// DeleteVersionByID deletes a package version and all associated files -func DeleteVersionByID(doer *models.User, owner *models.User, versionID int64) error { - return deletePackageVersion( - doer, - owner, - func(ctx context.Context) (*packages_models.PackageVersion, error) { - return packages_models.GetVersionByID(ctx, versionID) - }, - ) + return DeletePackageVersion(doer, pv) } -func deletePackageVersion(doer *models.User, owner *models.User, cb func(context.Context) (*packages_models.PackageVersion, error)) error { +// DeletePackageVersion deletes the package version and all associated files +func DeletePackageVersion(doer *models.User, pv *packages_models.PackageVersion) error { ctx, committer, err := db.TxContext() if err != nil { return err } defer committer.Close() - pv, err := cb(ctx) - if err != nil { - if err != packages_models.ErrPackageNotExist { - log.Error("Error getting package: %v", err) - } - return err - } - pd, err := packages_models.GetPackageDescriptorCtx(ctx, pv) if err != nil { return err } - log.Trace("Deleting package: %v, %v", owner.ID, pv.ID) + log.Trace("Deleting package: %v", pv.ID) if err := packages_models.DeleteVersionPropertiesByVersionID(ctx, pv.ID); err != nil { return err diff --git a/templates/package/content/generic.tmpl b/templates/package/content/generic.tmpl new file mode 100644 index 0000000000000..4d6d4ae4a1f41 --- /dev/null +++ b/templates/package/content/generic.tmpl @@ -0,0 +1,14 @@ +{{if eq .Package.Type 0}} +

    {{.i18n.Tr "packages.installation"}}

    +
    +
    +
    + + +
    +
    + +
    +
    +
    +{{end}} diff --git a/templates/repo/packages/content/maven.tmpl b/templates/package/content/maven.tmpl similarity index 64% rename from templates/repo/packages/content/maven.tmpl rename to templates/package/content/maven.tmpl index 3f9e703358b7e..956a9ceb0729a 100644 --- a/templates/repo/packages/content/maven.tmpl +++ b/templates/package/content/maven.tmpl @@ -1,30 +1,30 @@ {{if eq .PackageDescriptor.Package.Type "maven"}} -

    {{.i18n.Tr "repo.packages.installation"}}

    +

    {{.i18n.Tr "packages.installation"}}

    - +
    - +
    - +
    - - + +
    - +
    {{if .PackageDescriptor.Metadata.Description}} -

    {{.i18n.Tr "repo.packages.about"}}

    +

    {{.i18n.Tr "packages.about"}}

    {{.PackageDescriptor.Metadata.Description}}
    {{end}} {{if .PackageDescriptor.Metadata.Dependencies}} -

    {{.i18n.Tr "repo.packages.dependencies"}}

    +

    {{.i18n.Tr "packages.dependencies"}}

    {{range .PackageDescriptor.Metadata.Dependencies}} diff --git a/templates/repo/packages/content/npm.tmpl b/templates/package/content/npm.tmpl similarity index 57% rename from templates/repo/packages/content/npm.tmpl rename to templates/package/content/npm.tmpl index 5d862a4a50fe7..624e4ca338aa5 100644 --- a/templates/repo/packages/content/npm.tmpl +++ b/templates/package/content/npm.tmpl @@ -1,27 +1,27 @@ {{if eq .PackageDescriptor.Package.Type "npm"}} -

    {{.i18n.Tr "repo.packages.installation"}}

    +

    {{.i18n.Tr "packages.installation"}}

    - - + +
    - +
    - +
    - +
    {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.Readme}} -

    {{.i18n.Tr "repo.packages.about"}}

    +

    {{.i18n.Tr "packages.about"}}

    {{if .PackageDescriptor.Metadata.Readme}}
    @@ -34,19 +34,19 @@ {{end}} {{if or .PackageDescriptor.Metadata.Dependencies .PackageDescriptor.Metadata.DevelopmentDependencies .PackageDescriptor.Metadata.PeerDependencies .PackageDescriptor.Metadata.OptionalDependencies}} -

    {{.i18n.Tr "repo.packages.dependencies"}}

    +

    {{.i18n.Tr "packages.dependencies"}}

    - {{template "repo/packages/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.Dependencies "title" (.i18n.Tr "repo.packages.npm.dependencies")}} - {{template "repo/packages/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.development")}} - {{template "repo/packages/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.PeerDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.peer")}} - {{template "repo/packages/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.OptionalDependencies "title" (.i18n.Tr "repo.packages.npm.dependencies.optional")}} + {{template "package/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.Dependencies "title" (.i18n.Tr "packages.npm.dependencies")}} + {{template "package/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "packages.npm.dependencies.development")}} + {{template "package/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.PeerDependencies "title" (.i18n.Tr "packages.npm.dependencies.peer")}} + {{template "package/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.OptionalDependencies "title" (.i18n.Tr "packages.npm.dependencies.optional")}}
    {{end}} {{if or .PackageDescriptor.Metadata.Keywords}} -

    {{.i18n.Tr "repo.packages.keywords"}}

    +

    {{.i18n.Tr "packages.keywords"}}

    {{range .PackageDescriptor.Metadata.Keywords}} {{.}} diff --git a/templates/repo/packages/content/npm_dependencies.tmpl b/templates/package/content/npm_dependencies.tmpl similarity index 100% rename from templates/repo/packages/content/npm_dependencies.tmpl rename to templates/package/content/npm_dependencies.tmpl diff --git a/templates/repo/packages/content/nuget.tmpl b/templates/package/content/nuget.tmpl similarity index 76% rename from templates/repo/packages/content/nuget.tmpl rename to templates/package/content/nuget.tmpl index b0ae1bdd2e27a..50aa7b742a998 100644 --- a/templates/repo/packages/content/nuget.tmpl +++ b/templates/package/content/nuget.tmpl @@ -1,23 +1,23 @@ {{if eq .PackageDescriptor.Package.Type "nuget"}} -

    {{.i18n.Tr "repo.packages.installation"}}

    +

    {{.i18n.Tr "packages.installation"}}

    - - + +
    - +
    - +
    {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.ReleaseNotes}} -

    {{.i18n.Tr "repo.packages.about"}}

    +

    {{.i18n.Tr "packages.about"}}

    {{if .PackageDescriptor.Metadata.Description}}{{.PackageDescriptor.Metadata.Description}}{{end}} {{if .PackageDescriptor.Metadata.ReleaseNotes}}{{Str2html .PackageDescriptor.Metadata.ReleaseNotes}}{{end}} @@ -25,7 +25,7 @@ {{end}} {{if .PackageDescriptor.Metadata.Dependencies}} -

    {{.i18n.Tr "repo.packages.dependencies"}}

    +

    {{.i18n.Tr "packages.dependencies"}}

    {{range $framework, $dependencies := .PackageDescriptor.Metadata.Dependencies}} diff --git a/templates/repo/packages/content/pypi.tmpl b/templates/package/content/pypi.tmpl similarity index 61% rename from templates/repo/packages/content/pypi.tmpl rename to templates/package/content/pypi.tmpl index d2e11f3ca0b4c..70bcf0ad9cd70 100644 --- a/templates/repo/packages/content/pypi.tmpl +++ b/templates/package/content/pypi.tmpl @@ -1,18 +1,18 @@ {{if eq .PackageDescriptor.Package.Type "pypi"}} -

    {{.i18n.Tr "repo.packages.installation"}}

    +

    {{.i18n.Tr "packages.installation"}}

    - - + +
    - +
    {{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.LongDescription .PackageDescriptor.Metadata.Summary}} -

    {{.i18n.Tr "repo.packages.about"}}

    +

    {{.i18n.Tr "packages.about"}}

    {{if .PackageDescriptor.Metadata.Summary}}{{.PackageDescriptor.Metadata.Summary}}{{end}}

    {{if .PackageDescriptor.Metadata.LongDescription}} @@ -23,9 +23,9 @@
    {{end}} {{if .PackageDescriptor.Metadata.RequiresPython}} -

    {{.i18n.Tr "repo.packages.requirements"}}

    +

    {{.i18n.Tr "packages.requirements"}}

    - {{.i18n.Tr "repo.packages.pypi.requires"}}: {{.PackageDescriptor.Metadata.RequiresPython}} + {{.i18n.Tr "packages.pypi.requires"}}: {{.PackageDescriptor.Metadata.RequiresPython}}
    {{end}} {{end}} diff --git a/templates/package/content/rubygems.tmpl b/templates/package/content/rubygems.tmpl new file mode 100644 index 0000000000000..f22f9bc50dfd8 --- /dev/null +++ b/templates/package/content/rubygems.tmpl @@ -0,0 +1,40 @@ +{{if eq .PackageDescriptor.Package.Type "rubygems"}} +

    {{.i18n.Tr "packages.installation"}}

    +
    +
    +
    + + +
    +
    + + +
    +
    + +
    +
    +
    + {{if .PackageDescriptor.Metadata.Description}} +

    {{.i18n.Tr "packages.about"}}

    +
    {{.PackageDescriptor.Metadata.Description}}
    + {{end}} + {{if or .PackageDescriptor.Metadata.RequiredRubyVersion .PackageDescriptor.Metadata.RequiredRubygemsVersion}} +

    {{.i18n.Tr "packages.requirements"}}

    +
    + {{if .PackageDescriptor.Metadata.RequiredRubyVersion}}

    {{.i18n.Tr "packages.rubygems.required.ruby"}}: {{range $i, $v := .PackageDescriptor.Metadata.RequiredRubyVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}} + {{if .PackageDescriptor.Metadata.RequiredRubygemsVersion}}

    {{.i18n.Tr "packages.rubygems.required.rubygems"}}: {{range $i, $v := .PackageDescriptor.Metadata.RequiredRubygemsVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}} +
    + {{end}} + {{if or .PackageDescriptor.Metadata.RuntimeDependencies .PackageDescriptor.Metadata.DevelopmentDependencies}} +

    {{.i18n.Tr "packages.dependencies"}}

    +
    +
    + {{template "package/content/rubygems_dependencies" dict "dependencies" .PackageDescriptor.Metadata.RuntimeDependencies "title" (.i18n.Tr "packages.rubygems.dependencies.runtime")}} + {{template "package/content/rubygems_dependencies" dict "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "packages.rubygems.dependencies.development")}} +
    +
    + {{end}} +{{end}} diff --git a/templates/repo/packages/content/rubygems_dependencies.tmpl b/templates/package/content/rubygems_dependencies.tmpl similarity index 100% rename from templates/repo/packages/content/rubygems_dependencies.tmpl rename to templates/package/content/rubygems_dependencies.tmpl diff --git a/templates/repo/packages/metadata/generic.tmpl b/templates/package/metadata/generic.tmpl similarity index 100% rename from templates/repo/packages/metadata/generic.tmpl rename to templates/package/metadata/generic.tmpl diff --git a/templates/repo/packages/metadata/maven.tmpl b/templates/package/metadata/maven.tmpl similarity index 71% rename from templates/repo/packages/metadata/maven.tmpl rename to templates/package/metadata/maven.tmpl index cf52b21c2e5c4..14a613be475b7 100644 --- a/templates/repo/packages/metadata/maven.tmpl +++ b/templates/package/metadata/maven.tmpl @@ -1,5 +1,5 @@ {{if eq .PackageDescriptor.Package.Type "maven"}} {{if .PackageDescriptor.Metadata.Name}}
    {{svg "octicon-note" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Name}}
    {{end}} - {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{range .PackageDescriptor.Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.details.project_site"}}
    {{end}} + {{range .PackageDescriptor.Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/npm.tmpl b/templates/package/metadata/npm.tmpl similarity index 51% rename from templates/repo/packages/metadata/npm.tmpl rename to templates/package/metadata/npm.tmpl index 7a17d81ea72f8..7a9548499e5f1 100644 --- a/templates/repo/packages/metadata/npm.tmpl +++ b/templates/package/metadata/npm.tmpl @@ -1,5 +1,5 @@ {{if eq .PackageDescriptor.Package.Type "npm"}} - {{if .PackageDescriptor.Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
    {{end}} - {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{if .PackageDescriptor.Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
    {{end}} + {{if .PackageDescriptor.Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.details.project_site"}}
    {{end}} + {{if .PackageDescriptor.Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/nuget.tmpl b/templates/package/metadata/nuget.tmpl similarity index 59% rename from templates/repo/packages/metadata/nuget.tmpl rename to templates/package/metadata/nuget.tmpl index 3357e68467340..d5a3e909b9573 100644 --- a/templates/repo/packages/metadata/nuget.tmpl +++ b/templates/package/metadata/nuget.tmpl @@ -1,4 +1,4 @@ {{if eq .PackageDescriptor.Package.Type "nuget"}} - {{if .PackageDescriptor.Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Authors}}
    {{end}} - {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} + {{if .PackageDescriptor.Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Authors}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.details.project_site"}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/pypi.tmpl b/templates/package/metadata/pypi.tmpl similarity index 51% rename from templates/repo/packages/metadata/pypi.tmpl rename to templates/package/metadata/pypi.tmpl index 33f940a79b5d4..5cdfbdfe669a0 100644 --- a/templates/repo/packages/metadata/pypi.tmpl +++ b/templates/package/metadata/pypi.tmpl @@ -1,5 +1,5 @@ {{if eq .PackageDescriptor.Package.Type "pypi"}} - {{if .PackageDescriptor.Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
    {{end}} - {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{if .PackageDescriptor.Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
    {{end}} + {{if .PackageDescriptor.Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.details.project_site"}}
    {{end}} + {{if .PackageDescriptor.Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
    {{end}} {{end}} diff --git a/templates/repo/packages/metadata/rubygems.tmpl b/templates/package/metadata/rubygems.tmpl similarity index 56% rename from templates/repo/packages/metadata/rubygems.tmpl rename to templates/package/metadata/rubygems.tmpl index c2fa7b8d82438..dff6830df374b 100644 --- a/templates/repo/packages/metadata/rubygems.tmpl +++ b/templates/package/metadata/rubygems.tmpl @@ -1,5 +1,5 @@ {{if eq .PackageDescriptor.Package.Type "rubygems"}} - {{range .PackageDescriptor.Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.}}
    {{end}} - {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "repo.packages.details.project_site"}}
    {{end}} - {{range .PackageDescriptor.Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} + {{range .PackageDescriptor.Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.details.project_site"}}
    {{end}} + {{range .PackageDescriptor.Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} {{end}} diff --git a/templates/package/settings.tmpl b/templates/package/settings.tmpl new file mode 100644 index 0000000000000..97159112edff6 --- /dev/null +++ b/templates/package/settings.tmpl @@ -0,0 +1,72 @@ +{{template "base/head" .}} +
    + {{template "user/overview/header" .}} +
    + {{template "base/alert" .}} +

    {{.PackageDescriptor.Package.Name}} ({{.PackageDescriptor.Version.Version}}) / {{.i18n.Tr "repo.settings"}}

    +

    + {{.i18n.Tr "packages.settings.link"}} +

    +
    +

    {{.i18n.Tr "packages.settings.link.description"}}

    + + {{template "base/disable_form_autofill"}} + {{.CsrfTokenHtml}} + +
    +
    + {{$repoID := 0}} + {{if .PackageDescriptor.Repository}} + {{$repoID = .PackageDescriptor.Repository.ID}} + {{end}} + + +
    {{.i18n.Tr "packages.settings.link.select"}}
    +
    + {{range .Repos}} +
    {{.Name}}
    + {{end}} +
    +
    +
    +
    + +
    + +
    + +

    + {{.i18n.Tr "repo.settings.danger_zone"}} +

    +
    +
    +
    + +
    +
    +
    {{.i18n.Tr "packages.settings.delete"}}
    +

    {{.i18n.Tr "packages.settings.delete.description"}}

    +
    + +
    +
    +
    +
    +{{template "base/footer" .}} diff --git a/templates/package/shared/list.tmpl b/templates/package/shared/list.tmpl new file mode 100644 index 0000000000000..697819f324d84 --- /dev/null +++ b/templates/package/shared/list.tmpl @@ -0,0 +1,46 @@ +
    + {{template "base/alert" .}} +
    +
    + + + +
    +
    +
    + {{range .PackageDescriptors}} +
  • +
    +
    + {{.Package.Name}} + {{svg .Package.Type.SVGName 16}} {{.Package.Type.Name}} +
    +
    + {{ $timeStr := TimeSinceUnix .Version.CreatedUnix $.Lang }} + {{$.i18n.Tr "packages.published_by" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) | Safe}} +
    +
    +
    • + {{else}} + {{if not .HasPackages}} +
    + {{svg "octicon-package" 32}} +

    {{.i18n.Tr "packages.empty"}}

    +

    {{.i18n.Tr "packages.empty.documentation" | Safe}}

    +
    + {{else}} +

    {{.i18n.Tr "packages.filter.no_result"}}

    + {{end}} + {{end}} + {{template "base/paginate" .}} +
    +
    diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl new file mode 100644 index 0000000000000..7cd15f0114a0e --- /dev/null +++ b/templates/package/view.tmpl @@ -0,0 +1,85 @@ +{{template "base/head" .}} +
    + {{template "user/overview/header" .}} +
    +
    +
    +
    +
    +

    {{.PackageDescriptor.Package.Name}} ({{.PackageDescriptor.Version.Version}})

    +
    +
    + {{$timeStr := TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.Lang}} + {{if .PackageDescriptor.Repository}} + {{.i18n.Tr "packages.published_by_in" $timeStr .PackageDescriptor.Creator.HomeLink (.PackageDescriptor.Creator.GetDisplayName | Escape) .PackageDescriptor.Repository.HTMLURL (.PackageDescriptor.Repository.FullName | Escape) | Safe}} + {{else}} + {{.i18n.Tr "packages.published_by" $timeStr .PackageDescriptor.Creator.HomeLink (.PackageDescriptor.Creator.GetDisplayName | Escape) | Safe}} + {{end}} +
    +
    +
    +
    + {{template "package/content/generic" .}} + {{template "package/content/nuget" .}} + {{template "package/content/npm" .}} + {{template "package/content/maven" .}} + {{template "package/content/pypi" .}} + {{template "package/content/rubygems" .}} +
    +
    +
    + {{.i18n.Tr "packages.details"}} +
    +
    {{svg .PackageDescriptor.Package.Type.SVGName 16 "mr-3"}} {{.PackageDescriptor.Package.Type.Name}}
    + {{if .PackageDescriptor.Repository}} +
    {{svg "octicon-repo" 16 "mr-3"}} {{.PackageDescriptor.Repository.FullName}}
    + {{end}} +
    {{svg "octicon-calendar" 16 "mr-3"}} {{.PackageDescriptor.Version.CreatedUnix.FormatDate}}
    +
    {{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
    + {{template "package/metadata/generic" .}} + {{template "package/metadata/nuget" .}} + {{template "package/metadata/npm" .}} + {{template "package/metadata/maven" .}} + {{template "package/metadata/pypi" .}} + {{template "package/metadata/rubygems" .}} +
    +
    + {{.i18n.Tr "packages.assets"}} ({{len .PackageDescriptor.Files}}) +
    + {{range .PackageDescriptor.Files}} +
    + {{.File.Name}} + {{FileSize .Blob.Size}} +
    + {{end}} +
    + {{if .OtherVersions}} +
    + {{.i18n.Tr "packages.versions"}} +
    + {{range .OtherVersions}} +
    + {{.Version}} + {{$.i18n.Tr "packages.versions.on"}} {{.CreatedUnix.FormatDate}} +
    + {{end}} +
    + {{end}} + {{if or .CanWritePackages .PackageDescriptor.Repository}} +
    +
    + {{if .PackageDescriptor.Repository}} +
    {{svg "octicon-issue-opened" 16 "mr-3"}} {{.i18n.Tr "repo.issues"}}
    + {{end}} + {{if .CanWritePackages}} +
    {{svg "octicon-tools" 16 "mr-3"}} {{.i18n.Tr "repo.settings"}}
    + {{end}} +
    + {{end}} +
    +
    +
    +
    +
    +
    +{{template "base/footer" .}} diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index fabb703744216..5dd15df33a852 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -132,11 +132,9 @@ {{end}} - {{ if and (not .UnitPackagesGlobalDisabled) (.Permission.CanRead $.UnitTypePackages)}} - - {{svg "octicon-package"}} {{.i18n.Tr "repo.packages"}} - - {{ end }} + + {{svg "octicon-package"}} {{.i18n.Tr "packages.title"}} + {{ if and (not .UnitProjectsGlobalDisabled) (.Permission.CanRead $.UnitTypeProjects)}} diff --git a/templates/repo/packages.tmpl b/templates/repo/packages.tmpl new file mode 100644 index 0000000000000..69bea014d7ec2 --- /dev/null +++ b/templates/repo/packages.tmpl @@ -0,0 +1,6 @@ +{{template "base/head" .}} +
    + {{template "repo/header" .}} + {{template "package/shared/list" .}} +
    +{{template "base/footer" .}} diff --git a/templates/repo/packages/content/generic.tmpl b/templates/repo/packages/content/generic.tmpl deleted file mode 100644 index f443edd17a483..0000000000000 --- a/templates/repo/packages/content/generic.tmpl +++ /dev/null @@ -1,14 +0,0 @@ -{{if eq .Package.Type 0}} -

    {{.i18n.Tr "repo.packages.installation"}}

    -
    -
    -
    - - -
    -
    - -
    -
    -
    -{{end}} diff --git a/templates/repo/packages/content/rubygems.tmpl b/templates/repo/packages/content/rubygems.tmpl deleted file mode 100644 index 214714a8457c6..0000000000000 --- a/templates/repo/packages/content/rubygems.tmpl +++ /dev/null @@ -1,40 +0,0 @@ -{{if eq .PackageDescriptor.Package.Type "rubygems"}} -

    {{.i18n.Tr "repo.packages.installation"}}

    -
    -
    -
    - - -
    -
    - - -
    -
    - -
    -
    -
    - {{if .PackageDescriptor.Metadata.Description}} -

    {{.i18n.Tr "repo.packages.about"}}

    -
    {{.PackageDescriptor.Metadata.Description}}
    - {{end}} - {{if or .PackageDescriptor.Metadata.RequiredRubyVersion .PackageDescriptor.Metadata.RequiredRubygemsVersion}} -

    {{.i18n.Tr "repo.packages.requirements"}}

    -
    - {{if .PackageDescriptor.Metadata.RequiredRubyVersion}}

    {{.i18n.Tr "repo.packages.rubygems.required.ruby"}}: {{range $i, $v := .PackageDescriptor.Metadata.RequiredRubyVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}} - {{if .PackageDescriptor.Metadata.RequiredRubygemsVersion}}

    {{.i18n.Tr "repo.packages.rubygems.required.rubygems"}}: {{range $i, $v := .PackageDescriptor.Metadata.RequiredRubygemsVersion}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}

    {{end}} -
    - {{end}} - {{if or .PackageDescriptor.Metadata.RuntimeDependencies .PackageDescriptor.Metadata.DevelopmentDependencies}} -

    {{.i18n.Tr "repo.packages.dependencies"}}

    -
    -
    - {{template "repo/packages/content/rubygems_dependencies" dict "dependencies" .PackageDescriptor.Metadata.RuntimeDependencies "title" (.i18n.Tr "repo.packages.rubygems.dependencies.runtime")}} - {{template "repo/packages/content/rubygems_dependencies" dict "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "repo.packages.rubygems.dependencies.development")}} -
    -
    - {{end}} -{{end}} diff --git a/templates/repo/packages/list.tmpl b/templates/repo/packages/list.tmpl deleted file mode 100644 index bac0d4afae318..0000000000000 --- a/templates/repo/packages/list.tmpl +++ /dev/null @@ -1,51 +0,0 @@ -{{template "base/head" .}} -
    - {{template "repo/header" .}} -
    - {{template "base/alert" .}} -
    -
    - - - -
    -
    -
    - {{range .PackageDescriptors}} -
  • -
    -
    - {{.Package.Name}} - {{svg .Package.Type.SVGName 16}} {{.Package.Type.Name}} -
    -
    - {{ $timeStr := TimeSinceUnix .Version.CreatedUnix $.Lang }} - {{$.i18n.Tr "repo.packages.published_by" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) | Safe}} -
    -
    -
    • - {{else}} - {{if not .HasPackages}} -
    - {{svg "octicon-package" 32}} -

    {{.i18n.Tr "repo.packages.empty"}}

    -

    {{.i18n.Tr "repo.packages.empty.documentation" | Safe}}

    -
    - {{else}} -

    {{.i18n.Tr "repo.packages.filter.no_result"}}

    - {{end}} - {{end}} - {{template "base/paginate" .}} -
    -
    -
    -{{template "base/footer" .}} diff --git a/templates/repo/packages/view.tmpl b/templates/repo/packages/view.tmpl deleted file mode 100644 index 28621913edbe4..0000000000000 --- a/templates/repo/packages/view.tmpl +++ /dev/null @@ -1,87 +0,0 @@ -{{template "base/head" .}} -
    - {{template "repo/header" .}} -
    -
    -
    -
    -
    -

    {{.PackageDescriptor.Package.Name}} ({{.PackageDescriptor.Version.Version}})

    -
    - {{ $timeStr := TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.Lang }} -
    {{.i18n.Tr "repo.packages.published_by" $timeStr .PackageDescriptor.Creator.HomeLink (.PackageDescriptor.Creator.GetDisplayName | Escape) | Safe}} {{svg .PackageDescriptor.Package.Type.SVGName}} {{.PackageDescriptor.Package.Type.Name}}
    -
    -
    -
    - {{template "repo/packages/content/generic" .}} - {{template "repo/packages/content/nuget" .}} - {{template "repo/packages/content/npm" .}} - {{template "repo/packages/content/maven" .}} - {{template "repo/packages/content/pypi" .}} - {{template "repo/packages/content/rubygems" .}} -
    -
    -
    - {{.i18n.Tr "repo.packages.details"}} -
    -
    {{svg "octicon-calendar" 16 "mr-3"}} {{.PackageDescriptor.Version.CreatedUnix.FormatDate}}
    -
    {{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
    - {{template "repo/packages/metadata/generic" .}} - {{template "repo/packages/metadata/nuget" .}} - {{template "repo/packages/metadata/npm" .}} - {{template "repo/packages/metadata/maven" .}} - {{template "repo/packages/metadata/pypi" .}} - {{template "repo/packages/metadata/rubygems" .}} -
    -
    - {{.i18n.Tr "repo.packages.assets"}} ({{len .PackageDescriptor.Files}}) -
    - {{range .PackageDescriptor.Files}} -
    - {{.File.Name}} - {{FileSize .Blob.Size}} -
    - {{end}} -
    - {{if .OtherVersions}} -
    - {{.i18n.Tr "repo.packages.versions"}} -
    - {{range .OtherVersions}} -
    - {{.Name}} - {{$.i18n.Tr "repo.packages.versions.on"}} {{.CreatedUnix.FormatDate}} -
    - {{end}} -
    - {{end}} - {{if .CanWritePackages}} -
    -
    - -
    - - {{end}} -
    -
    -
    -
    -
    -
    -{{template "base/footer" .}} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index a7be9f60ec199..211b7da8e7cfb 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -377,19 +377,6 @@
    - {{$isPackagesEnabled := .Repository.UnitEnabled $.UnitTypePackages}} -
    - - {{if .UnitTypePackages.UnitGlobalDisabled}} -
    - {{else}} -
    - {{end}} - - -
    -
    - {{$isProjectsEnabled := .Repository.UnitEnabled $.UnitTypeProjects}}
    diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index a705e55684ae9..d5f3c777b6d32 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -1870,6 +1870,169 @@ } } }, + "/packages/{owner}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "package" + ], + "summary": "Gets all packages of an owner", + "operationId": "listPackages", + "parameters": [ + { + "type": "string", + "description": "owner of the packages", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "page number of results to return (1-based)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "page size of results", + "name": "limit", + "in": "query" + }, + { + "enum": [ + "generic", + "nuget", + "npm", + "maven", + "pypi" + ], + "type": "string", + "description": "package type filter", + "name": "type", + "in": "query" + }, + { + "type": "string", + "description": "name filter", + "name": "q", + "in": "query" + } + ], + "responses": { + "200": { + "$ref": "#/responses/PackageList" + } + } + } + }, + "/packages/{owner}/{versionid}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "package" + ], + "summary": "Gets a package", + "operationId": "getPackage", + "parameters": [ + { + "type": "string", + "description": "owner of the package", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "integer", + "format": "int64", + "description": "id of the package version", + "name": "versionid", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/Package" + }, + "404": { + "$ref": "#/responses/notFound" + } + } + }, + "delete": { + "tags": [ + "package" + ], + "summary": "Delete a package", + "operationId": "deletePackage", + "parameters": [ + { + "type": "string", + "description": "owner of the package", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "integer", + "format": "int64", + "description": "id of the package version", + "name": "versionid", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "$ref": "#/responses/empty" + }, + "404": { + "$ref": "#/responses/notFound" + } + } + } + }, + "/packages/{owner}/{versionid}/files": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "package" + ], + "summary": "Gets all files of a package", + "operationId": "listPackageFiles", + "parameters": [ + { + "type": "string", + "description": "owner of the package", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "integer", + "format": "int64", + "description": "id of the package version", + "name": "versionid", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/PackageFileList" + }, + "404": { + "$ref": "#/responses/notFound" + } + } + } + }, "/repos/issues/search": { "get": { "produces": [ @@ -7134,197 +7297,6 @@ } } }, - "/repos/{owner}/{repo}/packages": { - "get": { - "produces": [ - "application/json" - ], - "tags": [ - "repository" - ], - "summary": "Gets all packages of a repository", - "operationId": "repoListPackages", - "parameters": [ - { - "type": "string", - "description": "owner of the repo", - "name": "owner", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "name of the repo", - "name": "repo", - "in": "path", - "required": true - }, - { - "type": "integer", - "description": "page number of results to return (1-based)", - "name": "page", - "in": "query" - }, - { - "type": "integer", - "description": "page size of results", - "name": "limit", - "in": "query" - }, - { - "enum": [ - "generic", - "nuget", - "npm", - "maven", - "pypi" - ], - "type": "string", - "description": "package type filter", - "name": "package_type", - "in": "query" - }, - { - "type": "string", - "description": "name filter", - "name": "q", - "in": "query" - } - ], - "responses": { - "200": { - "$ref": "#/responses/PackageList" - } - } - } - }, - "/repos/{owner}/{repo}/packages/{id}": { - "get": { - "produces": [ - "application/json" - ], - "tags": [ - "repository" - ], - "summary": "Gets a package", - "operationId": "repoGetPackage", - "parameters": [ - { - "type": "string", - "description": "owner of the repo", - "name": "owner", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "name of the repo", - "name": "repo", - "in": "path", - "required": true - }, - { - "type": "integer", - "format": "int64", - "description": "id of the package", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "$ref": "#/responses/Package" - }, - "404": { - "$ref": "#/responses/notFound" - } - } - }, - "delete": { - "tags": [ - "repository" - ], - "summary": "Delete a package from a repository", - "operationId": "repoDeletePackage", - "parameters": [ - { - "type": "string", - "description": "owner of the repo", - "name": "owner", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "name of the repo", - "name": "repo", - "in": "path", - "required": true - }, - { - "type": "integer", - "format": "int64", - "description": "id of the package", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "204": { - "$ref": "#/responses/empty" - }, - "404": { - "$ref": "#/responses/notFound" - } - } - } - }, - "/repos/{owner}/{repo}/packages/{id}/files": { - "get": { - "produces": [ - "application/json" - ], - "tags": [ - "repository" - ], - "summary": "Gets all files of a package", - "operationId": "repoListPackageFiles", - "parameters": [ - { - "type": "string", - "description": "owner of the repo", - "name": "owner", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "name of the repo", - "name": "repo", - "in": "path", - "required": true - }, - { - "type": "integer", - "format": "int64", - "description": "id of the package", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "$ref": "#/responses/PackageFileList" - }, - "404": { - "$ref": "#/responses/notFound" - } - } - } - }, "/repos/{owner}/{repo}/pulls": { "get": { "produces": [ @@ -16482,6 +16454,12 @@ "type": "string", "x-go-name": "Name" }, + "owner": { + "$ref": "#/definitions/User" + }, + "repository": { + "$ref": "#/definitions/Repository" + }, "type": { "type": "string", "x-go-name": "Type" diff --git a/templates/user/overview/header.tmpl b/templates/user/overview/header.tmpl new file mode 100644 index 0000000000000..666805c8f6847 --- /dev/null +++ b/templates/user/overview/header.tmpl @@ -0,0 +1,25 @@ +
    +
    +
    +
    +
    + {{avatar .ContextUser 32}} + {{.ContextUser.Name}} +
    +
    +
    +
    +
    +
    + + {{svg "octicon-repo"}} {{.i18n.Tr "user.repositories"}} + + {{if (not .UnitPackagesGlobalDisabled)}} + + {{svg "octicon-package"}} {{.i18n.Tr "packages.title"}} + + {{end}} +
    +
    +
    +
    diff --git a/templates/user/overview/packages.tmpl b/templates/user/overview/packages.tmpl new file mode 100644 index 0000000000000..8c3ca36c319aa --- /dev/null +++ b/templates/user/overview/packages.tmpl @@ -0,0 +1,6 @@ +{{template "base/head" .}} +
    + {{template "user/overview/header" .}} + {{template "package/shared/list" .}} +
    +{{template "base/footer" .}} diff --git a/templates/user/profile.tmpl b/templates/user/profile.tmpl index e5962db65ce68..be904bd647584 100644 --- a/templates/user/profile.tmpl +++ b/templates/user/profile.tmpl @@ -87,6 +87,9 @@ {{svg "octicon-repo"}} {{.i18n.Tr "user.repositories"}} + + {{svg "octicon-package"}} {{.i18n.Tr "packages.title"}} + {{svg "octicon-rss"}} {{.i18n.Tr "user.activity"}} From 77485e37ec5b6bf28d63f946599731ad5c86cd49 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 13 Nov 2021 00:05:49 +0000 Subject: [PATCH 063/130] Use GetUserByIDCtx. --- models/packages/descriptor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index e44d7c12173ad..37a6a311b826c 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -55,7 +55,7 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD if err != nil { return nil, err } - o, err := models.GetUserByID(p.OwnerID) + o, err := models.GetUserByIDCtx(ctx, p.OwnerID) if err != nil { return nil, err } @@ -63,7 +63,7 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD if err != nil && !models.IsErrRepoNotExist(err) { return nil, err } - creator, err := models.GetUserByID(pv.CreatorID) + creator, err := models.GetUserByIDCtx(ctx, pv.CreatorID) if err != nil { return nil, err } From e01b8a7798cfee86acbf5b1d198eb1dd65a387aa Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 13 Nov 2021 10:37:56 +0000 Subject: [PATCH 064/130] Use table for dependencies. --- options/locale/locale_en-US.ini | 5 ++- .../package/content/npm_dependencies.tmpl | 33 +++++++-------- templates/package/content/nuget.tmpl | 40 +++++++++---------- .../content/rubygems_dependencies.tmpl | 33 +++++++-------- templates/package/shared/list.tmpl | 6 ++- 5 files changed, 63 insertions(+), 54 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 461ae62c41534..4176ba5a314ec 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2854,7 +2854,7 @@ filter.type = Type filter.type.all = All filter.no_result = Your filter produced no results. published_by = Published %[1]s by %[3]s -published_by_in = Published %[1]s by %[3]s in %[5]s +published_by_in = Published %[1]s by %[3]s in %[5]s installation = Installation about = About this package requirements = Requirements @@ -2867,6 +2867,8 @@ details.license = License assets = Assets versions = Versions versions.on = on +dependency.id = Id +dependency.version = Version generic.download = Download package from the command line: generic.documentation = For more information on the generic registry, see the documentation. maven.registry = Setup this registry in your project pom.xml file: @@ -2877,6 +2879,7 @@ maven.documentation = For more information on the Maven registry, see the documentation. +nuget.dependency.framework = Target Framework npm.registry = Setup this registry in your project .npmrc file: npm.install = Install from the command line: npm.install2 = Install via package.json: diff --git a/templates/package/content/npm_dependencies.tmpl b/templates/package/content/npm_dependencies.tmpl index 764d1b68504e3..7d6afb84503fe 100644 --- a/templates/package/content/npm_dependencies.tmpl +++ b/templates/package/content/npm_dependencies.tmpl @@ -1,18 +1,19 @@ {{if .dependencies}} -
    -
    -
    {{.title}}
    -
    - {{range $dependency, $version := .dependencies}} -
    - {{svg "octicon-package-dependencies" 16 ""}} -
    -
    {{$dependency}}
    -
    {{$version}}
    -
    -
    - {{end}} -
    -
    -
    +

    {{.title}}

    + + + + + + + + + {{range $dependency, $version := .dependencies}} + + + + + {{end}} + +
    {{.i18n.Tr "packages.dependency.id"}}{{.i18n.Tr "packages.dependency.version"}}
    {{$dependency}}{{$version}}
    {{end}} diff --git a/templates/package/content/nuget.tmpl b/templates/package/content/nuget.tmpl index 50aa7b742a998..2df2222092e43 100644 --- a/templates/package/content/nuget.tmpl +++ b/templates/package/content/nuget.tmpl @@ -27,26 +27,26 @@ {{if .PackageDescriptor.Metadata.Dependencies}}

    {{.i18n.Tr "packages.dependencies"}}

    -
    - {{range $framework, $dependencies := .PackageDescriptor.Metadata.Dependencies}} -
    -
    -
    {{$framework}}
    -
    - {{range $dependencies}} -
    - {{svg "octicon-package-dependencies" 16 ""}} -
    -
    {{.ID}}
    -
    {{.Version}}
    -
    -
    - {{end}} -
    -
    -
    - {{end}} -
    + + + + + + + + + + {{range $framework, $dependencies := .PackageDescriptor.Metadata.Dependencies}} + {{range $dependencies}} + + + + + + {{end}} + {{end}} + +
    {{.i18n.Tr "packages.dependency.id"}}{{.i18n.Tr "packages.dependency.version"}}{{.i18n.Tr "packages.nuget.dependency.framework"}}
    {{.ID}}{{.Version}}{{$framework}}
    {{end}} {{end}} diff --git a/templates/package/content/rubygems_dependencies.tmpl b/templates/package/content/rubygems_dependencies.tmpl index 48f9e15ab8c02..d3114679ff812 100644 --- a/templates/package/content/rubygems_dependencies.tmpl +++ b/templates/package/content/rubygems_dependencies.tmpl @@ -1,18 +1,19 @@ {{if .dependencies}} -
    -
    -
    {{.title}}
    -
    - {{range .dependencies}} -
    - {{svg "octicon-package-dependencies" 16 ""}} -
    -
    {{.Name}}
    -
    {{range $i, $v := .Version}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}
    -
    -
    - {{end}} -
    -
    -
    +

    {{.title}}

    + + + + + + + + + {{range .dependencies}} + + + + + {{end}} + +
    {{.i18n.Tr "packages.dependency.id"}}{{.i18n.Tr "packages.dependency.version"}}
    {{.Name}}{{range $i, $v := .Version}}{{if gt $i 0}}, {{end}}{{$v.Restriction}}{{$v.Version}}{{end}}
    {{end}} diff --git a/templates/package/shared/list.tmpl b/templates/package/shared/list.tmpl index 697819f324d84..36bd57a4d68d3 100644 --- a/templates/package/shared/list.tmpl +++ b/templates/package/shared/list.tmpl @@ -26,7 +26,11 @@
    {{ $timeStr := TimeSinceUnix .Version.CreatedUnix $.Lang }} - {{$.i18n.Tr "packages.published_by" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) | Safe}} + {{if .Repository}} + {{$.i18n.Tr "packages.published_by_in" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) .Repository.HTMLURL (.Repository.FullName | Escape) | Safe}} + {{else}} + {{$.i18n.Tr "packages.published_by" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) | Safe}} + {{end}}
    From 82147e6449cfa9398c6dd6622c7439efbd755122 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 13 Nov 2021 11:10:28 +0000 Subject: [PATCH 065/130] make svg --- public/img/svg/gitea-nuget.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/img/svg/gitea-nuget.svg b/public/img/svg/gitea-nuget.svg index 519b60863807e..a5e38de3f6623 100644 --- a/public/img/svg/gitea-nuget.svg +++ b/public/img/svg/gitea-nuget.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file From 4d883cbf9b19356864e6f40a4f53f8b41cfba2e0 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 14 Nov 2021 21:33:20 +0000 Subject: [PATCH 066/130] Added support for unscoped npm packages. --- docs/content/doc/packages/npm.en-us.md | 9 ++- models/unittest/testdb.go | 2 + modules/packages/npm/creator.go | 23 +++---- modules/packages/npm/creator_test.go | 86 ++++++++++++++++++-------- routers/api/v1/api.go | 5 ++ routers/api/v1/packages/npm/npm.go | 16 ++++- templates/package/content/npm.tmpl | 6 +- 7 files changed, 105 insertions(+), 42 deletions(-) diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index 729e82bb5a54d..ce5cb2c526790 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -24,7 +24,7 @@ Publish [npm](https://www.npmjs.com/) packages for your user or organization. To work with the npm package registry, you need [Node.js and npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm/) or an other tool like [Yarn](https://classic.yarnpkg.com/en/docs/install). -Only [scoped](https://docs.npmjs.com/misc/scope/) packages are supported. +The registry supprots [scoped](https://docs.npmjs.com/misc/scope/) and unscoped packages. The following examples use the `npm` tool with the scope `@test`. @@ -50,6 +50,13 @@ npm config set @test:registry https://gitea.example.com/api/v1/packages/testuser npm config set -- '//gitea.example.com/api/v1/packages/testuser/npm/:_authToken' "personal_access_token" ``` +or without scope: + +```shell +npm config set registry https://gitea.example.com/api/v1/packages/testuser/npm/ +npm config set -- '//gitea.example.com/api/v1/packages/testuser/npm/:_authToken' "personal_access_token" +``` + ## Publish a package Publish a package by running the following command in your project: diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index b4f8c813af54e..cd4c47c623455 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -91,6 +91,8 @@ func MainTest(m *testing.M, pathToGiteaRoot string, fixtureFiles ...string) { setting.RepoArchive.Storage.Path = filepath.Join(setting.AppDataPath, "repo-archive") + setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages") + if err = storage.Init(); err != nil { fatalTestError("storage.Init: %v\n", err) } diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 72f5fb842640f..cc02874e55ee2 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -35,7 +35,7 @@ var ( ErrInvalidIntegrity = errors.New("Failed to validate integrity") ) -var nameMatch = regexp.MustCompile(`\A(@[^\/~'!\(\)\*]+?)[\/]([^_.][^\/~'!\(\)\*]+)\z`) +var nameMatch = regexp.MustCompile(`\A((@[^\s\/~'!\(\)\*]+?)[\/])?([^_.][^\s\/~'!\(\)\*]+)\z`) // Package represents a npm package type Package struct { @@ -153,17 +153,23 @@ func ParsePackage(r io.Reader) (*Package, error) { return nil, err } - for key, meta := range upload.Versions { + for _, meta := range upload.Versions { if !validateName(meta.Name) { return nil, ErrInvalidPackageName } - _, err := version.NewSemver(key) + v, err := version.NewSemver(meta.Version) if err != nil { return nil, ErrInvalidPackageVersion } + scope := "" + name := meta.Name nameParts := strings.SplitN(meta.Name, "/", 2) + if len(nameParts) == 2 { + scope = nameParts[0] + name = nameParts[1] + } if !validation.IsValidURL(meta.Homepage) { meta.Homepage = "" @@ -171,10 +177,10 @@ func ParsePackage(r io.Reader) (*Package, error) { p := &Package{ Name: meta.Name, - Version: meta.Version, + Version: v.String(), Metadata: Metadata{ - Scope: nameParts[0], - Name: nameParts[1], + Scope: scope, + Name: name, Description: meta.Description, Author: meta.Author.Name, License: meta.License, @@ -188,11 +194,6 @@ func ParsePackage(r io.Reader) (*Package, error) { }, } - names := strings.SplitN(p.Name, "/", 2) - name := names[0] - if len(names) == 2 { - name = names[1] - } p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version)) attachment := func() *PackageAttachment { diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index 04946cf310641..1d93134f161fd 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -17,7 +17,9 @@ import ( ) func TestParsePackage(t *testing.T) { - packageName := "@scope/test-package" + packageScope := "@scope" + packageName := "test-package" + packageFullName := packageScope+"/"+packageName packageVersion := "1.0.1-pre" packageAuthor := "KN4CK3R" packageDescription := "Test Description" @@ -57,20 +59,48 @@ func TestParsePackage(t *testing.T) { } test(t, " test ") + test(t, " test") + test(t, "test ") + test(t, "te st") test(t, "invalid/scope") test(t, "@invalid/_name") test(t, "@invalid/.name") }) + t.Run("ValidPackageName", func(t *testing.T) { + test := func(t *testing.T, name string) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: name, + Name: name, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: name, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidPackageVersion) + } + + test(t, "test") + test(t, "@scope/name") + test(t, packageFullName) + }) + t.Run("InvalidPackageVersion", func(t *testing.T) { version := "first-version" b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ - ID: packageName, - Name: packageName, + ID: packageFullName, + Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ version: { - Name: packageName, + Name: packageFullName, + Version: version, }, }, }, @@ -84,11 +114,12 @@ func TestParsePackage(t *testing.T) { t.Run("InvalidAttachment", func(t *testing.T) { b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ - ID: packageName, - Name: packageName, + ID: packageFullName, + Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ packageVersion: { - Name: packageName, + Name: packageFullName, + Version: packageVersion, }, }, }, @@ -103,14 +134,15 @@ func TestParsePackage(t *testing.T) { }) t.Run("InvalidData", func(t *testing.T) { - filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ - ID: packageName, - Name: packageName, + ID: packageFullName, + Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ packageVersion: { - Name: packageName, + Name: packageFullName, + Version: packageVersion, }, }, }, @@ -127,14 +159,15 @@ func TestParsePackage(t *testing.T) { }) t.Run("InvalidIntegrity", func(t *testing.T) { - filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ - ID: packageName, - Name: packageName, + ID: packageFullName, + Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ packageVersion: { - Name: packageName, + Name: packageFullName, + Version: packageVersion, Dist: PackageDistribution{ Integrity: "sha512-test==", }, @@ -154,14 +187,15 @@ func TestParsePackage(t *testing.T) { }) t.Run("InvalidIntegrity2", func(t *testing.T) { - filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ - ID: packageName, - Name: packageName, + ID: packageFullName, + Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ packageVersion: { - Name: packageName, + Name: packageFullName, + Version: packageVersion, Dist: PackageDistribution{ Integrity: integrity, }, @@ -181,14 +215,14 @@ func TestParsePackage(t *testing.T) { }) t.Run("Valid", func(t *testing.T) { - filename := fmt.Sprintf("%s-%s.tgz", packageName, packageVersion) + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) b, _ := json.Marshal(packageUpload{ PackageMetadata: PackageMetadata{ - ID: packageName, - Name: packageName, + ID: packageFullName, + Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ packageVersion: { - Name: packageName, + Name: packageFullName, Version: packageVersion, Description: packageDescription, Author: User{Name: packageAuthor}, @@ -215,11 +249,13 @@ func TestParsePackage(t *testing.T) { assert.NotNil(t, p) assert.NoError(t, err) - assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageFullName, p.Name) assert.Equal(t, packageVersion, p.Version) - assert.Equal(t, fmt.Sprintf("%s-%s.tgz", strings.Split(packageName, "/")[1], packageVersion), p.Filename) + assert.Equal(t, fmt.Sprintf("%s-%s.tgz", strings.Split(packageFullName, "/")[1], packageVersion), p.Filename) b, _ = base64.StdEncoding.DecodeString(data) assert.Equal(t, b, p.Data) + assert.Equal(t, packageName, p.Metadata.Name) + assert.Equal(t, packageScope, p.Metadata.Scope) assert.Equal(t, packageDescription, p.Metadata.Description) assert.Equal(t, packageDescription, p.Metadata.Readme) assert.Equal(t, packageAuthor, p.Metadata.Author) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c27d70894246d..c70a14ea005c0 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1039,6 +1039,11 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { }, reqBasicAuth(), reqPackageAccess(models.AccessModeWrite)) }) m.Group("/npm", func() { + m.Group("/@{scope}/{id}", func() { + m.Get("", npm.PackageMetadata) + m.Put("", reqToken(), reqPackageAccess(models.AccessModeWrite), npm.UploadPackage) + m.Get("/-/{version}/{filename}", npm.DownloadPackageFile) + }) m.Group("/{id}", func() { m.Get("", npm.PackageMetadata) m.Put("", reqToken(), reqPackageAccess(models.AccessModeWrite), npm.UploadPackage) diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 1f5da23a3bc16..86447fd07fb44 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -6,6 +6,7 @@ package npm import ( "bytes" + "fmt" "net/http" "net/url" @@ -26,9 +27,20 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { }) } +// packageNameFromParams gets the package name from the url parameters +// Variations: /name/, /@scope/name/, /@scope%2Fname/ +func packageNameFromParams(ctx *context.APIContext) (string, error) { + scope := ctx.Params("scope") + id := ctx.Params("id") + if scope != "" { + return fmt.Sprintf("@%s/%s", scope, id), nil + } + return url.QueryUnescape(id) +} + // PackageMetadata returns the metadata for a single package func PackageMetadata(ctx *context.APIContext) { - packageName, err := url.QueryUnescape(ctx.Params("id")) + packageName, err := packageNameFromParams(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) return @@ -60,7 +72,7 @@ func PackageMetadata(ctx *context.APIContext) { // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.APIContext) { - packageName, err := url.QueryUnescape(ctx.Params("id")) + packageName, err := packageNameFromParams(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) return diff --git a/templates/package/content/npm.tmpl b/templates/package/content/npm.tmpl index 624e4ca338aa5..6a54efa163ced 100644 --- a/templates/package/content/npm.tmpl +++ b/templates/package/content/npm.tmpl @@ -4,15 +4,15 @@
    - +
    - - + +
    From 6335e68ec660b8e774aa347ca449fe0f031293fd Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 15 Nov 2021 17:10:18 +0000 Subject: [PATCH 067/130] Add support for npm dist tags. --- docs/content/doc/packages/npm.en-us.md | 27 +++- models/migrations/v202.go | 2 +- models/packages/package_version_property.go | 62 +++++--- modules/packages/npm/creator.go | 10 +- modules/packages/npm/creator_test.go | 12 +- modules/packages/npm/metadata.go | 3 + options/locale/locale_en-US.ini | 1 + routers/api/v1/api.go | 14 ++ routers/api/v1/packages/npm/api.go | 10 +- routers/api/v1/packages/npm/npm.go | 160 +++++++++++++++++++- routers/web/user/package.go | 2 +- services/packages/packages.go | 8 +- templates/package/metadata/npm.tmpl | 3 + 13 files changed, 271 insertions(+), 43 deletions(-) diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index ce5cb2c526790..7f4a02b776bec 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -72,16 +72,37 @@ You cannot publish a package if a package of the same name and version already e To install a package from the package registry, execute the following command: ```shell -npm install {scope}/{package_name} +npm install {package_name} ``` | Parameter | Description | | -------------- | ----------- | -| `scope` | The scope of the packages. | | `package_name` | The package name. | For example: ```shell npm install @test/test_package -``` \ No newline at end of file +``` + +## Tag a package + +The registry supports [version tags](https://docs.npmjs.com/adding-dist-tags-to-packages/) which can be managed by `npm dist-tag`: + +```shell +npm dist-tag add {package_name}@{version} {tag} +``` + +| Parameter | Description | +| -------------- | ----------- | +| `package_name` | The package name. | +| `version` | The version of the package. | +| `tag` | The tag name. | + +For example: + +```shell +npm dist-tag add test_package@1.0.2 release +``` + +The tag name must not be a valid version. All tag names which are parsable as a version are rejected. diff --git a/models/migrations/v202.go b/models/migrations/v202.go index 68b66500bc5e0..3850bc85b1cdf 100644 --- a/models/migrations/v202.go +++ b/models/migrations/v202.go @@ -44,7 +44,7 @@ func addPackageTables(x *xorm.Engine) error { ID int64 `xorm:"pk autoincr"` VersionID int64 `xorm:"INDEX NOT NULL"` Name string `xorm:"INDEX NOT NULL"` - Value string + Value string `xorm:"INDEX NOT NULL"` } if err := x.Sync2(new(PackageVersionProperty)); err != nil { diff --git a/models/packages/package_version_property.go b/models/packages/package_version_property.go index 3590c1a168884..7f384e9f9d085 100644 --- a/models/packages/package_version_property.go +++ b/models/packages/package_version_property.go @@ -8,6 +8,8 @@ import ( "context" "code.gitea.io/gitea/models/db" + + "xorm.io/builder" ) func init() { @@ -17,34 +19,21 @@ func init() { // PackageVersionProperty represents a property of a package version type PackageVersionProperty struct { ID int64 `xorm:"pk autoincr"` - VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string `xorm:"UNIQUE(s) INDEX NOT NULL"` + VersionID int64 `xorm:"INDEX NOT NULL"` + Name string `xorm:"INDEX NOT NULL"` Value string `xorm:"INDEX NOT NULL"` } -// SetVersionProperties sets the properties of the version -func SetVersionProperties(ctx context.Context, versionID int64, properties map[string]string) error { - e := db.GetEngine(ctx) - - for name, value := range properties { - pvp := &PackageVersionProperty{ - VersionID: versionID, - Name: name, - Value: value, - } - - count, err := e.Where("version_id = ? AND name = ?", versionID, pvp.Name).Update(pvp) - if err != nil { - return err - } - if count != 1 { - if _, err = e.Insert(pvp); err != nil { - return err - } - } +// InsertVersionProperty creates a property +func InsertVersionProperty(ctx context.Context, versionID int64, name, value string) (*PackageVersionProperty, error) { + pvp := &PackageVersionProperty{ + VersionID: versionID, + Name: name, + Value: value, } - return nil + _, err := db.GetEngine(ctx).Insert(pvp) + return pvp, err } // GetVersionProperties gets all properties of the version @@ -53,8 +42,35 @@ func GetVersionProperties(ctx context.Context, versionID int64) ([]*PackageVersi return pvps, db.GetEngine(ctx).Where("version_id = ?", versionID).Find(&pvps) } +// GetVersionPropertiesByName gets all properties with a specific name of the version +func GetVersionPropertiesByName(ctx context.Context, versionID int64, name string) ([]*PackageVersionProperty, error) { + pvps := make([]*PackageVersionProperty, 0, 10) + return pvps, db.GetEngine(ctx).Where("version_id = ? AND name = ?", versionID, name).Find(&pvps) +} + +// FindVersionsByPropertyNameAndValue gets all package versions which are associated with a specific property + value +func FindVersionsByPropertyNameAndValue(ctx context.Context, packageID int64, name, value string) ([]*PackageVersion, error) { + var cond builder.Cond = builder.Eq{ + "package_version_property.name": name, + "package_version_property.value": value, + "package_version.package_id": packageID, + } + + pvs := make([]*PackageVersion, 0, 5) + return pvs, db.GetEngine(ctx). + Where(cond). + Join("INNER", "package_version_property", "package_version_property.version_id = package_version.id"). + Find(&pvs) +} + // DeleteVersionPropertiesByVersionID deletes all properties of the version func DeleteVersionPropertiesByVersionID(ctx context.Context, versionID int64) error { _, err := db.GetEngine(ctx).Where("version_id = ?", versionID).Delete(&PackageVersionProperty{}) return err } + +// DeleteVersionPropertyByID deletes a property +func DeleteVersionPropertyByID(ctx context.Context, propertyID int64) error { + _, err := db.GetEngine(ctx).ID(propertyID).Delete(&PackageVersionProperty{}) + return err +} diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index cc02874e55ee2..88ce55ecdbef8 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -41,6 +41,7 @@ var nameMatch = regexp.MustCompile(`\A((@[^\s\/~'!\(\)\*]+?)[\/])?([^_.][^\s\/~' type Package struct { Name string Version string + DistTags []string Metadata Metadata Filename string Data []byte @@ -176,8 +177,9 @@ func ParsePackage(r io.Reader) (*Package, error) { } p := &Package{ - Name: meta.Name, - Version: v.String(), + Name: meta.Name, + Version: v.String(), + DistTags: make([]string, 0, 1), Metadata: Metadata{ Scope: scope, Name: name, @@ -194,6 +196,10 @@ func ParsePackage(r io.Reader) (*Package, error) { }, } + for tag := range upload.DistTags { + p.DistTags = append(p.DistTags, tag) + } + p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version)) attachment := func() *PackageAttachment { diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index 1d93134f161fd..d34ebbb1c06fe 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -19,7 +19,7 @@ import ( func TestParsePackage(t *testing.T) { packageScope := "@scope" packageName := "test-package" - packageFullName := packageScope+"/"+packageName + packageFullName := packageScope + "/" + packageName packageVersion := "1.0.1-pre" packageAuthor := "KN4CK3R" packageDescription := "Test Description" @@ -99,7 +99,7 @@ func TestParsePackage(t *testing.T) { Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ version: { - Name: packageFullName, + Name: packageFullName, Version: version, }, }, @@ -118,7 +118,7 @@ func TestParsePackage(t *testing.T) { Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ packageVersion: { - Name: packageFullName, + Name: packageFullName, Version: packageVersion, }, }, @@ -141,7 +141,7 @@ func TestParsePackage(t *testing.T) { Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ packageVersion: { - Name: packageFullName, + Name: packageFullName, Version: packageVersion, }, }, @@ -166,7 +166,7 @@ func TestParsePackage(t *testing.T) { Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ packageVersion: { - Name: packageFullName, + Name: packageFullName, Version: packageVersion, Dist: PackageDistribution{ Integrity: "sha512-test==", @@ -194,7 +194,7 @@ func TestParsePackage(t *testing.T) { Name: packageFullName, Versions: map[string]*PackageMetadataVersion{ packageVersion: { - Name: packageFullName, + Name: packageFullName, Version: packageVersion, Dist: PackageDistribution{ Integrity: integrity, diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index cb6589ae7895a..9543e94f9b646 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -4,6 +4,9 @@ package npm +// TagProperty is the name of the property for tag management +const TagProperty = "npm.tag" + // Metadata represents the metadata of a npm package type Metadata struct { Scope string `json:"scope"` diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 4176ba5a314ec..1695578f4febb 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2888,6 +2888,7 @@ npm.dependencies = Dependencies npm.dependencies.development = Development Dependencies npm.dependencies.peer = Peer Dependencies npm.dependencies.optional = Optional Dependencies +npm.details.tag = Tag pypi.requires = Requires Python pypi.install = Install from the command line: pypi.documentation = For more information on the PyPI registry, see the documentation. diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c70a14ea005c0..9ef016e14d6d9 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1049,6 +1049,20 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { m.Put("", reqToken(), reqPackageAccess(models.AccessModeWrite), npm.UploadPackage) m.Get("/-/{version}/{filename}", npm.DownloadPackageFile) }) + m.Group("/-/package/@{scope}/{id}/dist-tags", func() { + m.Get("", npm.ListPackageTags) + m.Group("/{tag}", func() { + m.Put("", npm.AddPackageTag) + m.Delete("", npm.DeletePackageTag) + }, reqToken(), reqPackageAccess(models.AccessModeWrite)) + }) + m.Group("/-/package/{id}/dist-tags", func() { + m.Get("", npm.ListPackageTags) + m.Group("/{tag}", func() { + m.Put("", npm.AddPackageTag) + m.Delete("", npm.DeletePackageTag) + }, reqToken(), reqPackageAccess(models.AccessModeWrite)) + }) }) m.Group("/pypi", func() { m.Post("/", reqBasicAuth(), reqPackageAccess(models.AccessModeWrite), pypi.UploadPackageFile) diff --git a/routers/api/v1/packages/npm/api.go b/routers/api/v1/packages/npm/api.go index fea3827ab4e89..a95b60a8caac7 100644 --- a/routers/api/v1/packages/npm/api.go +++ b/routers/api/v1/packages/npm/api.go @@ -21,15 +21,19 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_models.Pa }) versions := make(map[string]*npm_module.PackageMetadataVersion) + distTags := make(map[string]string) for _, pd := range pds { versions[pd.SemVer.String()] = createPackageMetadataVersion(registryURL, pd) + + for _, pvp := range pd.Properties { + if pvp.Name == npm_module.TagProperty { + distTags[pvp.Value] = pd.Version.Version + } + } } latest := pds[len(pds)-1] - distTags := make(map[string]string) - distTags["latest"] = latest.Version.Version - metadata := latest.Metadata.(*npm_module.Metadata) return &npm_module.PackageMetadata{ diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 86447fd07fb44..502c11cf89065 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -6,10 +6,14 @@ package npm import ( "bytes" + "errors" "fmt" + "io/ioutil" "net/http" "net/url" + "strings" + "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" package_module "code.gitea.io/gitea/modules/packages" @@ -17,6 +21,13 @@ import ( "code.gitea.io/gitea/modules/setting" package_router "code.gitea.io/gitea/routers/api/v1/packages" packages_service "code.gitea.io/gitea/services/packages" + + "github.com/hashicorp/go-version" +) + +var ( + // errInvalidTagName indicates an invalid tag name + errInvalidTagName = errors.New("The tag name is invalid") ) func apiError(ctx *context.APIContext, status int, obj interface{}) { @@ -117,7 +128,7 @@ func UploadPackage(ctx *context.APIContext) { } defer buf.Close() - _, _, err = packages_service.CreatePackageAndAddFile( + pv, _, err := packages_service.CreatePackageAndAddFile( &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, @@ -144,5 +155,152 @@ func UploadPackage(ctx *context.APIContext) { return } + for _, tag := range npmPackage.DistTags { + if err := setPackageTag(tag, pv, false); err != nil { + if err == errInvalidTagName { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + ctx.PlainText(http.StatusCreated, nil) } + +// ListPackageTags returns all tags for a package +func ListPackageTags(ctx *context.APIContext) { + packageName, err := packageNameFromParams(ctx) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNpm, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + tags := make(map[string]string) + for _, pv := range pvs { + pvps, err := packages.GetVersionPropertiesByName(ctx, pv.ID, npm_module.TagProperty) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + for _, pvp := range pvps { + tags[pvp.Value] = pv.Version + } + } + + ctx.JSON(http.StatusOK, tags) +} + +// AddPackageTag adds a tag to the package +func AddPackageTag(ctx *context.APIContext) { + packageName, err := packageNameFromParams(ctx) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + body, err := ioutil.ReadAll(ctx.Req.Body) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + version := strings.Trim(string(body), "\"") // is as "version" in the body + + pv, err := packages.GetVersionByNameAndVersion(ctx, ctx.ContextUser.ID, packages.TypeNpm, packageName, version) + if err != nil { + if err == packages.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if err := setPackageTag(ctx.Params("tag"), pv, false); err != nil { + if err == errInvalidTagName { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } +} + +// DeletePackageTag deletes a package tag +func DeletePackageTag(ctx *context.APIContext) { + packageName, err := packageNameFromParams(ctx) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + pvs, err := packages.GetVersionsByPackageName(ctx.ContextUser.ID, packages.TypeNpm, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pvs) != 0 { + if err := setPackageTag(ctx.Params("tag"), pvs[0], true); err != nil { + if err == errInvalidTagName { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + } +} + +func setPackageTag(tag string, pv *packages.PackageVersion, deleteOnly bool) error { + if tag == "" { + return errInvalidTagName + } + _, err := version.NewVersion(tag) + if err == nil { + return errInvalidTagName + } + + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + pvs, err := packages.FindVersionsByPropertyNameAndValue(ctx, pv.PackageID, npm_module.TagProperty, tag) + if err != nil { + return err + } + + if len(pvs) == 1 { + pvps, err := packages.GetVersionPropertiesByName(ctx, pvs[0].ID, npm_module.TagProperty) + if err != nil { + return err + } + + for _, pvp := range pvps { + if pvp.Value == tag { + if err := packages.DeleteVersionPropertyByID(ctx, pvp.ID); err != nil { + return err + } + break + } + } + } + + if !deleteOnly { + _, err = packages.InsertVersionProperty(ctx, pv.ID, npm_module.TagProperty, tag) + if err != nil { + return err + } + } + + return committer.Commit() +} diff --git a/routers/web/user/package.go b/routers/web/user/package.go index de079cfd45598..c2c1fa3f0e9c4 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -98,7 +98,7 @@ func ViewPackage(ctx *context.Context) { return } sort.Slice(otherVersions, func(i, j int) bool { - return otherVersions[i].Version > otherVersions[j].Version + return otherVersions[i].CreatedUnix > otherVersions[j].CreatedUnix }) ctx.Data["OtherVersions"] = otherVersions diff --git a/services/packages/packages.go b/services/packages/packages.go index 61da546ae3eca..9cc8d7db04792 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -138,9 +138,11 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } } - if err := packages_models.SetVersionProperties(ctx, pv.ID, pvci.Properties); err != nil { - log.Error("Error setting package version properties: %v", err) - return nil, false, err + for name, value := range pvci.Properties { + if _, err := packages_models.InsertVersionProperty(ctx, pv.ID, name, value); err != nil { + log.Error("Error setting package version property: %v", err) + return nil, false, err + } } return pv, created, nil diff --git a/templates/package/metadata/npm.tmpl b/templates/package/metadata/npm.tmpl index 7a9548499e5f1..3279f9edbf197 100644 --- a/templates/package/metadata/npm.tmpl +++ b/templates/package/metadata/npm.tmpl @@ -2,4 +2,7 @@ {{if .PackageDescriptor.Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
    {{end}} {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.details.project_site"}}
    {{end}} {{if .PackageDescriptor.Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
    {{end}} + {{range .PackageDescriptor.Properties}} + {{if eq .Name "npm.tag"}}
    {{svg "octicon-versions" 16 "mr-3"}} {{.Value}}
    {{end}} + {{end}} {{end}} From d16c8940af45fa5d969a3e698aa69c28fd52505a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 15 Nov 2021 23:06:51 +0000 Subject: [PATCH 068/130] Added tests for npm tags. --- integrations/api_packages_npm_test.go | 76 ++++++++++++++++++++++++++- modules/packages/npm/creator_test.go | 5 ++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index 0dd473e89fbfb..71f1442abaf8e 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -27,6 +27,8 @@ func TestPackageNpm(t *testing.T) { packageName := "@scope/test-package" packageVersion := "1.0.1-pre" + packageTag := "latest" + packageTag2 := "release" packageAuthor := "KN4CK3R" packageDescription := "Test Description" @@ -35,6 +37,9 @@ func TestPackageNpm(t *testing.T) { "_id": "` + packageName + `", "name": "` + packageName + `", "description": "` + packageDescription + `", + "dist-tags": { + "` + packageTag + `": "` + packageVersion + `" + }, "versions": { "` + packageVersion + `": { "name": "` + packageName + `", @@ -57,6 +62,7 @@ func TestPackageNpm(t *testing.T) { }` root := fmt.Sprintf("/api/v1/packages/%s/npm/%s", user.Name, url.QueryEscape(packageName)) + tagsRoot := fmt.Sprintf("/api/v1/packages/%s/npm/-/package/%s/dist-tags", user.Name, url.QueryEscape(packageName)) filename := fmt.Sprintf("%s-%s.tgz", strings.Split(packageName, "/")[1], packageVersion) t.Run("Upload", func(t *testing.T) { @@ -76,6 +82,9 @@ func TestPackageNpm(t *testing.T) { assert.IsType(t, &npm.Metadata{}, pd.Metadata) assert.Equal(t, packageName, pd.Package.Name) assert.Equal(t, packageVersion, pd.Version.Version) + assert.Len(t, pd.Properties, 1) + assert.Equal(t, npm.TagProperty, pd.Properties[0].Name) + assert.Equal(t, packageTag, pd.Properties[0].Value) pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) @@ -125,8 +134,8 @@ func TestPackageNpm(t *testing.T) { assert.Equal(t, packageName, result.ID) assert.Equal(t, packageName, result.Name) assert.Equal(t, packageDescription, result.Description) - assert.Contains(t, result.DistTags, "latest") - assert.Equal(t, packageVersion, result.DistTags["latest"]) + assert.Contains(t, result.DistTags, packageTag) + assert.Equal(t, packageVersion, result.DistTags[packageTag]) assert.Equal(t, packageAuthor, result.Author.Name) assert.Contains(t, result.Versions, packageVersion) pmv := result.Versions[packageVersion] @@ -138,4 +147,67 @@ func TestPackageNpm(t *testing.T) { assert.Equal(t, "aaa7eaf852a948b0aa05afeda35b1badca155d90", pmv.Dist.Shasum) assert.Equal(t, fmt.Sprintf("%s%s/-/%s/%s", setting.AppURL, root[1:], packageVersion, filename), pmv.Dist.Tarball) }) + + t.Run("AddTag", func(t *testing.T) { + defer PrintCurrentTest(t)() + + test := func(t *testing.T, status int, tag, version string) { + req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/%s", tagsRoot, tag), strings.NewReader(`"`+version+`"`)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, status) + } + + test(t, http.StatusBadRequest, "1.0", packageVersion) + test(t, http.StatusBadRequest, "v1.0", packageVersion) + test(t, http.StatusNotFound, packageTag2, "1.2") + test(t, http.StatusOK, packageTag, packageVersion) + test(t, http.StatusOK, packageTag2, packageVersion) + }) + + t.Run("ListTags", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", tagsRoot) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result map[string]string + DecodeJSON(t, resp, &result) + + assert.Len(t, result, 2) + assert.Contains(t, result, packageTag) + assert.Equal(t, packageVersion, result[packageTag]) + assert.Contains(t, result, packageTag2) + assert.Equal(t, packageVersion, result[packageTag2]) + }) + + t.Run("PackageMetadataDistTags", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", root) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result npm.PackageMetadata + DecodeJSON(t, resp, &result) + + assert.Len(t, result.DistTags, 2) + assert.Contains(t, result.DistTags, packageTag) + assert.Equal(t, packageVersion, result.DistTags[packageTag]) + assert.Contains(t, result.DistTags, packageTag2) + assert.Equal(t, packageVersion, result.DistTags[packageTag2]) + }) + + t.Run("DeleteTag", func(t *testing.T) { + defer PrintCurrentTest(t)() + + test := func(t *testing.T, status int, tag string) { + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/%s", tagsRoot, tag)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, status) + } + + test(t, http.StatusOK, "dummy") + test(t, http.StatusOK, packageTag2) + }) } diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index d34ebbb1c06fe..64ae6238f3baf 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -21,6 +21,7 @@ func TestParsePackage(t *testing.T) { packageName := "test-package" packageFullName := packageScope + "/" + packageName packageVersion := "1.0.1-pre" + packageTag := "latest" packageAuthor := "KN4CK3R" packageDescription := "Test Description" data := "H4sIAAAAAAAA/ytITM5OTE/VL4DQelnF+XkMVAYGBgZmJiYK2MRBwNDcSIHB2NTMwNDQzMwAqA7IMDUxA9LUdgg2UFpcklgEdAql5kD8ogCnhwio5lJQUMpLzE1VslJQcihOzi9I1S9JLS7RhSYIJR2QgrLUouLM/DyQGkM9Az1D3YIiqExKanFyUWZBCVQ2BKhVwQVJDKwosbQkI78IJO/tZ+LsbRykxFXLNdA+HwWjYBSMgpENACgAbtAACAAA" @@ -220,6 +221,9 @@ func TestParsePackage(t *testing.T) { PackageMetadata: PackageMetadata{ ID: packageFullName, Name: packageFullName, + DistTags: map[string]string{ + packageTag: packageVersion, + }, Versions: map[string]*PackageMetadataVersion{ packageVersion: { Name: packageFullName, @@ -251,6 +255,7 @@ func TestParsePackage(t *testing.T) { assert.Equal(t, packageFullName, p.Name) assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, []string{packageTag}, p.DistTags) assert.Equal(t, fmt.Sprintf("%s-%s.tgz", strings.Split(packageFullName, "/")[1], packageVersion), p.Filename) b, _ = base64.StdEncoding.DecodeString(data) assert.Equal(t, b, p.Data) From 9e1c5c13929908c1e11e40e0eb7c98fd54e8ae35 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 16 Nov 2021 16:15:00 +0200 Subject: [PATCH 069/130] Unlink packages if repository gets deleted. --- models/repo.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/models/repo.go b/models/repo.go index 614b86c597a93..6c69f870b62af 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1599,6 +1599,12 @@ func DeleteRepository(doer *User, uid, repoID int64) error { return err } + // Unlink packages + // TODO Currently raw sql because of import cycle + if _, err := sess.Exec("UPDATE `package` SET `repo_id` = 0 WHERE `repo_id` = ?", repo.ID); err != nil { + return err + } + // Remove archives var archives []*RepoArchiver if err = sess.Where("repo_id=?", repoID).Find(&archives); err != nil { From bd9a5167f006933c8fb2b43b06f2906b218cd4fb Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 16 Nov 2021 15:45:53 +0000 Subject: [PATCH 070/130] Prevent user/org delete if a packages exist. --- models/error.go | 15 +++++++++++++++ models/org.go | 11 ++++++++++- models/user.go | 11 ++++++++++- options/locale/locale_en-US.ini | 3 +++ routers/api/v1/admin/user.go | 3 ++- routers/web/admin/users.go | 5 +++++ routers/web/org/setting.go | 3 +++ routers/web/user/setting/account.go | 3 +++ 8 files changed, 51 insertions(+), 3 deletions(-) diff --git a/models/error.go b/models/error.go index db0fce0ce1e5a..fda0aebee3888 100644 --- a/models/error.go +++ b/models/error.go @@ -200,6 +200,21 @@ func (err ErrUserHasOrgs) Error() string { return fmt.Sprintf("user still has membership of organizations [uid: %d]", err.UID) } +// ErrUserOwnPackages represents a "UserOwnPackages" kind of error. +type ErrUserOwnPackages struct { + UID int64 +} + +// IsErrUserOwnPackages checks if an error is an ErrUserOwnPackages. +func IsErrUserOwnPackages(err error) bool { + _, ok := err.(ErrUserOwnPackages) + return ok +} + +func (err ErrUserOwnPackages) Error() string { + return fmt.Sprintf("user still has ownership of packages [uid: %d]", err.UID) +} + // ErrUserNotAllowedCreateOrg represents a "UserNotAllowedCreateOrg" kind of error. type ErrUserNotAllowedCreateOrg struct{} diff --git a/models/org.go b/models/org.go index 1840cae9fa0d3..ea9f475d0df2f 100644 --- a/models/org.go +++ b/models/org.go @@ -268,7 +268,7 @@ func DeleteOrganization(org *User) (err error) { } if err = deleteOrg(sess, org); err != nil { - if IsErrUserOwnRepos(err) { + if IsErrUserOwnRepos(err) || IsErrUserOwnPackages(err) { return err } else if err != nil { return fmt.Errorf("deleteOrg: %v", err) @@ -287,6 +287,15 @@ func deleteOrg(e *xorm.Session, u *User) error { return ErrUserOwnRepos{UID: u.ID} } + // Check ownership of packages. + // TODO: SQL because of import cycle + count, err = e.Table("package").Where("owner_id = ?", u.ID).Count() + if err != nil { + return fmt.Errorf("Get Package Count: %v", err) + } else if count > 0 { + return ErrUserOwnPackages{UID: u.ID} + } + if err := deleteBeans(e, &Team{OrgID: u.ID}, &OrgUser{OrgID: u.ID}, diff --git a/models/user.go b/models/user.go index 12035dbe42375..4e0c94b85f3f2 100644 --- a/models/user.go +++ b/models/user.go @@ -1166,6 +1166,15 @@ func deleteUser(e db.Engine, u *User) error { return ErrUserHasOrgs{UID: u.ID} } + // Check ownership of packages. + // TODO: SQL because of import cycle + count, err = e.Table("package").Where("owner_id = ?", u.ID).Count() + if err != nil { + return fmt.Errorf("Get Package Count: %v", err) + } else if count > 0 { + return ErrUserOwnPackages{UID: u.ID} + } + // ***** START: Watch ***** watchedRepoIDs := make([]int64, 0, 10) if err = e.Table("watch").Cols("watch.repo_id"). @@ -1365,7 +1374,7 @@ func DeleteInactiveUsers(ctx context.Context, olderThan time.Duration) (err erro } if err = DeleteUser(u); err != nil { // Ignore users that were set inactive by admin. - if IsErrUserOwnRepos(err) || IsErrUserHasOrgs(err) { + if IsErrUserOwnRepos(err) || IsErrUserHasOrgs(err) || IsErrUserOwnPackages(err) { continue } return err diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 1695578f4febb..e3a55fd067d37 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -463,7 +463,9 @@ auth_failed = Authentication failed: %v still_own_repo = "Your account owns one or more repositories; delete or transfer them first." still_has_org = "Your account is a member of one or more organizations; leave them first." +still_own_packages = Your account owns one or more packages; delete them first. org_still_own_repo = "This organization still owns one or more repositories; delete or transfer them first." +org_still_own_packages = This organization still owns one or more packages; delete them first. target_branch_not_exist = Target branch does not exist. @@ -2382,6 +2384,7 @@ users.update_profile = Update User Account users.delete_account = Delete User Account users.still_own_repo = This user still owns one or more repositories. Delete or transfer these repositories first. users.still_has_org = This user is a member of an organization. Remove the user from any organizations first. +users.still_own_packages = This user still owns one or more packages. Delete these packages first. users.deletion_success = The user account has been deleted. users.reset_2fa = Reset 2FA users.list_status_filter.menu_text = Filter diff --git a/routers/api/v1/admin/user.go b/routers/api/v1/admin/user.go index 1e4a2851c216b..761e2cc050aab 100644 --- a/routers/api/v1/admin/user.go +++ b/routers/api/v1/admin/user.go @@ -291,7 +291,8 @@ func DeleteUser(ctx *context.APIContext) { if err := models.DeleteUser(u); err != nil { if models.IsErrUserOwnRepos(err) || - models.IsErrUserHasOrgs(err) { + models.IsErrUserHasOrgs(err) || + models.IsErrUserOwnPackages(err) { ctx.Error(http.StatusUnprocessableEntity, "", err) } else { ctx.Error(http.StatusInternalServerError, "DeleteUser", err) diff --git a/routers/web/admin/users.go b/routers/web/admin/users.go index db7fe7b36f6ee..9800186411b6d 100644 --- a/routers/web/admin/users.go +++ b/routers/web/admin/users.go @@ -389,6 +389,11 @@ func DeleteUser(ctx *context.Context) { ctx.JSON(http.StatusOK, map[string]interface{}{ "redirect": setting.AppSubURL + "/admin/users/" + ctx.Params(":userid"), }) + case models.IsErrUserOwnPackages(err): + ctx.Flash.Error(ctx.Tr("admin.users.still_own_packages")) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "redirect": setting.AppSubURL + "/admin/users/" + ctx.Params(":userid"), + }) default: ctx.ServerError("DeleteUser", err) } diff --git a/routers/web/org/setting.go b/routers/web/org/setting.go index d8add77f66000..31a175d576b9c 100644 --- a/routers/web/org/setting.go +++ b/routers/web/org/setting.go @@ -167,6 +167,9 @@ func SettingsDelete(ctx *context.Context) { if models.IsErrUserOwnRepos(err) { ctx.Flash.Error(ctx.Tr("form.org_still_own_repo")) ctx.Redirect(ctx.Org.OrgLink + "/settings/delete") + } else if models.IsErrUserOwnPackages(err) { + ctx.Flash.Error(ctx.Tr("form.org_still_own_packages")) + ctx.Redirect(ctx.Org.OrgLink + "/settings/delete") } else { ctx.ServerError("DeleteOrganization", err) } diff --git a/routers/web/user/setting/account.go b/routers/web/user/setting/account.go index 5ef1c3bdc840b..a60cc11e12e09 100644 --- a/routers/web/user/setting/account.go +++ b/routers/web/user/setting/account.go @@ -249,6 +249,9 @@ func DeleteAccount(ctx *context.Context) { case models.IsErrUserHasOrgs(err): ctx.Flash.Error(ctx.Tr("form.still_has_org")) ctx.Redirect(setting.AppSubURL + "/user/settings/account") + case models.IsErrUserOwnPackages(err): + ctx.Flash.Error(ctx.Tr("form.still_own_packages")) + ctx.Redirect(setting.AppSubURL + "/user/settings/account") default: ctx.ServerError("DeleteUser", err) } From 28b1658e219b9774a23f3798a77876e644b6c716 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 19 Nov 2021 16:04:00 +0100 Subject: [PATCH 071/130] Use package unlink in repository service. --- models/packages/package.go | 6 ++ models/repo.go | 6 -- routers/api/v1/packages/maven/api.go | 6 +- routers/api/v1/packages/npm/api.go | 6 +- routers/api/v1/packages/nuget/api.go | 18 ++-- routers/api/v1/packages/rubygems/rubygems.go | 1 - services/packages/packages.go | 88 ++++++++++---------- services/repository/repository.go | 13 ++- 8 files changed, 76 insertions(+), 68 deletions(-) diff --git a/models/packages/package.go b/models/packages/package.go index 1fb362dd6387e..7adb25cf31031 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -113,6 +113,12 @@ func SetRepositoryLink(packageID, repoID int64) error { return err } +// UnlinkRepositoryFromAllPackages unlinks every package from the repository +func UnlinkRepositoryFromAllPackages(ctx context.Context, repoID int64) error { + _, err := db.GetEngine(ctx).Where("repo_id = ?", repoID).Cols("repo_id").Update(&Package{}) + return err +} + // GetPackageByID gets a package by id func GetPackageByID(ctx context.Context, packageID int64) (*Package, error) { p := &Package{} diff --git a/models/repo.go b/models/repo.go index fcb3e20dc186d..8adfea46e492d 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1596,12 +1596,6 @@ func DeleteRepository(doer *User, uid, repoID int64) error { return err } - // Unlink packages - // TODO Currently raw sql because of import cycle - if _, err := sess.Exec("UPDATE `package` SET `repo_id` = 0 WHERE `repo_id` = ?", repo.ID); err != nil { - return err - } - // Remove archives var archives []*RepoArchiver if err = sess.Where("repo_id=?", repoID).Find(&archives); err != nil { diff --git a/routers/api/v1/packages/maven/api.go b/routers/api/v1/packages/maven/api.go index 8cc5711aa4d6d..b60a317814b4a 100644 --- a/routers/api/v1/packages/maven/api.go +++ b/routers/api/v1/packages/maven/api.go @@ -9,7 +9,7 @@ import ( "sort" "strings" - packages_models "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" maven_module "code.gitea.io/gitea/modules/packages/maven" ) @@ -23,13 +23,13 @@ type MetadataResponse struct { Version []string `xml:"versioning>versions>version"` } -func createMetadataResponse(pds []*packages_models.PackageDescriptor) *MetadataResponse { +func createMetadataResponse(pds []*packages_model.PackageDescriptor) *MetadataResponse { sort.Slice(pds, func(i, j int) bool { // Maven and Gradle order packages by their creation timestamp and not by their version string return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix }) - var release *packages_models.PackageDescriptor + var release *packages_model.PackageDescriptor versions := make([]string, 0, len(pds)) for _, pd := range pds { diff --git a/routers/api/v1/packages/npm/api.go b/routers/api/v1/packages/npm/api.go index a95b60a8caac7..d168048ef5ebb 100644 --- a/routers/api/v1/packages/npm/api.go +++ b/routers/api/v1/packages/npm/api.go @@ -11,11 +11,11 @@ import ( "net/url" "sort" - packages_models "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" npm_module "code.gitea.io/gitea/modules/packages/npm" ) -func createPackageMetadataResponse(registryURL string, pds []*packages_models.PackageDescriptor) *npm_module.PackageMetadata { +func createPackageMetadataResponse(registryURL string, pds []*packages_model.PackageDescriptor) *npm_module.PackageMetadata { sort.Slice(pds, func(i, j int) bool { return pds[i].SemVer.LessThan(pds[j].SemVer) }) @@ -49,7 +49,7 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_models.Pa } } -func createPackageMetadataVersion(registryURL string, pd *packages_models.PackageDescriptor) *npm_module.PackageMetadataVersion { +func createPackageMetadataVersion(registryURL string, pd *packages_model.PackageDescriptor) *npm_module.PackageMetadataVersion { hashBytes, _ := hex.DecodeString(pd.Files[0].Blob.HashSHA512) metadata := pd.Metadata.(*npm_module.Metadata) diff --git a/routers/api/v1/packages/nuget/api.go b/routers/api/v1/packages/nuget/api.go index c6db443056ca6..b449cfc5bb3ae 100644 --- a/routers/api/v1/packages/nuget/api.go +++ b/routers/api/v1/packages/nuget/api.go @@ -10,7 +10,7 @@ import ( "sort" "time" - packages_models "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" nuget_module "code.gitea.io/gitea/modules/packages/nuget" "github.com/hashicorp/go-version" @@ -95,7 +95,7 @@ type PackageDependency struct { Range string `json:"range"` } -func createRegistrationIndexResponse(l *linkBuilder, pds []*packages_models.PackageDescriptor) *RegistrationIndexResponse { +func createRegistrationIndexResponse(l *linkBuilder, pds []*packages_model.PackageDescriptor) *RegistrationIndexResponse { sort.Slice(pds, func(i, j int) bool { return pds[i].SemVer.LessThan(pds[j].SemVer) }) @@ -121,7 +121,7 @@ func createRegistrationIndexResponse(l *linkBuilder, pds []*packages_models.Pack } } -func createRegistrationIndexPageItem(l *linkBuilder, pd *packages_models.PackageDescriptor) *RegistrationIndexPageItem { +func createRegistrationIndexPageItem(l *linkBuilder, pd *packages_model.PackageDescriptor) *RegistrationIndexPageItem { metadata := pd.Metadata.(*nuget_module.Metadata) return &RegistrationIndexPageItem{ @@ -141,7 +141,7 @@ func createRegistrationIndexPageItem(l *linkBuilder, pd *packages_models.Package } } -func createDependencyGroups(pd *packages_models.PackageDescriptor) []*PackageDependencyGroup { +func createDependencyGroups(pd *packages_model.PackageDescriptor) []*PackageDependencyGroup { metadata := pd.Metadata.(*nuget_module.Metadata) dependencyGroups := make([]*PackageDependencyGroup, 0, len(metadata.Dependencies)) @@ -172,7 +172,7 @@ type RegistrationLeafResponse struct { RegistrationIndexURL string `json:"registration"` } -func createRegistrationLeafResponse(l *linkBuilder, pd *packages_models.PackageDescriptor) *RegistrationLeafResponse { +func createRegistrationLeafResponse(l *linkBuilder, pd *packages_model.PackageDescriptor) *RegistrationLeafResponse { return &RegistrationLeafResponse{ Type: []string{"Package", "http://schema.nuget.org/catalog#Permalink"}, Listed: true, @@ -188,7 +188,7 @@ type PackageVersionsResponse struct { Versions []string `json:"versions"` } -func createPackageVersionsResponse(pds []*packages_models.PackageDescriptor) *PackageVersionsResponse { +func createPackageVersionsResponse(pds []*packages_model.PackageDescriptor) *PackageVersionsResponse { versions := make([]string, 0, len(pds)) for _, pd := range pds { versions = append(versions, normalizeVersion(pd.SemVer)) @@ -223,12 +223,12 @@ type SearchResultVersion struct { Downloads int64 `json:"downloads"` } -func createSearchResultResponse(l *linkBuilder, totalHits int64, pds []*packages_models.PackageDescriptor) *SearchResultResponse { +func createSearchResultResponse(l *linkBuilder, totalHits int64, pds []*packages_model.PackageDescriptor) *SearchResultResponse { data := make([]*SearchResult, 0, len(pds)) if len(pds) > 0 { groupID := pds[0].Package.Name - group := make([]*packages_models.PackageDescriptor, 0, 10) + group := make([]*packages_model.PackageDescriptor, 0, 10) for i := 0; i < len(pds); i++ { if groupID != pds[i].Package.Name { @@ -247,7 +247,7 @@ func createSearchResultResponse(l *linkBuilder, totalHits int64, pds []*packages } } -func createSearchResult(l *linkBuilder, pds []*packages_models.PackageDescriptor) *SearchResult { +func createSearchResult(l *linkBuilder, pds []*packages_model.PackageDescriptor) *SearchResult { latest := pds[0] versions := make([]*SearchResultVersion, 0, len(pds)) for _, pd := range pds { diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/v1/packages/rubygems/rubygems.go index d77aa3695462f..519a3f1464103 100644 --- a/routers/api/v1/packages/rubygems/rubygems.go +++ b/routers/api/v1/packages/rubygems/rubygems.go @@ -14,7 +14,6 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" - //"code.gitea.io/gitea/modules/log" package_module "code.gitea.io/gitea/modules/packages" rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" package_router "code.gitea.io/gitea/routers/api/v1/packages" diff --git a/services/packages/packages.go b/services/packages/packages.go index 9cc8d7db04792..1bca8d134da17 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -12,7 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" - packages_models "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" @@ -22,7 +22,7 @@ import ( // PackageInfo describes a package type PackageInfo struct { Owner *models.User - PackageType packages_models.Type + PackageType packages_model.Type Name string Version string } @@ -44,16 +44,16 @@ type PackageFileInfo struct { } // CreatePackageAndAddFile creates a package with a file. If the same package exists already, ErrDuplicatePackageVersion is returned -func CreatePackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo) (*packages_models.PackageVersion, *packages_models.PackageFile, error) { +func CreatePackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { return createPackageAndAddFile(pvci, pfi, false) } // CreatePackageOrAddFileToExisting creates a package with a file or adds the file if the package exists already -func CreatePackageOrAddFileToExisting(pvci *PackageCreationInfo, pfi *PackageFileInfo) (*packages_models.PackageVersion, *packages_models.PackageFile, error) { +func CreatePackageOrAddFileToExisting(pvci *PackageCreationInfo, pfi *PackageFileInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { return createPackageAndAddFile(pvci, pfi, true) } -func createPackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo, allowDuplicate bool) (*packages_models.PackageVersion, *packages_models.PackageFile, error) { +func createPackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo, allowDuplicate bool) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { ctx, committer, err := db.TxContext() if err != nil { return nil, nil, err @@ -86,7 +86,7 @@ func createPackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo, al } if created { - pd, err := packages_models.GetPackageDescriptor(pv) + pd, err := packages_model.GetPackageDescriptor(pv) if err != nil { return nil, nil, err } @@ -97,10 +97,10 @@ func createPackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo, al return pv, pf, nil } -func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, allowDuplicate bool) (*packages_models.PackageVersion, bool, error) { +func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, bool, error) { log.Trace("Creating package: %v, %v, %v, %s, %s, %+v, %v", pvci.Creator.ID, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, pvci.Properties, allowDuplicate) - p := &packages_models.Package{ + p := &packages_model.Package{ OwnerID: pvci.Owner.ID, Type: pvci.PackageType, Name: pvci.Name, @@ -108,8 +108,8 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all SemverCompatible: pvci.SemverCompatible, } var err error - if p, err = packages_models.TryInsertPackage(ctx, p); err != nil { - if err != packages_models.ErrDuplicatePackage { + if p, err = packages_model.TryInsertPackage(ctx, p); err != nil { + if err != packages_model.ErrDuplicatePackage { log.Error("Error inserting package: %v", err) return nil, false, err } @@ -121,25 +121,25 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } created := true - pv := &packages_models.PackageVersion{ + pv := &packages_model.PackageVersion{ PackageID: p.ID, CreatorID: pvci.Creator.ID, Version: pvci.Version, LowerVersion: strings.ToLower(pvci.Version), MetadataJSON: string(metadataJSON), } - if pv, err = packages_models.GetOrInsertVersion(ctx, pv); err != nil { - if err == packages_models.ErrDuplicatePackageVersion { + if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil { + if err == packages_model.ErrDuplicatePackageVersion { created = false } - if err != packages_models.ErrDuplicatePackageVersion || !allowDuplicate { + if err != packages_model.ErrDuplicatePackageVersion || !allowDuplicate { log.Error("Error inserting package: %v", err) return nil, false, err } } for name, value := range pvci.Properties { - if _, err := packages_models.InsertVersionProperty(ctx, pv.ID, name, value); err != nil { + if _, err := packages_model.InsertVersionProperty(ctx, pv.ID, name, value); err != nil { log.Error("Error setting package version property: %v", err) return nil, false, err } @@ -149,14 +149,14 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } // AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned -func AddFileToExistingPackage(pvi *PackageInfo, pfi *PackageFileInfo) (*packages_models.PackageVersion, *packages_models.PackageFile, error) { +func AddFileToExistingPackage(pvi *PackageInfo, pfi *PackageFileInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { ctx, committer, err := db.TxContext() if err != nil { return nil, nil, err } defer committer.Close() - pv, err := packages_models.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { return nil, nil, err } @@ -183,18 +183,18 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfi *PackageFileInfo) (*packages return pv, pf, nil } -func addFileToPackageVersion(ctx context.Context, pv *packages_models.PackageVersion, pfi *PackageFileInfo) (*packages_models.PackageFile, bool, error) { +func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (*packages_model.PackageFile, bool, error) { log.Trace("Adding package file: %v, %s", pv.ID, pfi.Filename) hashMD5, hashSHA1, hashSHA256, hashSHA512 := pfi.Data.Sums() - pb := &packages_models.PackageBlob{ + pb := &packages_model.PackageBlob{ Size: pfi.Data.Size(), HashMD5: fmt.Sprintf("%x", hashMD5), HashSHA1: fmt.Sprintf("%x", hashSHA1), HashSHA256: fmt.Sprintf("%x", hashSHA256), HashSHA512: fmt.Sprintf("%x", hashSHA512), } - pb, exists, err := packages_models.GetOrInsertBlob(ctx, pb) + pb, exists, err := packages_model.GetOrInsertBlob(ctx, pb) if err != nil { log.Error("Error inserting package blob: %v", err) return nil, false, err @@ -207,15 +207,15 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_models.PackageVer } } - pf := &packages_models.PackageFile{ + pf := &packages_model.PackageFile{ VersionID: pv.ID, BlobID: pb.ID, Name: pfi.Filename, LowerName: strings.ToLower(pfi.Filename), IsLead: pfi.IsLead, } - if pf, err = packages_models.TryInsertFile(ctx, pf); err != nil { - if err != packages_models.ErrDuplicatePackageFile { + if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil { + if err != packages_model.ErrDuplicatePackageFile { log.Error("Error inserting package file: %v", err) } return nil, !exists, err @@ -226,7 +226,7 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_models.PackageVer // DeletePackageVersionByNameAndVersion deletes a package version and all associated files func DeletePackageVersionByNameAndVersion(doer *models.User, pvi *PackageInfo) error { - pv, err := packages_models.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { return err } @@ -235,33 +235,33 @@ func DeletePackageVersionByNameAndVersion(doer *models.User, pvi *PackageInfo) e } // DeletePackageVersion deletes the package version and all associated files -func DeletePackageVersion(doer *models.User, pv *packages_models.PackageVersion) error { +func DeletePackageVersion(doer *models.User, pv *packages_model.PackageVersion) error { ctx, committer, err := db.TxContext() if err != nil { return err } defer committer.Close() - pd, err := packages_models.GetPackageDescriptorCtx(ctx, pv) + pd, err := packages_model.GetPackageDescriptorCtx(ctx, pv) if err != nil { return err } log.Trace("Deleting package: %v", pv.ID) - if err := packages_models.DeleteVersionPropertiesByVersionID(ctx, pv.ID); err != nil { + if err := packages_model.DeleteVersionPropertiesByVersionID(ctx, pv.ID); err != nil { return err } - if err := packages_models.DeleteFilesByVersionID(ctx, pv.ID); err != nil { + if err := packages_model.DeleteFilesByVersionID(ctx, pv.ID); err != nil { return err } - if err := packages_models.DeleteVersionByID(ctx, pv.ID); err != nil { + if err := packages_model.DeleteVersionByID(ctx, pv.ID); err != nil { return err } - if err := packages_models.DeletePackageByIDIfUnreferenced(ctx, pv.PackageID); err != nil { + if err := packages_model.DeletePackageByIDIfUnreferenced(ctx, pv.PackageID); err != nil { return err } @@ -282,13 +282,13 @@ func DeleteUnreferencedBlobs() error { } defer committer.Close() - pbs, err := packages_models.GetUnreferencedBlobs(ctx) + pbs, err := packages_model.GetUnreferencedBlobs(ctx) if err != nil { return err } for _, pb := range pbs { - if err := packages_models.DeleteBlobByID(ctx, pb.ID); err != nil { + if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil { return err } } @@ -308,12 +308,12 @@ func DeleteUnreferencedBlobs() error { } // GetFileStreamByPackageNameAndVersion returns the content of the specific package file -func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { +func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, filename string) (io.ReadCloser, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, filename) - pv, err := packages_models.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { - if err == packages_models.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { return nil, nil, err } log.Error("Error getting package: %v", err) @@ -324,34 +324,34 @@ func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, filename string) (io } // GetFileStreamByPackageVersionID returns the content of the specific package file -func GetFileStreamByPackageVersionID(owner *models.User, versionID int64, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { +func GetFileStreamByPackageVersionID(owner *models.User, versionID int64, filename string) (io.ReadCloser, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s", owner.ID, versionID, filename) - pv, err := packages_models.GetVersionByID(db.DefaultContext, versionID) + pv, err := packages_model.GetVersionByID(db.DefaultContext, versionID) if err != nil { - if err == packages_models.ErrPackageVersionNotExist { - return nil, nil, packages_models.ErrPackageNotExist + if err == packages_model.ErrPackageVersionNotExist { + return nil, nil, packages_model.ErrPackageNotExist } log.Error("Error getting package version: %v", err) return nil, nil, err } - p, err := packages_models.GetPackageByID(db.DefaultContext, pv.PackageID) + p, err := packages_model.GetPackageByID(db.DefaultContext, pv.PackageID) if err != nil { log.Error("Error getting package: %v", err) return nil, nil, err } if p.OwnerID != owner.ID { - return nil, nil, packages_models.ErrPackageNotExist + return nil, nil, packages_model.ErrPackageNotExist } return GetPackageFileStream(pv, filename) } // GetPackageFileStream returns the cotent of the specific package file -func GetPackageFileStream(pv *packages_models.PackageVersion, filename string) (io.ReadCloser, *packages_models.PackageFile, error) { - pf, err := packages_models.GetFileForVersionByName(db.DefaultContext, pv.ID, filename) +func GetPackageFileStream(pv *packages_model.PackageVersion, filename string) (io.ReadCloser, *packages_model.PackageFile, error) { + pf, err := packages_model.GetFileForVersionByName(db.DefaultContext, pv.ID, filename) if err != nil { return nil, nil, err } @@ -359,7 +359,7 @@ func GetPackageFileStream(pv *packages_models.PackageVersion, filename string) ( s, err := packages_module.NewContentStore().Get(pf.BlobID) if err == nil { if pf.IsLead { - if err := packages_models.IncrementDownloadCounter(pv.ID); err != nil { + if err := packages_model.IncrementDownloadCounter(pv.ID); err != nil { log.Error("Error incrementing download counter: %v", err) } } diff --git a/services/repository/repository.go b/services/repository/repository.go index 98d160c223903..050631dafa229 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -8,6 +8,8 @@ import ( "fmt" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" @@ -37,8 +39,15 @@ func DeleteRepository(doer *models.User, repo *models.Repository) error { // If the repo itself has webhooks, we need to trigger them before deleting it... notification.NotifyDeleteRepository(doer, repo) - err := models.DeleteRepository(doer, repo.OwnerID, repo.ID) - return err + if err := models.DeleteRepository(doer, repo.OwnerID, repo.ID); err != nil { + return err + } + + if err := packages_model.UnlinkRepositoryFromAllPackages(db.DefaultContext, repo.ID); err != nil { + return err + } + + return nil } // PushCreateRepo creates a repository when a new repository is pushed to an appropriate namespace From 1bf71067b339ceda32d5173aefdf1bf1b8ee252d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Nov 2021 16:10:52 +0000 Subject: [PATCH 072/130] Added support for composer packages. --- docs/content/doc/packages/composer.en-us.md | 120 +++++++++ docs/content/doc/packages/generic.en-us.md | 4 +- docs/content/doc/packages/maven.en-us.md | 2 +- docs/content/doc/packages/npm.en-us.md | 2 +- docs/content/doc/packages/overview.en-us.md | 1 + integrations/api_packages_composer_test.go | 223 ++++++++++++++++ models/migrations/v202.go | 2 +- models/packages/descriptor.go | 3 + models/packages/package.go | 20 ++ models/packages/package_version.go | 33 ++- modules/packages/composer/metadata.go | 147 +++++++++++ modules/packages/composer/metadata_test.go | 130 ++++++++++ options/locale/locale_en-US.ini | 17 +- public/img/svg/gitea-composer.svg | 1 + routers/api/v1/api.go | 10 + routers/api/v1/packages/composer/api.go | 117 +++++++++ routers/api/v1/packages/composer/composer.go | 244 ++++++++++++++++++ routers/api/v1/repo/package.go | 2 +- templates/package/content/composer.tmpl | 50 ++++ .../content/composer_dependencies.tmpl | 19 ++ templates/package/content/npm.tmpl | 10 +- .../package/content/npm_dependencies.tmpl | 4 +- templates/package/content/rubygems.tmpl | 4 +- .../content/rubygems_dependencies.tmpl | 4 +- templates/package/metadata/composer.tmpl | 5 + templates/package/view.tmpl | 2 + web_src/svg/gitea-composer.svg | 50 ++++ 27 files changed, 1194 insertions(+), 32 deletions(-) create mode 100644 docs/content/doc/packages/composer.en-us.md create mode 100644 integrations/api_packages_composer_test.go create mode 100644 modules/packages/composer/metadata.go create mode 100644 modules/packages/composer/metadata_test.go create mode 100644 public/img/svg/gitea-composer.svg create mode 100644 routers/api/v1/packages/composer/api.go create mode 100644 routers/api/v1/packages/composer/composer.go create mode 100644 templates/package/content/composer.tmpl create mode 100644 templates/package/content/composer_dependencies.tmpl create mode 100644 templates/package/metadata/composer.tmpl create mode 100644 web_src/svg/gitea-composer.svg diff --git a/docs/content/doc/packages/composer.en-us.md b/docs/content/doc/packages/composer.en-us.md new file mode 100644 index 0000000000000..542ad7b0983ad --- /dev/null +++ b/docs/content/doc/packages/composer.en-us.md @@ -0,0 +1,120 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "Composer Packages Repository" +slug: "composer" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Composer" + weight: 15 + identifier: "composer" +--- + +# Composer Packages Repository + +Publish [Composer](https://getcomposer.org/) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Composer package registry, you can use [Composer](hhttps://getcomposer.org/download/) to consume and a HTTP upload client like `curl` to publish packages. + +## Publish a package + +To publish a Composer package perform a HTTP PUT operation with the package content in the request body. +The package content must be the zipped PHP project with the `composer.json` file. +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +``` +PUT https://gitea.example.com/api/v1/packages/{owner}/composer +``` + +| Parameter | Description | +| ---------- | ----------- | +| `owner` | The owner of the package. | + +If the `composer.json` file does not contain a `version` property, you must provide it as a query parameter: + +``` +PUT https://gitea.example.com/api/v1/packages/{owner}/composer?version={x.y.z} +``` + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_password_or_token \ + --upload-file path/to/project.zip \ + https://gitea.example.com/api/v1/packages/testuser/composer +``` + +Or specify the package version as query parameter: + +```shell +curl --user your_username:your_password_or_token \ + --upload-file path/to/project.zip \ + https://gitea.example.com/api/v1/packages/testuser/composer?version=1.0.3 +``` + +The server reponds with the following HTTP Status codes. + +| HTTP Status Code | Meaning | +| ----------------- | ------- | +| `201 Created` | The package has been published. | +| `400 Bad Request` | The package name and/or version are invalid or a package with the same name and version already exist. | + +## Configuring the package registry + +To register the package registry you need to add it to the Composer `config.json` file (which can usually be found under `/.composer/config.json`): + +```json +{ + "repositories": [{ + "type": "composer", + "url": "https://gitea.example.com/api/v1/packages/{owner}/composer" + } + ] +} +``` + +To access the package registry using credentials, you must specify them in the `auth.json` file as follows: + +```json +{ + "http-basic": { + "gitea.example.com": { + "username": "{username}", + "password": "{password}" + } + } +} +``` + +| Parameter | Description | +| ---------- | ----------- | +| `owner` | The owner of the package. | +| `username` | Your Gitea username. | +| `password` | Your Gitea password or a personal access token. | + +## Install a package + +To install a package from the package registry, execute the following command: + +```shell +composer require {package_name} +``` + +Optional you can specify the package version: + +```shell +composer require {package_name}:{package_version} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `package_name` | The package name. | +| `package_version` | The package version. | diff --git a/docs/content/doc/packages/generic.en-us.md b/docs/content/doc/packages/generic.en-us.md index 72652cd8b3562..bda5e4fc13041 100644 --- a/docs/content/doc/packages/generic.en-us.md +++ b/docs/content/doc/packages/generic.en-us.md @@ -45,7 +45,7 @@ Example request using HTTP Basic authentication: ```shell curl --user your_username:your_password_or_token \ --upload-file path/to/file.bin \ - "https://gitea.example.com/api/v1/packages/testuser/generic/test_package/1.0.0/file.bin" + https://gitea.example.com/api/v1/packages/testuser/generic/test_package/1.0.0/file.bin ``` The server reponds with the following HTTP Status codes. @@ -76,5 +76,5 @@ Example request using HTTP Basic authentication: ```shell curl --user your_username:your_token_or_password \ - "https://gitea.example.com/api/v1/packages/testuser/generic/test_package/1.0.0/file.bin" + https://gitea.example.com/api/v1/packages/testuser/generic/test_package/1.0.0/file.bin ``` diff --git a/docs/content/doc/packages/maven.en-us.md b/docs/content/doc/packages/maven.en-us.md index c8f627528d49f..e717a927a4735 100644 --- a/docs/content/doc/packages/maven.en-us.md +++ b/docs/content/doc/packages/maven.en-us.md @@ -27,7 +27,7 @@ The following examples use `Maven`. ## Configuring the package registry -To register the project’s package registry you first need to add your access token to the [`settings.xml`](https://maven.apache.org/settings.html) file: +To register the package registry you first need to add your access token to the [`settings.xml`](https://maven.apache.org/settings.html) file: ```xml diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index 7f4a02b776bec..e3dfa3f2c94d7 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -30,7 +30,7 @@ The following examples use the `npm` tool with the scope `@test`. ## Configuring the package registry -To register the project’s package registry you need to configure a new package source. +To register the package registry you need to configure a new package source. ```shell npm config set {scope}:registry https://gitea.example.com/api/v1/packages/{owner}/npm/ diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 3995dc86396e4..35be6b3b08db3 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -19,6 +19,7 @@ The Package Registry can be used as a public or private registry for common pack These package types are supported: - [Generic]({{< relref "doc/packages/generic.en-us.md" >}}) +- [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) - [NuGet]({{< relref "doc/packages/nuget.en-us.md" >}}) - [npm]({{< relref "doc/packages/npm.en-us.md" >}}) - [Maven]({{< relref "doc/packages/maven.en-us.md" >}}) diff --git a/integrations/api_packages_composer_test.go b/integrations/api_packages_composer_test.go new file mode 100644 index 0000000000000..7d4b8ff45497d --- /dev/null +++ b/integrations/api_packages_composer_test.go @@ -0,0 +1,223 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "archive/zip" + "bytes" + "fmt" + "net/http" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + composer_module "code.gitea.io/gitea/modules/packages/composer" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/v1/packages/composer" + + "github.com/stretchr/testify/assert" +) + +func TestPackageComposer(t *testing.T) { + defer prepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) + + vendorName := "gitea" + projectName := "composer-package" + packageName := vendorName + "/" + projectName + packageVersion := "1.0.3" + packageDescription := "Package Description" + packageType := "composer-plugin" + packageAuthor := "Gitea Authors" + packageLicense := "MIT" + + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create("composer.json") + w.Write([]byte(`{ + "name": "` + packageName + `", + "description": "` + packageDescription + `", + "type": "` + packageType + `", + "license": "` + packageLicense + `", + "authors": [ + { + "name": "` + packageAuthor + `" + } + ] + }`)) + archive.Close() + content := buf.Bytes() + + url := fmt.Sprintf("%sapi/v1/packages/%s/composer", setting.AppURL, user.Name) + + t.Run("ServiceIndex", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/packages.json", url)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result composer.ServiceIndexResponse + DecodeJSON(t, resp, &result) + + assert.Equal(t, url+"/search.json?q=%query%&type=%type%", result.SearchTemplate) + assert.Equal(t, url+"/p2/%package%.json", result.MetadataTemplate) + assert.Equal(t, url+"/list.json", result.PackageList) + }) + + t.Run("Upload", func(t *testing.T) { + t.Run("MissingVersion", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) + + t.Run("Valid", func(t *testing.T) { + defer PrintCurrentTest(t)() + + uploadURL := url + "?version=" + packageVersion + + req := NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeComposer) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &composer_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, fmt.Sprintf("%s-%s.%s.zip", vendorName, projectName, packageVersion), pfs[0].Name) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), pb.Size) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeComposer) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(0), pvs[0].DownloadCount) + + req := NewRequest(t, "GET", fmt.Sprintf("%s/files/%d/%s-%s.%s.zip", url, pvs[0].ID, vendorName, projectName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, content, resp.Body.Bytes()) + + pvs, err = packages.GetVersionsByPackageType(user.ID, packages.TypeComposer) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, int64(1), pvs[0].DownloadCount) + }) + + t.Run("SearchService", func(t *testing.T) { + defer PrintCurrentTest(t)() + + cases := []struct { + Query string + Type string + Page int + PerPage int + ExpectedTotal int64 + ExpectedResults int + }{ + {"", "", 0, 0, 1, 1}, + {"", "", 1, 1, 1, 1}, + {"test", "", 1, 0, 0, 0}, + {"gitea", "", 1, 1, 1, 1}, + {"gitea", "", 2, 1, 1, 0}, + {"", packageType, 1, 1, 1, 1}, + {"gitea", packageType, 1, 1, 1, 1}, + {"gitea", "dummy", 1, 1, 0, 0}, + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s/search.json?q=%s&type=%s&page=%d&per_page=%d", url, c.Query, c.Type, c.Page, c.PerPage)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result composer.SearchResultResponse + DecodeJSON(t, resp, &result) + + assert.Equal(t, c.ExpectedTotal, result.Total, "case %d: unexpected total hits", i) + assert.Len(t, result.Results, c.ExpectedResults, "case %d: unexpected result count", i) + } + }) + + t.Run("EnumeratePackages", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", url+"/list.json") + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result map[string][]string + DecodeJSON(t, resp, &result) + + assert.Contains(t, result, "packageNames") + names := result["packageNames"] + assert.Len(t, names, 1) + assert.Equal(t, packageName, names[0]) + }) + + t.Run("PackageMetadata", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/p2/%s/%s.json", url, vendorName, projectName)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result composer.PackageMetadataResponse + DecodeJSON(t, resp, &result) + + assert.Contains(t, result.Packages, packageName) + pkgs := result.Packages[packageName] + assert.Len(t, pkgs, 1) + assert.Equal(t, packageName, pkgs[0].Name) + assert.Equal(t, packageVersion, pkgs[0].Version) + assert.Equal(t, packageType, pkgs[0].Type) + assert.Equal(t, packageDescription, pkgs[0].Description) + assert.Len(t, pkgs[0].Authors, 1) + assert.Equal(t, packageAuthor, pkgs[0].Authors[0].Name) + assert.Equal(t, "zip", pkgs[0].Dist.Type) + assert.Equal(t, "7b40bfd6da811b2b78deec1e944f156dbb2c747b", pkgs[0].Dist.Checksum) + }) + + /*t.Run("PackageService", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/index.json", url, packageName)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result nuget.PackageVersionsResponse + DecodeJSON(t, resp, &result) + + assert.Len(t, result.Versions, 1) + assert.Equal(t, packageVersion, result.Versions[0]) + })*/ +} diff --git a/models/migrations/v202.go b/models/migrations/v202.go index 3850bc85b1cdf..f001bdd110dfa 100644 --- a/models/migrations/v202.go +++ b/models/migrations/v202.go @@ -31,7 +31,7 @@ func addPackageTables(x *xorm.Engine) error { CreatorID int64 Version string LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` + CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` MetadataJSON string `xorm:"TEXT metadata_json"` DownloadCount int64 } diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 37a6a311b826c..4293e10896cc4 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/packages/nuget" @@ -97,6 +98,8 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD var metadata interface{} switch p.Type { + case TypeComposer: + metadata = &composer.Metadata{} case TypeNuGet: metadata = &nuget.Metadata{} case TypeNpm: diff --git a/models/packages/package.go b/models/packages/package.go index 7adb25cf31031..d53b7f324dce0 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -9,6 +9,8 @@ import ( "errors" "code.gitea.io/gitea/models/db" + + "xorm.io/builder" ) func init() { @@ -27,6 +29,7 @@ type Type string // List of supported packages const ( + TypeComposer Type = "composer" TypeGeneric Type = "generic" TypeNuGet Type = "nuget" TypeNpm Type = "npm" @@ -38,6 +41,8 @@ const ( // Name gets the name of the package type func (pt Type) Name() string { switch pt { + case TypeComposer: + return "Composer" case TypeGeneric: return "Generic" case TypeNuGet: @@ -57,6 +62,8 @@ func (pt Type) Name() string { // SVGName gets the name of the package type svg image func (pt Type) SVGName() string { switch pt { + case TypeComposer: + return "gitea-composer" case TypeGeneric: return "octicon-package" case TypeNuGet: @@ -133,6 +140,19 @@ func GetPackageByID(ctx context.Context, packageID int64) (*Package, error) { return p, nil } +// GetPackagesByType gets all packages of a specific type +func GetPackagesByType(ctx context.Context, ownerID int64, packageType Type) ([]*Package, error) { + var cond builder.Cond = builder.Eq{ + "package.owner_id": ownerID, + "package.type": packageType, + } + + ps := make([]*Package, 0, 10) + return ps, db.GetEngine(ctx). + Where(cond). + Find(&ps) +} + // DeletePackageByIDIfUnreferenced deletes a package if there are no associated versions func DeletePackageByIDIfUnreferenced(ctx context.Context, packageID int64) error { e := db.GetEngine(ctx) diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 826bbbe9bf2de..e3b535635eaf5 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -33,7 +33,7 @@ type PackageVersion struct { CreatorID int64 Version string LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` + CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` MetadataJSON string `xorm:"TEXT metadata_json"` DownloadCount int64 } @@ -163,10 +163,11 @@ func DeleteVersionByID(ctx context.Context, versionID int64) error { // PackageSearchOptions are options for SearchXXX methods type PackageSearchOptions struct { - OwnerID int64 - RepoID int64 - Type string - Query string + OwnerID int64 + RepoID int64 + Type string + Query string + Properties map[string]string db.Paginator } @@ -183,14 +184,27 @@ func (opts *PackageSearchOptions) toConds() builder.Cond { cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.Query)}) } + if len(opts.Properties) != 0 { + propsCond := builder.NewCond() + for name, value := range opts.Properties { + propsCond = propsCond.Or(builder.Eq{ + "package_version_property.name": name, + "package_version_property.value": value, + }) + } + cond = cond.And(builder.In("package_version.id", builder.Select("package_version_property.version_id").Where(propsCond).From("package_version_property"))) + } + return cond } // SearchVersions gets all versions of packages matching the search options func SearchVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, error) { - sess := db.GetEngine(db.DefaultContext).Where(opts.toConds()). + sess := db.GetEngine(db.DefaultContext). + Where(opts.toConds()). Table("package_version"). - Join("INNER", "package", "package.id = package_version.package_id") + Join("INNER", "package", "package.id = package_version.package_id"). + Desc("created_unix") if opts.Paginator != nil { sess = db.SetSessionPagination(sess, opts) @@ -208,9 +222,10 @@ func SearchLatestVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, sess := db.GetEngine(db.DefaultContext). Table("package_version"). - Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND package_version.version < pv2.version"). + Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND package_version.created_unix < pv2.created_unix"). Join("INNER", "package", "package.id = package_version.package_id"). - Where(cond) + Where(cond). + Desc("created_unix") if opts.Paginator != nil { sess = db.SetSessionPagination(sess, opts) diff --git a/modules/packages/composer/metadata.go b/modules/packages/composer/metadata.go new file mode 100644 index 0000000000000..797576b1e7de4 --- /dev/null +++ b/modules/packages/composer/metadata.go @@ -0,0 +1,147 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package composer + +import ( + "archive/zip" + "errors" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +// TypeProperty is the name of the property for Composer package types +const TypeProperty = "composer.type" + +var ( + // ErrMissingComposerFile indicates a missing composer.json file + ErrMissingComposerFile = errors.New("composer.json file is missing") + // ErrInvalidName indicates an invalid package name + ErrInvalidName = errors.New("package name is invalid") + // ErrInvalidVersion indicates an invalid package version + ErrInvalidVersion = errors.New("package version is invalid") +) + +// Package represents a Composer package +type Package struct { + Name string + Version string + Type string + Metadata *Metadata +} + +// Metadata represents the metadata of a Composer package +type Metadata struct { + Description string `json:"description,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Homepage string `json:"homepage,omitempty"` + License Licenses `json:"license,omitempty"` + Authors []Author `json:"authors,omitempty"` + Autoload map[string]interface{} `json:"autoload,omitempty"` + AutoloadDev map[string]interface{} `json:"autoload-dev,omitempty"` + Extra map[string]interface{} `json:"extra,omitempty"` + Require map[string]string `json:"require,omitempty"` + RequireDev map[string]string `json:"require-dev,omitempty"` + Suggest map[string]string `json:"suggest,omitempty"` + Provide map[string]string `json:"provide,omitempty"` +} + +// Licenses represents the licenses of a Composer package +type Licenses []string + +// UnmarshalJSON reads from a string or array +func (l *Licenses) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + *l = Licenses{value} + case '[': + values := make([]string, 0, 5) + if err := json.Unmarshal(data, &values); err != nil { + return err + } + *l = Licenses(values) + } + return nil +} + +// Author represents an author +type Author struct { + Name string `json:"name,omitempty"` + Email string `json:"email,omitempty"` + Homepage string `json:"homepage,omitempty"` +} + +var nameMatch = regexp.MustCompile(`\A[a-z0-9]([_\.-]?[a-z0-9]+)*/[a-z0-9](([_\.]?|-{0,2})[a-z0-9]+)*\z`) + +// ParsePackage parses the metadata of a Composer package file +func ParsePackage(r io.ReaderAt, size int64) (*Package, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range archive.File { + if strings.Count(file.Name, "/") > 1 { + continue + } + if strings.HasSuffix(strings.ToLower(file.Name), "composer.json") { + f, err := archive.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + return ParseComposerFile(f) + } + } + return nil, ErrMissingComposerFile +} + +// ParseComposerFile parses a composer.json file to retrieve the metadata of a Composer package +func ParseComposerFile(r io.Reader) (*Package, error) { + var cj struct { + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + Metadata + } + if err := json.NewDecoder(r).Decode(&cj); err != nil { + return nil, err + } + + if !nameMatch.MatchString(cj.Name) { + return nil, ErrInvalidName + } + + if cj.Version != "" { + if _, err := version.NewSemver(cj.Version); err != nil { + return nil, ErrInvalidVersion + } + } + + if !validation.IsValidURL(cj.Homepage) { + cj.Homepage = "" + } + + if cj.Type == "" { + cj.Type = "library" + } + + return &Package{ + Name: cj.Name, + Version: cj.Version, + Type: cj.Type, + Metadata: &cj.Metadata, + }, nil +} diff --git a/modules/packages/composer/metadata_test.go b/modules/packages/composer/metadata_test.go new file mode 100644 index 0000000000000..feadc18b6a187 --- /dev/null +++ b/modules/packages/composer/metadata_test.go @@ -0,0 +1,130 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package composer + +import ( + "archive/zip" + "bytes" + "strings" + "testing" + + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/assert" +) + +const ( + name = "gitea/composer-package" + description = "Package Description" + packageType = "composer-plugin" + author = "Gitea Authors" + email = "no.reply@gitea.io" + homepage = "https://gitea.io" + license = "MIT" +) + +const composerContent = `{ + "name": "` + name + `", + "description": "` + description + `", + "type": "` + packageType + `", + "license": "` + license + `", + "authors": [ + { + "name": "` + author + `", + "email": "` + email + `" + } + ], + "homepage": "` + homepage + `", + "autoload": { + "psr-4": {"Gitea\\ComposerPackage\\": "src/"} + }, + "require": { + "php": ">=7.2 || ^8.0" + } +}` + +func TestLicenseUnmarshal(t *testing.T) { + var l Licenses + assert.NoError(t, json.NewDecoder(strings.NewReader(`["MIT"]`)).Decode(&l)) + assert.Len(t, l, 1) + assert.Equal(t, "MIT", l[0]) + assert.NoError(t, json.NewDecoder(strings.NewReader(`"MIT"`)).Decode(&l)) + assert.Len(t, l, 1) + assert.Equal(t, "MIT", l[0]) +} + +func TestParsePackage(t *testing.T) { + createArchive := func(name, content string) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create(name) + w.Write([]byte(content)) + archive.Close() + return buf.Bytes() + } + + t.Run("MissingComposerFile", func(t *testing.T) { + data := createArchive("dummy.txt", "") + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrMissingComposerFile) + }) + + t.Run("MissingComposerFileInRoot", func(t *testing.T) { + data := createArchive("sub/sub/composer.json", "") + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrMissingComposerFile) + }) + + t.Run("InvalidComposerFile", func(t *testing.T) { + data := createArchive("composer.json", "") + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + assert.Error(t, err) + }) + + t.Run("Valid", func(t *testing.T) { + data := createArchive("composer.json", composerContent) + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.NoError(t, err) + assert.NotNil(t, cp) + }) +} + +func TestParseComposerFile(t *testing.T) { + t.Run("InvalidPackageName", func(t *testing.T) { + cp, err := ParseComposerFile(strings.NewReader(`{}`)) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidName) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + cp, err := ParseComposerFile(strings.NewReader(`{"name": "gitea/composer-package", "version": "1.a.3"}`)) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidVersion) + }) + + t.Run("Valid", func(t *testing.T) { + cp, err := ParseComposerFile(strings.NewReader(composerContent)) + assert.NoError(t, err) + assert.NotNil(t, cp) + + assert.Equal(t, name, cp.Name) + assert.Empty(t, cp.Version) + assert.Equal(t, description, cp.Metadata.Description) + assert.Len(t, cp.Metadata.Authors, 1) + assert.Equal(t, author, cp.Metadata.Authors[0].Name) + assert.Equal(t, email, cp.Metadata.Authors[0].Email) + assert.Equal(t, homepage, cp.Metadata.Homepage) + assert.Equal(t, packageType, cp.Type) + assert.Len(t, cp.Metadata.License, 1) + assert.Equal(t, license, cp.Metadata.License[0]) + }) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 60df7e6ef7d30..0f1cc20eabbf9 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2878,6 +2878,11 @@ versions = Versions versions.on = on dependency.id = Id dependency.version = Version +composer.registry = Setup this registry in your ~/.composer/config.json file: +composer.install = To install the package using Composer, run the following command: +composer.documentation = For more information on the Composer registry, see the documentation. +composer.dependencies = Dependencies +composer.dependencies.development = Development Dependencies generic.download = Download package from the command line: generic.documentation = For more information on the generic registry, see the documentation. maven.registry = Setup this registry in your project pom.xml file: @@ -2886,12 +2891,12 @@ maven.install2 = Run via command line: maven.download = To download the dependency, run via command line: maven.documentation = For more information on the Maven registry, see the documentation. nuget.registry = Setup the registry from the command line: -nuget.install = Install from the command line: +nuget.install = To install the package using NuGet, run the following command: nuget.documentation = For more information on the NuGet registry, see the documentation. nuget.dependency.framework = Target Framework npm.registry = Setup this registry in your project .npmrc file: -npm.install = Install from the command line: -npm.install2 = Install via package.json: +npm.install = To install the package using npm, run the following command: +npm.install2 = or add it to the package.json file: npm.documentation = For more information on the npm registry, see the documentation. npm.dependencies = Dependencies npm.dependencies.development = Development Dependencies @@ -2899,10 +2904,10 @@ npm.dependencies.peer = Peer Dependencies npm.dependencies.optional = Optional Dependencies npm.details.tag = Tag pypi.requires = Requires Python -pypi.install = Install from the command line: +pypi.install = To install the package using pip, run the following command: pypi.documentation = For more information on the PyPI registry, see the documentation. -rubygems.install = Install from the command line: -rubygems.install2 = Install via Gemfile: +rubygems.install = To install the package using gem, run the following command: +rubygems.install2 = or add it to the Gemfile: rubygems.dependencies.runtime = Runtime Dependencies rubygems.dependencies.development = Development Dependencies rubygems.required.ruby = Requires Ruby version diff --git a/public/img/svg/gitea-composer.svg b/public/img/svg/gitea-composer.svg new file mode 100644 index 0000000000000..1285b1bf913f8 --- /dev/null +++ b/public/img/svg/gitea-composer.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 651357ea02176..fb4f53320cfb0 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -81,6 +81,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/misc" "code.gitea.io/gitea/routers/api/v1/notify" "code.gitea.io/gitea/routers/api/v1/org" + "code.gitea.io/gitea/routers/api/v1/packages/composer" "code.gitea.io/gitea/routers/api/v1/packages/generic" "code.gitea.io/gitea/routers/api/v1/packages/maven" "code.gitea.io/gitea/routers/api/v1/packages/npm" @@ -1012,6 +1013,15 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { }) m.Group("/packages/{username}", func() { + m.Group("/composer", func() { + m.Get("/packages.json", composer.ServiceIndex) + m.Get("/search.json", composer.SearchPackages) + m.Get("/list.json", composer.EnumeratePackages) + m.Get("/p2/{vendorname}/{projectname}~dev.json", composer.PackageMetadata) + m.Get("/p2/{vendorname}/{projectname}.json", composer.PackageMetadata) + m.Get("/files/{versionid}/{filename}", composer.DownloadPackageFile) + m.Put("", reqToken(), reqPackageAccess(models.AccessModeWrite), composer.UploadPackage) + }) m.Group("/generic", func() { m.Group("/{packagename}/{packageversion}/{filename}", func() { m.Get("", generic.DownloadPackageFile) diff --git a/routers/api/v1/packages/composer/api.go b/routers/api/v1/packages/composer/api.go new file mode 100644 index 0000000000000..ea06a16de3338 --- /dev/null +++ b/routers/api/v1/packages/composer/api.go @@ -0,0 +1,117 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package composer + +import ( + "strconv" + "time" + + packages_model "code.gitea.io/gitea/models/packages" + composer_module "code.gitea.io/gitea/modules/packages/composer" +) + +// ServiceIndexResponse contains registry endpoints +type ServiceIndexResponse struct { + SearchTemplate string `json:"search"` + MetadataTemplate string `json:"metadata-url"` + PackageList string `json:"list"` +} + +func createServiceIndexResponse(registryURL string) *ServiceIndexResponse { + return &ServiceIndexResponse{ + SearchTemplate: registryURL + "/search.json?q=%query%&type=%type%", + MetadataTemplate: registryURL + "/p2/%package%.json", + PackageList: registryURL + "/list.json", + } +} + +// SearchResultResponse contains search results +type SearchResultResponse struct { + Total int64 `json:"total"` + Results []*SearchResult `json:"results"` + NextLink string `json:"next,omitempty"` +} + +// SearchResult contains a search result +type SearchResult struct { + Name string `json:"name"` + Description string `json:"description"` + Downloads int64 `json:"downloads"` +} + +func createSearchResultResponse(total int64, pds []*packages_model.PackageDescriptor, nextLink string) *SearchResultResponse { + results := make([]*SearchResult, 0, len(pds)) + + for _, pd := range pds { + results = append(results, &SearchResult{ + Name: pd.Package.Name, + Description: pd.Metadata.(*composer_module.Metadata).Description, + Downloads: pd.Version.DownloadCount, + }) + } + + return &SearchResultResponse{ + Total: total, + Results: results, + NextLink: nextLink, + } +} + +// PackageMetadataResponse contains packages metadata +type PackageMetadataResponse struct { + Minified string `json:"minified"` + Packages map[string][]*PackageVersionMetadata `json:"packages"` +} + +// PackageVersionMetadata contains package metadata +type PackageVersionMetadata struct { + *composer_module.Metadata + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + Created time.Time `json:"time"` + Dist Dist `json:"dist"` +} + +// Dist contains package download informations +type Dist struct { + Type string `json:"type"` + URL string `json:"url"` + Checksum string `json:"shasum"` +} + +func createPackageMetadataResponse(registryURL string, pds []*packages_model.PackageDescriptor) *PackageMetadataResponse { + versions := make([]*PackageVersionMetadata, 0, len(pds)) + + for _, pd := range pds { + packageType := "" + for _, pvp := range pd.Properties { + if pvp.Name == composer_module.TypeProperty { + packageType = pvp.Value + break + } + } + + versions = append(versions, &PackageVersionMetadata{ + Name: pd.Package.Name, + Version: pd.Version.Version, + Type: packageType, + Created: time.Unix(int64(pd.Version.CreatedUnix), 0), + Metadata: pd.Metadata.(*composer_module.Metadata), + Dist: Dist{ + Type: "zip", + URL: registryURL + "/files/" + strconv.FormatInt(pd.Version.ID, 10) + "/" + pd.Files[0].File.Name, + Checksum: pd.Files[0].Blob.HashSHA1, + }, + }) + } + + return &PackageMetadataResponse{ + Minified: "composer/2.0", + Packages: map[string][]*PackageVersionMetadata{ + pds[0].Package.Name: versions, + }, + } +} diff --git a/routers/api/v1/packages/composer/composer.go b/routers/api/v1/packages/composer/composer.go new file mode 100644 index 0000000000000..d44680fdedf03 --- /dev/null +++ b/routers/api/v1/packages/composer/composer.go @@ -0,0 +1,244 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package composer + +import ( + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/convert" + package_module "code.gitea.io/gitea/modules/packages" + composer_module "code.gitea.io/gitea/modules/packages/composer" + "code.gitea.io/gitea/modules/setting" + package_router "code.gitea.io/gitea/routers/api/v1/packages" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/hashicorp/go-version" +) + +func apiError(ctx *context.APIContext, status int, obj interface{}) { + package_router.LogAndProcessError(ctx, status, obj, func(message string) { + type Error struct { + Status int `json:"status"` + Message string `json:"message"` + } + ctx.JSON(status, struct { + Errors []Error `json:"errors"` + }{ + Errors: []Error{ + {Status: status, Message: message}, + }, + }) + }) +} + +// ServiceIndex displays registry endpoints +func ServiceIndex(ctx *context.APIContext) { + resp := createServiceIndexResponse(setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/composer") + + ctx.JSON(http.StatusOK, resp) +} + +// SearchPackages searches packages, only "q" is supported +// https://packagist.org/apidoc#search-packages +func SearchPackages(ctx *context.APIContext) { + page := ctx.FormInt("page") + if page < 1 { + page = 1 + } + perPage := ctx.FormInt("per_page") + paginator := db.ListOptions{ + Page: page, + PageSize: convert.ToCorrectPageSize(perPage), + } + + opts := &packages.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: string(packages.TypeComposer), + Query: ctx.FormTrim("q"), + Paginator: &paginator, + } + if ctx.FormTrim("type") != "" { + opts.Properties = map[string]string{ + composer_module.TypeProperty: ctx.FormTrim("type"), + } + } + + pvs, total, err := packages.SearchLatestVersions(opts) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + nextLink := "" + if len(pvs) == paginator.PageSize { + u, err := url.Parse(setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/composer/search.json") + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + q := u.Query() + q.Set("q", ctx.FormTrim("q")) + q.Set("type", ctx.FormTrim("type")) + q.Set("page", strconv.Itoa(page+1)) + if perPage != 0 { + q.Set("per_page", strconv.Itoa(perPage)) + } + u.RawQuery = q.Encode() + + nextLink = u.String() + } + + pds, err := packages.GetPackageDescriptors(pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createSearchResultResponse(total, pds, nextLink) + + ctx.JSON(http.StatusOK, resp) +} + +// EnumeratePackages lists all package names +// https://packagist.org/apidoc#list-packages +func EnumeratePackages(ctx *context.APIContext) { + ps, err := packages.GetPackagesByType(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeComposer) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + names := make([]string, 0, len(ps)) + for _, p := range ps { + names = append(names, p.Name) + } + + ctx.JSON(http.StatusOK, map[string][]string{ + "packageNames": names, + }) +} + +// PackageMetadata returns the metadata for a single package +// https://packagist.org/apidoc#get-package-data +func PackageMetadata(ctx *context.APIContext) { + vendorName := ctx.Params("vendorname") + projectName := ctx.Params("projectname") + + pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeComposer, vendorName+"/"+projectName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, packages.ErrPackageNotExist) + return + } + + pds, err := packages.GetPackageDescriptors(pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + resp := createPackageMetadataResponse( + setting.AppURL+"api/v1/packages/"+ctx.Package.Owner.Name+"/composer", + pds, + ) + + ctx.JSON(http.StatusOK, resp) +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.APIContext) { + versionID := ctx.ParamsInt64("versionid") + filename := ctx.Params("filename") + + s, pf, err := packages_service.GetFileStreamByPackageVersionID( + ctx.Package.Owner, + versionID, + filename, + ) + if err != nil { + if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// UploadPackage creates a new package +func UploadPackage(ctx *context.APIContext) { + buf, err := package_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + cp, err := composer_module.ParsePackage(buf, buf.Size()) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if cp.Version == "" { + v, err := version.NewVersion(ctx.FormTrim("version")) + if err != nil { + apiError(ctx, http.StatusBadRequest, composer_module.ErrInvalidVersion) + return + } + cp.Version = v.String() + } + + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages.TypeComposer, + Name: cp.Name, + Version: cp.Version, + }, + SemverCompatible: true, + Creator: ctx.User, + Metadata: cp.Metadata, + Properties: map[string]string{ + composer_module.TypeProperty: cp.Type, + }, + }, + &packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)), + Data: buf, + IsLead: true, + }, + ) + if err != nil { + if err == packages.ErrDuplicatePackageVersion { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.PlainText(http.StatusCreated, nil) +} diff --git a/routers/api/v1/repo/package.go b/routers/api/v1/repo/package.go index 2c8382ccfc251..848b5f7084221 100644 --- a/routers/api/v1/repo/package.go +++ b/routers/api/v1/repo/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [generic, nuget, npm, maven, pypi] + // enum: [composer, generic, nuget, npm, maven, pypi] // - name: q // in: query // description: name filter diff --git a/templates/package/content/composer.tmpl b/templates/package/content/composer.tmpl new file mode 100644 index 0000000000000..5a10ec0759cf2 --- /dev/null +++ b/templates/package/content/composer.tmpl @@ -0,0 +1,50 @@ +{{if eq .PackageDescriptor.Package.Type "composer"}} +

    {{.i18n.Tr "packages.installation"}}

    +
    +
    +
    + + +
    +
    + + +
    +
    + +
    +
    +
    + + {{if .PackageDescriptor.Metadata.Description}} +

    {{.i18n.Tr "packages.about"}}

    +
    + {{.PackageDescriptor.Metadata.Description}} +
    + {{end}} + + {{if or .PackageDescriptor.Metadata.Require .PackageDescriptor.Metadata.RequireDev}} +

    {{.i18n.Tr "packages.dependencies"}}

    +
    +
    + {{template "package/content/composer_dependencies" dict "root" $ "dependencies" .PackageDescriptor.Metadata.Require "title" (.i18n.Tr "packages.composer.dependencies")}} + {{template "package/content/composer_dependencies" dict "root" $ "dependencies" .PackageDescriptor.Metadata.RequireDev "title" (.i18n.Tr "packages.composer.dependencies.development")}} +
    +
    + {{end}} + + {{if or .PackageDescriptor.Metadata.Keywords}} +

    {{.i18n.Tr "packages.keywords"}}

    +
    + {{range .PackageDescriptor.Metadata.Keywords}} + {{.}} + {{end}} +
    + {{end}} +{{end}} diff --git a/templates/package/content/composer_dependencies.tmpl b/templates/package/content/composer_dependencies.tmpl new file mode 100644 index 0000000000000..1ab644f417412 --- /dev/null +++ b/templates/package/content/composer_dependencies.tmpl @@ -0,0 +1,19 @@ +{{if .dependencies}} +

    {{.title}}

    + + + + + + + + + {{range $dependency, $version := .dependencies}} + + + + + {{end}} + +
    {{.root.i18n.Tr "packages.dependency.id"}}{{.root.i18n.Tr "packages.dependency.version"}}
    {{$dependency}}{{$version}}
    +{{end}} diff --git a/templates/package/content/npm.tmpl b/templates/package/content/npm.tmpl index 6a54efa163ced..fe33ae4dc97eb 100644 --- a/templates/package/content/npm.tmpl +++ b/templates/package/content/npm.tmpl @@ -11,7 +11,7 @@
    - +
    @@ -37,10 +37,10 @@

    {{.i18n.Tr "packages.dependencies"}}

    - {{template "package/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.Dependencies "title" (.i18n.Tr "packages.npm.dependencies")}} - {{template "package/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "packages.npm.dependencies.development")}} - {{template "package/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.PeerDependencies "title" (.i18n.Tr "packages.npm.dependencies.peer")}} - {{template "package/content/npm_dependencies" dict "dependencies" .PackageDescriptor.Metadata.OptionalDependencies "title" (.i18n.Tr "packages.npm.dependencies.optional")}} + {{template "package/content/npm_dependencies" dict "root" $ "dependencies" .PackageDescriptor.Metadata.Dependencies "title" (.i18n.Tr "packages.npm.dependencies")}} + {{template "package/content/npm_dependencies" dict "root" $ "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "packages.npm.dependencies.development")}} + {{template "package/content/npm_dependencies" dict "root" $ "dependencies" .PackageDescriptor.Metadata.PeerDependencies "title" (.i18n.Tr "packages.npm.dependencies.peer")}} + {{template "package/content/npm_dependencies" dict "root" $ "dependencies" .PackageDescriptor.Metadata.OptionalDependencies "title" (.i18n.Tr "packages.npm.dependencies.optional")}}
    {{end}} diff --git a/templates/package/content/npm_dependencies.tmpl b/templates/package/content/npm_dependencies.tmpl index 7d6afb84503fe..1ab644f417412 100644 --- a/templates/package/content/npm_dependencies.tmpl +++ b/templates/package/content/npm_dependencies.tmpl @@ -3,8 +3,8 @@ - - + + diff --git a/templates/package/content/rubygems.tmpl b/templates/package/content/rubygems.tmpl index f22f9bc50dfd8..8f1d4e083b2a8 100644 --- a/templates/package/content/rubygems.tmpl +++ b/templates/package/content/rubygems.tmpl @@ -32,8 +32,8 @@ end

    {{.i18n.Tr "packages.dependencies"}}

    - {{template "package/content/rubygems_dependencies" dict "dependencies" .PackageDescriptor.Metadata.RuntimeDependencies "title" (.i18n.Tr "packages.rubygems.dependencies.runtime")}} - {{template "package/content/rubygems_dependencies" dict "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "packages.rubygems.dependencies.development")}} + {{template "package/content/rubygems_dependencies" dict "root" $ "dependencies" .PackageDescriptor.Metadata.RuntimeDependencies "title" (.i18n.Tr "packages.rubygems.dependencies.runtime")}} + {{template "package/content/rubygems_dependencies" dict "root" $ "dependencies" .PackageDescriptor.Metadata.DevelopmentDependencies "title" (.i18n.Tr "packages.rubygems.dependencies.development")}}
    {{end}} diff --git a/templates/package/content/rubygems_dependencies.tmpl b/templates/package/content/rubygems_dependencies.tmpl index d3114679ff812..79f66ad3f9d6b 100644 --- a/templates/package/content/rubygems_dependencies.tmpl +++ b/templates/package/content/rubygems_dependencies.tmpl @@ -3,8 +3,8 @@
    {{.i18n.Tr "packages.dependency.id"}}{{.i18n.Tr "packages.dependency.version"}}{{.root.i18n.Tr "packages.dependency.id"}}{{.root.i18n.Tr "packages.dependency.version"}}
    - - + + diff --git a/templates/package/metadata/composer.tmpl b/templates/package/metadata/composer.tmpl new file mode 100644 index 0000000000000..1178d00e0d15c --- /dev/null +++ b/templates/package/metadata/composer.tmpl @@ -0,0 +1,5 @@ +{{if eq .PackageDescriptor.Package.Type "composer"}} + {{range .PackageDescriptor.Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.Name}}
    {{end}} + {{if .PackageDescriptor.Metadata.Homepage}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.details.project_site"}}
    {{end}} + {{range .PackageDescriptor.Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.}}
    {{end}} +{{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 7cd15f0114a0e..f34baf392cdc0 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -19,6 +19,7 @@
    + {{template "package/content/composer" .}} {{template "package/content/generic" .}} {{template "package/content/nuget" .}} {{template "package/content/npm" .}} @@ -36,6 +37,7 @@ {{end}}
    {{svg "octicon-calendar" 16 "mr-3"}} {{.PackageDescriptor.Version.CreatedUnix.FormatDate}}
    {{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
    + {{template "package/metadata/composer" .}} {{template "package/metadata/generic" .}} {{template "package/metadata/nuget" .}} {{template "package/metadata/npm" .}} diff --git a/web_src/svg/gitea-composer.svg b/web_src/svg/gitea-composer.svg new file mode 100644 index 0000000000000..79925d3349963 --- /dev/null +++ b/web_src/svg/gitea-composer.svg @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 1addc8ea1fcd7d52f1b08324f70a186d5011d028 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Nov 2021 16:11:46 +0000 Subject: [PATCH 073/130] Restructured package docs. --- docs/content/doc/packages/composer.en-us.md | 4 ++-- docs/content/doc/packages/generic.en-us.md | 4 ++-- docs/content/doc/packages/maven.en-us.md | 4 ++-- docs/content/doc/packages/npm.en-us.md | 4 ++-- docs/content/doc/packages/nuget.en-us.md | 4 ++-- docs/content/doc/packages/overview.en-us.md | 25 ++++++++++++--------- docs/content/doc/packages/pypi.en-us.md | 4 ++-- docs/content/doc/packages/rubygems.en-us.md | 4 ++-- 8 files changed, 28 insertions(+), 25 deletions(-) diff --git a/docs/content/doc/packages/composer.en-us.md b/docs/content/doc/packages/composer.en-us.md index 542ad7b0983ad..382206a51efd7 100644 --- a/docs/content/doc/packages/composer.en-us.md +++ b/docs/content/doc/packages/composer.en-us.md @@ -1,14 +1,14 @@ --- date: "2021-07-20T00:00:00+00:00" title: "Composer Packages Repository" -slug: "composer" +slug: "packages/composer" draft: false toc: false menu: sidebar: parent: "packages" name: "Composer" - weight: 15 + weight: 10 identifier: "composer" --- diff --git a/docs/content/doc/packages/generic.en-us.md b/docs/content/doc/packages/generic.en-us.md index bda5e4fc13041..1d973bd6d034e 100644 --- a/docs/content/doc/packages/generic.en-us.md +++ b/docs/content/doc/packages/generic.en-us.md @@ -1,14 +1,14 @@ --- date: "2021-07-20T00:00:00+00:00" title: "Generic Packages Repository" -slug: "generic" +slug: "packages/generic" draft: false toc: false menu: sidebar: parent: "packages" name: "Generic" - weight: 10 + weight: 20 identifier: "generic" --- diff --git a/docs/content/doc/packages/maven.en-us.md b/docs/content/doc/packages/maven.en-us.md index e717a927a4735..8798730161f78 100644 --- a/docs/content/doc/packages/maven.en-us.md +++ b/docs/content/doc/packages/maven.en-us.md @@ -1,14 +1,14 @@ --- date: "2021-07-20T00:00:00+00:00" title: "Maven Packages Repository" -slug: "maven" +slug: "packages/maven" draft: false toc: false menu: sidebar: parent: "packages" name: "Maven" - weight: 40 + weight: 30 identifier: "maven" --- diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index e3dfa3f2c94d7..96999c31d9b8e 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -1,14 +1,14 @@ --- date: "2021-07-20T00:00:00+00:00" title: "npm Packages Repository" -slug: "npm" +slug: "packages/npm" draft: false toc: false menu: sidebar: parent: "packages" name: "npm" - weight: 30 + weight: 40 identifier: "npm" --- diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md index d68384758d287..3a326ec8aa593 100644 --- a/docs/content/doc/packages/nuget.en-us.md +++ b/docs/content/doc/packages/nuget.en-us.md @@ -1,14 +1,14 @@ --- date: "2021-07-20T00:00:00+00:00" title: "NuGet Packages Repository" -slug: "nuget" +slug: "packages/nuget" draft: false toc: false menu: sidebar: parent: "packages" name: "NuGet" - weight: 20 + weight: 50 identifier: "nuget" --- diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 35be6b3b08db3..ec3c73b1d668a 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -1,7 +1,7 @@ --- date: "2021-07-20T00:00:00+00:00" title: "Package Registry" -slug: "overview" +slug: "packages/overview" draft: false toc: false menu: @@ -16,20 +16,23 @@ menu: The Package Registry can be used as a public or private registry for common package managers. -These package types are supported: - -- [Generic]({{< relref "doc/packages/generic.en-us.md" >}}) -- [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) -- [NuGet]({{< relref "doc/packages/nuget.en-us.md" >}}) -- [npm]({{< relref "doc/packages/npm.en-us.md" >}}) -- [Maven]({{< relref "doc/packages/maven.en-us.md" >}}) -- [PyPI]({{< relref "doc/packages/pypi.en-us.md" >}}) -- [RubyGems]({{< relref "doc/packages/rubygems.en-us.md" >}}) - **Table of Contents** {{< toc >}} +## Supported package managers + +The following package managers are currently supported: + +| Name | Language | Package client | +| ---- | -------- | -------------- | +| [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` | +| [Generic]({{< relref "doc/packages/generic.en-us.md" >}}) | - | any HTTP client | +| [Maven]({{< relref "doc/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` | +| [npm]({{< relref "doc/packages/npm.en-us.md" >}}) | JavaScript | `npm` | +| [NuGet]({{< relref "doc/packages/nuget.en-us.md" >}}) | .NET | `nuget` | +| [RubyGems]({{< relref "doc/packages/rubygems.en-us.md" >}}) | Ruby | `gem` | + ## View packages You can view the packages of a repository on the repository page. diff --git a/docs/content/doc/packages/pypi.en-us.md b/docs/content/doc/packages/pypi.en-us.md index 34058562b57a9..90ee42fa728f6 100644 --- a/docs/content/doc/packages/pypi.en-us.md +++ b/docs/content/doc/packages/pypi.en-us.md @@ -1,14 +1,14 @@ --- date: "2021-07-20T00:00:00+00:00" title: "PyPI Packages Repository" -slug: "pypi" +slug: "packages/pypi" draft: false toc: false menu: sidebar: parent: "packages" name: "PyPI" - weight: 50 + weight: 60 identifier: "pypi" --- diff --git a/docs/content/doc/packages/rubygems.en-us.md b/docs/content/doc/packages/rubygems.en-us.md index c02e367f771ed..872b13c6abbf8 100644 --- a/docs/content/doc/packages/rubygems.en-us.md +++ b/docs/content/doc/packages/rubygems.en-us.md @@ -1,14 +1,14 @@ --- date: "2021-07-20T00:00:00+00:00" title: "RubyGems Packages Repository" -slug: "rubygems" +slug: "packages/rubygems" draft: false toc: false menu: sidebar: parent: "packages" name: "RubyGems" - weight: 60 + weight: 70 identifier: "rubygems" --- From f7d2484e7b10a3b9e6571d689aa933cfaaccdeff Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Nov 2021 20:24:02 +0000 Subject: [PATCH 074/130] Added missing tests. --- integrations/api_packages_maven_test.go | 45 ++++++++++++++++++++ integrations/api_packages_npm_test.go | 2 + integrations/api_packages_nuget_test.go | 3 ++ routers/api/v1/api.go | 9 ++-- routers/api/v1/{repo => packages}/package.go | 2 +- 5 files changed, 56 insertions(+), 5 deletions(-) rename routers/api/v1/{repo => packages}/package.go (99%) diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go index 3a8e9431f125d..bb4b184be7b57 100644 --- a/integrations/api_packages_maven_test.go +++ b/integrations/api_packages_maven_test.go @@ -157,4 +157,49 @@ func TestPackageMaven(t *testing.T) { assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) }) + + t.Run("DownloadChecksums", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/1.2.3/%s", root, filename)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusNotFound) + + for key, checksum := range map[string]string{ + "md5": "098f6bcd4621d373cade4e832627b4f6", + "sha1": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", + "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "sha512": "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff", + } { + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s.%s", root, packageVersion, filename, key)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, checksum, resp.Body.String()) + } + }) + + t.Run("DownloadMetadata", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", root+"/maven-metadata.xml") + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + expectedMetadata := `` + "\ncom.giteatest-project1.0.11.0.11.0.1" + assert.Equal(t, expectedMetadata, resp.Body.String()) + + for key, checksum := range map[string]string{ + "md5": "6bee0cebaaa686d658adf3e7e16371a0", + "sha1": "8696abce499fe84d9ea93e5492abe7147e195b6c", + "sha256": "3f48322f81c4b2c3bb8649ae1e5c9801476162b520e1c2734ac06b2c06143208", + "sha512": "cb075aa2e2ef1a83cdc14dd1e08c505b72d633399b39e73a21f00f0deecb39a3e2c79f157c1163f8a3854828750706e0dec3a0f5e4778e91f8ec2cf351a855f2", + } { + req := NewRequest(t, "GET", fmt.Sprintf("%s/maven-metadata.xml.%s", root, key)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, checksum, resp.Body.String()) + } + }) } diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index 46d0527b65a31..4e1918fee5f9c 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -208,6 +208,8 @@ func TestPackageNpm(t *testing.T) { MakeRequest(t, req, status) } + test(t, http.StatusBadRequest, "v1.0") + test(t, http.StatusBadRequest, "1.0") test(t, http.StatusOK, "dummy") test(t, http.StatusOK, packageTag2) }) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index 42e3fe5e430e2..6c1f1681f9a57 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -43,6 +43,9 @@ func TestPackageNuGet(t *testing.T) { ` + packageVersion + ` ` + packageAuthors + ` ` + packageDescription + ` + + + `)) archive.Close() diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 6bf981e6df525..980b5674d5a3d 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -82,6 +82,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/misc" "code.gitea.io/gitea/routers/api/v1/notify" "code.gitea.io/gitea/routers/api/v1/org" + "code.gitea.io/gitea/routers/api/v1/packages" "code.gitea.io/gitea/routers/api/v1/packages/composer" "code.gitea.io/gitea/routers/api/v1/packages/generic" "code.gitea.io/gitea/routers/api/v1/packages/maven" @@ -1096,11 +1097,11 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { }, reqToken(), reqPackageAccess(models.AccessModeWrite)) }) m.Group("/{versionid}", func() { - m.Get("", repo.GetPackage) - m.Delete("", reqPackageAccess(models.AccessModeWrite), repo.DeletePackage) - m.Get("/files", repo.ListPackageFiles) + m.Get("", packages.GetPackage) + m.Delete("", reqPackageAccess(models.AccessModeWrite), packages.DeletePackage) + m.Get("/files", packages.ListPackageFiles) }) - m.Get("/", repo.ListPackages) + m.Get("/", packages.ListPackages) }, context.UserAssignmentAPI(), context.PackageAssignmentAPI(), reqPackageAccess(models.AccessModeRead)) // Organizations diff --git a/routers/api/v1/repo/package.go b/routers/api/v1/packages/package.go similarity index 99% rename from routers/api/v1/repo/package.go rename to routers/api/v1/packages/package.go index 848b5f7084221..38b399315b8ca 100644 --- a/routers/api/v1/repo/package.go +++ b/routers/api/v1/packages/package.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package repo +package packages import ( "net/http" From 3cb36d5c3fa44cd5dbd767663b0654dde83a9e3e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Nov 2021 21:27:54 +0000 Subject: [PATCH 075/130] Fixed generic content page. --- templates/package/content/generic.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/package/content/generic.tmpl b/templates/package/content/generic.tmpl index 4d6d4ae4a1f41..bfaae85b507fb 100644 --- a/templates/package/content/generic.tmpl +++ b/templates/package/content/generic.tmpl @@ -1,10 +1,10 @@ -{{if eq .Package.Type 0}} +{{if eq .PackageDescriptor.Package.Type "generic"}}

    {{.i18n.Tr "packages.installation"}}

    - +
    From 289278298c659428d65193bc36d82d808d179c84 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Nov 2021 23:04:00 +0100 Subject: [PATCH 076/130] Fixed docs. --- docs/content/doc/packages/nuget.en-us.md | 4 ++-- docs/content/doc/packages/pypi.en-us.md | 2 +- docs/content/doc/packages/rubygems.en-us.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md index 3a326ec8aa593..ad10cfba9098a 100644 --- a/docs/content/doc/packages/nuget.en-us.md +++ b/docs/content/doc/packages/nuget.en-us.md @@ -22,13 +22,13 @@ Publish [NuGet](https://www.nuget.org/) packages for your user or organization. ## Requirements -To work with the NuGet package registry, you can use command-line interface (CLI) tools as well as NuGet features in various IDEs like Visual Studio. +To work with the NuGet package registry, you can use command-line interface tools as well as NuGet features in various IDEs like Visual Studio. More informations about NuGet clients can be found in [the official documentation](https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools). The following examples use the `dotnet nuget` tool. ## Configuring the package registry -To register the project’s package registry you need to configure a new NuGet feed source: +To register the package registry you need to configure a new NuGet feed source: ```shell dotnet nuget add source --name {source_name} --username {username} --password {password} https://gitea.example.com/api/v1/packages/{owner}/nuget/index.json diff --git a/docs/content/doc/packages/pypi.en-us.md b/docs/content/doc/packages/pypi.en-us.md index 90ee42fa728f6..8d527fd8e1d01 100644 --- a/docs/content/doc/packages/pypi.en-us.md +++ b/docs/content/doc/packages/pypi.en-us.md @@ -26,7 +26,7 @@ To work with the PyPI package registry, you need to use the tools [pip](https:// ## Configuring the package registry -To register the project’s package registry you need to edit your local `~/.pypirc` file. Add +To register the package registry you need to edit your local `~/.pypirc` file. Add ```ini [distutils] diff --git a/docs/content/doc/packages/rubygems.en-us.md b/docs/content/doc/packages/rubygems.en-us.md index 872b13c6abbf8..d672bbc4fd662 100644 --- a/docs/content/doc/packages/rubygems.en-us.md +++ b/docs/content/doc/packages/rubygems.en-us.md @@ -26,7 +26,7 @@ To work with the RubyGems package registry, you need to use the [gem](https://gu ## Configuring the package registry -To register the project’s package registry edit the `~/.gem/credentials` file and add: +To register the package registry edit the `~/.gem/credentials` file and add: ```ini --- @@ -40,7 +40,7 @@ https://gitea.example.com/api/v1/packages/{owner}/rubygems: Bearer {token} For example: -```ini +``` --- https://gitea.example.com/api/v1/packages/testuser/rubygems: Bearer 3bd626f84b01cd26b873931eace1e430a5773cc4 ``` From e84812cc83a25571af525757d672664cd29f9bcb Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 23 Nov 2021 09:49:36 +0000 Subject: [PATCH 077/130] Fixed swagger. --- templates/swagger/v1_json.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index d5f3c777b6d32..dafde478fe81a 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -1902,6 +1902,7 @@ }, { "enum": [ + "composer", "generic", "nuget", "npm", From 87ac83612e60e13da920e412dfa5f3d0b85b800c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 23 Nov 2021 16:14:00 +0100 Subject: [PATCH 078/130] Added missing type. --- routers/api/v1/packages/package.go | 2 +- templates/swagger/v1_json.tmpl | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 38b399315b8ca..e9a47695a5c6f 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [composer, generic, nuget, npm, maven, pypi] + // enum: [composer, generic, maven, npm, nuget, pypi, rubygems] // - name: q // in: query // description: name filter diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index dafde478fe81a..65b10d57eaef5 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -1904,10 +1904,11 @@ "enum": [ "composer", "generic", - "nuget", - "npm", "maven", - "pypi" + "npm", + "nuget", + "pypi", + "rubygems" ], "type": "string", "description": "package type filter", From 89342141e4bc84bc74603fd55b7639349adf7ba2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 23 Nov 2021 16:14:30 +0100 Subject: [PATCH 079/130] Fixed ambiguous column. --- models/packages/package_version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/packages/package_version.go b/models/packages/package_version.go index e3b535635eaf5..8298da69ab765 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -204,7 +204,7 @@ func SearchVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, error Where(opts.toConds()). Table("package_version"). Join("INNER", "package", "package.id = package_version.package_id"). - Desc("created_unix") + Desc("package_version.created_unix") if opts.Paginator != nil { sess = db.SetSessionPagination(sess, opts) @@ -225,7 +225,7 @@ func SearchLatestVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND package_version.created_unix < pv2.created_unix"). Join("INNER", "package", "package.id = package_version.package_id"). Where(cond). - Desc("created_unix") + Desc("package_version.created_unix") if opts.Paginator != nil { sess = db.SetSessionPagination(sess, opts) From abf508f7fbab10cc1115854c0ba937cf748b8004 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 27 Nov 2021 13:46:09 +0000 Subject: [PATCH 080/130] Organize content store by sha256 hash. --- modules/packages/content_store.go | 22 ++++++++++------ routers/api/v1/packages/maven/maven.go | 14 +++++------ services/packages/packages.go | 35 ++++++++++++++++---------- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index 26471fdd38cba..64c3eedc2328e 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -6,11 +6,14 @@ package packages import ( "io" - "strconv" + "path" "code.gitea.io/gitea/modules/storage" ) +// BlobHash256Key is the key to address a blob content +type BlobHash256Key string + // ContentStore is a wrapper around ObjectStorage type ContentStore struct { store storage.ObjectStorage @@ -23,17 +26,22 @@ func NewContentStore() *ContentStore { } // Get gets a package blob -func (s *ContentStore) Get(packageBlobID int64) (storage.Object, error) { - return s.store.Open(strconv.FormatInt(packageBlobID, 10)) +func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { + return s.store.Open(keyToRelativePath(key)) } // Save stores a package blob -func (s *ContentStore) Save(packageBlobID int64, r io.Reader, size int64) error { - _, err := s.store.Save(strconv.FormatInt(packageBlobID, 10), r, size) +func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error { + _, err := s.store.Save(keyToRelativePath(key), r, size) return err } // Delete deletes a package blob -func (s *ContentStore) Delete(packageBlobID int64) error { - return s.store.Delete(strconv.FormatInt(packageBlobID, 10)) +func (s *ContentStore) Delete(key BlobHash256Key) error { + return s.store.Delete(keyToRelativePath(key)) +} + +// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000... +func keyToRelativePath(key BlobHash256Key) string { + return path.Join(string(key)[0:2], string(key)[2:4], string(key)) } diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 784213881420e..dfec572a3035c 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -145,13 +145,13 @@ func servePackageFile(ctx *context.APIContext, params parameters) { return } - if isChecksumExtension(ext) { - pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if isChecksumExtension(ext) { var hash string switch ext { case extensionMD5: @@ -167,7 +167,7 @@ func servePackageFile(ctx *context.APIContext, params parameters) { return } - s, err := packages_module.NewContentStore().Get(pf.BlobID) + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) } diff --git a/services/packages/packages.go b/services/packages/packages.go index a599536b0ccd1..72bba9fd7b7c4 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -65,12 +65,12 @@ func createPackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo, al return nil, nil, err } - pf, blobCreated, err := addFileToPackageVersion(ctx, pv, pfi) + pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfi) removeBlob := false defer func() { if blobCreated && removeBlob { contentStore := packages_module.NewContentStore() - if err := contentStore.Delete(pf.BlobID); err != nil { + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { log.Error("Error deleting package blob from content store: %v", err) } } @@ -161,12 +161,12 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfi *PackageFileInfo) (*packages return nil, nil, err } - pf, blobCreated, err := addFileToPackageVersion(ctx, pv, pfi) + pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfi) removeBlob := false defer func() { if blobCreated && removeBlob { contentStore := packages_module.NewContentStore() - if err := contentStore.Delete(pf.BlobID); err != nil { + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { log.Error("Error deleting package blob from content store: %v", err) } } @@ -177,33 +177,37 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfi *PackageFileInfo) (*packages } if err := committer.Commit(); err != nil { + removeBlob = true return nil, nil, err } return pv, pf, nil } -func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (*packages_model.PackageFile, bool, error) { +func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { log.Trace("Adding package file: %v, %s", pv.ID, pfi.Filename) hashMD5, hashSHA1, hashSHA256, hashSHA512 := pfi.Data.Sums() + + blobKey := fmt.Sprintf("%x", hashSHA256) + pb := &packages_model.PackageBlob{ Size: pfi.Data.Size(), HashMD5: fmt.Sprintf("%x", hashMD5), HashSHA1: fmt.Sprintf("%x", hashSHA1), - HashSHA256: fmt.Sprintf("%x", hashSHA256), + HashSHA256: blobKey, HashSHA512: fmt.Sprintf("%x", hashSHA512), } pb, exists, err := packages_model.GetOrInsertBlob(ctx, pb) if err != nil { log.Error("Error inserting package blob: %v", err) - return nil, false, err + return nil, nil, false, err } if !exists { contentStore := packages_module.NewContentStore() - if err := contentStore.Save(pb.ID, pfi.Data, pfi.Data.Size()); err != nil { + if err := contentStore.Save(packages_module.BlobHash256Key(blobKey), pfi.Data, pfi.Data.Size()); err != nil { log.Error("Error saving package blob in content store: %v", err) - return nil, false, err + return nil, nil, false, err } } @@ -218,10 +222,10 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers if err != packages_model.ErrDuplicatePackageFile { log.Error("Error inserting package file: %v", err) } - return nil, !exists, err + return nil, nil, !exists, err } - return pf, !exists, nil + return pf, pb, !exists, nil } // DeletePackageVersionByNameAndVersion deletes a package version and all associated files @@ -299,7 +303,7 @@ func DeleteUnreferencedBlobs() error { contentStore := packages_module.NewContentStore() for _, pb := range pbs { - if err := contentStore.Delete(pb.ID); err != nil { + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { log.Error("Error deleting package blob [%v]: %v", pb.ID, err) } } @@ -356,7 +360,12 @@ func GetPackageFileStream(pv *packages_model.PackageVersion, filename string) (i return nil, nil, err } - s, err := packages_module.NewContentStore().Get(pf.BlobID) + pb, err := packages_model.GetBlobByID(db.DefaultContext, pf.BlobID) + if err != nil { + return nil, nil, err + } + + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256)) if err == nil { if pf.IsLead { if err := packages_model.IncrementDownloadCounter(pv.ID); err != nil { From 1239f82e6f20ca149b4cba1c5edfcfefa2633de9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 30 Nov 2021 22:13:31 +0000 Subject: [PATCH 081/130] Added admin package management. --- models/packages/descriptor.go | 9 +++ models/packages/package_blob.go | 6 ++ models/packages/package_version.go | 7 ++- options/locale/locale_en-US.ini | 11 ++++ routers/web/admin/packages.go | 91 +++++++++++++++++++++++++++++ routers/web/web.go | 5 ++ templates/admin/navbar.tmpl | 3 + templates/admin/packages/list.tmpl | 92 ++++++++++++++++++++++++++++++ templates/package/shared/list.tmpl | 5 +- 9 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 routers/web/admin/packages.go create mode 100644 templates/admin/packages/list.tmpl diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 11b0a2703a6fe..bc1b7f1366b79 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -46,6 +46,15 @@ func (pd *PackageDescriptor) WebLink() string { return pd.Owner.HTMLURL() + "/-/packages/" + strconv.FormatInt(pd.Version.ID, 10) } +// CalculateBlobSize returns the total blobs size in bytes +func (pd *PackageDescriptor) CalculateBlobSize() int64 { + size := int64(0) + for _, f := range pd.Files { + size += f.Blob.Size + } + return size +} + // GetPackageDescriptor gets the package description for a version func GetPackageDescriptor(pv *PackageVersion) (*PackageDescriptor, error) { return GetPackageDescriptorCtx(db.DefaultContext, pv) diff --git a/models/packages/package_blob.go b/models/packages/package_blob.go index 4f4414bdad4fc..8d5279d933dbc 100644 --- a/models/packages/package_blob.go +++ b/models/packages/package_blob.go @@ -76,3 +76,9 @@ func DeleteBlobByID(ctx context.Context, blobID int64) error { _, err := db.GetEngine(ctx).ID(blobID).Delete(&PackageBlob{}) return err } + +// GetTotalBlobSize returns the total blobs size in bytes +func GetTotalBlobSize() (int64, error) { + return db.GetEngine(db.DefaultContext). + SumInt(&PackageBlob{}, "size") +} diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 8298da69ab765..7087c6baa4859 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -172,12 +172,15 @@ type PackageSearchOptions struct { } func (opts *PackageSearchOptions) toConds() builder.Cond { - var cond builder.Cond = builder.Eq{"package.owner_id": opts.OwnerID} + cond := builder.NewCond() + if opts.OwnerID != 0 { + cond = cond.And(builder.Eq{"package.owner_id": opts.OwnerID}) + } if opts.RepoID != 0 { cond = cond.And(builder.Eq{"package.repo_id": opts.RepoID}) } - if opts.Type != "" { + if opts.Type != "" && opts.Type != "all" { cond = cond.And(builder.Eq{"package.type": opts.Type}) } if opts.Query != "" { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index c4e3380c3e758..402d2039b8054 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2439,6 +2439,17 @@ repos.forks = Forks repos.issues = Issues repos.size = Size +packages.package_manage_panel = Package Management +packages.total_size = Total Blob Size: %s +packages.owner = Owner +packages.creator = Creator +packages.name = Name +packages.version = Version +packages.type = Type +packages.repository = Repository +packages.size = Size +packages.published = Published + defaulthooks = Default Webhooks defaulthooks.desc = Webhooks automatically make HTTP POST requests to a server when certain Gitea events trigger. Webhooks defined here are defaults and will be copied into all new repositories. Read more in the webhooks guide. defaulthooks.add_webhook = Add Default Webhook diff --git a/routers/web/admin/packages.go b/routers/web/admin/packages.go new file mode 100644 index 0000000000000..2b8ee8436331d --- /dev/null +++ b/routers/web/admin/packages.go @@ -0,0 +1,91 @@ +// Copyright 2014 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package admin + +import ( + "net/http" + "net/url" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + packages_service "code.gitea.io/gitea/services/packages" +) + +const ( + tplPackagesList base.TplName = "admin/packages/list" +) + +// Packages shows all packages +func Packages(ctx *context.Context) { + page := ctx.FormInt("page") + if page <= 1 { + page = 1 + } + query := ctx.FormTrim("q") + packageType := ctx.FormTrim("type") + + pvs, total, err := packages_model.SearchVersions(&packages_model.PackageSearchOptions{ + Query: query, + Type: packageType, + Paginator: &db.ListOptions{ + PageSize: setting.UI.PackagesPagingNum, + Page: page, + }, + }) + if err != nil { + ctx.ServerError("SearchLatestVersions", err) + return + } + + pds, err := packages_model.GetPackageDescriptors(pvs) + if err != nil { + ctx.ServerError("GetPackageDescriptors", err) + return + } + + totalBlobSize, err := packages_model.GetTotalBlobSize() + if err != nil { + ctx.ServerError("GetTotalBlobSize", err) + return + } + + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminPackages"] = true + ctx.Data["Query"] = query + ctx.Data["PackageType"] = packageType + ctx.Data["PackageDescriptors"] = pds + ctx.Data["Total"] = total + ctx.Data["TotalBlobSize"] = totalBlobSize + + pager := context.NewPagination(int(total), setting.UI.PackagesPagingNum, page, 5) + pager.AddParam(ctx, "q", "Query") + pager.AddParam(ctx, "type", "PackageType") + ctx.Data["Page"] = pager + + ctx.HTML(http.StatusOK, tplPackagesList) +} + +// DeletePackageVersion deletes a package version +func DeletePackageVersion(ctx *context.Context) { + pv, err := packages_model.GetVersionByID(db.DefaultContext, ctx.FormInt64("id")) + if err != nil { + ctx.ServerError("GetRepositoryByID", err) + return + } + + if err := packages_service.DeletePackageVersion(ctx.User, pv); err != nil { + ctx.ServerError("DeletePackageVersion", err) + return + } + + ctx.Flash.Success(ctx.Tr("packages.settings.delete.success")) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "redirect": setting.AppSubURL + "/admin/packages?page=" + url.QueryEscape(ctx.FormString("page")) + "&q=" + url.QueryEscape(ctx.FormString("q")) + "&type=" + url.QueryEscape(ctx.FormString("type")), + }) +} diff --git a/routers/web/web.go b/routers/web/web.go index 0543c9b36e451..9cc8c410923d0 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -433,6 +433,11 @@ func RegisterRoutes(m *web.Route) { m.Post("/delete", admin.DeleteRepo) }) + m.Group("/packages", func() { + m.Get("", admin.Packages) + m.Post("/delete", admin.DeletePackageVersion) + }) + m.Group("/hooks", func() { m.Get("", admin.DefaultOrSystemWebhooks) m.Post("/delete", admin.DeleteDefaultOrSystemWebhook) diff --git a/templates/admin/navbar.tmpl b/templates/admin/navbar.tmpl index c656d0619b339..24a0a093a6b28 100644 --- a/templates/admin/navbar.tmpl +++ b/templates/admin/navbar.tmpl @@ -12,6 +12,9 @@ {{.i18n.Tr "admin.repositories"}} + + {{.i18n.Tr "packages.title"}} + {{if not DisableWebhooks}} {{.i18n.Tr "admin.hooks"}} diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl new file mode 100644 index 0000000000000..2e0d2c9230b29 --- /dev/null +++ b/templates/admin/packages/list.tmpl @@ -0,0 +1,92 @@ +{{template "base/head" .}} +
    + {{template "admin/navbar" .}} +
    + {{template "base/alert" .}} +

    + {{.i18n.Tr "admin.packages.package_manage_panel"}} ({{.i18n.Tr "admin.total" .Total}}, {{.i18n.Tr "admin.packages.total_size" (FileSize .TotalBlobSize)}}) +

    +
    +
    +
    + + + +
    + +
    +
    +
    {{.i18n.Tr "packages.dependency.id"}}{{.i18n.Tr "packages.dependency.version"}}{{.root.i18n.Tr "packages.dependency.id"}}{{.root.i18n.Tr "packages.dependency.version"}}
    + + + + + + + + + + + + + + + + {{range .PackageDescriptors}} + + + + + + + + + + + + + {{end}} + +
    ID{{SortArrow "oldest" "newest" "$.SortType" false}}{{.i18n.Tr "admin.packages.owner"}}{{.i18n.Tr "admin.packages.type"}} + {{.i18n.Tr "admin.packages.name"}} + {{SortArrow "alphabetically" "reversealphabetically" "$.SortType" false}} + {{.i18n.Tr "admin.packages.version"}}{{.i18n.Tr "admin.packages.creator"}}{{.i18n.Tr "admin.packages.repository"}} + {{.i18n.Tr "admin.packages.size"}} + {{SortArrow "size" "reversesize" "$.SortType" false}} + {{.i18n.Tr "admin.packages.published"}}{{.i18n.Tr "admin.notices.op"}}
    {{.Version.ID}} + {{.Owner.Name}} + {{if .Owner.Visibility.IsPrivate}} + {{svg "octicon-lock"}} + {{end}} + {{.Package.Type.Name}}{{.Package.Name}}{{.Version.Version}}{{.Creator.Name}} + {{if .Repository}} + {{.Repository.Name}} + {{end}} + {{FileSize .CalculateBlobSize}}{{.Version.CreatedUnix.FormatShort}}{{svg "octicon-trash"}}
    +
    + + {{template "base/paginate" .}} +
    +
    + +
    +
    + {{svg "octicon-trash"}} + {{.i18n.Tr "packages.settings.delete"}} +
    +
    + {{.i18n.Tr "packages.settings.delete.notice" `` `` | Safe}} +
    + {{template "base/delete_modal_actions" .}} +
    +{{template "base/footer" .}} diff --git a/templates/package/shared/list.tmpl b/templates/package/shared/list.tmpl index 36bd57a4d68d3..e30a1920ab66a 100644 --- a/templates/package/shared/list.tmpl +++ b/templates/package/shared/list.tmpl @@ -6,10 +6,11 @@ From 16edc81e44bfa1978f108a75c338d3e9e5d62ef8 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 8 Dec 2021 21:37:43 +0000 Subject: [PATCH 082/130] Added support for sorting. --- models/packages/package_version.go | 28 ++++++++++++++++++++++++---- routers/web/admin/packages.go | 8 ++++++-- templates/admin/packages/list.tmpl | 17 ++++++++++------- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 7087c6baa4859..7518306ecae72 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -168,6 +168,7 @@ type PackageSearchOptions struct { Type string Query string Properties map[string]string + Sort string db.Paginator } @@ -201,13 +202,31 @@ func (opts *PackageSearchOptions) toConds() builder.Cond { return cond } +func (opts *PackageSearchOptions) configureOrderBy(e db.Engine) { + switch opts.Sort { + case "alphabetically": + e.Asc("package.name") + case "reversealphabetically": + e.Desc("package.name") + case "highestversion": + e.Desc("package_version.version") + case "lowestversion": + e.Asc("package_version.version") + case "oldest": + e.Asc("package_version.created_unix") + default: + e.Desc("package_version.created_unix") + } +} + // SearchVersions gets all versions of packages matching the search options func SearchVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, error) { sess := db.GetEngine(db.DefaultContext). Where(opts.toConds()). Table("package_version"). - Join("INNER", "package", "package.id = package_version.package_id"). - Desc("package_version.created_unix") + Join("INNER", "package", "package.id = package_version.package_id") + + opts.configureOrderBy(sess) if opts.Paginator != nil { sess = db.SetSessionPagination(sess, opts) @@ -227,8 +246,9 @@ func SearchLatestVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, Table("package_version"). Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND package_version.created_unix < pv2.created_unix"). Join("INNER", "package", "package.id = package_version.package_id"). - Where(cond). - Desc("package_version.created_unix") + Where(cond) + + opts.configureOrderBy(sess) if opts.Paginator != nil { sess = db.SetSessionPagination(sess, opts) diff --git a/routers/web/admin/packages.go b/routers/web/admin/packages.go index 2b8ee8436331d..86c26db62af43 100644 --- a/routers/web/admin/packages.go +++ b/routers/web/admin/packages.go @@ -28,10 +28,12 @@ func Packages(ctx *context.Context) { } query := ctx.FormTrim("q") packageType := ctx.FormTrim("type") + sort := ctx.FormTrim("sort") pvs, total, err := packages_model.SearchVersions(&packages_model.PackageSearchOptions{ Query: query, Type: packageType, + Sort: sort, Paginator: &db.ListOptions{ PageSize: setting.UI.PackagesPagingNum, Page: page, @@ -59,13 +61,15 @@ func Packages(ctx *context.Context) { ctx.Data["PageIsAdminPackages"] = true ctx.Data["Query"] = query ctx.Data["PackageType"] = packageType + ctx.Data["SortType"] = sort ctx.Data["PackageDescriptors"] = pds ctx.Data["Total"] = total ctx.Data["TotalBlobSize"] = totalBlobSize pager := context.NewPagination(int(total), setting.UI.PackagesPagingNum, page, 5) - pager.AddParam(ctx, "q", "Query") - pager.AddParam(ctx, "type", "PackageType") + pager.AddParamString("q", query) + pager.AddParamString("type", packageType) + pager.AddParamString("sort", sort) ctx.Data["Page"] = pager ctx.HTML(http.StatusOK, tplPackagesList) diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index 2e0d2c9230b29..874740f35c01c 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -29,21 +29,24 @@ - + + - - + - From 842308cf81a2d8bc95a4a1eb1136fe2331c33365 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 13 Jan 2022 22:17:17 +0000 Subject: [PATCH 083/130] Add support for multiple identical versions/files. --- integrations/api_packages_composer_test.go | 20 ++--- integrations/api_packages_pypi_test.go | 2 +- models/migrations/v207.go | 14 +-- models/packages/package_file.go | 44 ++++++--- models/packages/package_version.go | 8 +- routers/api/v1/api.go | 2 +- routers/api/v1/packages/composer/api.go | 2 +- routers/api/v1/packages/composer/composer.go | 16 ++-- routers/api/v1/packages/generic/generic.go | 14 +-- routers/api/v1/packages/maven/maven.go | 20 +++-- routers/api/v1/packages/npm/npm.go | 16 ++-- routers/api/v1/packages/nuget/nuget.go | 26 +++--- routers/api/v1/packages/pypi/pypi.go | 14 +-- routers/api/v1/packages/rubygems/rubygems.go | 17 ++-- routers/web/user/package.go | 8 +- routers/web/web.go | 2 +- services/packages/packages.go | 95 ++++++++++++-------- templates/package/view.tmpl | 2 +- 18 files changed, 197 insertions(+), 125 deletions(-) diff --git a/integrations/api_packages_composer_test.go b/integrations/api_packages_composer_test.go index 5c45b12e5def9..ea7de6cac458c 100644 --- a/integrations/api_packages_composer_test.go +++ b/integrations/api_packages_composer_test.go @@ -122,7 +122,11 @@ func TestPackageComposer(t *testing.T) { assert.Len(t, pvs, 1) assert.Equal(t, int64(0), pvs[0].DownloadCount) - req := NewRequest(t, "GET", fmt.Sprintf("%s/files/%d/%s-%s.%s.zip", url, pvs[0].ID, vendorName, projectName, packageVersion)) + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + + req := NewRequest(t, "GET", fmt.Sprintf("%s/files/%d/%d", url, pvs[0].ID, pfs[0].ID)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) @@ -206,18 +210,4 @@ func TestPackageComposer(t *testing.T) { assert.Equal(t, "zip", pkgs[0].Dist.Type) assert.Equal(t, "7b40bfd6da811b2b78deec1e944f156dbb2c747b", pkgs[0].Dist.Checksum) }) - - /*t.Run("PackageService", func(t *testing.T) { - defer PrintCurrentTest(t)() - - req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/index.json", url, packageName)) - req = AddBasicAuthHeader(req, user.Name) - resp := MakeRequest(t, req, http.StatusOK) - - var result nuget.PackageVersionsResponse - DecodeJSON(t, resp, &result) - - assert.Len(t, result.Versions, 1) - assert.Equal(t, packageVersion, result.Versions[0]) - })*/ } diff --git a/integrations/api_packages_pypi_test.go b/integrations/api_packages_pypi_test.go index af489d53c1280..b21c2b7023c14 100644 --- a/integrations/api_packages_pypi_test.go +++ b/integrations/api_packages_pypi_test.go @@ -108,7 +108,7 @@ func TestPackagePyPI(t *testing.T) { assert.NoError(t, err) assert.Len(t, pfs, 2) - pf, err := packages.GetFileForVersionByName(db.DefaultContext, pvs[0].ID, filename) + pf, err := packages.GetFileForVersionByName(db.DefaultContext, pvs[0].ID, filename, packages.EmptyFileKey) assert.NoError(t, err) assert.Equal(t, filename, pf.Name) assert.True(t, pf.IsLead) diff --git a/models/migrations/v207.go b/models/migrations/v207.go index f001bdd110dfa..c5436c9958a87 100644 --- a/models/migrations/v207.go +++ b/models/migrations/v207.go @@ -31,6 +31,7 @@ func addPackageTables(x *xorm.Engine) error { CreatorID int64 Version string LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` + CompositeKey string `xorm:"UNIQUE(s) INDEX"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` MetadataJSON string `xorm:"TEXT metadata_json"` DownloadCount int64 @@ -52,12 +53,13 @@ func addPackageTables(x *xorm.Engine) error { } type PackageFile struct { - ID int64 `xorm:"pk autoincr"` - VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - BlobID int64 `xorm:"INDEX NOT NULL"` - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - IsLead bool + ID int64 `xorm:"pk autoincr"` + VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + BlobID int64 `xorm:"INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + CompositeKey string `xorm:"UNIQUE(s) INDEX"` + IsLead bool } if err := x.Sync2(new(PackageFile)); err != nil { diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 20c0593aa3c4d..9c8982c37037f 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -23,14 +23,18 @@ var ( ErrPackageFileNotExist = errors.New("Package file does not exist") ) +// EmptyFileKey is a named constant for an empty file key +const EmptyFileKey = "" + // PackageFile represents a package file type PackageFile struct { - ID int64 `xorm:"pk autoincr"` - VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - BlobID int64 `xorm:"INDEX NOT NULL"` - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - IsLead bool + ID int64 `xorm:"pk autoincr"` + VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + BlobID int64 `xorm:"INDEX NOT NULL"` + Name string + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + CompositeKey string `xorm:"UNIQUE(s) INDEX"` + IsLead bool } // TryInsertFile inserts a file. If the file exists already ErrDuplicatePackageFile is returned @@ -38,8 +42,9 @@ func TryInsertFile(ctx context.Context, pf *PackageFile) (*PackageFile, error) { e := db.GetEngine(ctx) key := &PackageFile{ - VersionID: pf.VersionID, - LowerName: pf.LowerName, + VersionID: pf.VersionID, + LowerName: pf.LowerName, + CompositeKey: pf.CompositeKey, } has, err := e.Get(key) @@ -61,15 +66,32 @@ func GetFilesByVersionID(ctx context.Context, versionID int64) ([]*PackageFile, return pfs, db.GetEngine(ctx).Where("version_id = ?", versionID).Find(&pfs) } +// GetFileForVersionByID gets a file of a version by id +func GetFileForVersionByID(ctx context.Context, versionID, fileID int64) (*PackageFile, error) { + pf := &PackageFile{ + VersionID: versionID, + } + + has, err := db.GetEngine(ctx).ID(fileID).Get(pf) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageFileNotExist + } + return pf, nil +} + // GetFileForVersionByName gets a file of a version by name -func GetFileForVersionByName(ctx context.Context, versionID int64, name string) (*PackageFile, error) { +func GetFileForVersionByName(ctx context.Context, versionID int64, name, key string) (*PackageFile, error) { if name == "" { return nil, ErrPackageFileNotExist } pf := &PackageFile{ - VersionID: versionID, - LowerName: strings.ToLower(name), + VersionID: versionID, + LowerName: strings.ToLower(name), + CompositeKey: key, } has, err := db.GetEngine(ctx).Get(pf) diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 7518306ecae72..b54686f7184a6 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -22,6 +22,9 @@ var ( ErrPackageVersionNotExist = errors.New("Package version does not exist") ) +// EmptyVersionKey is a named constant for an empty version key +const EmptyVersionKey = "" + func init() { db.RegisterModel(new(PackageVersion)) } @@ -33,6 +36,7 @@ type PackageVersion struct { CreatorID int64 Version string LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` + CompositeKey string `xorm:"UNIQUE(s) INDEX"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` MetadataJSON string `xorm:"TEXT metadata_json"` DownloadCount int64 @@ -45,6 +49,7 @@ func GetOrInsertVersion(ctx context.Context, pv *PackageVersion) (*PackageVersio key := &PackageVersion{ PackageID: pv.PackageID, LowerVersion: pv.LowerVersion, + CompositeKey: pv.CompositeKey, } has, err := e.Get(key) @@ -87,7 +92,7 @@ func GetVersionByID(ctx context.Context, versionID int64) (*PackageVersion, erro } // GetVersionByNameAndVersion gets a version by name and version number -func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType Type, name, version string) (*PackageVersion, error) { +func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType Type, name, version, key string) (*PackageVersion, error) { var cond builder.Cond = builder.Eq{ "package.owner_id": ownerID, "package.type": packageType, @@ -95,6 +100,7 @@ func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType } pv := &PackageVersion{ LowerVersion: strings.ToLower(version), + CompositeKey: key, } has, err := db.GetEngine(ctx). Join("INNER", "package", "package.id = package_version.package_id"). diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 74c053274fde2..95ca4b05cf06d 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1026,7 +1026,7 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { m.Get("/list.json", composer.EnumeratePackages) m.Get("/p2/{vendorname}/{projectname}~dev.json", composer.PackageMetadata) m.Get("/p2/{vendorname}/{projectname}.json", composer.PackageMetadata) - m.Get("/files/{versionid}/{filename}", composer.DownloadPackageFile) + m.Get("/files/{versionid}/{fileid}", composer.DownloadPackageFile) m.Put("", reqToken(), reqPackageAccess(perm.AccessModeWrite), composer.UploadPackage) }) m.Group("/generic", func() { diff --git a/routers/api/v1/packages/composer/api.go b/routers/api/v1/packages/composer/api.go index ea06a16de3338..70aa336993a0d 100644 --- a/routers/api/v1/packages/composer/api.go +++ b/routers/api/v1/packages/composer/api.go @@ -102,7 +102,7 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_model.Pac Metadata: pd.Metadata.(*composer_module.Metadata), Dist: Dist{ Type: "zip", - URL: registryURL + "/files/" + strconv.FormatInt(pd.Version.ID, 10) + "/" + pd.Files[0].File.Name, + URL: registryURL + "/files/" + strconv.FormatInt(pd.Version.ID, 10) + "/" + strconv.FormatInt(pd.Files[0].File.ID, 10), Checksum: pd.Files[0].Blob.HashSHA1, }, }) diff --git a/routers/api/v1/packages/composer/composer.go b/routers/api/v1/packages/composer/composer.go index 7a15acb00d030..47d7a95a734c8 100644 --- a/routers/api/v1/packages/composer/composer.go +++ b/routers/api/v1/packages/composer/composer.go @@ -161,12 +161,12 @@ func PackageMetadata(ctx *context.APIContext) { // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.APIContext) { versionID := ctx.ParamsInt64("versionid") - filename := ctx.Params("filename") + fileID := ctx.ParamsInt64("fileid") - s, pf, err := packages_service.GetFileStreamByPackageVersionID( + s, pf, err := packages_service.GetFileStreamByPackageVersionAndFileID( ctx.Package.Owner, versionID, - filename, + fileID, ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { @@ -225,10 +225,12 @@ func UploadPackage(ctx *context.APIContext) { composer_module.TypeProperty: cp.Type, }, }, - &packages_service.PackageFileInfo{ - Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)), - Data: buf, - IsLead: true, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)), + }, + Data: buf, + IsLead: true, }, ) if err != nil { diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 13592782b9751..9102f5e938b7b 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -43,7 +43,9 @@ func DownloadPackageFile(ctx *context.APIContext) { Name: packageName, Version: packageVersion, }, - filename, + &package_service.PackageFileInfo{ + Filename: filename, + }, ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { @@ -95,10 +97,12 @@ func UploadPackage(ctx *context.APIContext) { SemverCompatible: true, Creator: ctx.User, }, - &package_service.PackageFileInfo{ - Filename: filename, - Data: buf, - IsLead: true, + &package_service.PackageFileCreationInfo{ + PackageFileInfo: package_service.PackageFileInfo{ + Filename: filename, + }, + Data: buf, + IsLead: true, }, ) if err != nil { diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 22f4794189dbc..7c28cd64141fb 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -118,7 +118,7 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { func servePackageFile(ctx *context.APIContext, params parameters) { packageName := params.GroupID + "-" + params.ArtifactID - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeMaven, packageName, params.Version) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeMaven, packageName, params.Version, packages.EmptyVersionKey) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -135,7 +135,7 @@ func servePackageFile(ctx *context.APIContext, params parameters) { filename = filename[:len(filename)-len(ext)] } - pf, err := packages.GetFileForVersionByName(db.DefaultContext, pv.ID, filename) + pf, err := packages.GetFileForVersionByName(db.DefaultContext, pv.ID, filename, packages.EmptyFileKey) if err != nil { if err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -222,7 +222,7 @@ func UploadPackageFile(ctx *context.APIContext) { // Do not upload checksum files but compare the hashes. if isChecksumExtension(ext) { - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, packages.EmptyVersionKey) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -231,7 +231,7 @@ func UploadPackageFile(ctx *context.APIContext) { apiError(ctx, http.StatusInternalServerError, err) return } - pf, err := packages.GetFileForVersionByName(db.DefaultContext, pv.ID, params.Filename[:len(params.Filename)-len(ext)]) + pf, err := packages.GetFileForVersionByName(db.DefaultContext, pv.ID, params.Filename[:len(params.Filename)-len(ext)], packages.EmptyFileKey) if err != nil { if err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -264,10 +264,12 @@ func UploadPackageFile(ctx *context.APIContext) { return } - pfi := &packages_service.PackageFileInfo{ - Filename: params.Filename, - Data: buf, - IsLead: false, + pfi := &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: params.Filename, + }, + Data: buf, + IsLead: false, } // If it's the package pom file extract the metadata @@ -281,7 +283,7 @@ func UploadPackageFile(ctx *context.APIContext) { } if pvci.Metadata != nil { - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, packages.EmptyVersionKey) if err != nil && err != packages.ErrPackageNotExist { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 2731db0031ca3..80acb0c08a42a 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -98,7 +98,9 @@ func DownloadPackageFile(ctx *context.APIContext) { Name: packageName, Version: packageVersion, }, - filename, + &packages_service.PackageFileInfo{ + Filename: filename, + }, ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { @@ -140,10 +142,12 @@ func UploadPackage(ctx *context.APIContext) { Creator: ctx.User, Metadata: npmPackage.Metadata, }, - &packages_service.PackageFileInfo{ - Filename: npmPackage.Filename, - Data: buf, - IsLead: true, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: npmPackage.Filename, + }, + Data: buf, + IsLead: true, }, ) if err != nil { @@ -213,7 +217,7 @@ func AddPackageTag(ctx *context.APIContext) { } version := strings.Trim(string(body), "\"") // is as "version" in the body - pv, err := packages.GetVersionByNameAndVersion(ctx, ctx.ContextUser.ID, packages.TypeNpm, packageName, version) + pv, err := packages.GetVersionByNameAndVersion(ctx, ctx.ContextUser.ID, packages.TypeNpm, packageName, version, packages.EmptyVersionKey) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 098fd198e23e0..891d27ede9549 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -100,7 +100,7 @@ func RegistrationLeaf(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeNuGet, packageName, packageVersion) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeNuGet, packageName, packageVersion, packages.EmptyVersionKey) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -162,7 +162,9 @@ func DownloadPackageFile(ctx *context.APIContext) { Name: packageName, Version: packageVersion, }, - filename, + &packages_service.PackageFileInfo{ + Filename: filename, + }, ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { @@ -202,10 +204,12 @@ func UploadPackage(ctx *context.APIContext) { Creator: ctx.User, Metadata: np.Metadata, }, - &packages_service.PackageFileInfo{ - Filename: strings.ToLower(fmt.Sprintf("%s.%s.nupkg", np.ID, np.Version)), - Data: buf, - IsLead: true, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.nupkg", np.ID, np.Version)), + }, + Data: buf, + IsLead: true, }, ) if err != nil { @@ -240,10 +244,12 @@ func UploadSymbolPackage(ctx *context.APIContext) { Name: np.ID, Version: np.Version, }, - &packages_service.PackageFileInfo{ - Filename: strings.ToLower(fmt.Sprintf("%s.%s.snupkg", np.ID, np.Version)), - Data: buf, - IsLead: false, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.%s.snupkg", np.ID, np.Version)), + }, + Data: buf, + IsLead: false, }, ) if err != nil { diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index b494dc2e5fef2..2ec1585d787f8 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -75,7 +75,9 @@ func DownloadPackageFile(ctx *context.APIContext) { Name: packageName, Version: packageVersion, }, - filename, + &packages_service.PackageFileInfo{ + Filename: filename, + }, ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { @@ -150,10 +152,12 @@ func UploadPackageFile(ctx *context.APIContext) { RequiresPython: ctx.Req.FormValue("requires_python"), }, }, - &packages_service.PackageFileInfo{ - Filename: fileHeader.Filename, - Data: buf, - IsLead: true, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: fileHeader.Filename, + }, + Data: buf, + IsLead: true, }, ) if err != nil { diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/v1/packages/rubygems/rubygems.go index 8e65f36f870f3..97567c19cb168 100644 --- a/routers/api/v1/packages/rubygems/rubygems.go +++ b/routers/api/v1/packages/rubygems/rubygems.go @@ -169,7 +169,12 @@ func DownloadPackageFile(ctx *context.APIContext) { return } - s, pf, err := packages_service.GetPackageFileStream(pvs[0], filename) + s, pf, err := packages_service.GetFileStreamByPackageVersion( + pvs[0], + &packages_service.PackageFileInfo{ + Filename: filename, + }, + ) if err != nil { if err == packages.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) @@ -230,10 +235,12 @@ func UploadPackageFile(ctx *context.APIContext) { Creator: ctx.User, Metadata: rp.Metadata, }, - &packages_service.PackageFileInfo{ - Filename: filename, - Data: buf, - IsLead: true, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: filename, + }, + Data: buf, + IsLead: true, }, ) if err != nil { diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 16ccb33774f0d..0abd8409a69d0 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -187,13 +187,17 @@ func PackageSettingsPost(ctx *context.Context) { // DownloadPackageFile serves the content of a package file func DownloadPackageFile(ctx *context.Context) { - s, pf, err := packages_service.GetFileStreamByPackageVersionID(ctx.ContextUser, ctx.ParamsInt64(":versionid"), ctx.Params(":filename")) + s, pf, err := packages_service.GetFileStreamByPackageVersionAndFileID( + ctx.ContextUser, + ctx.ParamsInt64(":versionid"), + ctx.ParamsInt64(":fileid"), + ) if err != nil { if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { ctx.NotFound("", err) return } - ctx.ServerError("GetFileStreamByPackageVersionID", err) + ctx.ServerError("GetFileStreamByPackageVersionAndFileID", err) return } defer s.Close() diff --git a/routers/web/web.go b/routers/web/web.go index fae701955d87b..4298268218d32 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -623,7 +623,7 @@ func RegisterRoutes(m *web.Route) { m.Get("", user.Packages) m.Group("/{versionid}", func() { m.Get("", user.ViewPackage) - m.Get("/files/{filename}", user.DownloadPackageFile) + m.Get("/files/{fileid}", user.DownloadPackageFile) m.Group("/settings", func() { m.Get("", user.PackageSettings) m.Post("", bindIgnErr(forms.PackageSettingForm{}), user.PackageSettingsPost) diff --git a/services/packages/packages.go b/services/packages/packages.go index 72bba9fd7b7c4..4e326ee0ecc8e 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -21,10 +21,11 @@ import ( // PackageInfo describes a package type PackageInfo struct { - Owner *user_model.User - PackageType packages_model.Type - Name string - Version string + Owner *user_model.User + PackageType packages_model.Type + Name string + Version string + CompositeKey string } // PackageCreationInfo describes a package to create @@ -38,22 +39,28 @@ type PackageCreationInfo struct { // PackageFileInfo describes a package file type PackageFileInfo struct { - Filename string - Data *packages_module.HashedBuffer - IsLead bool + Filename string + CompositeKey string +} + +// PackageFileCreationInfo describes a package file to create +type PackageFileCreationInfo struct { + PackageFileInfo + Data *packages_module.HashedBuffer + IsLead bool } // CreatePackageAndAddFile creates a package with a file. If the same package exists already, ErrDuplicatePackageVersion is returned -func CreatePackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { - return createPackageAndAddFile(pvci, pfi, false) +func CreatePackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { + return createPackageAndAddFile(pvci, pfci, false) } // CreatePackageOrAddFileToExisting creates a package with a file or adds the file if the package exists already -func CreatePackageOrAddFileToExisting(pvci *PackageCreationInfo, pfi *PackageFileInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { - return createPackageAndAddFile(pvci, pfi, true) +func CreatePackageOrAddFileToExisting(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { + return createPackageAndAddFile(pvci, pfci, true) } -func createPackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo, allowDuplicate bool) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { +func createPackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreationInfo, allowDuplicate bool) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { ctx, committer, err := db.TxContext() if err != nil { return nil, nil, err @@ -65,7 +72,7 @@ func createPackageAndAddFile(pvci *PackageCreationInfo, pfi *PackageFileInfo, al return nil, nil, err } - pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfi) + pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci) removeBlob := false defer func() { if blobCreated && removeBlob { @@ -149,19 +156,19 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } // AddFileToExistingPackage adds a file to an existing package. If the package does not exist, ErrPackageNotExist is returned -func AddFileToExistingPackage(pvi *PackageInfo, pfi *PackageFileInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { +func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageVersion, *packages_model.PackageFile, error) { ctx, committer, err := db.TxContext() if err != nil { return nil, nil, err } defer committer.Close() - pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pvi.CompositeKey) if err != nil { return nil, nil, err } - pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfi) + pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci) removeBlob := false defer func() { if blobCreated && removeBlob { @@ -184,15 +191,15 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfi *PackageFileInfo) (*packages return pv, pf, nil } -func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { - log.Trace("Adding package file: %v, %s", pv.ID, pfi.Filename) +func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { + log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) - hashMD5, hashSHA1, hashSHA256, hashSHA512 := pfi.Data.Sums() + hashMD5, hashSHA1, hashSHA256, hashSHA512 := pfci.Data.Sums() blobKey := fmt.Sprintf("%x", hashSHA256) pb := &packages_model.PackageBlob{ - Size: pfi.Data.Size(), + Size: pfci.Data.Size(), HashMD5: fmt.Sprintf("%x", hashMD5), HashSHA1: fmt.Sprintf("%x", hashSHA1), HashSHA256: blobKey, @@ -205,18 +212,19 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers } if !exists { contentStore := packages_module.NewContentStore() - if err := contentStore.Save(packages_module.BlobHash256Key(blobKey), pfi.Data, pfi.Data.Size()); err != nil { + if err := contentStore.Save(packages_module.BlobHash256Key(blobKey), pfci.Data, pfci.Data.Size()); err != nil { log.Error("Error saving package blob in content store: %v", err) return nil, nil, false, err } } pf := &packages_model.PackageFile{ - VersionID: pv.ID, - BlobID: pb.ID, - Name: pfi.Filename, - LowerName: strings.ToLower(pfi.Filename), - IsLead: pfi.IsLead, + VersionID: pv.ID, + BlobID: pb.ID, + Name: pfci.Filename, + LowerName: strings.ToLower(pfci.Filename), + CompositeKey: pfci.CompositeKey, + IsLead: pfci.IsLead, } if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil { if err != packages_model.ErrDuplicatePackageFile { @@ -230,7 +238,7 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers // DeletePackageVersionByNameAndVersion deletes a package version and all associated files func DeletePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error { - pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pvi.CompositeKey) if err != nil { return err } @@ -312,10 +320,10 @@ func DeleteUnreferencedBlobs() error { } // GetFileStreamByPackageNameAndVersion returns the content of the specific package file -func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, filename string) (io.ReadCloser, *packages_model.PackageFile, error) { - log.Trace("Getting package file stream: %v, %v, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, filename) +func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) { + log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey) - pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pvi.CompositeKey) if err != nil { if err == packages_model.ErrPackageNotExist { return nil, nil, err @@ -324,12 +332,12 @@ func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, filename string) (io return nil, nil, err } - return GetPackageFileStream(pv, filename) + return GetFileStreamByPackageVersion(pv, pfi) } -// GetFileStreamByPackageVersionID returns the content of the specific package file -func GetFileStreamByPackageVersionID(owner *user_model.User, versionID int64, filename string) (io.ReadCloser, *packages_model.PackageFile, error) { - log.Trace("Getting package file stream: %v, %v, %s", owner.ID, versionID, filename) +// GetFileStreamByPackageVersionAndFileID returns the content of the specific package file +func GetFileStreamByPackageVersionAndFileID(owner *user_model.User, versionID, fileID int64) (io.ReadCloser, *packages_model.PackageFile, error) { + log.Trace("Getting package file stream: %v, %v, %v", owner.ID, versionID, fileID) pv, err := packages_model.GetVersionByID(db.DefaultContext, versionID) if err != nil { @@ -350,16 +358,27 @@ func GetFileStreamByPackageVersionID(owner *user_model.User, versionID int64, fi return nil, nil, packages_model.ErrPackageNotExist } - return GetPackageFileStream(pv, filename) + pf, err := packages_model.GetFileForVersionByID(db.DefaultContext, versionID, fileID) + if err != nil { + log.Error("Error getting file: %v", err) + return nil, nil, err + } + + return GetPackageFileStream(pv, pf) } -// GetPackageFileStream returns the cotent of the specific package file -func GetPackageFileStream(pv *packages_model.PackageVersion, filename string) (io.ReadCloser, *packages_model.PackageFile, error) { - pf, err := packages_model.GetFileForVersionByName(db.DefaultContext, pv.ID, filename) +// GetFileStreamByPackageVersion returns the content of the specific package file +func GetFileStreamByPackageVersion(pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) { + pf, err := packages_model.GetFileForVersionByName(db.DefaultContext, pv.ID, pfi.Filename, pfi.CompositeKey) if err != nil { return nil, nil, err } + return GetPackageFileStream(pv, pf) +} + +// GetPackageFileStream returns the cotent of the specific package file +func GetPackageFileStream(pv *packages_model.PackageVersion, pf *packages_model.PackageFile) (io.ReadCloser, *packages_model.PackageFile, error) { pb, err := packages_model.GetBlobByID(db.DefaultContext, pf.BlobID) if err != nil { return nil, nil, err diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index f34baf392cdc0..35cc14067f369 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -50,7 +50,7 @@
    {{range .PackageDescriptor.Files}}
    - {{.File.Name}} + {{.File.Name}} {{FileSize .Blob.Size}}
    {{end}} From da48b2e3289e6eba2c399b3b63de0d129d72b65d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 13 Jan 2022 22:22:14 +0000 Subject: [PATCH 084/130] Added missing repository unlink. --- routers/web/user/package.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 0abd8409a69d0..1cc71c5a7c399 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -145,17 +145,22 @@ func PackageSettingsPost(ctx *context.Context) { switch form.Action { case "link": success := func() bool { - repo, err := repo_model.GetRepositoryByID(form.RepoID) - if err != nil { - log.Error("Error getting repository: %v", err) - return false - } - - if repo.OwnerID != pd.Owner.ID { - return false + repoID := int64(0) + if form.RepoID != 0 { + repo, err := repo_model.GetRepositoryByID(form.RepoID) + if err != nil { + log.Error("Error getting repository: %v", err) + return false + } + + if repo.OwnerID != pd.Owner.ID { + return false + } + + repoID = repo.ID } - if err = packages.SetRepositoryLink(pd.Package.ID, repo.ID); err != nil { + if err := packages.SetRepositoryLink(pd.Package.ID, repoID); err != nil { log.Error("Error updating package: %v", err) return false } From df81183c9aad8c40ce13434360a00cb1fe0766d6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 21 Jan 2022 23:50:11 +0000 Subject: [PATCH 085/130] Added file properties. --- models/migrations/v207.go | 13 ++-- models/packages/descriptor.go | 14 ++-- models/packages/package_property.go | 70 +++++++++++++++++++ models/packages/package_version.go | 23 ++++++- models/packages/package_version_property.go | 76 --------------------- routers/api/v1/packages/npm/npm.go | 8 +-- services/packages/packages.go | 4 +- 7 files changed, 113 insertions(+), 95 deletions(-) create mode 100644 models/packages/package_property.go delete mode 100644 models/packages/package_version_property.go diff --git a/models/migrations/v207.go b/models/migrations/v207.go index c5436c9958a87..2656e26799627 100644 --- a/models/migrations/v207.go +++ b/models/migrations/v207.go @@ -41,14 +41,15 @@ func addPackageTables(x *xorm.Engine) error { return err } - type PackageVersionProperty struct { - ID int64 `xorm:"pk autoincr"` - VersionID int64 `xorm:"INDEX NOT NULL"` - Name string `xorm:"INDEX NOT NULL"` - Value string `xorm:"INDEX NOT NULL"` + type PackageProperty struct { + ID int64 `xorm:"pk autoincr"` + RefType int64 `xorm:"INDEX NOT NULL"` + RefID int64 `xorm:"INDEX NOT NULL"` + Name string `xorm:"INDEX NOT NULL"` + Value string `xorm:"INDEX NOT NULL"` } - if err := x.Sync2(new(PackageVersionProperty)); err != nil { + if err := x.Sync2(new(PackageProperty)); err != nil { return err } diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 1c18f10719c73..4cf313a4418ff 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -30,15 +30,16 @@ type PackageDescriptor struct { Version *PackageVersion SemVer *version.Version Creator *user_model.User - Properties []*PackageVersionProperty + Properties []*PackageProperty Metadata interface{} Files []PackageFileDescriptor } // PackageFileDescriptor describes a package file type PackageFileDescriptor struct { - File *PackageFile - Blob *PackageBlob + File *PackageFile + Blob *PackageBlob + Properties []*PackageProperty } // WebLink returns the package's web link @@ -85,7 +86,7 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD return nil, err } } - pvps, err := GetVersionProperties(ctx, pv.ID) + pvps, err := GetProperties(ctx, PropertyTypeVersion, pv.ID) if err != nil { return nil, err } @@ -100,9 +101,14 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD if err != nil { return nil, err } + pfps, err := GetProperties(ctx, PropertyTypeFile, pf.ID) + if err != nil { + return nil, err + } pfds = append(pfds, PackageFileDescriptor{ pf, pb, + pfps, }) } diff --git a/models/packages/package_property.go b/models/packages/package_property.go new file mode 100644 index 0000000000000..514222a374a30 --- /dev/null +++ b/models/packages/package_property.go @@ -0,0 +1,70 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + + "code.gitea.io/gitea/models/db" +) + +func init() { + db.RegisterModel(new(PackageProperty)) +} + +type PropertyType int64 + +const ( + // PropertyTypeVersion means the reference is a package version + PropertyTypeVersion PropertyType = iota // 0 + // PropertyTypeFile means the reference is a package file + PropertyTypeFile // 1 +) + +// PackageProperty represents a property of a package version or file +type PackageProperty struct { + ID int64 `xorm:"pk autoincr"` + RefType PropertyType `xorm:"INDEX NOT NULL"` + RefID int64 `xorm:"INDEX NOT NULL"` + Name string `xorm:"INDEX NOT NULL"` + Value string `xorm:"INDEX NOT NULL"` +} + +// InsertProperty creates a property +func InsertProperty(ctx context.Context, refType PropertyType, refID int64, name, value string) (*PackageProperty, error) { + pp := &PackageProperty{ + RefType: refType, + RefID: refID, + Name: name, + Value: value, + } + + _, err := db.GetEngine(ctx).Insert(pp) + return pp, err +} + +// GetProperties gets all properties +func GetProperties(ctx context.Context, refType PropertyType, refID int64) ([]*PackageProperty, error) { + pps := make([]*PackageProperty, 0, 10) + return pps, db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ?", refType, refID).Find(&pps) +} + +// GetPropertiesByName gets all properties with a specific name +func GetPropertiesByName(ctx context.Context, refType PropertyType, refID int64, name string) ([]*PackageProperty, error) { + pps := make([]*PackageProperty, 0, 10) + return pps, db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Find(&pps) +} + +// DeleteAllProperties deletes all properties of a ref +func DeleteAllProperties(ctx context.Context, refType PropertyType, refID int64) error { + _, err := db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ?", refType, refID).Delete(&PackageProperty{}) + return err +} + +// DeletePropertyByID deletes a property +func DeletePropertyByID(ctx context.Context, propertyID int64) error { + _, err := db.GetEngine(ctx).ID(propertyID).Delete(&PackageProperty{}) + return err +} diff --git a/models/packages/package_version.go b/models/packages/package_version.go index b54686f7184a6..ca07ae6b52a1d 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -198,11 +198,12 @@ func (opts *PackageSearchOptions) toConds() builder.Cond { propsCond := builder.NewCond() for name, value := range opts.Properties { propsCond = propsCond.Or(builder.Eq{ - "package_version_property.name": name, - "package_version_property.value": value, + "package_property.ref_type": PropertyTypeVersion, + "package_property.name": name, + "package_property.value": value, }) } - cond = cond.And(builder.In("package_version.id", builder.Select("package_version_property.version_id").Where(propsCond).From("package_version_property"))) + cond = cond.And(builder.In("package_version.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property"))) } return cond @@ -264,3 +265,19 @@ func SearchLatestVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, count, err := sess.FindAndCount(&pvs) return pvs, count, err } + +// FindVersionsByPropertyNameAndValue gets all package versions which are associated with a specific property + value +func FindVersionsByPropertyNameAndValue(ctx context.Context, packageID int64, name, value string) ([]*PackageVersion, error) { + var cond builder.Cond = builder.Eq{ + "package_property.ref_type": PropertyTypeVersion, + "package_property.name": name, + "package_property.value": value, + "package_version.package_id": packageID, + } + + pvs := make([]*PackageVersion, 0, 5) + return pvs, db.GetEngine(ctx). + Where(cond). + Join("INNER", "package_property", "package_property.ref_id = package_version.id"). + Find(&pvs) +} diff --git a/models/packages/package_version_property.go b/models/packages/package_version_property.go deleted file mode 100644 index 7f384e9f9d085..0000000000000 --- a/models/packages/package_version_property.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package packages - -import ( - "context" - - "code.gitea.io/gitea/models/db" - - "xorm.io/builder" -) - -func init() { - db.RegisterModel(new(PackageVersionProperty)) -} - -// PackageVersionProperty represents a property of a package version -type PackageVersionProperty struct { - ID int64 `xorm:"pk autoincr"` - VersionID int64 `xorm:"INDEX NOT NULL"` - Name string `xorm:"INDEX NOT NULL"` - Value string `xorm:"INDEX NOT NULL"` -} - -// InsertVersionProperty creates a property -func InsertVersionProperty(ctx context.Context, versionID int64, name, value string) (*PackageVersionProperty, error) { - pvp := &PackageVersionProperty{ - VersionID: versionID, - Name: name, - Value: value, - } - - _, err := db.GetEngine(ctx).Insert(pvp) - return pvp, err -} - -// GetVersionProperties gets all properties of the version -func GetVersionProperties(ctx context.Context, versionID int64) ([]*PackageVersionProperty, error) { - pvps := make([]*PackageVersionProperty, 0, 10) - return pvps, db.GetEngine(ctx).Where("version_id = ?", versionID).Find(&pvps) -} - -// GetVersionPropertiesByName gets all properties with a specific name of the version -func GetVersionPropertiesByName(ctx context.Context, versionID int64, name string) ([]*PackageVersionProperty, error) { - pvps := make([]*PackageVersionProperty, 0, 10) - return pvps, db.GetEngine(ctx).Where("version_id = ? AND name = ?", versionID, name).Find(&pvps) -} - -// FindVersionsByPropertyNameAndValue gets all package versions which are associated with a specific property + value -func FindVersionsByPropertyNameAndValue(ctx context.Context, packageID int64, name, value string) ([]*PackageVersion, error) { - var cond builder.Cond = builder.Eq{ - "package_version_property.name": name, - "package_version_property.value": value, - "package_version.package_id": packageID, - } - - pvs := make([]*PackageVersion, 0, 5) - return pvs, db.GetEngine(ctx). - Where(cond). - Join("INNER", "package_version_property", "package_version_property.version_id = package_version.id"). - Find(&pvs) -} - -// DeleteVersionPropertiesByVersionID deletes all properties of the version -func DeleteVersionPropertiesByVersionID(ctx context.Context, versionID int64) error { - _, err := db.GetEngine(ctx).Where("version_id = ?", versionID).Delete(&PackageVersionProperty{}) - return err -} - -// DeleteVersionPropertyByID deletes a property -func DeleteVersionPropertyByID(ctx context.Context, propertyID int64) error { - _, err := db.GetEngine(ctx).ID(propertyID).Delete(&PackageVersionProperty{}) - return err -} diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 80acb0c08a42a..41d08b2aaeb49 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -189,7 +189,7 @@ func ListPackageTags(ctx *context.APIContext) { tags := make(map[string]string) for _, pv := range pvs { - pvps, err := packages.GetVersionPropertiesByName(ctx, pv.ID, npm_module.TagProperty) + pvps, err := packages.GetPropertiesByName(ctx, packages.PropertyTypeVersion, pv.ID, npm_module.TagProperty) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -284,14 +284,14 @@ func setPackageTag(tag string, pv *packages.PackageVersion, deleteOnly bool) err } if len(pvs) == 1 { - pvps, err := packages.GetVersionPropertiesByName(ctx, pvs[0].ID, npm_module.TagProperty) + pvps, err := packages.GetPropertiesByName(ctx, packages.PropertyTypeVersion, pvs[0].ID, npm_module.TagProperty) if err != nil { return err } for _, pvp := range pvps { if pvp.Value == tag { - if err := packages.DeleteVersionPropertyByID(ctx, pvp.ID); err != nil { + if err := packages.DeletePropertyByID(ctx, pvp.ID); err != nil { return err } break @@ -300,7 +300,7 @@ func setPackageTag(tag string, pv *packages.PackageVersion, deleteOnly bool) err } if !deleteOnly { - _, err = packages.InsertVersionProperty(ctx, pv.ID, npm_module.TagProperty, tag) + _, err = packages.InsertProperty(ctx, packages.PropertyTypeVersion, pv.ID, npm_module.TagProperty, tag) if err != nil { return err } diff --git a/services/packages/packages.go b/services/packages/packages.go index 4e326ee0ecc8e..f5ba60c15a37a 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -146,7 +146,7 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } for name, value := range pvci.Properties { - if _, err := packages_model.InsertVersionProperty(ctx, pv.ID, name, value); err != nil { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil { log.Error("Error setting package version property: %v", err) return nil, false, err } @@ -261,7 +261,7 @@ func DeletePackageVersion(doer *user_model.User, pv *packages_model.PackageVersi log.Trace("Deleting package: %v", pv.ID) - if err := packages_model.DeleteVersionPropertiesByVersionID(ctx, pv.ID); err != nil { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { return err } From b4bf28e9f24f4dc38b7f197f470e993e2bc5cb7d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 10 Feb 2022 21:44:12 +0000 Subject: [PATCH 086/130] make fmt --- models/packages/package_blob.go | 6 ++---- modules/notification/base/notifier.go | 1 - modules/notification/webhook/webhook.go | 2 +- modules/packages/rubygems/metadata_test.go | 2 +- routers/api/v1/packages/generic/generic.go | 6 ++++-- routers/api/v1/packages/npm/npm.go | 6 ++---- 6 files changed, 10 insertions(+), 13 deletions(-) diff --git a/models/packages/package_blob.go b/models/packages/package_blob.go index 8d5279d933dbc..a3b9c7bf50f2c 100644 --- a/models/packages/package_blob.go +++ b/models/packages/package_blob.go @@ -11,10 +11,8 @@ import ( "code.gitea.io/gitea/models/db" ) -var ( - // ErrPackageBlobNotExist indicates a package blob not exist error - ErrPackageBlobNotExist = errors.New("Package blob does not exist") -) +// ErrPackageBlobNotExist indicates a package blob not exist error +var ErrPackageBlobNotExist = errors.New("Package blob does not exist") func init() { db.RegisterModel(new(PackageBlob)) diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index fc516e4289a91..7a5e8f502de78 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -54,7 +54,6 @@ type Notifier interface { NotifySyncCreateRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName, refID string) NotifySyncDeleteRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName string) NotifyRepoPendingTransfer(doer, newOwner *user_model.User, repo *repo_model.Repository) - NotifyPackageCreate(doer *user_model.User, pd *packages_model.PackageDescriptor) NotifyPackageDelete(doer *user_model.User, pd *packages_model.PackageDescriptor) } diff --git a/modules/notification/webhook/webhook.go b/modules/notification/webhook/webhook.go index 88f0e456d10ad..a8469f1112ba8 100644 --- a/modules/notification/webhook/webhook.go +++ b/modules/notification/webhook/webhook.go @@ -865,7 +865,7 @@ func (m *webhookNotifier) NotifyPackageDelete(doer *user_model.User, pd *package func notifyPackage(sender *user_model.User, pd *packages_model.PackageDescriptor, action api.HookPackageAction) { if pd.Repository == nil { - //TODO + // TODO return } diff --git a/modules/packages/rubygems/metadata_test.go b/modules/packages/rubygems/metadata_test.go index f3f248021e5c8..dbefa9c23670e 100644 --- a/modules/packages/rubygems/metadata_test.go +++ b/modules/packages/rubygems/metadata_test.go @@ -20,7 +20,7 @@ func TestParsePackageMetaData(t *testing.T) { tw := tar.NewWriter(&buf) hdr := &tar.Header{ Name: filename, - Mode: 0600, + Mode: 0o600, Size: int64(len(content)), } tw.WriteHeader(hdr) diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 9102f5e938b7b..8d425bda2da94 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -19,8 +19,10 @@ import ( "github.com/hashicorp/go-version" ) -var packageNameRegex = regexp.MustCompile(`\A[A-Za-z0-9\.\_\-\+]+\z`) -var filenameRegex = packageNameRegex +var ( + packageNameRegex = regexp.MustCompile(`\A[A-Za-z0-9\.\_\-\+]+\z`) + filenameRegex = packageNameRegex +) func apiError(ctx *context.APIContext, status int, obj interface{}) { package_router.LogAndProcessError(ctx, status, obj, func(message string) { diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 41d08b2aaeb49..e9659cac6da04 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -25,10 +25,8 @@ import ( "github.com/hashicorp/go-version" ) -var ( - // errInvalidTagName indicates an invalid tag name - errInvalidTagName = errors.New("The tag name is invalid") -) +// errInvalidTagName indicates an invalid tag name +var errInvalidTagName = errors.New("The tag name is invalid") func apiError(ctx *context.APIContext, status int, obj interface{}) { package_router.LogAndProcessError(ctx, status, obj, func(message string) { From ac237970977d88483a452507b6b01d9f81c89068 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 10 Feb 2022 21:53:51 +0000 Subject: [PATCH 087/130] lint --- modules/util/filebuffer/file_backed_buffer.go | 3 +-- routers/api/v1/packages/maven/maven.go | 9 ++++----- routers/api/v1/packages/npm/npm.go | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index 3339f373bf072..ac3990219b737 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -8,7 +8,6 @@ import ( "bytes" "errors" "io" - "io/ioutil" "os" ) @@ -76,7 +75,7 @@ func (b *FileBackedBuffer) Write(p []byte) (int, error) { n, err = b.file.Write(p) } else { if b.size+int64(len(p)) > b.maxMemorySize { - b.file, err = ioutil.TempFile("", "buffer-") + b.file, err = os.CreateTemp("", "buffer-") if err != nil { return 0, err } diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 7c28cd64141fb..8bb38339b9e61 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -13,7 +13,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "path/filepath" "regexp" @@ -246,7 +245,7 @@ func UploadPackageFile(ctx *context.APIContext) { return } - hash, err := ioutil.ReadAll(buf) + hash, err := io.ReadAll(buf) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -264,7 +263,7 @@ func UploadPackageFile(ctx *context.APIContext) { return } - pfi := &packages_service.PackageFileCreationInfo{ + pfci := &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ Filename: params.Filename, }, @@ -274,7 +273,7 @@ func UploadPackageFile(ctx *context.APIContext) { // If it's the package pom file extract the metadata if ext == ".pom" { - pfi.IsLead = true + pfci.IsLead = true var err error pvci.Metadata, err = maven_module.ParsePackageMetaData(buf) @@ -310,7 +309,7 @@ func UploadPackageFile(ctx *context.APIContext) { _, _, err = packages_service.CreatePackageOrAddFileToExisting( pvci, - pfi, + pfci, ) if err != nil { if err == packages.ErrDuplicatePackageFile { diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index e9659cac6da04..e9a8501e1c892 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -8,7 +8,7 @@ import ( "bytes" "errors" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "strings" @@ -208,7 +208,7 @@ func AddPackageTag(ctx *context.APIContext) { return } - body, err := ioutil.ReadAll(ctx.Req.Body) + body, err := io.ReadAll(ctx.Req.Body) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return From 80a69aaf3a17a484f0f963cc8f442710f72ae7de Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 14 Feb 2022 21:20:24 +0000 Subject: [PATCH 088/130] Added Conan package registry. --- docs/content/doc/packages/conan.en-us.md | 101 +++ docs/content/doc/packages/overview.en-us.md | 1 + integrations/api_packages_conan_test.go | 724 ++++++++++++++++ models/migrations/v209.go | 14 +- models/packages/conan/references.go | 170 ++++ models/packages/conan/search.go | 150 ++++ models/packages/descriptor.go | 3 + models/packages/package.go | 5 + models/packages/package_blob.go | 14 +- models/packages/package_file.go | 70 +- models/packages/package_version.go | 28 +- modules/packages/conan/conanfile_parser.go | 68 ++ .../packages/conan/conanfile_parser_test.go | 51 ++ modules/packages/conan/conaninfo_parser.go | 123 +++ .../packages/conan/conaninfo_parser_test.go | 85 ++ modules/packages/conan/metadata.go | 24 + modules/packages/conan/reference.go | 155 ++++ modules/packages/conan/reference_test.go | 147 ++++ options/locale/locale_en-US.ini | 6 +- public/img/svg/gitea-conan.svg | 1 + routers/api/v1/api.go | 91 +- routers/api/v1/packages/conan/auth.go | 87 ++ routers/api/v1/packages/conan/conan.go | 816 ++++++++++++++++++ routers/api/v1/packages/conan/search.go | 163 ++++ routers/api/v1/packages/helper.go | 2 +- services/packages/packages.go | 69 +- templates/admin/packages/list.tmpl | 1 + templates/package/content/conan.tmpl | 34 + templates/package/metadata/conan.tmpl | 6 + templates/package/shared/list.tmpl | 3 +- templates/package/view.tmpl | 4 +- web_src/svg/gitea-conan.svg | 9 + 32 files changed, 3182 insertions(+), 43 deletions(-) create mode 100644 docs/content/doc/packages/conan.en-us.md create mode 100644 integrations/api_packages_conan_test.go create mode 100644 models/packages/conan/references.go create mode 100644 models/packages/conan/search.go create mode 100644 modules/packages/conan/conanfile_parser.go create mode 100644 modules/packages/conan/conanfile_parser_test.go create mode 100644 modules/packages/conan/conaninfo_parser.go create mode 100644 modules/packages/conan/conaninfo_parser_test.go create mode 100644 modules/packages/conan/metadata.go create mode 100644 modules/packages/conan/reference.go create mode 100644 modules/packages/conan/reference_test.go create mode 100644 public/img/svg/gitea-conan.svg create mode 100644 routers/api/v1/packages/conan/auth.go create mode 100644 routers/api/v1/packages/conan/conan.go create mode 100644 routers/api/v1/packages/conan/search.go create mode 100644 templates/package/content/conan.tmpl create mode 100644 templates/package/metadata/conan.tmpl create mode 100644 web_src/svg/gitea-conan.svg diff --git a/docs/content/doc/packages/conan.en-us.md b/docs/content/doc/packages/conan.en-us.md new file mode 100644 index 0000000000000..fe24e87e78594 --- /dev/null +++ b/docs/content/doc/packages/conan.en-us.md @@ -0,0 +1,101 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "Conan Packages Repository" +slug: "packages/conan" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Conan" + weight: 20 + identifier: "conan" +--- + +# Conan Packages Repository + +Publish [Conan](https://conan.io/) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Conan package registry, you need to use the [conan](https://conan.io/downloads.html) command line tool to consume and publish packages. + +## Configuring the package registry + +To register the package registry you need to configure a new Conan remote: + +```shell +conan remote add {remote} https://gitea.example.com/api/v1/packages/{owner}/conan +conan user --remote {remote} --password {password} {username} +``` + +| Parameter | Description | +| -----------| ----------- | +| `remote` | The remote name. | +| `username` | Your Gitea username. | +| `password` | Your Gitea password or a personal access token. | +| `owner` | The owner of the package. | + +For example: + +```shell +conan remote add gitea https://gitea.example.com/api/v1/packages/testuser/conan +conan user --remote gitea --password password123 testuser +``` + +## Publish a package + +Publish a Conan package by running the following command: + +```shell +conan upload --remote={remote} {recipe} +``` + +| Parameter | Description | +| ----------| ----------- | +| `remote` | The remote name. | +| `recipe` | The recipe to upload. | + +For example: + +```shell +conan upload --remote=gitea ConanPackage/1.2@gitea/final +``` + +The Gitea Conan package registry has full [revision](https://docs.conan.io/en/latest/versioning/revisions.html) support. + +## Install a package + +To install a Conan package from the package registry, execute the following command: + +```shell +conan install --remote={remote} {recipe} +``` + +| Parameter | Description | +| ----------| ----------- | +| `remote` | The remote name. | +| `recipe` | The recipe to upload. | + +For example: + +```shell +conan install --remote=gitea ConanPackage/1.2@gitea/final +``` + +## Supported commands + +``` +conan install +conan get +conan info +conan search +conan upload +conan user +conan download +conan remove +``` diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index ec3c73b1d668a..53ccddf130a58 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -27,6 +27,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | | [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` | +| [Conan]({{< relref "doc/packages/conan.en-us.md" >}}) | C++ | `conan` | | [Generic]({{< relref "doc/packages/generic.en-us.md" >}}) | - | any HTTP client | | [Maven]({{< relref "doc/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` | | [npm]({{< relref "doc/packages/npm.en-us.md" >}}) | JavaScript | `npm` | diff --git a/integrations/api_packages_conan_test.go b/integrations/api_packages_conan_test.go new file mode 100644 index 0000000000000..4988a365cd8f7 --- /dev/null +++ b/integrations/api_packages_conan_test.go @@ -0,0 +1,724 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "fmt" + "net/http" + stdurl "net/url" + "strings" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + conan_model "code.gitea.io/gitea/models/packages/conan" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + conan_module "code.gitea.io/gitea/modules/packages/conan" + "code.gitea.io/gitea/modules/setting" + conan_router "code.gitea.io/gitea/routers/api/v1/packages/conan" + + "github.com/stretchr/testify/assert" +) + +const ( + conanfileName = "conanfile.py" + conaninfoName = "conaninfo.txt" + + conanLicense = "MIT" + conanAuthor = "Gitea " + conanHomepage = "https://gitea.io/" + conanURL = "https://gitea.com/" + conanDescription = "Description of ConanPackage" + conanTopic = "gitea" + + conanPackageReference = "dummyreference" + + contentConaninfo = `[settings] + arch=x84_64 + +[requires] + fmt/7.1.3 + +[options] + shared=False + +[full_settings] + arch=x84_64 + +[full_requires] + fmt/7.1.3 + +[full_options] + shared=False + +[recipe_hash] + 74714915a51073acb548ca1ce29afbac + +[env] +CC=gcc-10` +) + +func addTokenAuthHeader(request *http.Request, token string) *http.Request { + request.Header.Set("Authorization", token) + return request +} + +func buildConanfileContent(name, version string) string { + return `from conans import ConanFile, CMake, tools + +class ConanPackageConan(ConanFile): + name = "` + name + `" + version = "` + version + `" + license = "` + conanLicense + `" + author = "` + conanAuthor + `" + homepage = "` + conanHomepage + `" + url = "` + conanURL + `" + description = "` + conanDescription + `" + topics = ("` + conanTopic + `") + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + generators = "cmake"` +} + +func uploadConanPackageV1(t *testing.T, baseURL, token, name, version, user, channel string) { + contentConanfile := buildConanfileContent(name, version) + + recipeURL := fmt.Sprintf("%s/v1/conans/%s/%s/%s/%s", baseURL, name, version, user, channel) + + req := NewRequest(t, "GET", recipeURL) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/digest", recipeURL)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/download_urls", recipeURL)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "POST", fmt.Sprintf("%s/upload_urls", recipeURL)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithJSON(t, "POST", fmt.Sprintf("%s/upload_urls", recipeURL), map[string]int64{ + conanfileName: int64(len(contentConanfile)), + "removed.txt": 0, + }) + req = addTokenAuthHeader(req, token) + resp := MakeRequest(t, req, http.StatusOK) + + uploadURLs := make(map[string]string) + DecodeJSON(t, resp, &uploadURLs) + + assert.Contains(t, uploadURLs, conanfileName) + assert.NotContains(t, uploadURLs, "removed.txt") + + uploadURL := uploadURLs[conanfileName] + assert.NotEmpty(t, uploadURL) + + req = NewRequestWithBody(t, "PUT", uploadURL, strings.NewReader(contentConanfile)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusCreated) + + packageURL := fmt.Sprintf("%s/packages/%s", recipeURL, conanPackageReference) + + req = NewRequest(t, "GET", packageURL) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/digest", packageURL)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/download_urls", packageURL)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "POST", fmt.Sprintf("%s/upload_urls", packageURL)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithJSON(t, "POST", fmt.Sprintf("%s/upload_urls", packageURL), map[string]int64{ + conaninfoName: int64(len(contentConaninfo)), + "removed.txt": 0, + }) + req = addTokenAuthHeader(req, token) + resp = MakeRequest(t, req, http.StatusOK) + + uploadURLs = make(map[string]string) + DecodeJSON(t, resp, &uploadURLs) + + assert.Contains(t, uploadURLs, conaninfoName) + assert.NotContains(t, uploadURLs, "removed.txt") + + uploadURL = uploadURLs[conaninfoName] + assert.NotEmpty(t, uploadURL) + + req = NewRequestWithBody(t, "PUT", uploadURL, strings.NewReader(contentConaninfo)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusCreated) +} + +func uploadConanPackageV2(t *testing.T, baseURL, token, name, version, user, channel, recipeRevision, packageRevision string) { + contentConanfile := buildConanfileContent(name, version) + + recipeURL := fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s/revisions/%s", baseURL, name, version, user, channel, recipeRevision) + + req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/files/%s", recipeURL, conanfileName), strings.NewReader(contentConanfile)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/files", recipeURL)) + req = addTokenAuthHeader(req, token) + resp := MakeRequest(t, req, http.StatusOK) + + var list *struct { + Files map[string]interface{} `json:"files"` + } + DecodeJSON(t, resp, &list) + assert.Len(t, list.Files, 1) + assert.Contains(t, list.Files, conanfileName) + + packageURL := fmt.Sprintf("%s/packages/%s/revisions/%s", recipeURL, conanPackageReference, packageRevision) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/files", packageURL)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/files/%s", packageURL, conaninfoName), strings.NewReader(contentConaninfo)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/files", packageURL)) + req = addTokenAuthHeader(req, token) + resp = MakeRequest(t, req, http.StatusOK) + + list = nil + DecodeJSON(t, resp, &list) + assert.Len(t, list.Files, 1) + assert.Contains(t, list.Files, conaninfoName) +} + +func TestPackageConan(t *testing.T) { + defer prepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User) + + name := "ConanPackage" + version1 := "1.2" + version2 := "1.3" + user1 := "dummy" + user2 := "gitea" + channel1 := "test" + channel2 := "final" + revision1 := "rev1" + revision2 := "rev2" + + url := fmt.Sprintf("%sapi/v1/packages/%s/conan", setting.AppURL, user.Name) + + t.Run("v1", func(t *testing.T) { + t.Run("Ping", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/v1/ping", url)) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "revisions", resp.Header().Get("X-Conan-Server-Capabilities")) + }) + + token := "" + + t.Run("Authenticate", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/v1/users/authenticate", url)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + body := resp.Body.String() + assert.NotEmpty(t, body) + + token = fmt.Sprintf("Bearer %s", body) + }) + + t.Run("CheckCredentials", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/v1/users/check_credentials", url)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + uploadConanPackageV1(t, url, token, name, version1, user1, channel1) + + t.Run("Validate", func(t *testing.T) { + defer PrintCurrentTest(t)() + + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeConan) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.Equal(t, name, pd.Package.Name) + assert.Equal(t, version1, pd.Version.Version) + assert.IsType(t, &conan_module.Metadata{}, pd.Metadata) + metadata := pd.Metadata.(*conan_module.Metadata) + assert.Equal(t, conanLicense, metadata.License) + assert.Equal(t, conanAuthor, metadata.Author) + assert.Equal(t, conanHomepage, metadata.ProjectURL) + assert.Equal(t, conanURL, metadata.RepositoryURL) + assert.Equal(t, conanDescription, metadata.Description) + assert.Equal(t, []string{conanTopic}, metadata.Keywords) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 2) + + for _, pf := range pfs { + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + + if pf.Name == conanfileName { + assert.True(t, pf.IsLead) + + assert.Equal(t, int64(len(buildConanfileContent(name, version1))), pb.Size) + } else if pf.Name == conaninfoName { + assert.False(t, pf.IsLead) + + assert.Equal(t, int64(len(contentConaninfo)), pb.Size) + } else { + assert.Fail(t, "unknown file: %s", pf.Name) + } + } + }) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + recipeURL := fmt.Sprintf("%s/v1/conans/%s/%s/%s/%s", url, name, version1, user1, channel1) + + req := NewRequest(t, "GET", recipeURL) + resp := MakeRequest(t, req, http.StatusOK) + + fileHashes := make(map[string]string) + DecodeJSON(t, resp, &fileHashes) + assert.Len(t, fileHashes, 1) + assert.Contains(t, fileHashes, conanfileName) + assert.Equal(t, "7abc52241c22090782c54731371847a8", fileHashes[conanfileName]) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/digest", recipeURL)) + resp = MakeRequest(t, req, http.StatusOK) + + downloadURLs := make(map[string]string) + DecodeJSON(t, resp, &downloadURLs) + assert.Contains(t, downloadURLs, conanfileName) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/download_urls", recipeURL)) + resp = MakeRequest(t, req, http.StatusOK) + + DecodeJSON(t, resp, &downloadURLs) + assert.Contains(t, downloadURLs, conanfileName) + + req = NewRequest(t, "GET", downloadURLs[conanfileName]) + resp = MakeRequest(t, req, http.StatusOK) + assert.Equal(t, buildConanfileContent(name, version1), resp.Body.String()) + + packageURL := fmt.Sprintf("%s/packages/%s", recipeURL, conanPackageReference) + + req = NewRequest(t, "GET", packageURL) + resp = MakeRequest(t, req, http.StatusOK) + + fileHashes = make(map[string]string) + DecodeJSON(t, resp, &fileHashes) + assert.Len(t, fileHashes, 1) + assert.Contains(t, fileHashes, conaninfoName) + assert.Equal(t, "7628bfcc5b17f1470c468621a78df394", fileHashes[conaninfoName]) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/digest", packageURL)) + resp = MakeRequest(t, req, http.StatusOK) + + downloadURLs = make(map[string]string) + DecodeJSON(t, resp, &downloadURLs) + assert.Contains(t, downloadURLs, conaninfoName) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/download_urls", packageURL)) + resp = MakeRequest(t, req, http.StatusOK) + + DecodeJSON(t, resp, &downloadURLs) + assert.Contains(t, downloadURLs, conaninfoName) + + req = NewRequest(t, "GET", downloadURLs[conaninfoName]) + resp = MakeRequest(t, req, http.StatusOK) + assert.Equal(t, contentConaninfo, resp.Body.String()) + }) + + t.Run("Search", func(t *testing.T) { + uploadConanPackageV1(t, url, token, name, version2, user1, channel1) + uploadConanPackageV1(t, url, token, name, version1, user1, channel2) + uploadConanPackageV1(t, url, token, name, version1, user2, channel1) + uploadConanPackageV1(t, url, token, name, version1, user2, channel2) + + t.Run("Recipe", func(t *testing.T) { + defer PrintCurrentTest(t)() + + cases := []struct { + Query string + Expected []string + }{ + {"ConanPackage", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1.2", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1.1", []string{}}, + {"Conan*", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/*", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1*", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/*2", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1*2", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1.2@", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1.2@du*", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@dummy/final"}}, + {"ConanPackage/1.2@du*/", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@dummy/final"}}, + {"ConanPackage/1.2@du*/*test", []string{"ConanPackage/1.2@dummy/test"}}, + {"ConanPackage/1.2@du*/*st", []string{"ConanPackage/1.2@dummy/test"}}, + {"ConanPackage/1.2@gitea/*", []string{"ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"*/*@dummy", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@dummy/final"}}, + {"*/*@*/final", []string{"ConanPackage/1.2@dummy/final", "ConanPackage/1.2@gitea/final"}}, + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s/v1/conans/search?q=%s", url, stdurl.QueryEscape(c.Query))) + resp := MakeRequest(t, req, http.StatusOK) + + var result *conan_router.SearchResult + DecodeJSON(t, resp, &result) + + assert.ElementsMatch(t, c.Expected, result.Results, "case %d: unexpected result", i) + } + }) + + t.Run("Package", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/v1/conans/%s/%s/%s/%s/search", url, name, version1, user1, channel2)) + resp := MakeRequest(t, req, http.StatusOK) + + var result map[string]*conan_module.Conaninfo + DecodeJSON(t, resp, &result) + + assert.Contains(t, result, conanPackageReference) + info := result[conanPackageReference] + assert.NotEmpty(t, info.Settings) + }) + }) + + t.Run("Delete", func(t *testing.T) { + t.Run("Package", func(t *testing.T) { + defer PrintCurrentTest(t)() + + cases := []struct { + Channel string + References []string + }{ + {channel1, []string{conanPackageReference}}, + {channel2, []string{}}, + } + + for i, c := range cases { + rref, _ := conan_module.NewRecipeReference(name, version1, user1, c.Channel, conan_module.DefaultRevision) + references, err := conan_model.GetPackageReferences(db.DefaultContext, user.ID, rref) + assert.NoError(t, err) + assert.NotEmpty(t, references) + + req := NewRequestWithJSON(t, "POST", fmt.Sprintf("%s/v1/conans/%s/%s/%s/%s/packages/delete", url, name, version1, user1, c.Channel), map[string][]string{ + "package_ids": c.References, + }) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + + references, err = conan_model.GetPackageReferences(db.DefaultContext, user.ID, rref) + assert.NoError(t, err) + assert.Empty(t, references, "case %d: should be empty", i) + } + }) + + t.Run("Recipe", func(t *testing.T) { + defer PrintCurrentTest(t)() + + cases := []struct { + Channel string + }{ + {channel1}, + {channel2}, + } + + for i, c := range cases { + rref, _ := conan_module.NewRecipeReference(name, version1, user1, c.Channel, conan_module.DefaultRevision) + revisions, err := conan_model.GetRecipeRevisions(db.DefaultContext, user.ID, rref) + assert.NoError(t, err) + assert.NotEmpty(t, revisions) + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/v1/conans/%s/%s/%s/%s", url, name, version1, user1, c.Channel)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + + revisions, err = conan_model.GetRecipeRevisions(db.DefaultContext, user.ID, rref) + assert.NoError(t, err) + assert.Empty(t, revisions, "case %d: should be empty", i) + } + }) + }) + }) + + t.Run("v2", func(t *testing.T) { + t.Run("Ping", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/v2/ping", url)) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "revisions", resp.Header().Get("X-Conan-Server-Capabilities")) + }) + + token := "" + + t.Run("Authenticate", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/v2/users/authenticate", url)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + body := resp.Body.String() + assert.NotEmpty(t, body) + + token = fmt.Sprintf("Bearer %s", body) + }) + + t.Run("CheckCredentials", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/v2/users/check_credentials", url)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + uploadConanPackageV2(t, url, token, name, version1, user1, channel1, revision1, revision1) + + t.Run("Validate", func(t *testing.T) { + defer PrintCurrentTest(t)() + + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeConan) + assert.NoError(t, err) + assert.Len(t, pvs, 2) + }) + }) + + t.Run("Latest", func(t *testing.T) { + defer PrintCurrentTest(t)() + + recipeURL := fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s", url, name, version1, user1, channel1) + + req := NewRequest(t, "GET", fmt.Sprintf("%s/latest", recipeURL)) + resp := MakeRequest(t, req, http.StatusOK) + + obj := make(map[string]string) + DecodeJSON(t, resp, &obj) + assert.Contains(t, obj, "revision") + assert.Equal(t, revision1, obj["revision"]) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/revisions/%s/packages/%s/latest", recipeURL, revision1, conanPackageReference)) + resp = MakeRequest(t, req, http.StatusOK) + + obj = make(map[string]string) + DecodeJSON(t, resp, &obj) + assert.Contains(t, obj, "revision") + assert.Equal(t, revision1, obj["revision"]) + }) + + t.Run("ListRevisions", func(t *testing.T) { + defer PrintCurrentTest(t)() + + uploadConanPackageV2(t, url, token, name, version1, user1, channel1, revision1, revision2) + uploadConanPackageV2(t, url, token, name, version1, user1, channel1, revision2, revision1) + uploadConanPackageV2(t, url, token, name, version1, user1, channel1, revision2, revision2) + + recipeURL := fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s/revisions", url, name, version1, user1, channel1) + + req := NewRequest(t, "GET", recipeURL) + resp := MakeRequest(t, req, http.StatusOK) + + type RevisionInfo struct { + Revision string `json:"revision"` + Time time.Time `json:"time"` + } + + type RevisionList struct { + Revisions []*RevisionInfo `json:"revisions"` + } + + var list *RevisionList + DecodeJSON(t, resp, &list) + assert.Len(t, list.Revisions, 2) + revs := make([]string, 0, len(list.Revisions)) + for _, rev := range list.Revisions { + revs = append(revs, rev.Revision) + } + assert.ElementsMatch(t, []string{revision1, revision2}, revs) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/packages/%s/revisions", recipeURL, revision1, conanPackageReference)) + resp = MakeRequest(t, req, http.StatusOK) + + DecodeJSON(t, resp, &list) + assert.Len(t, list.Revisions, 2) + revs = make([]string, 0, len(list.Revisions)) + for _, rev := range list.Revisions { + revs = append(revs, rev.Revision) + } + assert.ElementsMatch(t, []string{revision1, revision2}, revs) + }) + + t.Run("Search", func(t *testing.T) { + t.Run("Recipe", func(t *testing.T) { + defer PrintCurrentTest(t)() + + cases := []struct { + Query string + Expected []string + }{ + {"ConanPackage", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1.2", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1.1", []string{}}, + {"Conan*", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/*", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1*", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/*2", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1*2", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1.2@", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"ConanPackage/1.2@du*", []string{"ConanPackage/1.2@dummy/test"}}, + {"ConanPackage/1.2@du*/", []string{"ConanPackage/1.2@dummy/test"}}, + {"ConanPackage/1.2@du*/*test", []string{"ConanPackage/1.2@dummy/test"}}, + {"ConanPackage/1.2@du*/*st", []string{"ConanPackage/1.2@dummy/test"}}, + {"ConanPackage/1.2@gitea/*", []string{"ConanPackage/1.2@gitea/test", "ConanPackage/1.2@gitea/final"}}, + {"*/*@dummy", []string{"ConanPackage/1.2@dummy/test", "ConanPackage/1.3@dummy/test"}}, + {"*/*@*/final", []string{"ConanPackage/1.2@gitea/final"}}, + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s/v2/conans/search?q=%s", url, stdurl.QueryEscape(c.Query))) + resp := MakeRequest(t, req, http.StatusOK) + + var result *conan_router.SearchResult + DecodeJSON(t, resp, &result) + + assert.ElementsMatch(t, c.Expected, result.Results, "case %d: unexpected result", i) + } + }) + + t.Run("Package", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s/search", url, name, version1, user1, channel1)) + resp := MakeRequest(t, req, http.StatusOK) + + var result map[string]*conan_module.Conaninfo + DecodeJSON(t, resp, &result) + + assert.Contains(t, result, conanPackageReference) + info := result[conanPackageReference] + assert.NotEmpty(t, info.Settings) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s/revisions/%s/search", url, name, version1, user1, channel1, revision1)) + resp = MakeRequest(t, req, http.StatusOK) + + result = make(map[string]*conan_module.Conaninfo) + DecodeJSON(t, resp, &result) + + assert.Contains(t, result, conanPackageReference) + info = result[conanPackageReference] + assert.NotEmpty(t, info.Settings) + }) + }) + + t.Run("Delete", func(t *testing.T) { + t.Run("Package", func(t *testing.T) { + defer PrintCurrentTest(t)() + + rref, _ := conan_module.NewRecipeReference(name, version1, user1, channel1, revision1) + pref, _ := conan_module.NewPackageReference(rref, conanPackageReference, conan_module.DefaultRevision) + + checkPackageRevisionCount := func(count int) { + revisions, err := conan_model.GetPackageRevisions(db.DefaultContext, user.ID, pref) + assert.NoError(t, err) + assert.Len(t, revisions, count) + } + checkPackageReferenceCount := func(count int) { + references, err := conan_model.GetPackageReferences(db.DefaultContext, user.ID, rref) + assert.NoError(t, err) + assert.Len(t, references, count) + } + + checkPackageRevisionCount(2) + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s/revisions/%s/packages/%s/revisions/%s", url, name, version1, user1, channel1, revision1, conanPackageReference, revision1)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + + checkPackageRevisionCount(1) + + req = NewRequest(t, "DELETE", fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s/revisions/%s/packages/%s", url, name, version1, user1, channel1, revision1, conanPackageReference)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + + checkPackageRevisionCount(0) + + rref = rref.WithRevision(revision2) + + checkPackageReferenceCount(1) + + req = NewRequest(t, "DELETE", fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s/revisions/%s/packages", url, name, version1, user1, channel1, revision2)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + + checkPackageReferenceCount(0) + }) + + t.Run("Recipe", func(t *testing.T) { + defer PrintCurrentTest(t)() + + rref, _ := conan_module.NewRecipeReference(name, version1, user1, channel1, conan_module.DefaultRevision) + + checkRecipeRevisionCount := func(count int) { + revisions, err := conan_model.GetRecipeRevisions(db.DefaultContext, user.ID, rref) + assert.NoError(t, err) + assert.Len(t, revisions, count) + } + + checkRecipeRevisionCount(2) + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s/revisions/%s", url, name, version1, user1, channel1, revision1)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + + checkRecipeRevisionCount(1) + + req = NewRequest(t, "DELETE", fmt.Sprintf("%s/v2/conans/%s/%s/%s/%s", url, name, version1, user1, channel1)) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + + checkRecipeRevisionCount(0) + }) + }) + }) +} diff --git a/models/migrations/v209.go b/models/migrations/v209.go index 2656e26799627..da420d1f7f124 100644 --- a/models/migrations/v209.go +++ b/models/migrations/v209.go @@ -61,6 +61,7 @@ func addPackageTables(x *xorm.Engine) error { LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` CompositeKey string `xorm:"UNIQUE(s) INDEX"` IsLead bool + CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` } if err := x.Sync2(new(PackageFile)); err != nil { @@ -68,12 +69,13 @@ func addPackageTables(x *xorm.Engine) error { } type PackageBlob struct { - ID int64 `xorm:"pk autoincr"` - Size int64 - HashMD5 string `xorm:"hash_md5 char(32) UNIQUE(md5) INDEX NOT NULL"` - HashSHA1 string `xorm:"hash_sha1 char(40) UNIQUE(sha1) INDEX NOT NULL"` - HashSHA256 string `xorm:"hash_sha256 char(64) UNIQUE(sha256) INDEX NOT NULL"` - HashSHA512 string `xorm:"hash_sha512 char(128) UNIQUE(sha512) INDEX NOT NULL"` + ID int64 `xorm:"pk autoincr"` + Size int64 + HashMD5 string `xorm:"hash_md5 char(32) UNIQUE(md5) INDEX NOT NULL"` + HashSHA1 string `xorm:"hash_sha1 char(40) UNIQUE(sha1) INDEX NOT NULL"` + HashSHA256 string `xorm:"hash_sha256 char(64) UNIQUE(sha256) INDEX NOT NULL"` + HashSHA512 string `xorm:"hash_sha512 char(128) UNIQUE(sha512) INDEX NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` } return x.Sync2(new(PackageBlob)) diff --git a/models/packages/conan/references.go b/models/packages/conan/references.go new file mode 100644 index 0000000000000..492d8b9e42187 --- /dev/null +++ b/models/packages/conan/references.go @@ -0,0 +1,170 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "context" + "errors" + "strconv" + "strings" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + conan_module "code.gitea.io/gitea/modules/packages/conan" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" +) + +var ( + ErrRecipeReferenceNotExist = errors.New("Recipe reference does not exist") + ErrPackageReferenceNotExist = errors.New("Package reference does not exist") +) + +// RecipeExists checks if a recipe exists +func RecipeExists(ctx context.Context, ownerID int64, ref *conan_module.RecipeReference) (bool, error) { + revisions, err := GetRecipeRevisions(ctx, ownerID, ref) + if err != nil { + return false, err + } + + return len(revisions) != 0, nil +} + +type PropertyValue struct { + Value string + CreatedUnix timeutil.TimeStamp +} + +func findPropertyValues(ctx context.Context, propertyName string, ownerID int64, name, version string, propertyFilter map[string]string) ([]*PropertyValue, error) { + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeFile, + } + propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id")) + + propsCondBlock := builder.NewCond() + for name, value := range propertyFilter { + propsCondBlock = propsCondBlock.Or(builder.Eq{ + "package_property.name": name, + "package_property.value": value, + }) + } + propsCond = propsCond.And(propsCondBlock) + + var cond builder.Cond = builder.Eq{ + "package.type": packages.TypeConan, + "package.owner_id": ownerID, + "package.lower_name": strings.ToLower(name), + "package_version.lower_version": strings.ToLower(version), + strconv.Itoa(len(propertyFilter)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"), + } + + in2 := builder. + Select("package_file.id"). + From("package_file"). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond) + + query := builder. + Select("package_property.value, package_file.created_unix"). + From("package_property"). + Join("INNER", "package_file", "package_file.id = package_property.ref_id"). + Where(builder.Eq{"package_property.name": propertyName}.And(builder.In("package_property.ref_id", in2))). + GroupBy("package_property.value"). + OrderBy("package_file.created_unix DESC") + + var values []*PropertyValue + return values, db.GetEngine(ctx).SQL(query).Find(&values) +} + +// GetRecipeRevisions gets all revisions of a recipe +func GetRecipeRevisions(ctx context.Context, ownerID int64, ref *conan_module.RecipeReference) ([]*PropertyValue, error) { + values, err := findPropertyValues( + ctx, + conan_module.PropertyRecipeRevision, + ownerID, + ref.Name, + ref.Version, + map[string]string{ + conan_module.PropertyRecipeUser: ref.User, + conan_module.PropertyRecipeChannel: ref.Channel, + }, + ) + if err != nil { + return nil, err + } + + return values, nil +} + +// GetLastRecipeRevision gets the latest recipe revision +func GetLastRecipeRevision(ctx context.Context, ownerID int64, ref *conan_module.RecipeReference) (*PropertyValue, error) { + revisions, err := GetRecipeRevisions(ctx, ownerID, ref) + if err != nil { + return nil, err + } + + if len(revisions) == 0 { + return nil, ErrRecipeReferenceNotExist + } + return revisions[0], nil +} + +// GetPackageReferences gets all package references of a recipe +func GetPackageReferences(ctx context.Context, ownerID int64, ref *conan_module.RecipeReference) ([]*PropertyValue, error) { + values, err := findPropertyValues( + ctx, + conan_module.PropertyPackageReference, + ownerID, + ref.Name, + ref.Version, + map[string]string{ + conan_module.PropertyRecipeUser: ref.User, + conan_module.PropertyRecipeChannel: ref.Channel, + conan_module.PropertyRecipeRevision: ref.Revision, + }, + ) + if err != nil { + return nil, err + } + + return values, nil +} + +// GetPackageRevisions gets all revision of a package +func GetPackageRevisions(ctx context.Context, ownerID int64, ref *conan_module.PackageReference) ([]*PropertyValue, error) { + values, err := findPropertyValues( + ctx, + conan_module.PropertyPackageRevision, + ownerID, + ref.Recipe.Name, + ref.Recipe.Version, + map[string]string{ + conan_module.PropertyRecipeUser: ref.Recipe.User, + conan_module.PropertyRecipeChannel: ref.Recipe.Channel, + conan_module.PropertyRecipeRevision: ref.Recipe.Revision, + conan_module.PropertyPackageReference: ref.Reference, + }, + ) + if err != nil { + return nil, err + } + + return values, nil +} + +// GetLastPackageRevision gets the latest package revision +func GetLastPackageRevision(ctx context.Context, ownerID int64, ref *conan_module.PackageReference) (*PropertyValue, error) { + revisions, err := GetPackageRevisions(ctx, ownerID, ref) + if err != nil { + return nil, err + } + + if len(revisions) == 0 { + return nil, ErrPackageReferenceNotExist + } + return revisions[0], nil +} diff --git a/models/packages/conan/search.go b/models/packages/conan/search.go new file mode 100644 index 0000000000000..36f3915b2762a --- /dev/null +++ b/models/packages/conan/search.go @@ -0,0 +1,150 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "context" + "fmt" + "strconv" + "strings" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + conan_module "code.gitea.io/gitea/modules/packages/conan" + + "xorm.io/builder" +) + +// buildCondition creates a Like condition if a wildcard is present. Otherwise Eq is used. +func buildCondition(name, value string) builder.Cond { + if strings.Contains(value, "*") { + return builder.Like{name, strings.ReplaceAll(value, "*", "%")} + } + return builder.Eq{name: value} +} + +type RecipeSearchOptions struct { + OwnerID int64 + Name string + Version string + User string + Channel string +} + +// SearchRecipes gets all recipes matching the search options +func SearchRecipes(ctx context.Context, opts *RecipeSearchOptions) ([]string, error) { + cond := builder.NewCond() + + if opts.Name != "" { + cond = cond.And(buildCondition("package.lower_name", strings.ToLower(opts.Name))) + } + if opts.Version != "" { + cond = cond.And(buildCondition("package_version.lower_version", strings.ToLower(opts.Version))) + } + if opts.User != "" || opts.Channel != "" { + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeFile, + } + propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id")) + + count := 0 + propsCondBlock := builder.NewCond() + if opts.User != "" { + count++ + propsCondBlock = propsCondBlock.Or(builder.Eq{"package_property.name": conan_module.PropertyRecipeUser}.And(buildCondition("package_property.value", opts.User))) + } + if opts.Channel != "" { + count++ + propsCondBlock = propsCondBlock.Or(builder.Eq{"package_property.name": conan_module.PropertyRecipeChannel}.And(buildCondition("package_property.value", opts.Channel))) + } + propsCond = propsCond.And(propsCondBlock) + + cond = cond.And(builder.Eq{ + strconv.Itoa(count): builder.Select("COUNT(*)").Where(propsCond).From("package_property"), + }) + } + + cond = cond.And(builder.Eq{ + "package_file.is_lead": true, + "package.type": packages.TypeConan, + "package.owner_id": opts.OwnerID, + }) + + query := builder. + Select("package.name, package_version.version, package_file.id"). + From("package_file"). + Join("INNER", "package", "package.id = package_version.package_id"). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Where(cond) + + results := make([]struct { + Name string + Version string + ID int64 + }, 0, 5) + err := db.GetEngine(ctx).SQL(query).Find(&results) + if err != nil { + return nil, err + } + + unique := make(map[string]bool) + for _, info := range results { + recipe := fmt.Sprintf("%s/%s", info.Name, info.Version) + + props, _ := packages.GetProperties(ctx, packages.PropertyTypeFile, info.ID) + if len(props) > 0 { + var ( + user = "" + channel = "" + ) + for _, prop := range props { + if prop.Name == conan_module.PropertyRecipeUser { + user = prop.Value + } + if prop.Name == conan_module.PropertyRecipeChannel { + channel = prop.Value + } + } + if user != "" && channel != "" { + recipe = fmt.Sprintf("%s@%s/%s", recipe, user, channel) + } + } + + unique[recipe] = true + } + + recipes := make([]string, 0, len(unique)) + for recipe := range unique { + recipes = append(recipes, recipe) + } + return recipes, nil +} + +// GetPackageInfo gets the Conaninfo for a package +func GetPackageInfo(ctx context.Context, ownerID int64, ref *conan_module.PackageReference) (string, error) { + values, err := findPropertyValues( + ctx, + conan_module.PropertyPackageInfo, + ownerID, + ref.Recipe.Name, + ref.Recipe.Version, + map[string]string{ + conan_module.PropertyRecipeUser: ref.Recipe.User, + conan_module.PropertyRecipeChannel: ref.Recipe.Channel, + conan_module.PropertyRecipeRevision: ref.Recipe.Revision, + conan_module.PropertyPackageReference: ref.Reference, + conan_module.PropertyPackageRevision: ref.Revision, + }, + ) + if err != nil { + return "", err + } + + if len(values) == 0 { + return "", ErrPackageReferenceNotExist + } + + return values[0].Value, nil +} diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 4cf313a4418ff..12887e76b328c 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -13,6 +13,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/composer" + "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/packages/nuget" @@ -116,6 +117,8 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD switch p.Type { case TypeComposer: metadata = &composer.Metadata{} + case TypeConan: + metadata = &conan.Metadata{} case TypeNuGet: metadata = &nuget.Metadata{} case TypeNpm: diff --git a/models/packages/package.go b/models/packages/package.go index d53b7f324dce0..2ef03953b5276 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -30,6 +30,7 @@ type Type string // List of supported packages const ( TypeComposer Type = "composer" + TypeConan Type = "conan" TypeGeneric Type = "generic" TypeNuGet Type = "nuget" TypeNpm Type = "npm" @@ -43,6 +44,8 @@ func (pt Type) Name() string { switch pt { case TypeComposer: return "Composer" + case TypeConan: + return "Conan" case TypeGeneric: return "Generic" case TypeNuGet: @@ -64,6 +67,8 @@ func (pt Type) SVGName() string { switch pt { case TypeComposer: return "gitea-composer" + case TypeConan: + return "gitea-conan" case TypeGeneric: return "octicon-package" case TypeNuGet: diff --git a/models/packages/package_blob.go b/models/packages/package_blob.go index a3b9c7bf50f2c..233b4a26488b6 100644 --- a/models/packages/package_blob.go +++ b/models/packages/package_blob.go @@ -9,6 +9,7 @@ import ( "errors" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" ) // ErrPackageBlobNotExist indicates a package blob not exist error @@ -20,12 +21,13 @@ func init() { // PackageBlob represents a package blob type PackageBlob struct { - ID int64 `xorm:"pk autoincr"` - Size int64 - HashMD5 string `xorm:"hash_md5 char(32) UNIQUE(md5) INDEX NOT NULL"` - HashSHA1 string `xorm:"hash_sha1 char(40) UNIQUE(sha1) INDEX NOT NULL"` - HashSHA256 string `xorm:"hash_sha256 char(64) UNIQUE(sha256) INDEX NOT NULL"` - HashSHA512 string `xorm:"hash_sha512 char(128) UNIQUE(sha512) INDEX NOT NULL"` + ID int64 `xorm:"pk autoincr"` + Size int64 + HashMD5 string `xorm:"hash_md5 char(32) UNIQUE(md5) INDEX NOT NULL"` + HashSHA1 string `xorm:"hash_sha1 char(40) UNIQUE(sha1) INDEX NOT NULL"` + HashSHA256 string `xorm:"hash_sha256 char(64) UNIQUE(sha256) INDEX NOT NULL"` + HashSHA512 string `xorm:"hash_sha512 char(128) UNIQUE(sha512) INDEX NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` } // GetOrInsertBlob inserts a blob. If the blob exists already the existing blob is returned diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 9c8982c37037f..9404956a8223c 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -7,9 +7,13 @@ package packages import ( "context" "errors" + "strconv" "strings" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" ) func init() { @@ -35,6 +39,7 @@ type PackageFile struct { LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` CompositeKey string `xorm:"UNIQUE(s) INDEX"` IsLead bool + CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` } // TryInsertFile inserts a file. If the file exists already ErrDuplicatePackageFile is returned @@ -104,8 +109,67 @@ func GetFileForVersionByName(ctx context.Context, versionID int64, name, key str return pf, nil } -// DeleteFilesByVersionID deletes all files of a version -func DeleteFilesByVersionID(ctx context.Context, versionID int64) error { - _, err := db.GetEngine(ctx).Where("version_id = ?", versionID).Delete(&PackageFile{}) +// DeleteFileByID deletes a file +func DeleteFileByID(ctx context.Context, fileID int64) error { + _, err := db.GetEngine(ctx).ID(fileID).Delete(&PackageFile{}) return err } + +// PackageFileSearchOptions are options for SearchXXX methods +type PackageFileSearchOptions struct { + VersionID int64 + Query string + CompositeKey string + Properties map[string]string + db.Paginator +} + +func (opts *PackageFileSearchOptions) toConds() builder.Cond { + cond := builder.NewCond() + + if opts.VersionID != 0 { + cond = cond.And(builder.Eq{"package_file.version_id": opts.VersionID}) + } + if opts.CompositeKey != "" { + cond = cond.And(builder.Eq{"package_file.composite_key": opts.CompositeKey}) + } + if opts.Query != "" { + cond = cond.And(builder.Like{"package_file.lower_name", strings.ToLower(opts.Query)}) + } + + if len(opts.Properties) != 0 { + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": PropertyTypeFile, + } + propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_file.id")) + + propsCondBlock := builder.NewCond() + for name, value := range opts.Properties { + propsCondBlock = propsCondBlock.Or(builder.Eq{ + "package_property.name": name, + "package_property.value": value, + }) + } + propsCond = propsCond.And(propsCondBlock) + + cond = cond.And(builder.Eq{ + strconv.Itoa(len(opts.Properties)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"), + }) + } + + return cond +} + +// SearchFiles gets all files of packages matching the search options +func SearchFiles(ctx context.Context, opts *PackageFileSearchOptions) ([]*PackageFile, int64, error) { + sess := db.GetEngine(ctx). + Where(opts.toConds()) + + if opts.Paginator != nil { + sess = db.SetSessionPagination(sess, opts) + } + + pfs := make([]*PackageFile, 0, 10) + count, err := sess.FindAndCount(&pfs) + return pfs, count, err +} diff --git a/models/packages/package_version.go b/models/packages/package_version.go index ca07ae6b52a1d..735937cd76def 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -7,6 +7,7 @@ package packages import ( "context" "errors" + "strconv" "strings" "code.gitea.io/gitea/models/db" @@ -167,6 +168,13 @@ func DeleteVersionByID(ctx context.Context, versionID int64) error { return err } +// HasVersionFileReferences checks if there are associated files +func HasVersionFileReferences(ctx context.Context, versionID int64) (bool, error) { + return db.GetEngine(ctx).Get(&PackageFile{ + VersionID: versionID, + }) +} + // PackageSearchOptions are options for SearchXXX methods type PackageSearchOptions struct { OwnerID int64 @@ -195,15 +203,23 @@ func (opts *PackageSearchOptions) toConds() builder.Cond { } if len(opts.Properties) != 0 { - propsCond := builder.NewCond() + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": PropertyTypeVersion, + } + propsCond = propsCond.And(builder.Expr("package_property.ref_id = package_version.id")) + + propsCondBlock := builder.NewCond() for name, value := range opts.Properties { - propsCond = propsCond.Or(builder.Eq{ - "package_property.ref_type": PropertyTypeVersion, - "package_property.name": name, - "package_property.value": value, + propsCondBlock = propsCondBlock.Or(builder.Eq{ + "package_property.name": name, + "package_property.value": value, }) } - cond = cond.And(builder.In("package_version.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property"))) + propsCond = propsCond.And(propsCondBlock) + + cond = cond.And(builder.Eq{ + strconv.Itoa(len(opts.Properties)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"), + }) } return cond diff --git a/modules/packages/conan/conanfile_parser.go b/modules/packages/conan/conanfile_parser.go new file mode 100644 index 0000000000000..960e813533d14 --- /dev/null +++ b/modules/packages/conan/conanfile_parser.go @@ -0,0 +1,68 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "io" + "regexp" + "strings" +) + +var ( + patternAuthor = compilePattern("author") + patternHomepage = compilePattern("homepage") + patternURL = compilePattern("url") + patternLicense = compilePattern("license") + patternDescription = compilePattern("description") + patternTopics = regexp.MustCompile(`(?im)^\s*topics\s*=\s*\((.+)\)`) + patternTopicList = regexp.MustCompile(`\s*['"](.+?)['"]\s*,?`) +) + +func compilePattern(name string) *regexp.Regexp { + return regexp.MustCompile(`(?im)^\s*` + name + `\s*=\s*['"\(](.+)['"\)]`) +} + +func ParseConanfile(r io.Reader) (*Metadata, error) { + buf, err := io.ReadAll(io.LimitReader(r, 1<<20)) + if err != nil { + return nil, err + } + + metadata := &Metadata{} + + m := patternAuthor.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.Author = string(m[1]) + } + m = patternHomepage.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.ProjectURL = string(m[1]) + } + m = patternURL.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.RepositoryURL = string(m[1]) + } + m = patternLicense.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.License = strings.ReplaceAll(strings.ReplaceAll(string(m[1]), "'", ""), "\"", "") + } + m = patternDescription.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.Description = string(m[1]) + } + m = patternTopics.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + m2 := patternTopicList.FindAllSubmatch(m[1], -1) + if len(m2) > 0 { + metadata.Keywords = make([]string, 0, len(m2)) + for _, g := range m2 { + if len(g) > 1 { + metadata.Keywords = append(metadata.Keywords, string(g[1])) + } + } + } + } + return metadata, nil +} diff --git a/modules/packages/conan/conanfile_parser_test.go b/modules/packages/conan/conanfile_parser_test.go new file mode 100644 index 0000000000000..0ac9c87b14511 --- /dev/null +++ b/modules/packages/conan/conanfile_parser_test.go @@ -0,0 +1,51 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + name = "ConanPackage" + version = "1.2" + license = "MIT" + author = "Gitea " + homepage = "https://gitea.io/" + url = "https://gitea.com/" + description = "Description of ConanPackage" + topic1 = "gitea" + topic2 = "conan" + contentConanfile = `from conans import ConanFile, CMake, tools + +class ConanPackageConan(ConanFile): + name = "` + name + `" + version = "` + version + `" + license = "` + license + `" + author = "` + author + `" + homepage = "` + homepage + `" + url = "` + url + `" + description = "` + description + `" + topics = ("` + topic1 + `", "` + topic2 + `") + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + generators = "cmake" +` +) + +func TestParseConanfile(t *testing.T) { + metadata, err := ParseConanfile(strings.NewReader(contentConanfile)) + assert.Nil(t, err) + assert.Equal(t, license, metadata.License) + assert.Equal(t, author, metadata.Author) + assert.Equal(t, homepage, metadata.ProjectURL) + assert.Equal(t, url, metadata.RepositoryURL) + assert.Equal(t, description, metadata.Description) + assert.Equal(t, []string{topic1, topic2}, metadata.Keywords) +} diff --git a/modules/packages/conan/conaninfo_parser.go b/modules/packages/conan/conaninfo_parser.go new file mode 100644 index 0000000000000..bb228e0207a51 --- /dev/null +++ b/modules/packages/conan/conaninfo_parser.go @@ -0,0 +1,123 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "bufio" + "errors" + "io" + "strings" +) + +// Conaninfo represents infos of a Conan package +type Conaninfo struct { + Settings map[string]string `json:"settings"` + FullSettings map[string]string `json:"full_settings"` + Requires []string `json:"requires"` + FullRequires []string `json:"full_requires"` + Options map[string]string `json:"options"` + FullOptions map[string]string `json:"full_options"` + RecipeHash string `json:"recipe_hash"` + Environment map[string][]string `json:"environment"` +} + +func ParseConaninfo(r io.Reader) (*Conaninfo, error) { + sections, err := readSections(io.LimitReader(r, 1<<20)) + if err != nil { + return nil, err + } + + info := &Conaninfo{} + for section, lines := range sections { + if len(lines) == 0 { + continue + } + switch section { + case "settings": + info.Settings = toMap(lines) + case "full_settings": + info.FullSettings = toMap(lines) + case "options": + info.Options = toMap(lines) + case "full_options": + info.FullOptions = toMap(lines) + case "requires": + info.Requires = lines + case "full_requires": + info.FullRequires = lines + case "recipe_hash": + info.RecipeHash = lines[0] + case "env": + info.Environment = toMapArray(lines) + } + } + return info, nil +} + +func readSections(r io.Reader) (map[string][]string, error) { + sections := make(map[string][]string) + + section := "" + lines := make([]string, 0, 5) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + if section != "" { + sections[section] = lines + } + section = line[1 : len(line)-1] + lines = make([]string, 0, 5) + continue + } + if section != "" { + if line != "" { + lines = append(lines, line) + } + continue + } + if line != "" { + return nil, errors.New("Invalid conaninfo.txt") + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + if section != "" { + sections[section] = lines + } + return sections, nil +} + +func toMap(lines []string) map[string]string { + result := make(map[string]string) + for _, line := range lines { + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { + continue + } + result[parts[0]] = parts[1] + } + return result +} + +func toMapArray(lines []string) map[string][]string { + result := make(map[string][]string) + for _, line := range lines { + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { + continue + } + var items []string + if strings.HasPrefix(parts[1], "[") && strings.HasSuffix(parts[1], "]") { + items = strings.Split(parts[1], ",") + } else { + items = []string{parts[1]} + } + result[parts[0]] = items + } + return result +} diff --git a/modules/packages/conan/conaninfo_parser_test.go b/modules/packages/conan/conaninfo_parser_test.go new file mode 100644 index 0000000000000..3e28191b06261 --- /dev/null +++ b/modules/packages/conan/conaninfo_parser_test.go @@ -0,0 +1,85 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + settingsKey = "arch" + settingsValue = "x84_64" + optionsKey = "shared" + optionsValue = "False" + requires = "fmt/7.1.3" + hash = "74714915a51073acb548ca1ce29afbac" + envKey = "CC" + envValue = "gcc-10" + + contentConaninfo = `[settings] + ` + settingsKey + `=` + settingsValue + ` + +[requires] + ` + requires + ` + +[options] + ` + optionsKey + `=` + optionsValue + ` + +[full_settings] + ` + settingsKey + `=` + settingsValue + ` + +[full_requires] + ` + requires + ` + +[full_options] + ` + optionsKey + `=` + optionsValue + ` + +[recipe_hash] + ` + hash + ` + +[env] +` + envKey + `=` + envValue + ` + +` +) + +func TestParseConaninfo(t *testing.T) { + info, err := ParseConaninfo(strings.NewReader(contentConaninfo)) + assert.NotNil(t, info) + assert.Nil(t, err) + assert.Equal( + t, + map[string]string{ + settingsKey: settingsValue, + }, + info.Settings, + ) + assert.Equal(t, info.Settings, info.FullSettings) + assert.Equal( + t, + map[string]string{ + optionsKey: optionsValue, + }, + info.Options, + ) + assert.Equal(t, info.Options, info.FullOptions) + assert.Equal( + t, + []string{requires}, + info.Requires, + ) + assert.Equal(t, info.Requires, info.FullRequires) + assert.Equal(t, hash, info.RecipeHash) + assert.Equal( + t, + map[string][]string{ + envKey: {envValue}, + }, + info.Environment, + ) +} diff --git a/modules/packages/conan/metadata.go b/modules/packages/conan/metadata.go new file mode 100644 index 0000000000000..fc82d87fe328c --- /dev/null +++ b/modules/packages/conan/metadata.go @@ -0,0 +1,24 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +const ( + PropertyRecipeUser = "conan.recipe.user" + PropertyRecipeChannel = "conan.recipe.channel" + PropertyRecipeRevision = "conan.recipe.revision" + PropertyPackageReference = "conan.package.reference" + PropertyPackageRevision = "conan.package.revision" + PropertyPackageInfo = "conan.package.info" +) + +// Metadata represents the metadata of a Conan package +type Metadata struct { + Author string `json:"author"` + License string `json:"license"` + ProjectURL string `json:"project_url"` + RepositoryURL string `json:"repository_url"` + Description string `json:"description"` + Keywords []string `json:"keywords"` +} diff --git a/modules/packages/conan/reference.go b/modules/packages/conan/reference.go new file mode 100644 index 0000000000000..c43446e6e5bda --- /dev/null +++ b/modules/packages/conan/reference.go @@ -0,0 +1,155 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "errors" + "fmt" + "regexp" + + "code.gitea.io/gitea/modules/log" + + goversion "github.com/hashicorp/go-version" +) + +const ( + // taken from https://github.com/conan-io/conan/blob/develop/conans/model/ref.py + minChars = 2 + maxChars = 51 + + // DefaultRevision if no revision is specified + DefaultRevision = "0" +) + +var ( + namePattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{%d,%d}$`, minChars-1, maxChars-1)) + revisionPattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9]{1,%d}$`, maxChars)) + + ErrValidation = errors.New("Could not validate one or more reference fields") +) + +// RecipeReference represents a recipe /@/# +type RecipeReference struct { + Name string + Version string + User string + Channel string + Revision string +} + +func NewRecipeReference(name, version, user, channel, revision string) (*RecipeReference, error) { + log.Trace("Conan Recipe: %s/%s(@%s/%s(#%s))", name, version, user, channel, revision) + + if user == "_" { + user = "" + } + if channel == "_" { + channel = "" + } + + if (user != "" && channel == "") || (user == "" && channel != "") { + return nil, ErrValidation + } + + if !namePattern.MatchString(name) { + return nil, ErrValidation + } + if _, err := goversion.NewSemver(version); err != nil { + return nil, ErrValidation + } + if user != "" && !namePattern.MatchString(user) { + return nil, ErrValidation + } + if channel != "" && !namePattern.MatchString(channel) { + return nil, ErrValidation + } + if revision != "" && !revisionPattern.MatchString(revision) { + return nil, ErrValidation + } + + return &RecipeReference{name, version, user, channel, revision}, nil +} + +func (r *RecipeReference) RevisionOrDefault() string { + if r.Revision == "" { + return DefaultRevision + } + return r.Revision +} + +func (r *RecipeReference) String() string { + rev := "" + if r.Revision != "" { + rev = "#" + r.Revision + } + if r.User == "" || r.Channel == "" { + return fmt.Sprintf("%s/%s%s", r.Name, r.Version, rev) + } + return fmt.Sprintf("%s/%s@%s/%s%s", r.Name, r.Version, r.User, r.Channel, rev) +} + +func (r *RecipeReference) LinkName() string { + user := r.User + if user == "" { + user = "_" + } + channel := r.Channel + if channel == "" { + channel = "_" + } + return fmt.Sprintf("%s/%s/%s/%s/%s", r.Name, r.Version, user, channel, r.RevisionOrDefault()) +} + +func (r *RecipeReference) WithRevision(revision string) *RecipeReference { + return &RecipeReference{r.Name, r.Version, r.User, r.Channel, revision} +} + +// AsKey builds the additional key for the package file +func (r *RecipeReference) AsKey() string { + return fmt.Sprintf("%s|%s|%s", r.User, r.Channel, r.RevisionOrDefault()) +} + +// PackageReference represents a package of a recipe /@/# # +type PackageReference struct { + Recipe *RecipeReference + Reference string + Revision string +} + +func NewPackageReference(recipe *RecipeReference, reference, revision string) (*PackageReference, error) { + log.Trace("Conan Package: %v %s(#%s)", recipe, reference, revision) + + if recipe == nil { + return nil, ErrValidation + } + if reference == "" || !revisionPattern.MatchString(reference) { + return nil, ErrValidation + } + if revision != "" && !revisionPattern.MatchString(revision) { + return nil, ErrValidation + } + + return &PackageReference{recipe, reference, revision}, nil +} + +func (r *PackageReference) RevisionOrDefault() string { + if r.Revision == "" { + return DefaultRevision + } + return r.Revision +} + +func (r *PackageReference) LinkName() string { + return fmt.Sprintf("%s/%s", r.Reference, r.RevisionOrDefault()) +} + +func (r *PackageReference) WithRevision(revision string) *PackageReference { + return &PackageReference{r.Recipe, r.Reference, revision} +} + +// AsKey builds the additional key for the package file +func (r *PackageReference) AsKey() string { + return fmt.Sprintf("%s|%s|%s|%s|%s", r.Recipe.User, r.Recipe.Channel, r.Recipe.RevisionOrDefault(), r.Reference, r.RevisionOrDefault()) +} diff --git a/modules/packages/conan/reference_test.go b/modules/packages/conan/reference_test.go new file mode 100644 index 0000000000000..29ba3a543bf2d --- /dev/null +++ b/modules/packages/conan/reference_test.go @@ -0,0 +1,147 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewRecipeReference(t *testing.T) { + cases := []struct { + Name string + Version string + User string + Channel string + Revision string + IsValid bool + }{ + {"", "", "", "", "", false}, + {"name", "", "", "", "", false}, + {"", "1.0", "", "", "", false}, + {"", "", "user", "", "", false}, + {"", "", "", "channel", "", false}, + {"", "", "", "", "0", false}, + {"name", "1.0", "", "", "", true}, + {"name", "1.0", "user", "", "", false}, + {"name", "1.0", "", "channel", "", false}, + {"name", "1.0", "user", "channel", "", true}, + {"name", "1.0", "_", "", "", true}, + {"name", "1.0", "", "_", "", true}, + {"name", "1.0", "_", "_", "", true}, + {"name", "1.0", "_", "_", "0", true}, + {"name", "1.0", "", "", "0", true}, + {"name", "1.0", "", "", "000000000000000000000000000000000000000000000000000000000000", false}, + } + + for i, c := range cases { + rref, err := NewRecipeReference(c.Name, c.Version, c.User, c.Channel, c.Revision) + if c.IsValid { + assert.NoError(t, err, "case %d, should be invalid", i) + assert.NotNil(t, rref, "case %d, should not be nil", i) + } else { + assert.Error(t, err, "case %d, should be valid", i) + } + } +} + +func TestRecipeReferenceRevisionOrDefault(t *testing.T) { + rref, err := NewRecipeReference("name", "1.0", "", "", "") + assert.NoError(t, err) + assert.Equal(t, DefaultRevision, rref.RevisionOrDefault()) + + rref, err = NewRecipeReference("name", "1.0", "", "", DefaultRevision) + assert.NoError(t, err) + assert.Equal(t, DefaultRevision, rref.RevisionOrDefault()) + + rref, err = NewRecipeReference("name", "1.0", "", "", "Az09") + assert.NoError(t, err) + assert.Equal(t, "Az09", rref.RevisionOrDefault()) +} + +func TestRecipeReferenceString(t *testing.T) { + rref, err := NewRecipeReference("name", "1.0", "", "", "") + assert.NoError(t, err) + assert.Equal(t, "name/1.0", rref.String()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "") + assert.NoError(t, err) + assert.Equal(t, "name/1.0@user/channel", rref.String()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09") + assert.NoError(t, err) + assert.Equal(t, "name/1.0@user/channel#Az09", rref.String()) +} + +func TestRecipeReferenceLinkName(t *testing.T) { + rref, err := NewRecipeReference("name", "1.0", "", "", "") + assert.NoError(t, err) + assert.Equal(t, "name/1.0/_/_/0", rref.LinkName()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "") + assert.NoError(t, err) + assert.Equal(t, "name/1.0/user/channel/0", rref.LinkName()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09") + assert.NoError(t, err) + assert.Equal(t, "name/1.0/user/channel/Az09", rref.LinkName()) +} + +func TestNewPackageReference(t *testing.T) { + rref, _ := NewRecipeReference("name", "1.0", "", "", "") + + cases := []struct { + Recipe *RecipeReference + Reference string + Revision string + IsValid bool + }{ + {nil, "", "", false}, + {rref, "", "", false}, + {nil, "aZ09", "", false}, + {rref, "aZ09", "", true}, + {rref, "", "Az09", false}, + {rref, "aZ09", "Az09", true}, + } + + for i, c := range cases { + pref, err := NewPackageReference(c.Recipe, c.Reference, c.Revision) + if c.IsValid { + assert.NoError(t, err, "case %d, should be invalid", i) + assert.NotNil(t, pref, "case %d, should not be nil", i) + } else { + assert.Error(t, err, "case %d, should be valid", i) + } + } +} + +func TestPackageReferenceRevisionOrDefault(t *testing.T) { + rref, _ := NewRecipeReference("name", "1.0", "", "", "") + + pref, err := NewPackageReference(rref, "ref", "") + assert.NoError(t, err) + assert.Equal(t, DefaultRevision, pref.RevisionOrDefault()) + + pref, err = NewPackageReference(rref, "ref", DefaultRevision) + assert.NoError(t, err) + assert.Equal(t, DefaultRevision, pref.RevisionOrDefault()) + + pref, err = NewPackageReference(rref, "ref", "Az09") + assert.NoError(t, err) + assert.Equal(t, "Az09", pref.RevisionOrDefault()) +} + +func TestPackageReferenceLinkName(t *testing.T) { + rref, _ := NewRecipeReference("name", "1.0", "", "", "") + + pref, err := NewPackageReference(rref, "ref", "") + assert.NoError(t, err) + assert.Equal(t, "ref/0", pref.LinkName()) + + pref, err = NewPackageReference(rref, "ref", "Az09") + assert.NoError(t, err) + assert.Equal(t, "ref/Az09", pref.LinkName()) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ba11a92f6da19..324dd25fb2c40 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3018,6 +3018,10 @@ composer.install = To install the package using Composer, run the following comm composer.documentation = For more information on the Composer registry, see the documentation. composer.dependencies = Dependencies composer.dependencies.development = Development Dependencies +conan.details.repository = Repository +conan.registry = Setup this registry from the command line: +conan.install = To install the package using Conan, run the following command: +conan.documentation = For more information on the Conan registry, see the documentation. generic.download = Download package from the command line: generic.documentation = For more information on the generic registry, see the documentation. maven.registry = Setup this registry in your project pom.xml file: @@ -3025,7 +3029,7 @@ maven.install = To use the package include the following in the dependenci maven.install2 = Run via command line: maven.download = To download the dependency, run via command line: maven.documentation = For more information on the Maven registry, see the documentation. -nuget.registry = Setup the registry from the command line: +nuget.registry = Setup this registry from the command line: nuget.install = To install the package using NuGet, run the following command: nuget.documentation = For more information on the NuGet registry, see the documentation. nuget.dependency.framework = Target Framework diff --git a/public/img/svg/gitea-conan.svg b/public/img/svg/gitea-conan.svg new file mode 100644 index 0000000000000..d7d5ad5f184cf --- /dev/null +++ b/public/img/svg/gitea-conan.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c01651f0040a7..b1dcb7c027279 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -86,6 +86,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/org" "code.gitea.io/gitea/routers/api/v1/packages" "code.gitea.io/gitea/routers/api/v1/packages/composer" + "code.gitea.io/gitea/routers/api/v1/packages/conan" "code.gitea.io/gitea/routers/api/v1/packages/generic" "code.gitea.io/gitea/routers/api/v1/packages/maven" "code.gitea.io/gitea/routers/api/v1/packages/npm" @@ -1030,6 +1031,94 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { m.Get("/files/{versionid}/{fileid}", composer.DownloadPackageFile) m.Put("", reqToken(), reqPackageAccess(perm.AccessModeWrite), composer.UploadPackage) }) + m.Group("/conan", func() { + m.Group("/v1", func() { + m.Get("/ping", conan.Ping) + m.Group("/users", func() { + m.Get("/authenticate", conan.Authenticate) + m.Get("/check_credentials", conan.CheckCredentials) + }) + m.Group("/conans", func() { + m.Get("/search", conan.SearchRecipes) + m.Group("/{name}/{version}/{user}/{channel}", func() { + m.Get("", conan.RecipeSnapshot) + m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV1) + m.Get("/search", conan.SearchPackagesV1) + m.Get("/digest", conan.RecipeDownloadURLs) + m.Post("/upload_urls", reqPackageAccess(perm.AccessModeWrite), conan.RecipeUploadURLs) + m.Get("/download_urls", conan.RecipeDownloadURLs) + m.Group("/packages", func() { + m.Post("/delete", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV1) + m.Group("/{package_reference}", func() { + m.Get("", conan.PackageSnapshot) + m.Get("/digest", conan.PackageDownloadURLs) + m.Post("/upload_urls", reqPackageAccess(perm.AccessModeWrite), conan.PackageUploadURLs) + m.Get("/download_urls", conan.PackageDownloadURLs) + }) + }) + }, conan.ExtractPathParameters) + }) + m.Group("/files/{name}/{version}/{user}/{channel}/{recipe_revision}", func() { + m.Group("/recipe/{filename}", func() { + m.Get("", conan.DownloadRecipeFile) + m.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadRecipeFile) + }) + m.Group("/package/{package_reference}/{package_revision}/{filename}", func() { + m.Get("", conan.DownloadPackageFile) + m.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadPackageFile) + }) + }, conan.ExtractPathParameters) + }) + m.Group("/v2", func() { + m.Get("/ping", conan.Ping) + m.Group("/users", func() { + m.Get("/authenticate", conan.Authenticate) + m.Get("/check_credentials", conan.CheckCredentials) + }) + m.Group("/conans", func() { + m.Get("/search", conan.SearchRecipes) + m.Group("/{name}/{version}/{user}/{channel}", func() { + m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV2) + m.Get("/search", conan.SearchPackagesV2) + m.Get("/latest", conan.LatestRecipeRevision) + m.Group("/revisions", func() { + m.Get("", conan.ListRecipeRevisions) + m.Group("/{recipe_revision}", func() { + m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV2) + m.Get("/search", conan.SearchPackagesV2) + m.Group("/files", func() { + m.Get("", conan.ListRecipeRevisionFiles) + m.Group("/{filename}", func() { + m.Get("", conan.DownloadRecipeFile) + m.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadRecipeFile) + }) + }) + m.Group("/packages", func() { + m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) + m.Group("/{package_reference}", func() { + m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) + m.Get("/latest", conan.LatestPackageRevision) + m.Group("/revisions", func() { + m.Get("", conan.ListPackageRevisions) + m.Group("/{package_revision}", func() { + m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) + m.Group("/files", func() { + m.Get("", conan.ListPackageRevisionFiles) + m.Group("/{filename}", func() { + m.Get("", conan.DownloadPackageFile) + m.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadPackageFile) + }) + }) + }) + }) + }) + }) + }) + }) + }, conan.ExtractPathParameters) + }) + }) + }) m.Group("/generic", func() { m.Group("/{packagename}/{packageversion}/{filename}", func() { m.Get("", generic.DownloadPackageFile) @@ -1108,7 +1197,7 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { m.Get("/files", packages.ListPackageFiles) }) m.Get("/", packages.ListPackages) - }, context.UserAssignmentAPI(), context.PackageAssignmentAPI(), reqPackageAccess(perm.AccessModeRead)) + }, conan.CheckAuth, context.UserAssignmentAPI(), context.PackageAssignmentAPI(), reqPackageAccess(perm.AccessModeRead)) // Organizations m.Get("/user/orgs", reqToken(), org.ListMyOrgs) diff --git a/routers/api/v1/packages/conan/auth.go b/routers/api/v1/packages/conan/auth.go new file mode 100644 index 0000000000000..6c84ed89516b4 --- /dev/null +++ b/routers/api/v1/packages/conan/auth.go @@ -0,0 +1,87 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "fmt" + "strings" + "time" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "github.com/golang-jwt/jwt/v4" +) + +type conanClaims struct { + jwt.RegisteredClaims + UserID int64 +} + +// CheckAuth is a middleware which handles the custom Conan authorization token +func CheckAuth(ctx *context.APIContext) { + if ctx.User == nil { + token, err := parseAuthorizationToken(ctx) + if err != nil { + log.Trace("parseAuthorizationToken: %v", err) + return + } + + ctx.User, err = user_model.GetUserByIDCtx(ctx, token.UserID) + if err != nil { + if !user_model.IsErrUserNotExist(err) { + log.Error("GetUserByID: %v", err) + } + return + } + + ctx.IsSigned = ctx.User != nil + } +} + +func createAuthorizationToken(ctx *context.APIContext) (string, error) { + now := time.Now() + claims := conanClaims{ + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(now.Add(24 * time.Hour)), + NotBefore: jwt.NewNumericDate(now), + }, + UserID: ctx.User.ID, + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + + tokenString, err := token.SignedString([]byte(setting.SecretKey)) + if err != nil { + return "", err + } + + return tokenString, nil +} + +func parseAuthorizationToken(ctx *context.APIContext) (*conanClaims, error) { + parts := strings.SplitN(ctx.Req.Header.Get("Authorization"), " ", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("no token") + } + + token, err := jwt.ParseWithClaims(parts[1], &conanClaims{}, func(t *jwt.Token) (interface{}, error) { + if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"]) + } + return []byte(setting.SecretKey), nil + }) + if err != nil { + return nil, err + } + + c, ok := token.Claims.(*conanClaims) + if !token.Valid || !ok { + return nil, fmt.Errorf("invalid token claim") + } + + return c, nil +} diff --git a/routers/api/v1/packages/conan/conan.go b/routers/api/v1/packages/conan/conan.go new file mode 100644 index 0000000000000..3bff6533d8463 --- /dev/null +++ b/routers/api/v1/packages/conan/conan.go @@ -0,0 +1,816 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "fmt" + "io" + "net/http" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + package_model "code.gitea.io/gitea/models/packages" + conan_model "code.gitea.io/gitea/models/packages/conan" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/notification" + package_module "code.gitea.io/gitea/modules/packages" + conan_module "code.gitea.io/gitea/modules/packages/conan" + "code.gitea.io/gitea/modules/setting" + package_router "code.gitea.io/gitea/routers/api/v1/packages" + package_service "code.gitea.io/gitea/services/packages" +) + +const ( + conanfileFile = "conanfile.py" + conaninfoFile = "conaninfo.txt" + + recipeReferenceKey = "RecipeReference" + packageReferenceKey = "PackageReference" +) + +type stringSet map[string]struct{} + +var ( + recipeFileList = stringSet{ + conanfileFile: struct{}{}, + "conanmanifest.txt": struct{}{}, + "conan_sources.tgz": struct{}{}, + "conan_export.tgz": struct{}{}, + } + packageFileList = stringSet{ + conaninfoFile: struct{}{}, + "conanmanifest.txt": struct{}{}, + "conan_package.tgz": struct{}{}, + } +) + +func jsonResponse(ctx *context.APIContext, status int, obj interface{}) { + // https://github.com/conan-io/conan/issues/6613 + ctx.Resp.Header().Set("Content-Type", "application/json") + ctx.Status(status) + if err := json.NewEncoder(ctx.Resp).Encode(obj); err != nil { + log.Error("JSON encode: %v", err) + } +} + +func apiError(ctx *context.APIContext, status int, obj interface{}) { + package_router.LogAndProcessError(ctx, status, obj, func(message string) { + jsonResponse(ctx, status, map[string]string{ + "message": message, + }) + }) +} + +func baseURL(ctx *context.APIContext) string { + return setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/conan" +} + +// ExtractPathParameters is a middleware to extract common parameters from path +func ExtractPathParameters(ctx *context.APIContext) { + rref, err := conan_module.NewRecipeReference( + ctx.Params("name"), + ctx.Params("version"), + ctx.Params("user"), + ctx.Params("channel"), + ctx.Params("recipe_revision"), + ) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + ctx.Data[recipeReferenceKey] = rref + + reference := ctx.Params("package_reference") + + var pref *conan_module.PackageReference + if reference != "" { + pref, err = conan_module.NewPackageReference( + rref, + reference, + ctx.Params("package_revision"), + ) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + } + + ctx.Data[packageReferenceKey] = pref +} + +// Ping reports the server capabilities +func Ping(ctx *context.APIContext) { + ctx.RespHeader().Add("X-Conan-Server-Capabilities", "revisions") // complex_search,checksum_deploy,matrix_params + + ctx.Status(http.StatusOK) +} + +// Authenticate creates an authentication token for the user +func Authenticate(ctx *context.APIContext) { + token, err := createAuthorizationToken(ctx) + if err != nil { + ctx.Error(http.StatusInternalServerError, "", err) + return + } + + ctx.PlainText(http.StatusOK, token) +} + +// CheckCredentials tests if the provided authentication token is valid +func CheckCredentials(ctx *context.APIContext) { + if ctx.User == nil { + ctx.Status(http.StatusUnauthorized) + } else { + ctx.Status(http.StatusOK) + } +} + +// RecipeSnapshot displays the recipe files with their md5 hash +func RecipeSnapshot(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + serveSnapshot(ctx, rref.AsKey()) +} + +// RecipeSnapshot displays the package files with their md5 hash +func PackageSnapshot(ctx *context.APIContext) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + serveSnapshot(ctx, pref.AsKey()) +} + +func serveSnapshot(ctx *context.APIContext, fileKey string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + pv, err := package_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, package_model.TypeConan, rref.Name, rref.Version, package_model.EmptyVersionKey) + if err != nil { + if err == package_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + pfs, _, err := package_model.SearchFiles(ctx, &package_model.PackageFileSearchOptions{ + VersionID: pv.ID, + CompositeKey: fileKey, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pfs) == 0 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + files := make(map[string]string) + for _, pf := range pfs { + pb, err := package_model.GetBlobByID(ctx, pf.BlobID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + files[pf.Name] = pb.HashMD5 + } + + jsonResponse(ctx, http.StatusOK, files) +} + +// RecipeDownloadURLs displays the recipe files with their download url +func RecipeDownloadURLs(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + serveDownloadURLs( + ctx, + rref.AsKey(), + fmt.Sprintf(baseURL(ctx)+"/v1/files/%s/recipe", rref.LinkName()), + ) +} + +// PackageDownloadURLs displays the package files with their download url +func PackageDownloadURLs(ctx *context.APIContext) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + serveDownloadURLs( + ctx, + pref.AsKey(), + fmt.Sprintf(baseURL(ctx)+"/v1/files/%s/package/%s", pref.Recipe.LinkName(), pref.LinkName()), + ) +} + +func serveDownloadURLs(ctx *context.APIContext, fileKey, downloadURL string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + pv, err := package_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, package_model.TypeConan, rref.Name, rref.Version, package_model.EmptyVersionKey) + if err != nil { + if err == package_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + pfs, _, err := package_model.SearchFiles(ctx, &package_model.PackageFileSearchOptions{ + VersionID: pv.ID, + CompositeKey: fileKey, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pfs) == 0 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + urls := make(map[string]string) + for _, pf := range pfs { + urls[pf.Name] = fmt.Sprintf("%s/%s", downloadURL, pf.Name) + } + + jsonResponse(ctx, http.StatusOK, urls) +} + +// RecipeUploadURLs displays the upload urls for the provided recipe files +func RecipeUploadURLs(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + serveUploadURLs( + ctx, + recipeFileList, + fmt.Sprintf(baseURL(ctx)+"/v1/files/%s/recipe", rref.LinkName()), + ) +} + +// PackageUploadURLs displays the upload urls for the provided package files +func PackageUploadURLs(ctx *context.APIContext) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + serveUploadURLs( + ctx, + packageFileList, + fmt.Sprintf(baseURL(ctx)+"/v1/files/%s/package/%s", pref.Recipe.LinkName(), pref.LinkName()), + ) +} + +func serveUploadURLs(ctx *context.APIContext, fileFilter stringSet, uploadURL string) { + defer ctx.Req.Body.Close() + + var files map[string]int64 + if err := json.NewDecoder(ctx.Req.Body).Decode(&files); err != nil { + ctx.Error(http.StatusBadRequest, "", err) + return + } + + urls := make(map[string]string) + for file := range files { + if _, ok := fileFilter[file]; ok { + urls[file] = fmt.Sprintf("%s/%s", uploadURL, file) + } + } + + jsonResponse(ctx, http.StatusOK, urls) +} + +// UploadRecipeFile handles the upload of a recipe file +func UploadRecipeFile(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + uploadFile(ctx, recipeFileList, rref.AsKey()) +} + +// UploadPackageFile handles the upload of a package file +func UploadPackageFile(ctx *context.APIContext) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + uploadFile(ctx, packageFileList, pref.AsKey()) +} + +func uploadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + filename := ctx.Params("filename") + if _, ok := fileFilter[filename]; !ok { + apiError(ctx, http.StatusBadRequest, nil) + return + } + + upload, close, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + if close { + defer upload.Close() + } + + buf, err := package_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + if buf.Size() == 0 { + // ignore empty uploads, second request contains content + jsonResponse(ctx, http.StatusOK, nil) + return + } + + isConanfileFile := filename == conanfileFile + + pci := &package_service.PackageCreationInfo{ + PackageInfo: package_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: package_model.TypeConan, + Name: rref.Name, + Version: rref.Version, + }, + SemverCompatible: true, + Creator: ctx.User, + } + pfci := &package_service.PackageFileCreationInfo{ + PackageFileInfo: package_service.PackageFileInfo{ + Filename: strings.ToLower(filename), + CompositeKey: fileKey, + }, + Data: buf, + IsLead: isConanfileFile, + Properties: map[string]string{ + conan_module.PropertyRecipeUser: rref.User, + conan_module.PropertyRecipeChannel: rref.Channel, + conan_module.PropertyRecipeRevision: rref.RevisionOrDefault(), + }, + OverwriteExisting: true, + } + + if pref != nil { + pfci.Properties[conan_module.PropertyPackageReference] = pref.Reference + pfci.Properties[conan_module.PropertyPackageRevision] = pref.RevisionOrDefault() + } + + if isConanfileFile || filename == conaninfoFile { + if isConanfileFile { + metadata, err := conan_module.ParseConanfile(buf) + if err != nil { + log.Error("Error parsing package metadata: %v", err) + apiError(ctx, http.StatusInternalServerError, err) + return + } + pv, err := package_model.GetVersionByNameAndVersion(ctx, pci.Owner.ID, pci.PackageType, pci.Name, pci.Version, package_model.EmptyVersionKey) + if err != nil && err != package_model.ErrPackageNotExist { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if pv != nil { + raw, err := json.Marshal(metadata) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + pv.MetadataJSON = string(raw) + if err := package_model.UpdateVersion(pv); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } else { + pci.Metadata = metadata + } + } else { + info, err := conan_module.ParseConaninfo(buf) + if err != nil { + log.Error("Error parsing conan info: %v", err) + apiError(ctx, http.StatusInternalServerError, err) + return + } + raw, err := json.Marshal(info) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + pfci.Properties[conan_module.PropertyPackageInfo] = string(raw) + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + _, _, err = package_service.CreatePackageOrAddFileToExisting( + pci, + pfci, + ) + if err != nil { + if err == package_model.ErrDuplicatePackageFile { + apiError(ctx, http.StatusBadRequest, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} + +// DownloadRecipeFile serves the conent of the requested recipe file +func DownloadRecipeFile(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + downloadFile(ctx, recipeFileList, rref.AsKey()) +} + +// DownloadPackageFile serves the conent of the requested package file +func DownloadPackageFile(ctx *context.APIContext) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + downloadFile(ctx, packageFileList, pref.AsKey()) +} + +func downloadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + filename := ctx.Params("filename") + if _, ok := fileFilter[filename]; !ok { + apiError(ctx, http.StatusBadRequest, nil) + return + } + + s, pf, err := package_service.GetFileStreamByPackageNameAndVersion( + &package_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: package_model.TypeConan, + Name: rref.Name, + Version: rref.Version, + }, + &package_service.PackageFileInfo{ + Filename: filename, + CompositeKey: fileKey, + }, + ) + if err != nil { + if err == package_model.ErrPackageNotExist || err == package_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pf.Name) +} + +// DeleteRecipeV1 deletes the requested recipe(s) +func DeleteRecipeV1(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + if err := deleteRecipeOrPackage(ctx, rref, true, nil, false); err != nil { + if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + ctx.Status(http.StatusOK) +} + +// DeleteRecipeV2 deletes the requested recipe(s) respecting its revisions +func DeleteRecipeV2(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + if err := deleteRecipeOrPackage(ctx, rref, rref.Revision == "", nil, false); err != nil { + if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + ctx.Status(http.StatusOK) +} + +// DeletePackageV1 deletes the requested package(s) +func DeletePackageV1(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + type PackageReferences struct { + References []string `json:"package_ids"` + } + + var ids *PackageReferences + if err := json.NewDecoder(ctx.Req.Body).Decode(&ids); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + revisions, err := conan_model.GetRecipeRevisions(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + for _, revision := range revisions { + currentRref := rref.WithRevision(revision.Value) + + var references []*conan_model.PropertyValue + if len(ids.References) == 0 { + if references, err = conan_model.GetPackageReferences(ctx, ctx.Package.Owner.ID, currentRref); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } else { + for _, reference := range ids.References { + references = append(references, &conan_model.PropertyValue{Value: reference}) + } + } + + for _, reference := range references { + pref, _ := conan_module.NewPackageReference(currentRref, reference.Value, conan_module.DefaultRevision) + if err := deleteRecipeOrPackage(ctx, currentRref, true, pref, true); err != nil { + if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + } + } + ctx.Status(http.StatusOK) +} + +// DeletePackageV2 deletes the requested package(s) respecting its revisions +func DeletePackageV2(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + if pref != nil { // has package reference + if err := deleteRecipeOrPackage(ctx, rref, false, pref, pref.Revision == ""); err != nil { + if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + } else { + ctx.Status(http.StatusOK) + } + return + } + + references, err := conan_model.GetPackageReferences(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(references) == 0 { + apiError(ctx, http.StatusNotFound, conan_model.ErrPackageReferenceNotExist) + return + } + + for _, reference := range references { + pref, _ := conan_module.NewPackageReference(rref, reference.Value, conan_module.DefaultRevision) + + if err := deleteRecipeOrPackage(ctx, rref, false, pref, true); err != nil { + if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + } + + ctx.Status(http.StatusOK) +} + +func deleteRecipeOrPackage(apictx *context.APIContext, rref *conan_module.RecipeReference, ignoreRecipeRevision bool, pref *conan_module.PackageReference, ignorePackageRevision bool) error { + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + pv, err := package_model.GetVersionByNameAndVersion(ctx, apictx.Package.Owner.ID, package_model.TypeConan, rref.Name, rref.Version, package_model.EmptyVersionKey) + if err != nil { + return err + } + + pd, err := package_model.GetPackageDescriptorCtx(ctx, pv) + if err != nil { + return err + } + + filter := map[string]string{ + conan_module.PropertyRecipeUser: rref.User, + conan_module.PropertyRecipeChannel: rref.Channel, + } + if !ignoreRecipeRevision { + filter[conan_module.PropertyRecipeRevision] = rref.RevisionOrDefault() + } + if pref != nil { + filter[conan_module.PropertyPackageReference] = pref.Reference + if !ignorePackageRevision { + filter[conan_module.PropertyPackageRevision] = pref.RevisionOrDefault() + } + } + + pfs, _, err := package_model.SearchFiles(ctx, &package_model.PackageFileSearchOptions{ + VersionID: pv.ID, + Properties: filter, + }) + if err != nil { + return err + } + if len(pfs) == 0 { + return conan_model.ErrPackageReferenceNotExist + } + + for _, pf := range pfs { + if err := package_model.DeleteAllProperties(ctx, package_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + if err := package_model.DeleteFileByID(ctx, pf.ID); err != nil { + return err + } + } + + versionDeleted := false + has, err := package_model.HasVersionFileReferences(ctx, pv.ID) + if err != nil { + return err + } + if !has { + versionDeleted = true + + if err := package_model.DeleteAllProperties(ctx, package_model.PropertyTypeVersion, pv.ID); err != nil { + return err + } + + if err := package_model.DeleteVersionByID(ctx, pv.ID); err != nil { + return err + } + + if err := package_model.DeletePackageByIDIfUnreferenced(ctx, pv.PackageID); err != nil { + return err + } + } + + if err := committer.Commit(); err != nil { + return err + } + + if versionDeleted { + notification.NotifyPackageDelete(apictx.User, pd) + } + + return package_service.DeleteUnreferencedBlobs() +} + +// ListRecipeRevisions gets a list of all recipe revisions +func ListRecipeRevisions(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + revisions, err := conan_model.GetRecipeRevisions(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + listRevisions(ctx, revisions) +} + +// ListPackageRevisions gets a list of all package revisions +func ListPackageRevisions(ctx *context.APIContext) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + revisions, err := conan_model.GetPackageRevisions(ctx, ctx.Package.Owner.ID, pref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + listRevisions(ctx, revisions) +} + +type revisionInfo struct { + Revision string `json:"revision"` + Time time.Time `json:"time"` +} + +func listRevisions(ctx *context.APIContext, revisions []*conan_model.PropertyValue) { + if len(revisions) == 0 { + apiError(ctx, http.StatusNotFound, conan_model.ErrRecipeReferenceNotExist) + return + } + + type RevisionList struct { + Revisions []*revisionInfo `json:"revisions"` + } + + revs := make([]*revisionInfo, 0, len(revisions)) + for _, rev := range revisions { + revs = append(revs, &revisionInfo{Revision: rev.Value, Time: time.Unix(int64(rev.CreatedUnix), 0)}) + } + + jsonResponse(ctx, http.StatusOK, &RevisionList{revs}) +} + +// LatestRecipeRevision gets the latest recipe revision +func LatestRecipeRevision(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + revision, err := conan_model.GetLastRecipeRevision(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + jsonResponse(ctx, http.StatusOK, &revisionInfo{Revision: revision.Value, Time: time.Unix(int64(revision.CreatedUnix), 0)}) +} + +// LatestPackageRevision gets the latest package revision +func LatestPackageRevision(ctx *context.APIContext) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + revision, err := conan_model.GetLastPackageRevision(ctx, ctx.Package.Owner.ID, pref) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist || err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + jsonResponse(ctx, http.StatusOK, &revisionInfo{Revision: revision.Value, Time: time.Unix(int64(revision.CreatedUnix), 0)}) +} + +// ListRecipeRevisionFiles gets a list of all recipe revision files +func ListRecipeRevisionFiles(ctx *context.APIContext) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + listRevisionFiles(ctx, rref.AsKey()) +} + +// ListPackageRevisionFiles gets a list of all package revision files +func ListPackageRevisionFiles(ctx *context.APIContext) { + pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) + + listRevisionFiles(ctx, pref.AsKey()) +} + +func listRevisionFiles(ctx *context.APIContext, fileKey string) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + pv, err := package_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, package_model.TypeConan, rref.Name, rref.Version, package_model.EmptyVersionKey) + if err != nil { + if err == package_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + pfs, _, err := package_model.SearchFiles(ctx, &package_model.PackageFileSearchOptions{ + VersionID: pv.ID, + CompositeKey: fileKey, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pfs) == 0 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + files := make(map[string]interface{}) + for _, pf := range pfs { + files[pf.Name] = nil + } + + type FileList struct { + Files map[string]interface{} `json:"files"` + } + + jsonResponse(ctx, http.StatusOK, &FileList{ + Files: files, + }) +} diff --git a/routers/api/v1/packages/conan/search.go b/routers/api/v1/packages/conan/search.go new file mode 100644 index 0000000000000..ea8ed284be685 --- /dev/null +++ b/routers/api/v1/packages/conan/search.go @@ -0,0 +1,163 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package conan + +import ( + "net/http" + "strings" + + conan_model "code.gitea.io/gitea/models/packages/conan" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + conan_module "code.gitea.io/gitea/modules/packages/conan" +) + +// SearchResult contains the found recipe names +type SearchResult struct { + Results []string `json:"results"` +} + +// SearchRecipes searches all recipes matching the query +func SearchRecipes(ctx *context.APIContext) { + q := ctx.FormTrim("q") + + opts := parseQuery(ctx.Package.Owner, q) + + results, err := conan_model.SearchRecipes(ctx, opts) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + jsonResponse(ctx, http.StatusOK, &SearchResult{ + Results: results, + }) +} + +// parseQuery creates search options for the given query +func parseQuery(owner *user_model.User, query string) *conan_model.RecipeSearchOptions { + opts := &conan_model.RecipeSearchOptions{ + OwnerID: owner.ID, + } + + if query != "" { + parts := strings.Split(strings.ReplaceAll(query, "@", "/"), "/") + + opts.Name = parts[0] + if len(parts) > 1 && parts[1] != "*" { + opts.Version = parts[1] + } + if len(parts) > 2 && parts[2] != "*" { + opts.User = parts[2] + } + if len(parts) > 3 && parts[3] != "*" { + opts.Channel = parts[3] + } + } + + return opts +} + +// SearchPackagesV1 searches all packages of a recipe (Conan v1 endpoint) +func SearchPackagesV1(ctx *context.APIContext) { + searchPackages(ctx, true) +} + +// SearchPackagesV2 searches all packages of a recipe (Conan v2 endpoint) +func SearchPackagesV2(ctx *context.APIContext) { + searchPackages(ctx, false) +} + +func searchPackages(ctx *context.APIContext, searchAllRevisions bool) { + rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) + + if !searchAllRevisions && rref.Revision == "" { + lastRevision, err := conan_model.GetLastRecipeRevision(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + rref = rref.WithRevision(lastRevision.Value) + } + + has, err := conan_model.RecipeExists(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + if !has { + apiError(ctx, http.StatusNotFound, nil) + return + } + + recipeRevisions := []*conan_model.PropertyValue{{Value: rref.Revision}} + if searchAllRevisions { + recipeRevisions, err = conan_model.GetRecipeRevisions(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + result := make(map[string]*conan_module.Conaninfo) + + for _, recipeRevision := range recipeRevisions { + currentRef := rref + if recipeRevision.Value != "" { + currentRef = rref.WithRevision(recipeRevision.Value) + } + packageReferences, err := conan_model.GetPackageReferences(ctx, ctx.Package.Owner.ID, currentRef) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + for _, packageReference := range packageReferences { + if _, ok := result[packageReference.Value]; ok { + continue + } + pref, _ := conan_module.NewPackageReference(currentRef, packageReference.Value, "") + lastPackageRevision, err := conan_model.GetLastPackageRevision(ctx, ctx.Package.Owner.ID, pref) + if err != nil { + if err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + pref = pref.WithRevision(lastPackageRevision.Value) + infoRaw, err := conan_model.GetPackageInfo(ctx, ctx.Package.Owner.ID, pref) + if err != nil { + if err == conan_model.ErrPackageReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + var info *conan_module.Conaninfo + if err := json.Unmarshal([]byte(infoRaw), &info); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + result[pref.Reference] = info + } + } + + jsonResponse(ctx, http.StatusOK, result) +} diff --git a/routers/api/v1/packages/helper.go b/routers/api/v1/packages/helper.go index 3a0c5c522146f..e6133b1eeb806 100644 --- a/routers/api/v1/packages/helper.go +++ b/routers/api/v1/packages/helper.go @@ -19,7 +19,7 @@ func LogAndProcessError(ctx *context.APIContext, status int, obj interface{}, cb var message string if err, ok := obj.(error); ok { message = err.Error() - } else { + } else if obj != nil { message = fmt.Sprintf("%s", obj) } if status == http.StatusInternalServerError { diff --git a/services/packages/packages.go b/services/packages/packages.go index f5ba60c15a37a..a36d7ffe05895 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -21,11 +21,10 @@ import ( // PackageInfo describes a package type PackageInfo struct { - Owner *user_model.User - PackageType packages_model.Type - Name string - Version string - CompositeKey string + Owner *user_model.User + PackageType packages_model.Type + Name string + Version string } // PackageCreationInfo describes a package to create @@ -46,8 +45,10 @@ type PackageFileInfo struct { // PackageFileCreationInfo describes a package file to create type PackageFileCreationInfo struct { PackageFileInfo - Data *packages_module.HashedBuffer - IsLead bool + Data *packages_module.HashedBuffer + IsLead bool + Properties map[string]string + OverwriteExisting bool } // CreatePackageAndAddFile creates a package with a file. If the same package exists already, ErrDuplicatePackageVersion is returned @@ -145,10 +146,12 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } } - for name, value := range pvci.Properties { - if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil { - log.Error("Error setting package version property: %v", err) - return nil, false, err + if created { + for name, value := range pvci.Properties { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil { + log.Error("Error setting package version property: %v", err) + return nil, false, err + } } } @@ -163,7 +166,7 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) ( } defer committer.Close() - pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pvi.CompositeKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, packages_model.EmptyVersionKey) if err != nil { return nil, nil, err } @@ -218,6 +221,26 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers } } + if pfci.OverwriteExisting { + pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, pfci.Filename, pfci.CompositeKey) + if err != nil && err != packages_model.ErrPackageFileNotExist { + return nil, pb, !exists, err + } + if pf != nil { + // Short circuit if blob is the same + if pf.BlobID == pb.ID { + return pf, pb, !exists, nil + } + + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return nil, pb, !exists, err + } + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { + return nil, pb, !exists, err + } + } + } + pf := &packages_model.PackageFile{ VersionID: pv.ID, BlobID: pb.ID, @@ -230,7 +253,14 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers if err != packages_model.ErrDuplicatePackageFile { log.Error("Error inserting package file: %v", err) } - return nil, nil, !exists, err + return nil, pb, !exists, err + } + + for name, value := range pfci.Properties { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil { + log.Error("Error setting package file property: %v", err) + return pf, pb, !exists, err + } } return pf, pb, !exists, nil @@ -238,7 +268,7 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers // DeletePackageVersionByNameAndVersion deletes a package version and all associated files func DeletePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error { - pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pvi.CompositeKey) + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, packages_model.EmptyVersionKey) if err != nil { return err } @@ -265,8 +295,13 @@ func DeletePackageVersion(doer *user_model.User, pv *packages_model.PackageVersi return err } - if err := packages_model.DeleteFilesByVersionID(ctx, pv.ID); err != nil { - return err + for _, file := range pd.Files { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, file.File.ID); err != nil { + return err + } + if err := packages_model.DeleteFileByID(ctx, file.File.ID); err != nil { + return err + } } if err := packages_model.DeleteVersionByID(ctx, pv.ID); err != nil { @@ -323,7 +358,7 @@ func DeleteUnreferencedBlobs() error { func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey) - pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pvi.CompositeKey) + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, packages_model.EmptyVersionKey) if err != nil { if err == packages_model.ErrPackageNotExist { return nil, nil, err diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index 874740f35c01c..65181b67176e0 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -14,6 +14,7 @@ + diff --git a/templates/package/content/conan.tmpl b/templates/package/content/conan.tmpl new file mode 100644 index 0000000000000..25baada9e5396 --- /dev/null +++ b/templates/package/content/conan.tmpl @@ -0,0 +1,34 @@ +{{if eq .PackageDescriptor.Package.Type "conan"}} +

    {{.i18n.Tr "packages.installation"}}

    +
    +
    +
    + + +
    +
    + + +
    +
    + +
    +
    +
    + + {{if .PackageDescriptor.Metadata.Description}} +

    {{.i18n.Tr "packages.about"}}

    +
    + {{if .PackageDescriptor.Metadata.Description}}{{.PackageDescriptor.Metadata.Description}}{{end}} +
    + {{end}} + + {{if or .PackageDescriptor.Metadata.Keywords}} +

    {{.i18n.Tr "packages.keywords"}}

    +
    + {{range .PackageDescriptor.Metadata.Keywords}} + {{.}} + {{end}} +
    + {{end}} +{{end}} diff --git a/templates/package/metadata/conan.tmpl b/templates/package/metadata/conan.tmpl new file mode 100644 index 0000000000000..1ef82aea4e6ea --- /dev/null +++ b/templates/package/metadata/conan.tmpl @@ -0,0 +1,6 @@ +{{if eq .PackageDescriptor.Package.Type "conan"}} + {{if .PackageDescriptor.Metadata.Author}}
    {{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.details.project_site"}}
    {{end}} + {{if .PackageDescriptor.Metadata.License}}
    {{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}
    {{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.conan.details.repository"}}
    {{end}} +{{end}} diff --git a/templates/package/shared/list.tmpl b/templates/package/shared/list.tmpl index e30a1920ab66a..45b05f2bec515 100644 --- a/templates/package/shared/list.tmpl +++ b/templates/package/shared/list.tmpl @@ -7,6 +7,7 @@ + @@ -26,7 +27,7 @@ {{svg .Package.Type.SVGName 16}} {{.Package.Type.Name}}
    - {{ $timeStr := TimeSinceUnix .Version.CreatedUnix $.Lang }} + {{$timeStr := TimeSinceUnix .Version.CreatedUnix $.i18n.Lang}} {{if .Repository}} {{$.i18n.Tr "packages.published_by_in" $timeStr .Creator.HomeLink (.Creator.GetDisplayName | Escape) .Repository.HTMLURL (.Repository.FullName | Escape) | Safe}} {{else}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 35cc14067f369..86573a2db5120 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -9,7 +9,7 @@

    {{.PackageDescriptor.Package.Name}} ({{.PackageDescriptor.Version.Version}})

    - {{$timeStr := TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.Lang}} + {{$timeStr := TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.i18n.Lang}} {{if .PackageDescriptor.Repository}} {{.i18n.Tr "packages.published_by_in" $timeStr .PackageDescriptor.Creator.HomeLink (.PackageDescriptor.Creator.GetDisplayName | Escape) .PackageDescriptor.Repository.HTMLURL (.PackageDescriptor.Repository.FullName | Escape) | Safe}} {{else}} @@ -20,6 +20,7 @@
    {{template "package/content/composer" .}} + {{template "package/content/conan" .}} {{template "package/content/generic" .}} {{template "package/content/nuget" .}} {{template "package/content/npm" .}} @@ -38,6 +39,7 @@
    {{svg "octicon-calendar" 16 "mr-3"}} {{.PackageDescriptor.Version.CreatedUnix.FormatDate}}
    {{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
    {{template "package/metadata/composer" .}} + {{template "package/metadata/conan" .}} {{template "package/metadata/generic" .}} {{template "package/metadata/nuget" .}} {{template "package/metadata/npm" .}} diff --git a/web_src/svg/gitea-conan.svg b/web_src/svg/gitea-conan.svg new file mode 100644 index 0000000000000..f1719cece36a7 --- /dev/null +++ b/web_src/svg/gitea-conan.svg @@ -0,0 +1,9 @@ + + + + + + + + + From ba7f0501586d2dad28ae46b51ae0ec763bc4ec53 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 14 Feb 2022 21:20:48 +0000 Subject: [PATCH 089/130] Updated docs. --- docs/content/doc/packages/generic.en-us.md | 2 +- docs/content/doc/packages/maven.en-us.md | 10 +++++++++- docs/content/doc/packages/npm.en-us.md | 12 +++++++++++- docs/content/doc/packages/nuget.en-us.md | 12 ++++++++++-- docs/content/doc/packages/pypi.en-us.md | 9 ++++++++- docs/content/doc/packages/rubygems.en-us.md | 10 +++++++++- 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/docs/content/doc/packages/generic.en-us.md b/docs/content/doc/packages/generic.en-us.md index 1d973bd6d034e..15a714c9242c9 100644 --- a/docs/content/doc/packages/generic.en-us.md +++ b/docs/content/doc/packages/generic.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "Generic" - weight: 20 + weight: 30 identifier: "generic" --- diff --git a/docs/content/doc/packages/maven.en-us.md b/docs/content/doc/packages/maven.en-us.md index 8798730161f78..d55f4eb47397e 100644 --- a/docs/content/doc/packages/maven.en-us.md +++ b/docs/content/doc/packages/maven.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "Maven" - weight: 30 + weight: 40 identifier: "maven" --- @@ -99,4 +99,12 @@ Afterwards run: ```shell mvn install +``` + +## Supported commands + +``` +mvn install +mvn deploy +mvn dependency:get: ``` \ No newline at end of file diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index 96999c31d9b8e..818ded0fdb024 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "npm" - weight: 40 + weight: 50 identifier: "npm" --- @@ -106,3 +106,13 @@ npm dist-tag add test_package@1.0.2 release ``` The tag name must not be a valid version. All tag names which are parsable as a version are rejected. + +## Supported commands + +``` +npm install +npm ci +npm publish +npm dist-tag +npm view +``` diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md index ad10cfba9098a..b7974f06921d1 100644 --- a/docs/content/doc/packages/nuget.en-us.md +++ b/docs/content/doc/packages/nuget.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "NuGet" - weight: 50 + weight: 60 identifier: "nuget" --- @@ -86,4 +86,12 @@ For example: ```shell dotnet add package --source gitea --version 1.0.0 test_package -``` \ No newline at end of file +``` + +## Supported commands + +``` +dotnet add +dotnet nuget push +dotnet nuget delete +``` diff --git a/docs/content/doc/packages/pypi.en-us.md b/docs/content/doc/packages/pypi.en-us.md index 8d527fd8e1d01..4411bb1f760a5 100644 --- a/docs/content/doc/packages/pypi.en-us.md +++ b/docs/content/doc/packages/pypi.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "PyPI" - weight: 60 + weight: 70 identifier: "pypi" --- @@ -76,3 +76,10 @@ For example: ```shell pip install --index-url https://testuser:password123@gitea.example.com/api/v1/packages/testuser/pypi/simple --no-deps test_package ``` + +## Supported commands + +``` +pip install +twine upload +``` \ No newline at end of file diff --git a/docs/content/doc/packages/rubygems.en-us.md b/docs/content/doc/packages/rubygems.en-us.md index d672bbc4fd662..8a43dda407d4a 100644 --- a/docs/content/doc/packages/rubygems.en-us.md +++ b/docs/content/doc/packages/rubygems.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "RubyGems" - weight: 70 + weight: 80 identifier: "rubygems" --- @@ -116,4 +116,12 @@ For example: ```shell gem install --host https://gitea.example.com/api/v1/packages/testuser/rubygems test_package +``` + +## Supported commands + +``` +gem install +bundle install +gem push ``` \ No newline at end of file From b10ef7528359f58f1ea58d888dde7e6c65e26f80 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 14 Feb 2022 21:27:54 +0000 Subject: [PATCH 090/130] Unify package names. --- routers/api/v1/packages/composer/composer.go | 8 +- routers/api/v1/packages/conan/conan.go | 92 ++++++++++---------- routers/api/v1/packages/generic/generic.go | 30 +++---- routers/api/v1/packages/maven/maven.go | 4 +- routers/api/v1/packages/npm/npm.go | 8 +- routers/api/v1/packages/nuget/nuget.go | 10 +-- routers/api/v1/packages/package.go | 4 +- routers/api/v1/packages/pypi/pypi.go | 8 +- routers/api/v1/packages/rubygems/rubygems.go | 8 +- 9 files changed, 86 insertions(+), 86 deletions(-) diff --git a/routers/api/v1/packages/composer/composer.go b/routers/api/v1/packages/composer/composer.go index 47d7a95a734c8..54bd715a623fb 100644 --- a/routers/api/v1/packages/composer/composer.go +++ b/routers/api/v1/packages/composer/composer.go @@ -16,17 +16,17 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" - package_module "code.gitea.io/gitea/modules/packages" + packages_module "code.gitea.io/gitea/modules/packages" composer_module "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/setting" - package_router "code.gitea.io/gitea/routers/api/v1/packages" + packages_router "code.gitea.io/gitea/routers/api/v1/packages" packages_service "code.gitea.io/gitea/services/packages" "github.com/hashicorp/go-version" ) func apiError(ctx *context.APIContext, status int, obj interface{}) { - package_router.LogAndProcessError(ctx, status, obj, func(message string) { + packages_router.LogAndProcessError(ctx, status, obj, func(message string) { type Error struct { Status int `json:"status"` Message string `json:"message"` @@ -183,7 +183,7 @@ func DownloadPackageFile(ctx *context.APIContext) { // UploadPackage creates a new package func UploadPackage(ctx *context.APIContext) { - buf, err := package_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/conan/conan.go b/routers/api/v1/packages/conan/conan.go index 3bff6533d8463..1228a235bffbb 100644 --- a/routers/api/v1/packages/conan/conan.go +++ b/routers/api/v1/packages/conan/conan.go @@ -12,17 +12,17 @@ import ( "time" "code.gitea.io/gitea/models/db" - package_model "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" conan_model "code.gitea.io/gitea/models/packages/conan" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" - package_module "code.gitea.io/gitea/modules/packages" + packages_module "code.gitea.io/gitea/modules/packages" conan_module "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/setting" - package_router "code.gitea.io/gitea/routers/api/v1/packages" - package_service "code.gitea.io/gitea/services/packages" + packages_router "code.gitea.io/gitea/routers/api/v1/packages" + packages_service "code.gitea.io/gitea/services/packages" ) const ( @@ -59,7 +59,7 @@ func jsonResponse(ctx *context.APIContext, status int, obj interface{}) { } func apiError(ctx *context.APIContext, status int, obj interface{}) { - package_router.LogAndProcessError(ctx, status, obj, func(message string) { + packages_router.LogAndProcessError(ctx, status, obj, func(message string) { jsonResponse(ctx, status, map[string]string{ "message": message, }) @@ -148,9 +148,9 @@ func PackageSnapshot(ctx *context.APIContext) { func serveSnapshot(ctx *context.APIContext, fileKey string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) - pv, err := package_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, package_model.TypeConan, rref.Name, rref.Version, package_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version, packages_model.EmptyVersionKey) if err != nil { - if err == package_model.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -158,7 +158,7 @@ func serveSnapshot(ctx *context.APIContext, fileKey string) { return } - pfs, _, err := package_model.SearchFiles(ctx, &package_model.PackageFileSearchOptions{ + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ VersionID: pv.ID, CompositeKey: fileKey, }) @@ -173,7 +173,7 @@ func serveSnapshot(ctx *context.APIContext, fileKey string) { files := make(map[string]string) for _, pf := range pfs { - pb, err := package_model.GetBlobByID(ctx, pf.BlobID) + pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -209,9 +209,9 @@ func PackageDownloadURLs(ctx *context.APIContext) { func serveDownloadURLs(ctx *context.APIContext, fileKey, downloadURL string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) - pv, err := package_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, package_model.TypeConan, rref.Name, rref.Version, package_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version, packages_model.EmptyVersionKey) if err != nil { - if err == package_model.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -219,7 +219,7 @@ func serveDownloadURLs(ctx *context.APIContext, fileKey, downloadURL string) { return } - pfs, _, err := package_model.SearchFiles(ctx, &package_model.PackageFileSearchOptions{ + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ VersionID: pv.ID, CompositeKey: fileKey, }) @@ -315,7 +315,7 @@ func uploadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { defer upload.Close() } - buf, err := package_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -330,18 +330,18 @@ func uploadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { isConanfileFile := filename == conanfileFile - pci := &package_service.PackageCreationInfo{ - PackageInfo: package_service.PackageInfo{ + pci := &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: package_model.TypeConan, + PackageType: packages_model.TypeConan, Name: rref.Name, Version: rref.Version, }, SemverCompatible: true, Creator: ctx.User, } - pfci := &package_service.PackageFileCreationInfo{ - PackageFileInfo: package_service.PackageFileInfo{ + pfci := &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ Filename: strings.ToLower(filename), CompositeKey: fileKey, }, @@ -368,8 +368,8 @@ func uploadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { apiError(ctx, http.StatusInternalServerError, err) return } - pv, err := package_model.GetVersionByNameAndVersion(ctx, pci.Owner.ID, pci.PackageType, pci.Name, pci.Version, package_model.EmptyVersionKey) - if err != nil && err != package_model.ErrPackageNotExist { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pci.Owner.ID, pci.PackageType, pci.Name, pci.Version, packages_model.EmptyVersionKey) + if err != nil && err != packages_model.ErrPackageNotExist { apiError(ctx, http.StatusInternalServerError, err) return } @@ -380,7 +380,7 @@ func uploadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { return } pv.MetadataJSON = string(raw) - if err := package_model.UpdateVersion(pv); err != nil { + if err := packages_model.UpdateVersion(pv); err != nil { apiError(ctx, http.StatusInternalServerError, err) return } @@ -408,12 +408,12 @@ func uploadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { } } - _, _, err = package_service.CreatePackageOrAddFileToExisting( + _, _, err = packages_service.CreatePackageOrAddFileToExisting( pci, pfci, ) if err != nil { - if err == package_model.ErrDuplicatePackageFile { + if err == packages_model.ErrDuplicatePackageFile { apiError(ctx, http.StatusBadRequest, err) return } @@ -447,20 +447,20 @@ func downloadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) return } - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion( - &package_service.PackageInfo{ + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + &packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: package_model.TypeConan, + PackageType: packages_model.TypeConan, Name: rref.Name, Version: rref.Version, }, - &package_service.PackageFileInfo{ + &packages_service.PackageFileInfo{ Filename: filename, CompositeKey: fileKey, }, ) if err != nil { - if err == package_model.ErrPackageNotExist || err == package_model.ErrPackageFileNotExist { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -477,7 +477,7 @@ func DeleteRecipeV1(ctx *context.APIContext) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) if err := deleteRecipeOrPackage(ctx, rref, true, nil, false); err != nil { - if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -492,7 +492,7 @@ func DeleteRecipeV2(ctx *context.APIContext) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) if err := deleteRecipeOrPackage(ctx, rref, rref.Revision == "", nil, false); err != nil { - if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -539,7 +539,7 @@ func DeletePackageV1(ctx *context.APIContext) { for _, reference := range references { pref, _ := conan_module.NewPackageReference(currentRref, reference.Value, conan_module.DefaultRevision) if err := deleteRecipeOrPackage(ctx, currentRref, true, pref, true); err != nil { - if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -558,7 +558,7 @@ func DeletePackageV2(ctx *context.APIContext) { if pref != nil { // has package reference if err := deleteRecipeOrPackage(ctx, rref, false, pref, pref.Revision == ""); err != nil { - if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -583,7 +583,7 @@ func DeletePackageV2(ctx *context.APIContext) { pref, _ := conan_module.NewPackageReference(rref, reference.Value, conan_module.DefaultRevision) if err := deleteRecipeOrPackage(ctx, rref, false, pref, true); err != nil { - if err == package_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { + if err == packages_model.ErrPackageNotExist || err == conan_model.ErrPackageReferenceNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -602,12 +602,12 @@ func deleteRecipeOrPackage(apictx *context.APIContext, rref *conan_module.Recipe } defer committer.Close() - pv, err := package_model.GetVersionByNameAndVersion(ctx, apictx.Package.Owner.ID, package_model.TypeConan, rref.Name, rref.Version, package_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, apictx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version, packages_model.EmptyVersionKey) if err != nil { return err } - pd, err := package_model.GetPackageDescriptorCtx(ctx, pv) + pd, err := packages_model.GetPackageDescriptorCtx(ctx, pv) if err != nil { return err } @@ -626,7 +626,7 @@ func deleteRecipeOrPackage(apictx *context.APIContext, rref *conan_module.Recipe } } - pfs, _, err := package_model.SearchFiles(ctx, &package_model.PackageFileSearchOptions{ + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ VersionID: pv.ID, Properties: filter, }) @@ -638,31 +638,31 @@ func deleteRecipeOrPackage(apictx *context.APIContext, rref *conan_module.Recipe } for _, pf := range pfs { - if err := package_model.DeleteAllProperties(ctx, package_model.PropertyTypeFile, pf.ID); err != nil { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { return err } - if err := package_model.DeleteFileByID(ctx, pf.ID); err != nil { + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { return err } } versionDeleted := false - has, err := package_model.HasVersionFileReferences(ctx, pv.ID) + has, err := packages_model.HasVersionFileReferences(ctx, pv.ID) if err != nil { return err } if !has { versionDeleted = true - if err := package_model.DeleteAllProperties(ctx, package_model.PropertyTypeVersion, pv.ID); err != nil { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { return err } - if err := package_model.DeleteVersionByID(ctx, pv.ID); err != nil { + if err := packages_model.DeleteVersionByID(ctx, pv.ID); err != nil { return err } - if err := package_model.DeletePackageByIDIfUnreferenced(ctx, pv.PackageID); err != nil { + if err := packages_model.DeletePackageByIDIfUnreferenced(ctx, pv.PackageID); err != nil { return err } } @@ -675,7 +675,7 @@ func deleteRecipeOrPackage(apictx *context.APIContext, rref *conan_module.Recipe notification.NotifyPackageDelete(apictx.User, pd) } - return package_service.DeleteUnreferencedBlobs() + return packages_service.DeleteUnreferencedBlobs() } // ListRecipeRevisions gets a list of all recipe revisions @@ -778,9 +778,9 @@ func ListPackageRevisionFiles(ctx *context.APIContext) { func listRevisionFiles(ctx *context.APIContext, fileKey string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) - pv, err := package_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, package_model.TypeConan, rref.Name, rref.Version, package_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version, packages_model.EmptyVersionKey) if err != nil { - if err == package_model.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -788,7 +788,7 @@ func listRevisionFiles(ctx *context.APIContext, fileKey string) { return } - pfs, _, err := package_model.SearchFiles(ctx, &package_model.PackageFileSearchOptions{ + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ VersionID: pv.ID, CompositeKey: fileKey, }) diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index 8d425bda2da94..b8eba11b233f4 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -12,9 +12,9 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" - package_module "code.gitea.io/gitea/modules/packages" - package_router "code.gitea.io/gitea/routers/api/v1/packages" - package_service "code.gitea.io/gitea/services/packages" + packages_module "code.gitea.io/gitea/modules/packages" + packages_router "code.gitea.io/gitea/routers/api/v1/packages" + packages_service "code.gitea.io/gitea/services/packages" "github.com/hashicorp/go-version" ) @@ -25,7 +25,7 @@ var ( ) func apiError(ctx *context.APIContext, status int, obj interface{}) { - package_router.LogAndProcessError(ctx, status, obj, func(message string) { + packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } @@ -38,14 +38,14 @@ func DownloadPackageFile(ctx *context.APIContext) { return } - s, pf, err := package_service.GetFileStreamByPackageNameAndVersion( - &package_service.PackageInfo{ + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + &packages_service.PackageInfo{ Owner: ctx.Package.Owner, PackageType: packages.TypeGeneric, Name: packageName, Version: packageVersion, }, - &package_service.PackageFileInfo{ + &packages_service.PackageFileInfo{ Filename: filename, }, ) @@ -80,7 +80,7 @@ func UploadPackage(ctx *context.APIContext) { defer upload.Close() } - buf, err := package_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { log.Error("Error creating hashed buffer: %v", err) apiError(ctx, http.StatusInternalServerError, err) @@ -88,9 +88,9 @@ func UploadPackage(ctx *context.APIContext) { } defer buf.Close() - _, _, err = package_service.CreatePackageAndAddFile( - &package_service.PackageCreationInfo{ - PackageInfo: package_service.PackageInfo{ + _, _, err = packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, PackageType: packages.TypeGeneric, Name: packageName, @@ -99,8 +99,8 @@ func UploadPackage(ctx *context.APIContext) { SemverCompatible: true, Creator: ctx.User, }, - &package_service.PackageFileCreationInfo{ - PackageFileInfo: package_service.PackageFileInfo{ + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ Filename: filename, }, Data: buf, @@ -127,9 +127,9 @@ func DeletePackage(ctx *context.APIContext) { return } - err = package_service.DeletePackageVersionByNameAndVersion( + err = packages_service.DeletePackageVersionByNameAndVersion( ctx.User, - &package_service.PackageInfo{ + &packages_service.PackageInfo{ Owner: ctx.Package.Owner, PackageType: packages.TypeGeneric, Name: packageName, diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 8bb38339b9e61..85debd6ced8cd 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -25,7 +25,7 @@ import ( "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" maven_module "code.gitea.io/gitea/modules/packages/maven" - package_router "code.gitea.io/gitea/routers/api/v1/packages" + packages_router "code.gitea.io/gitea/routers/api/v1/packages" packages_service "code.gitea.io/gitea/services/packages" ) @@ -43,7 +43,7 @@ var ( ) func apiError(ctx *context.APIContext, status int, obj interface{}) { - package_router.LogAndProcessError(ctx, status, obj, func(message string) { + packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index e9a8501e1c892..704f8917395a3 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -16,10 +16,10 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" - package_module "code.gitea.io/gitea/modules/packages" + packages_module "code.gitea.io/gitea/modules/packages" npm_module "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/setting" - package_router "code.gitea.io/gitea/routers/api/v1/packages" + packages_router "code.gitea.io/gitea/routers/api/v1/packages" packages_service "code.gitea.io/gitea/services/packages" "github.com/hashicorp/go-version" @@ -29,7 +29,7 @@ import ( var errInvalidTagName = errors.New("The tag name is invalid") func apiError(ctx *context.APIContext, status int, obj interface{}) { - package_router.LogAndProcessError(ctx, status, obj, func(message string) { + packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.JSON(status, map[string]string{ "error": message, }) @@ -121,7 +121,7 @@ func UploadPackage(ctx *context.APIContext) { return } - buf, err := package_module.CreateHashedBufferFromReader(bytes.NewReader(npmPackage.Data), 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(bytes.NewReader(npmPackage.Data), 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 891d27ede9549..7f2745ae3778e 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -14,15 +14,15 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" - package_module "code.gitea.io/gitea/modules/packages" + packages_module "code.gitea.io/gitea/modules/packages" nuget_module "code.gitea.io/gitea/modules/packages/nuget" "code.gitea.io/gitea/modules/setting" - package_router "code.gitea.io/gitea/routers/api/v1/packages" + packages_router "code.gitea.io/gitea/routers/api/v1/packages" packages_service "code.gitea.io/gitea/services/packages" ) func apiError(ctx *context.APIContext, status int, obj interface{}) { - package_router.LogAndProcessError(ctx, status, obj, func(message string) { + packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.JSON(status, map[string]string{ "Message": message, }) @@ -267,7 +267,7 @@ func UploadSymbolPackage(ctx *context.APIContext) { ctx.Status(http.StatusCreated) } -func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.PackageType) (*nuget_module.Package, *package_module.HashedBuffer, []io.Closer) { +func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.PackageType) (*nuget_module.Package, *packages_module.HashedBuffer, []io.Closer) { closables := make([]io.Closer, 0, 2) upload, close, err := ctx.UploadStream() @@ -280,7 +280,7 @@ func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.Pack closables = append(closables, upload) } - buf, err := package_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return nil, nil, closables diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index e9a47695a5c6f..4468a871e596f 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -12,7 +12,7 @@ import ( "code.gitea.io/gitea/modules/convert" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/routers/api/v1/utils" - package_service "code.gitea.io/gitea/services/packages" + packages_service "code.gitea.io/gitea/services/packages" ) // ListPackages gets all packages of an owner @@ -132,7 +132,7 @@ func DeletePackage(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - err := package_service.DeletePackageVersion(ctx.User, ctx.Package.Descriptor.Version) + err := packages_service.DeletePackageVersion(ctx.User, ctx.Package.Descriptor.Version) if err != nil { ctx.Error(http.StatusInternalServerError, "DeletePackageVersion", err) return diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index 2ec1585d787f8..a9e4b929dc577 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -13,12 +13,12 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" - package_module "code.gitea.io/gitea/modules/packages" + packages_module "code.gitea.io/gitea/modules/packages" pypi_module "code.gitea.io/gitea/modules/packages/pypi" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/validation" - package_router "code.gitea.io/gitea/routers/api/v1/packages" + packages_router "code.gitea.io/gitea/routers/api/v1/packages" packages_service "code.gitea.io/gitea/services/packages" ) @@ -30,7 +30,7 @@ var nameMatcher = regexp.MustCompile(`\A[a-z0-9\.\-_]+\z`) var versionMatcher = regexp.MustCompile(`^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$`) func apiError(ctx *context.APIContext, status int, obj interface{}) { - package_router.LogAndProcessError(ctx, status, obj, func(message string) { + packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } @@ -101,7 +101,7 @@ func UploadPackageFile(ctx *context.APIContext) { } defer file.Close() - buf, err := package_module.CreateHashedBufferFromReader(file, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(file, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/v1/packages/rubygems/rubygems.go index 97567c19cb168..e6c8eba1e5673 100644 --- a/routers/api/v1/packages/rubygems/rubygems.go +++ b/routers/api/v1/packages/rubygems/rubygems.go @@ -14,14 +14,14 @@ import ( "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" - package_module "code.gitea.io/gitea/modules/packages" + packages_module "code.gitea.io/gitea/modules/packages" rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" - package_router "code.gitea.io/gitea/routers/api/v1/packages" + packages_router "code.gitea.io/gitea/routers/api/v1/packages" packages_service "code.gitea.io/gitea/services/packages" ) func apiError(ctx *context.APIContext, status int, obj interface{}) { - package_router.LogAndProcessError(ctx, status, obj, func(message string) { + packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } @@ -199,7 +199,7 @@ func UploadPackageFile(ctx *context.APIContext) { defer upload.Close() } - buf, err := package_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return From 83148f2b9dbb1299a6843ed5ed20aa19e6a0eff6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 14 Feb 2022 21:35:07 +0000 Subject: [PATCH 091/130] Added swagger enum. --- routers/api/v1/packages/package.go | 2 +- templates/swagger/v1_json.tmpl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 4468a871e596f..88c35c82b0f96 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [composer, generic, maven, npm, nuget, pypi, rubygems] + // enum: [composer, conan, generic, maven, npm, nuget, pypi, rubygems] // - name: q // in: query // description: name filter diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index b615bff20adfb..e4d5b6a132986 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -1903,6 +1903,7 @@ { "enum": [ "composer", + "conan", "generic", "maven", "npm", From c6eca22ad3f8c3803ab81a9169a53d6aa2f653d1 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 14 Feb 2022 22:00:20 +0000 Subject: [PATCH 092/130] Use longer TEXT column type. --- models/migrations/v210.go | 2 +- models/packages/package_property.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/v210.go b/models/migrations/v210.go index da420d1f7f124..35fe4675cc0ae 100644 --- a/models/migrations/v210.go +++ b/models/migrations/v210.go @@ -46,7 +46,7 @@ func addPackageTables(x *xorm.Engine) error { RefType int64 `xorm:"INDEX NOT NULL"` RefID int64 `xorm:"INDEX NOT NULL"` Name string `xorm:"INDEX NOT NULL"` - Value string `xorm:"INDEX NOT NULL"` + Value string `xorm:"TEXT INDEX NOT NULL"` } if err := x.Sync2(new(PackageProperty)); err != nil { diff --git a/models/packages/package_property.go b/models/packages/package_property.go index 514222a374a30..2686b6bc1585c 100644 --- a/models/packages/package_property.go +++ b/models/packages/package_property.go @@ -29,7 +29,7 @@ type PackageProperty struct { RefType PropertyType `xorm:"INDEX NOT NULL"` RefID int64 `xorm:"INDEX NOT NULL"` Name string `xorm:"INDEX NOT NULL"` - Value string `xorm:"INDEX NOT NULL"` + Value string `xorm:"TEXT INDEX NOT NULL"` } // InsertProperty creates a property From 417308dfe46d35e6b37613a2226756021e1737a2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 14 Feb 2022 22:01:10 +0000 Subject: [PATCH 093/130] Removed version composite key. --- models/migrations/v210.go | 1 - models/packages/package_version.go | 8 +------- routers/api/v1/packages/conan/conan.go | 10 +++++----- routers/api/v1/packages/maven/maven.go | 6 +++--- routers/api/v1/packages/npm/npm.go | 2 +- routers/api/v1/packages/nuget/nuget.go | 2 +- services/packages/packages.go | 6 +++--- 7 files changed, 14 insertions(+), 21 deletions(-) diff --git a/models/migrations/v210.go b/models/migrations/v210.go index 35fe4675cc0ae..327a66ac3f994 100644 --- a/models/migrations/v210.go +++ b/models/migrations/v210.go @@ -31,7 +31,6 @@ func addPackageTables(x *xorm.Engine) error { CreatorID int64 Version string LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` - CompositeKey string `xorm:"UNIQUE(s) INDEX"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` MetadataJSON string `xorm:"TEXT metadata_json"` DownloadCount int64 diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 735937cd76def..d7dc01fda2bee 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -23,9 +23,6 @@ var ( ErrPackageVersionNotExist = errors.New("Package version does not exist") ) -// EmptyVersionKey is a named constant for an empty version key -const EmptyVersionKey = "" - func init() { db.RegisterModel(new(PackageVersion)) } @@ -37,7 +34,6 @@ type PackageVersion struct { CreatorID int64 Version string LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` - CompositeKey string `xorm:"UNIQUE(s) INDEX"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` MetadataJSON string `xorm:"TEXT metadata_json"` DownloadCount int64 @@ -50,7 +46,6 @@ func GetOrInsertVersion(ctx context.Context, pv *PackageVersion) (*PackageVersio key := &PackageVersion{ PackageID: pv.PackageID, LowerVersion: pv.LowerVersion, - CompositeKey: pv.CompositeKey, } has, err := e.Get(key) @@ -93,7 +88,7 @@ func GetVersionByID(ctx context.Context, versionID int64) (*PackageVersion, erro } // GetVersionByNameAndVersion gets a version by name and version number -func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType Type, name, version, key string) (*PackageVersion, error) { +func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType Type, name, version string) (*PackageVersion, error) { var cond builder.Cond = builder.Eq{ "package.owner_id": ownerID, "package.type": packageType, @@ -101,7 +96,6 @@ func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType } pv := &PackageVersion{ LowerVersion: strings.ToLower(version), - CompositeKey: key, } has, err := db.GetEngine(ctx). Join("INNER", "package", "package.id = package_version.package_id"). diff --git a/routers/api/v1/packages/conan/conan.go b/routers/api/v1/packages/conan/conan.go index 1228a235bffbb..131f59b05c022 100644 --- a/routers/api/v1/packages/conan/conan.go +++ b/routers/api/v1/packages/conan/conan.go @@ -148,7 +148,7 @@ func PackageSnapshot(ctx *context.APIContext) { func serveSnapshot(ctx *context.APIContext, fileKey string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) - pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version, packages_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) if err != nil { if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -209,7 +209,7 @@ func PackageDownloadURLs(ctx *context.APIContext) { func serveDownloadURLs(ctx *context.APIContext, fileKey, downloadURL string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) - pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version, packages_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) if err != nil { if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -368,7 +368,7 @@ func uploadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { apiError(ctx, http.StatusInternalServerError, err) return } - pv, err := packages_model.GetVersionByNameAndVersion(ctx, pci.Owner.ID, pci.PackageType, pci.Name, pci.Version, packages_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pci.Owner.ID, pci.PackageType, pci.Name, pci.Version) if err != nil && err != packages_model.ErrPackageNotExist { apiError(ctx, http.StatusInternalServerError, err) return @@ -602,7 +602,7 @@ func deleteRecipeOrPackage(apictx *context.APIContext, rref *conan_module.Recipe } defer committer.Close() - pv, err := packages_model.GetVersionByNameAndVersion(ctx, apictx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version, packages_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, apictx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) if err != nil { return err } @@ -778,7 +778,7 @@ func ListPackageRevisionFiles(ctx *context.APIContext) { func listRevisionFiles(ctx *context.APIContext, fileKey string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) - pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version, packages_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) if err != nil { if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index 85debd6ced8cd..ed9617e49ea45 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -117,7 +117,7 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { func servePackageFile(ctx *context.APIContext, params parameters) { packageName := params.GroupID + "-" + params.ArtifactID - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeMaven, packageName, params.Version, packages.EmptyVersionKey) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeMaven, packageName, params.Version) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -221,7 +221,7 @@ func UploadPackageFile(ctx *context.APIContext) { // Do not upload checksum files but compare the hashes. if isChecksumExtension(ext) { - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, packages.EmptyVersionKey) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -282,7 +282,7 @@ func UploadPackageFile(ctx *context.APIContext) { } if pvci.Metadata != nil { - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version, packages.EmptyVersionKey) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) if err != nil && err != packages.ErrPackageNotExist { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 704f8917395a3..69c1ff6e07978 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -215,7 +215,7 @@ func AddPackageTag(ctx *context.APIContext) { } version := strings.Trim(string(body), "\"") // is as "version" in the body - pv, err := packages.GetVersionByNameAndVersion(ctx, ctx.ContextUser.ID, packages.TypeNpm, packageName, version, packages.EmptyVersionKey) + pv, err := packages.GetVersionByNameAndVersion(ctx, ctx.ContextUser.ID, packages.TypeNpm, packageName, version) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 7f2745ae3778e..086ed42b8ee7c 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -100,7 +100,7 @@ func RegistrationLeaf(ctx *context.APIContext) { packageName := ctx.Params("id") packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeNuGet, packageName, packageVersion, packages.EmptyVersionKey) + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeNuGet, packageName, packageVersion) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) diff --git a/services/packages/packages.go b/services/packages/packages.go index a36d7ffe05895..1b1625af02723 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -166,7 +166,7 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) ( } defer committer.Close() - pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, packages_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { return nil, nil, err } @@ -268,7 +268,7 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers // DeletePackageVersionByNameAndVersion deletes a package version and all associated files func DeletePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error { - pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, packages_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { return err } @@ -358,7 +358,7 @@ func DeleteUnreferencedBlobs() error { func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey) - pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, packages_model.EmptyVersionKey) + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { if err == packages_model.ErrPackageNotExist { return nil, nil, err From 6e128331bf172d95842cd4311ffd3f2581c2224c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 15 Feb 2022 17:10:57 +0000 Subject: [PATCH 094/130] Merged package and container registry. --- docs/content/doc/features/comparison.en-us.md | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/docs/content/doc/features/comparison.en-us.md b/docs/content/doc/features/comparison.en-us.md index b7b8d08af3eef..51ec6bf129c42 100644 --- a/docs/content/doc/features/comparison.en-us.md +++ b/docs/content/doc/features/comparison.en-us.md @@ -33,27 +33,26 @@ _Symbols used in table:_ ## General Features -| Feature | Gitea | Gogs | GitHub EE | GitLab CE | GitLab EE | BitBucket | RhodeCode CE | -| ----------------------------------- | -------------------------------------------------- | ---- | --------- | --------- | --------- | -------------- | ------------ | -| Open source and free | ✓ | ✓ | ✘ | ✓ | ✘ | ✘ | ✓ | -| Low resource usage (RAM/CPU) | ✓ | ✓ | ✘ | ✘ | ✘ | ✘ | ✘ | -| Multiple database support | ✓ | ✓ | ✘ | ⁄ | ⁄ | ✓ | ✓ | -| Multiple OS support | ✓ | ✓ | ✘ | ✘ | ✘ | ✘ | ✓ | -| Easy upgrade process | ✓ | ✓ | ✘ | ✓ | ✓ | ✘ | ✓ | -| Markdown support | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Orgmode support | ✓ | ✘ | ✓ | ✘ | ✘ | ✘ | ? | -| CSV support | ✓ | ✘ | ✓ | ✘ | ✘ | ✓ | ? | -| Third-party render tool support | ✓ | ✘ | ✘ | ✘ | ✘ | ✓ | ? | -| Static Git-powered pages | [✘](https://github.com/go-gitea/gitea/issues/302) | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | -| Integrated Git-powered wiki | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ (cloud only) | ✘ | -| Deploy Tokens | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Repository Tokens with write rights | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Built-in Container Registry | [✘](https://github.com/go-gitea/gitea/issues/2316) | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | -| Built-in Package Registry | ✓ | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | -| External git mirroring | ✓ | ✓ | ✘ | ✘ | ✓ | ✓ | ✓ | -| WebAuthn (2FA) | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ? | -| Built-in CI/CD | ✘ | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | -| Subgroups: groups within groups | ✘ | ✘ | ✘ | ✓ | ✓ | ✘ | ✓ | +| Feature | Gitea | Gogs | GitHub EE | GitLab CE | GitLab EE | BitBucket | RhodeCode CE | +| ----------------------------------- | ---------------------------------------------------- | ---- | --------- | --------- | --------- | -------------- | ------------ | +| Open source and free | ✓ | ✓ | ✘ | ✓ | ✘ | ✘ | ✓ | +| Low resource usage (RAM/CPU) | ✓ | ✓ | ✘ | ✘ | ✘ | ✘ | ✘ | +| Multiple database support | ✓ | ✓ | ✘ | ⁄ | ⁄ | ✓ | ✓ | +| Multiple OS support | ✓ | ✓ | ✘ | ✘ | ✘ | ✘ | ✓ | +| Easy upgrade process | ✓ | ✓ | ✘ | ✓ | ✓ | ✘ | ✓ | +| Markdown support | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Orgmode support | ✓ | ✘ | ✓ | ✘ | ✘ | ✘ | ? | +| CSV support | ✓ | ✘ | ✓ | ✘ | ✘ | ✓ | ? | +| Third-party render tool support | ✓ | ✘ | ✘ | ✘ | ✘ | ✓ | ? | +| Static Git-powered pages | [✘](https://github.com/go-gitea/gitea/issues/302) | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | +| Integrated Git-powered wiki | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ (cloud only) | ✘ | +| Deploy Tokens | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Repository Tokens with write rights | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Built-in Package/Container Registry | ✓/[✘](https://github.com/go-gitea/gitea/issues/2316) | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | +| External git mirroring | ✓ | ✓ | ✘ | ✘ | ✓ | ✓ | ✓ | +| WebAuthn (2FA) | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ? | +| Built-in CI/CD | ✘ | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | +| Subgroups: groups within groups | ✘ | ✘ | ✘ | ✓ | ✓ | ✘ | ✓ | ## Code management From bfae746ba3c63fe8620fcad418a72f1fdad62239 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 15 Feb 2022 17:11:13 +0000 Subject: [PATCH 095/130] Removed index. --- models/migrations/v210.go | 2 +- models/packages/package_property.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/v210.go b/models/migrations/v210.go index 327a66ac3f994..94b3ad708bf45 100644 --- a/models/migrations/v210.go +++ b/models/migrations/v210.go @@ -45,7 +45,7 @@ func addPackageTables(x *xorm.Engine) error { RefType int64 `xorm:"INDEX NOT NULL"` RefID int64 `xorm:"INDEX NOT NULL"` Name string `xorm:"INDEX NOT NULL"` - Value string `xorm:"TEXT INDEX NOT NULL"` + Value string `xorm:"TEXT NOT NULL"` } if err := x.Sync2(new(PackageProperty)); err != nil { diff --git a/models/packages/package_property.go b/models/packages/package_property.go index 2686b6bc1585c..bf7dc346c6c97 100644 --- a/models/packages/package_property.go +++ b/models/packages/package_property.go @@ -29,7 +29,7 @@ type PackageProperty struct { RefType PropertyType `xorm:"INDEX NOT NULL"` RefID int64 `xorm:"INDEX NOT NULL"` Name string `xorm:"INDEX NOT NULL"` - Value string `xorm:"TEXT INDEX NOT NULL"` + Value string `xorm:"TEXT NOT NULL"` } // InsertProperty creates a property From c6a45ffa2efafbec42900fcc04e4a331959fa7a3 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 15 Feb 2022 17:24:53 +0000 Subject: [PATCH 096/130] Use dedicated package router. --- integrations/api_packages_composer_test.go | 2 +- integrations/api_packages_conan_test.go | 2 +- integrations/api_packages_generic_test.go | 2 +- integrations/api_packages_maven_test.go | 2 +- integrations/api_packages_npm_test.go | 4 +- integrations/api_packages_nuget_test.go | 2 +- integrations/api_packages_pypi_test.go | 2 +- integrations/api_packages_rubygems_test.go | 2 +- integrations/api_packages_test.go | 2 +- modules/context/package.go | 18 +- routers/api/packages/api.go | 216 +++++++++++++++++++ routers/api/v1/api.go | 179 +-------------- routers/api/v1/packages/composer/composer.go | 20 +- routers/api/v1/packages/conan/auth.go | 49 +++-- routers/api/v1/packages/conan/conan.go | 76 +++---- routers/api/v1/packages/conan/search.go | 8 +- routers/api/v1/packages/generic/generic.go | 10 +- routers/api/v1/packages/helper.go | 2 +- routers/api/v1/packages/maven/maven.go | 12 +- routers/api/v1/packages/npm/npm.go | 22 +- routers/api/v1/packages/nuget/nuget.go | 30 +-- routers/api/v1/packages/pypi/pypi.go | 10 +- routers/api/v1/packages/rubygems/rubygems.go | 18 +- routers/init.go | 2 + 24 files changed, 376 insertions(+), 316 deletions(-) create mode 100644 routers/api/packages/api.go diff --git a/integrations/api_packages_composer_test.go b/integrations/api_packages_composer_test.go index ea7de6cac458c..4db7bab2577e5 100644 --- a/integrations/api_packages_composer_test.go +++ b/integrations/api_packages_composer_test.go @@ -52,7 +52,7 @@ func TestPackageComposer(t *testing.T) { archive.Close() content := buf.Bytes() - url := fmt.Sprintf("%sapi/v1/packages/%s/composer", setting.AppURL, user.Name) + url := fmt.Sprintf("%sapi/packages/%s/composer", setting.AppURL, user.Name) t.Run("ServiceIndex", func(t *testing.T) { defer PrintCurrentTest(t)() diff --git a/integrations/api_packages_conan_test.go b/integrations/api_packages_conan_test.go index 4988a365cd8f7..9dc0c2a3a396f 100644 --- a/integrations/api_packages_conan_test.go +++ b/integrations/api_packages_conan_test.go @@ -217,7 +217,7 @@ func TestPackageConan(t *testing.T) { revision1 := "rev1" revision2 := "rev2" - url := fmt.Sprintf("%sapi/v1/packages/%s/conan", setting.AppURL, user.Name) + url := fmt.Sprintf("%sapi/packages/%s/conan", setting.AppURL, user.Name) t.Run("v1", func(t *testing.T) { t.Run("Ping", func(t *testing.T) { diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index b16a6c8239fdc..34ea48c3e1dc5 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -29,7 +29,7 @@ func TestPackageGeneric(t *testing.T) { filename := "fi-le_na.me" content := []byte{1, 2, 3} - url := fmt.Sprintf("/api/v1/packages/%s/generic/%s/%s/%s?token=%s", user.Name, packageName, packageVersion, filename, token) + url := fmt.Sprintf("/api/packages/%s/generic/%s/%s/%s?token=%s", user.Name, packageName, packageVersion, filename, token) t.Run("Upload", func(t *testing.T) { defer PrintCurrentTest(t)() diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go index d33832fdedc55..63a64385b5788 100644 --- a/integrations/api_packages_maven_test.go +++ b/integrations/api_packages_maven_test.go @@ -29,7 +29,7 @@ func TestPackageMaven(t *testing.T) { packageVersion := "1.0.1" packageDescription := "Test Description" - root := fmt.Sprintf("/api/v1/packages/%s/maven/%s/%s", user.Name, strings.ReplaceAll(groupID, ".", "/"), artifactID) + root := fmt.Sprintf("/api/packages/%s/maven/%s/%s", user.Name, strings.ReplaceAll(groupID, ".", "/"), artifactID) filename := fmt.Sprintf("%s-%s.jar", packageName, packageVersion) putFile := func(t *testing.T, path, content string, expectedStatus int) { diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index 69974df4fe520..8e6cbce4df62c 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -62,8 +62,8 @@ func TestPackageNpm(t *testing.T) { } }` - root := fmt.Sprintf("/api/v1/packages/%s/npm/%s", user.Name, url.QueryEscape(packageName)) - tagsRoot := fmt.Sprintf("/api/v1/packages/%s/npm/-/package/%s/dist-tags", user.Name, url.QueryEscape(packageName)) + root := fmt.Sprintf("/api/packages/%s/npm/%s", user.Name, url.QueryEscape(packageName)) + tagsRoot := fmt.Sprintf("/api/packages/%s/npm/-/package/%s/dist-tags", user.Name, url.QueryEscape(packageName)) filename := fmt.Sprintf("%s-%s.tgz", strings.Split(packageName, "/")[1], packageVersion) t.Run("Upload", func(t *testing.T) { diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index 726781d2c2453..9069ac6222fb6 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -51,7 +51,7 @@ func TestPackageNuGet(t *testing.T) { archive.Close() content := buf.Bytes() - url := fmt.Sprintf("/api/v1/packages/%s/nuget", user.Name) + url := fmt.Sprintf("/api/packages/%s/nuget", user.Name) t.Run("ServiceIndex", func(t *testing.T) { defer PrintCurrentTest(t)() diff --git a/integrations/api_packages_pypi_test.go b/integrations/api_packages_pypi_test.go index b21c2b7023c14..170daa69456ee 100644 --- a/integrations/api_packages_pypi_test.go +++ b/integrations/api_packages_pypi_test.go @@ -35,7 +35,7 @@ func TestPackagePyPI(t *testing.T) { content := "test" hashSHA256 := "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" - root := fmt.Sprintf("/api/v1/packages/%s/pypi", user.Name) + root := fmt.Sprintf("/api/packages/%s/pypi", user.Name) uploadFile := func(t *testing.T, filename, content string, expectedStatus int) { body := &bytes.Buffer{} diff --git a/integrations/api_packages_rubygems_test.go b/integrations/api_packages_rubygems_test.go index 79a7f6ab83ded..32a374cd3f48d 100644 --- a/integrations/api_packages_rubygems_test.go +++ b/integrations/api_packages_rubygems_test.go @@ -111,7 +111,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) - root := fmt.Sprintf("/api/v1/packages/%s/rubygems", user.Name) + root := fmt.Sprintf("/api/packages/%s/rubygems", user.Name) uploadFile := func(t *testing.T, expectedStatus int) { req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/api/v1/gems", root), bytes.NewReader(gemContent)) diff --git a/integrations/api_packages_test.go b/integrations/api_packages_test.go index 84a2b69beef13..8a9cf16edf356 100644 --- a/integrations/api_packages_test.go +++ b/integrations/api_packages_test.go @@ -28,7 +28,7 @@ func TestPackageAPI(t *testing.T) { packageVersion := "1.0.3" filename := "file.bin" - url := fmt.Sprintf("/api/v1/packages/%s/generic/%s/%s/%s?token=%s", user.Name, packageName, packageVersion, filename, token) + url := fmt.Sprintf("/api/packages/%s/generic/%s/%s/%s?token=%s", user.Name, packageName, packageVersion, filename, token) req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{})) MakeRequest(t, req, http.StatusCreated) diff --git a/modules/context/package.go b/modules/context/package.go index b0b1998de599e..eb1a72fbb4006 100644 --- a/modules/context/package.go +++ b/modules/context/package.go @@ -51,7 +51,7 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) { Owner: ctx.ContextUser, } - if ctx.IsSigned && ctx.User.ID == ctx.ContextUser.ID { + if ctx.User != nil && ctx.User.ID == ctx.ContextUser.ID { ctx.Package.AccessMode = perm.AccessModeOwner } else { if ctx.Package.Owner.IsOrganization() { @@ -98,3 +98,19 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) { } } } + +// PackageContexter initializes a package context for a request. +func PackageContexter() func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + ctx := Context{ + Resp: NewResponse(resp), + Data: map[string]interface{}{}, + } + + ctx.Req = WithContext(req, &ctx) + + next.ServeHTTP(ctx.Resp, ctx.Req) + }) + } +} diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go new file mode 100644 index 0000000000000..0c8e9a1416a45 --- /dev/null +++ b/routers/api/packages/api.go @@ -0,0 +1,216 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "net/http" + + "code.gitea.io/gitea/models/perm" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/api/v1/packages/composer" + "code.gitea.io/gitea/routers/api/v1/packages/conan" + "code.gitea.io/gitea/routers/api/v1/packages/generic" + "code.gitea.io/gitea/routers/api/v1/packages/maven" + "code.gitea.io/gitea/routers/api/v1/packages/npm" + "code.gitea.io/gitea/routers/api/v1/packages/nuget" + "code.gitea.io/gitea/routers/api/v1/packages/pypi" + "code.gitea.io/gitea/routers/api/v1/packages/rubygems" + "code.gitea.io/gitea/services/auth" +) + +func reqPackageAccess(accessMode perm.AccessMode) func(ctx *context.Context) { + return func(ctx *context.Context) { + if ctx.Package.AccessMode < accessMode && !ctx.IsUserSiteAdmin() { + ctx.Error(http.StatusUnauthorized, "reqPackageAccess", "user should have specific permission or be a site admin") + return + } + } +} + +func Routes() *web.Route { + r := web.NewRoute() + + r.Use(context.PackageContexter()) + + authGroup := auth.NewGroup(&auth.OAuth2{}, &auth.Basic{}, &conan.Auth{}) + r.Use(func(ctx *context.Context) { + ctx.User = authGroup.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) + }) + + r.Group("/{username}", func() { + r.Group("/composer", func() { + r.Get("/packages.json", composer.ServiceIndex) + r.Get("/search.json", composer.SearchPackages) + r.Get("/list.json", composer.EnumeratePackages) + r.Get("/p2/{vendorname}/{projectname}~dev.json", composer.PackageMetadata) + r.Get("/p2/{vendorname}/{projectname}.json", composer.PackageMetadata) + r.Get("/files/{versionid}/{fileid}", composer.DownloadPackageFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), composer.UploadPackage) + }) + r.Group("/conan", func() { + r.Group("/v1", func() { + r.Get("/ping", conan.Ping) + r.Group("/users", func() { + r.Get("/authenticate", conan.Authenticate) + r.Get("/check_credentials", conan.CheckCredentials) + }) + r.Group("/conans", func() { + r.Get("/search", conan.SearchRecipes) + r.Group("/{name}/{version}/{user}/{channel}", func() { + r.Get("", conan.RecipeSnapshot) + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV1) + r.Get("/search", conan.SearchPackagesV1) + r.Get("/digest", conan.RecipeDownloadURLs) + r.Post("/upload_urls", reqPackageAccess(perm.AccessModeWrite), conan.RecipeUploadURLs) + r.Get("/download_urls", conan.RecipeDownloadURLs) + r.Group("/packages", func() { + r.Post("/delete", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV1) + r.Group("/{package_reference}", func() { + r.Get("", conan.PackageSnapshot) + r.Get("/digest", conan.PackageDownloadURLs) + r.Post("/upload_urls", reqPackageAccess(perm.AccessModeWrite), conan.PackageUploadURLs) + r.Get("/download_urls", conan.PackageDownloadURLs) + }) + }) + }, conan.ExtractPathParameters) + }) + r.Group("/files/{name}/{version}/{user}/{channel}/{recipe_revision}", func() { + r.Group("/recipe/{filename}", func() { + r.Get("", conan.DownloadRecipeFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadRecipeFile) + }) + r.Group("/package/{package_reference}/{package_revision}/{filename}", func() { + r.Get("", conan.DownloadPackageFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadPackageFile) + }) + }, conan.ExtractPathParameters) + }) + r.Group("/v2", func() { + r.Get("/ping", conan.Ping) + r.Group("/users", func() { + r.Get("/authenticate", conan.Authenticate) + r.Get("/check_credentials", conan.CheckCredentials) + }) + r.Group("/conans", func() { + r.Get("/search", conan.SearchRecipes) + r.Group("/{name}/{version}/{user}/{channel}", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV2) + r.Get("/search", conan.SearchPackagesV2) + r.Get("/latest", conan.LatestRecipeRevision) + r.Group("/revisions", func() { + r.Get("", conan.ListRecipeRevisions) + r.Group("/{recipe_revision}", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV2) + r.Get("/search", conan.SearchPackagesV2) + r.Group("/files", func() { + r.Get("", conan.ListRecipeRevisionFiles) + r.Group("/{filename}", func() { + r.Get("", conan.DownloadRecipeFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadRecipeFile) + }) + }) + r.Group("/packages", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) + r.Group("/{package_reference}", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) + r.Get("/latest", conan.LatestPackageRevision) + r.Group("/revisions", func() { + r.Get("", conan.ListPackageRevisions) + r.Group("/{package_revision}", func() { + r.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) + r.Group("/files", func() { + r.Get("", conan.ListPackageRevisionFiles) + r.Group("/{filename}", func() { + r.Get("", conan.DownloadPackageFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadPackageFile) + }) + }) + }) + }) + }) + }) + }) + }) + }, conan.ExtractPathParameters) + }) + }) + }) + r.Group("/generic", func() { + r.Group("/{packagename}/{packageversion}/{filename}", func() { + r.Get("", generic.DownloadPackageFile) + r.Group("", func() { + r.Put("", generic.UploadPackage) + r.Delete("", generic.DeletePackage) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + }) + r.Group("/maven", func() { + r.Put("/*", reqPackageAccess(perm.AccessModeWrite), maven.UploadPackageFile) + r.Get("/*", maven.DownloadPackageFile) + }) + r.Group("/nuget", func() { + r.Get("/index.json", nuget.ServiceIndex) + r.Get("/query", nuget.SearchService) + r.Group("/registration/{id}", func() { + r.Get("/index.json", nuget.RegistrationIndex) + r.Get("/{version}", nuget.RegistrationLeaf) + }) + r.Group("/package/{id}", func() { + r.Get("/index.json", nuget.EnumeratePackageVersions) + r.Get("/{version}/{filename}", nuget.DownloadPackageFile) + }) + r.Group("", func() { + r.Put("/", nuget.UploadPackage) + r.Put("/symbolpackage", nuget.UploadSymbolPackage) + r.Delete("/{id}/{version}", nuget.DeletePackage) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + r.Group("/npm", func() { + r.Group("/@{scope}/{id}", func() { + r.Get("", npm.PackageMetadata) + r.Put("", reqPackageAccess(perm.AccessModeWrite), npm.UploadPackage) + r.Get("/-/{version}/{filename}", npm.DownloadPackageFile) + }) + r.Group("/{id}", func() { + r.Get("", npm.PackageMetadata) + r.Put("", reqPackageAccess(perm.AccessModeWrite), npm.UploadPackage) + r.Get("/-/{version}/{filename}", npm.DownloadPackageFile) + }) + r.Group("/-/package/@{scope}/{id}/dist-tags", func() { + r.Get("", npm.ListPackageTags) + r.Group("/{tag}", func() { + r.Put("", npm.AddPackageTag) + r.Delete("", npm.DeletePackageTag) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + r.Group("/-/package/{id}/dist-tags", func() { + r.Get("", npm.ListPackageTags) + r.Group("/{tag}", func() { + r.Put("", npm.AddPackageTag) + r.Delete("", npm.DeletePackageTag) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + }) + r.Group("/pypi", func() { + r.Post("/", reqPackageAccess(perm.AccessModeWrite), pypi.UploadPackageFile) + r.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) + r.Get("/simple/{id}", pypi.PackageMetadata) + }) + r.Group("/rubygems", func() { + r.Get("/specs.4.8.gz", rubygems.EnumeratePackages) + r.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) + r.Get("/prerelease_specs.4.8.gz", rubygems.EnumeratePackagesPreRelease) + r.Get("/quick/Marshal.4.8/{filename}", rubygems.ServePackageSpecification) + r.Get("/gems/{filename}", rubygems.DownloadPackageFile) + r.Group("/api/v1/gems", func() { + r.Post("/", rubygems.UploadPackageFile) + r.Delete("/yank", rubygems.DeletePackage) + }, reqPackageAccess(perm.AccessModeWrite)) + }) + }, context.UserAssignment(), context.PackageAssignment(), reqPackageAccess(perm.AccessModeRead)) + + return r +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index b1dcb7c027279..1ce917854686d 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -85,14 +85,6 @@ import ( "code.gitea.io/gitea/routers/api/v1/notify" "code.gitea.io/gitea/routers/api/v1/org" "code.gitea.io/gitea/routers/api/v1/packages" - "code.gitea.io/gitea/routers/api/v1/packages/composer" - "code.gitea.io/gitea/routers/api/v1/packages/conan" - "code.gitea.io/gitea/routers/api/v1/packages/generic" - "code.gitea.io/gitea/routers/api/v1/packages/maven" - "code.gitea.io/gitea/routers/api/v1/packages/npm" - "code.gitea.io/gitea/routers/api/v1/packages/nuget" - "code.gitea.io/gitea/routers/api/v1/packages/pypi" - "code.gitea.io/gitea/routers/api/v1/packages/rubygems" "code.gitea.io/gitea/routers/api/v1/repo" "code.gitea.io/gitea/routers/api/v1/settings" "code.gitea.io/gitea/routers/api/v1/user" @@ -1022,182 +1014,13 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { }) m.Group("/packages/{username}", func() { - m.Group("/composer", func() { - m.Get("/packages.json", composer.ServiceIndex) - m.Get("/search.json", composer.SearchPackages) - m.Get("/list.json", composer.EnumeratePackages) - m.Get("/p2/{vendorname}/{projectname}~dev.json", composer.PackageMetadata) - m.Get("/p2/{vendorname}/{projectname}.json", composer.PackageMetadata) - m.Get("/files/{versionid}/{fileid}", composer.DownloadPackageFile) - m.Put("", reqToken(), reqPackageAccess(perm.AccessModeWrite), composer.UploadPackage) - }) - m.Group("/conan", func() { - m.Group("/v1", func() { - m.Get("/ping", conan.Ping) - m.Group("/users", func() { - m.Get("/authenticate", conan.Authenticate) - m.Get("/check_credentials", conan.CheckCredentials) - }) - m.Group("/conans", func() { - m.Get("/search", conan.SearchRecipes) - m.Group("/{name}/{version}/{user}/{channel}", func() { - m.Get("", conan.RecipeSnapshot) - m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV1) - m.Get("/search", conan.SearchPackagesV1) - m.Get("/digest", conan.RecipeDownloadURLs) - m.Post("/upload_urls", reqPackageAccess(perm.AccessModeWrite), conan.RecipeUploadURLs) - m.Get("/download_urls", conan.RecipeDownloadURLs) - m.Group("/packages", func() { - m.Post("/delete", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV1) - m.Group("/{package_reference}", func() { - m.Get("", conan.PackageSnapshot) - m.Get("/digest", conan.PackageDownloadURLs) - m.Post("/upload_urls", reqPackageAccess(perm.AccessModeWrite), conan.PackageUploadURLs) - m.Get("/download_urls", conan.PackageDownloadURLs) - }) - }) - }, conan.ExtractPathParameters) - }) - m.Group("/files/{name}/{version}/{user}/{channel}/{recipe_revision}", func() { - m.Group("/recipe/{filename}", func() { - m.Get("", conan.DownloadRecipeFile) - m.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadRecipeFile) - }) - m.Group("/package/{package_reference}/{package_revision}/{filename}", func() { - m.Get("", conan.DownloadPackageFile) - m.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadPackageFile) - }) - }, conan.ExtractPathParameters) - }) - m.Group("/v2", func() { - m.Get("/ping", conan.Ping) - m.Group("/users", func() { - m.Get("/authenticate", conan.Authenticate) - m.Get("/check_credentials", conan.CheckCredentials) - }) - m.Group("/conans", func() { - m.Get("/search", conan.SearchRecipes) - m.Group("/{name}/{version}/{user}/{channel}", func() { - m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV2) - m.Get("/search", conan.SearchPackagesV2) - m.Get("/latest", conan.LatestRecipeRevision) - m.Group("/revisions", func() { - m.Get("", conan.ListRecipeRevisions) - m.Group("/{recipe_revision}", func() { - m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeleteRecipeV2) - m.Get("/search", conan.SearchPackagesV2) - m.Group("/files", func() { - m.Get("", conan.ListRecipeRevisionFiles) - m.Group("/{filename}", func() { - m.Get("", conan.DownloadRecipeFile) - m.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadRecipeFile) - }) - }) - m.Group("/packages", func() { - m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) - m.Group("/{package_reference}", func() { - m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) - m.Get("/latest", conan.LatestPackageRevision) - m.Group("/revisions", func() { - m.Get("", conan.ListPackageRevisions) - m.Group("/{package_revision}", func() { - m.Delete("", reqPackageAccess(perm.AccessModeWrite), conan.DeletePackageV2) - m.Group("/files", func() { - m.Get("", conan.ListPackageRevisionFiles) - m.Group("/{filename}", func() { - m.Get("", conan.DownloadPackageFile) - m.Put("", reqPackageAccess(perm.AccessModeWrite), conan.UploadPackageFile) - }) - }) - }) - }) - }) - }) - }) - }) - }, conan.ExtractPathParameters) - }) - }) - }) - m.Group("/generic", func() { - m.Group("/{packagename}/{packageversion}/{filename}", func() { - m.Get("", generic.DownloadPackageFile) - m.Group("", func() { - m.Put("", generic.UploadPackage) - m.Delete("", generic.DeletePackage) - }, reqToken(), reqPackageAccess(perm.AccessModeWrite)) - }) - }) - m.Group("/maven", func() { - m.Put("/*", reqToken(), reqPackageAccess(perm.AccessModeWrite), maven.UploadPackageFile) - m.Get("/*", maven.DownloadPackageFile) - }) - m.Group("/nuget", func() { - m.Get("/index.json", nuget.ServiceIndex) - m.Get("/query", nuget.SearchService) - m.Group("/registration/{id}", func() { - m.Get("/index.json", nuget.RegistrationIndex) - m.Get("/{version}", nuget.RegistrationLeaf) - }) - m.Group("/package/{id}", func() { - m.Get("/index.json", nuget.EnumeratePackageVersions) - m.Get("/{version}/{filename}", nuget.DownloadPackageFile) - }) - m.Group("", func() { - m.Put("/", nuget.UploadPackage) - m.Put("/symbolpackage", nuget.UploadSymbolPackage) - m.Delete("/{id}/{version}", nuget.DeletePackage) - }, reqBasicOrRevProxyAuth(), reqPackageAccess(perm.AccessModeWrite)) - }) - m.Group("/npm", func() { - m.Group("/@{scope}/{id}", func() { - m.Get("", npm.PackageMetadata) - m.Put("", reqToken(), reqPackageAccess(perm.AccessModeWrite), npm.UploadPackage) - m.Get("/-/{version}/{filename}", npm.DownloadPackageFile) - }) - m.Group("/{id}", func() { - m.Get("", npm.PackageMetadata) - m.Put("", reqToken(), reqPackageAccess(perm.AccessModeWrite), npm.UploadPackage) - m.Get("/-/{version}/{filename}", npm.DownloadPackageFile) - }) - m.Group("/-/package/@{scope}/{id}/dist-tags", func() { - m.Get("", npm.ListPackageTags) - m.Group("/{tag}", func() { - m.Put("", npm.AddPackageTag) - m.Delete("", npm.DeletePackageTag) - }, reqToken(), reqPackageAccess(perm.AccessModeWrite)) - }) - m.Group("/-/package/{id}/dist-tags", func() { - m.Get("", npm.ListPackageTags) - m.Group("/{tag}", func() { - m.Put("", npm.AddPackageTag) - m.Delete("", npm.DeletePackageTag) - }, reqToken(), reqPackageAccess(perm.AccessModeWrite)) - }) - }) - m.Group("/pypi", func() { - m.Post("/", reqBasicOrRevProxyAuth(), reqPackageAccess(perm.AccessModeWrite), pypi.UploadPackageFile) - m.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) - m.Get("/simple/{id}", pypi.PackageMetadata) - }) - m.Group("/rubygems", func() { - m.Get("/specs.4.8.gz", rubygems.EnumeratePackages) - m.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) - m.Get("/prerelease_specs.4.8.gz", rubygems.EnumeratePackagesPreRelease) - m.Get("/quick/Marshal.4.8/{filename}", rubygems.ServePackageSpecification) - m.Get("/gems/{filename}", rubygems.DownloadPackageFile) - m.Group("/api/v1/gems", func() { - m.Post("/", rubygems.UploadPackageFile) - m.Delete("/yank", rubygems.DeletePackage) - }, reqToken(), reqPackageAccess(perm.AccessModeWrite)) - }) m.Group("/{versionid}", func() { m.Get("", packages.GetPackage) m.Delete("", reqPackageAccess(perm.AccessModeWrite), packages.DeletePackage) m.Get("/files", packages.ListPackageFiles) }) m.Get("/", packages.ListPackages) - }, conan.CheckAuth, context.UserAssignmentAPI(), context.PackageAssignmentAPI(), reqPackageAccess(perm.AccessModeRead)) + }, context.UserAssignmentAPI(), context.PackageAssignmentAPI(), reqPackageAccess(perm.AccessModeRead)) // Organizations m.Get("/user/orgs", reqToken(), org.ListMyOrgs) diff --git a/routers/api/v1/packages/composer/composer.go b/routers/api/v1/packages/composer/composer.go index 54bd715a623fb..183eea32c2e51 100644 --- a/routers/api/v1/packages/composer/composer.go +++ b/routers/api/v1/packages/composer/composer.go @@ -25,7 +25,7 @@ import ( "github.com/hashicorp/go-version" ) -func apiError(ctx *context.APIContext, status int, obj interface{}) { +func apiError(ctx *context.Context, status int, obj interface{}) { packages_router.LogAndProcessError(ctx, status, obj, func(message string) { type Error struct { Status int `json:"status"` @@ -42,15 +42,15 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { } // ServiceIndex displays registry endpoints -func ServiceIndex(ctx *context.APIContext) { - resp := createServiceIndexResponse(setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/composer") +func ServiceIndex(ctx *context.Context) { + resp := createServiceIndexResponse(setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/composer") ctx.JSON(http.StatusOK, resp) } // SearchPackages searches packages, only "q" is supported // https://packagist.org/apidoc#search-packages -func SearchPackages(ctx *context.APIContext) { +func SearchPackages(ctx *context.Context) { page := ctx.FormInt("page") if page < 1 { page = 1 @@ -81,7 +81,7 @@ func SearchPackages(ctx *context.APIContext) { nextLink := "" if len(pvs) == paginator.PageSize { - u, err := url.Parse(setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/composer/search.json") + u, err := url.Parse(setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/composer/search.json") if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -111,7 +111,7 @@ func SearchPackages(ctx *context.APIContext) { // EnumeratePackages lists all package names // https://packagist.org/apidoc#list-packages -func EnumeratePackages(ctx *context.APIContext) { +func EnumeratePackages(ctx *context.Context) { ps, err := packages.GetPackagesByType(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeComposer) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -130,7 +130,7 @@ func EnumeratePackages(ctx *context.APIContext) { // PackageMetadata returns the metadata for a single package // https://packagist.org/apidoc#get-package-data -func PackageMetadata(ctx *context.APIContext) { +func PackageMetadata(ctx *context.Context) { vendorName := ctx.Params("vendorname") projectName := ctx.Params("projectname") @@ -151,7 +151,7 @@ func PackageMetadata(ctx *context.APIContext) { } resp := createPackageMetadataResponse( - setting.AppURL+"api/v1/packages/"+ctx.Package.Owner.Name+"/composer", + setting.AppURL+"api/packages/"+ctx.Package.Owner.Name+"/composer", pds, ) @@ -159,7 +159,7 @@ func PackageMetadata(ctx *context.APIContext) { } // DownloadPackageFile serves the content of a package -func DownloadPackageFile(ctx *context.APIContext) { +func DownloadPackageFile(ctx *context.Context) { versionID := ctx.ParamsInt64("versionid") fileID := ctx.ParamsInt64("fileid") @@ -182,7 +182,7 @@ func DownloadPackageFile(ctx *context.APIContext) { } // UploadPackage creates a new package -func UploadPackage(ctx *context.APIContext) { +func UploadPackage(ctx *context.Context) { buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) if err != nil { apiError(ctx, http.StatusInternalServerError, err) diff --git a/routers/api/v1/packages/conan/auth.go b/routers/api/v1/packages/conan/auth.go index 6c84ed89516b4..cc5fa341add59 100644 --- a/routers/api/v1/packages/conan/auth.go +++ b/routers/api/v1/packages/conan/auth.go @@ -6,6 +6,7 @@ package conan import ( "fmt" + "net/http" "strings" "time" @@ -13,37 +14,39 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/auth" "github.com/golang-jwt/jwt/v4" ) -type conanClaims struct { - jwt.RegisteredClaims - UserID int64 -} +type Auth struct{} -// CheckAuth is a middleware which handles the custom Conan authorization token -func CheckAuth(ctx *context.APIContext) { - if ctx.User == nil { - token, err := parseAuthorizationToken(ctx) - if err != nil { - log.Trace("parseAuthorizationToken: %v", err) - return - } +func (a *Auth) Name() string { + return "conan" +} - ctx.User, err = user_model.GetUserByIDCtx(ctx, token.UserID) - if err != nil { - if !user_model.IsErrUserNotExist(err) { - log.Error("GetUserByID: %v", err) - } - return - } +func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) *user_model.User { + token, err := parseAuthorizationToken(req) + if err != nil { + log.Trace("parseAuthorizationToken: %v", err) + return nil + } - ctx.IsSigned = ctx.User != nil + u, err := user_model.GetUserByID(token.UserID) + if err != nil { + log.Error("GetUserByID: %v", err) + return nil } + + return u +} + +type conanClaims struct { + jwt.RegisteredClaims + UserID int64 } -func createAuthorizationToken(ctx *context.APIContext) (string, error) { +func createAuthorizationToken(ctx *context.Context) (string, error) { now := time.Now() claims := conanClaims{ RegisteredClaims: jwt.RegisteredClaims{ @@ -62,8 +65,8 @@ func createAuthorizationToken(ctx *context.APIContext) (string, error) { return tokenString, nil } -func parseAuthorizationToken(ctx *context.APIContext) (*conanClaims, error) { - parts := strings.SplitN(ctx.Req.Header.Get("Authorization"), " ", 2) +func parseAuthorizationToken(req *http.Request) (*conanClaims, error) { + parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2) if len(parts) != 2 { return nil, fmt.Errorf("no token") } diff --git a/routers/api/v1/packages/conan/conan.go b/routers/api/v1/packages/conan/conan.go index 131f59b05c022..76ecf4eb95223 100644 --- a/routers/api/v1/packages/conan/conan.go +++ b/routers/api/v1/packages/conan/conan.go @@ -49,7 +49,7 @@ var ( } ) -func jsonResponse(ctx *context.APIContext, status int, obj interface{}) { +func jsonResponse(ctx *context.Context, status int, obj interface{}) { // https://github.com/conan-io/conan/issues/6613 ctx.Resp.Header().Set("Content-Type", "application/json") ctx.Status(status) @@ -58,7 +58,7 @@ func jsonResponse(ctx *context.APIContext, status int, obj interface{}) { } } -func apiError(ctx *context.APIContext, status int, obj interface{}) { +func apiError(ctx *context.Context, status int, obj interface{}) { packages_router.LogAndProcessError(ctx, status, obj, func(message string) { jsonResponse(ctx, status, map[string]string{ "message": message, @@ -66,12 +66,12 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { }) } -func baseURL(ctx *context.APIContext) string { - return setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/conan" +func baseURL(ctx *context.Context) string { + return setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/conan" } // ExtractPathParameters is a middleware to extract common parameters from path -func ExtractPathParameters(ctx *context.APIContext) { +func ExtractPathParameters(ctx *context.Context) { rref, err := conan_module.NewRecipeReference( ctx.Params("name"), ctx.Params("version"), @@ -105,17 +105,17 @@ func ExtractPathParameters(ctx *context.APIContext) { } // Ping reports the server capabilities -func Ping(ctx *context.APIContext) { +func Ping(ctx *context.Context) { ctx.RespHeader().Add("X-Conan-Server-Capabilities", "revisions") // complex_search,checksum_deploy,matrix_params ctx.Status(http.StatusOK) } // Authenticate creates an authentication token for the user -func Authenticate(ctx *context.APIContext) { +func Authenticate(ctx *context.Context) { token, err := createAuthorizationToken(ctx) if err != nil { - ctx.Error(http.StatusInternalServerError, "", err) + apiError(ctx, http.StatusInternalServerError, err) return } @@ -123,7 +123,7 @@ func Authenticate(ctx *context.APIContext) { } // CheckCredentials tests if the provided authentication token is valid -func CheckCredentials(ctx *context.APIContext) { +func CheckCredentials(ctx *context.Context) { if ctx.User == nil { ctx.Status(http.StatusUnauthorized) } else { @@ -132,20 +132,20 @@ func CheckCredentials(ctx *context.APIContext) { } // RecipeSnapshot displays the recipe files with their md5 hash -func RecipeSnapshot(ctx *context.APIContext) { +func RecipeSnapshot(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) serveSnapshot(ctx, rref.AsKey()) } // RecipeSnapshot displays the package files with their md5 hash -func PackageSnapshot(ctx *context.APIContext) { +func PackageSnapshot(ctx *context.Context) { pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) serveSnapshot(ctx, pref.AsKey()) } -func serveSnapshot(ctx *context.APIContext, fileKey string) { +func serveSnapshot(ctx *context.Context, fileKey string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) @@ -185,7 +185,7 @@ func serveSnapshot(ctx *context.APIContext, fileKey string) { } // RecipeDownloadURLs displays the recipe files with their download url -func RecipeDownloadURLs(ctx *context.APIContext) { +func RecipeDownloadURLs(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) serveDownloadURLs( @@ -196,7 +196,7 @@ func RecipeDownloadURLs(ctx *context.APIContext) { } // PackageDownloadURLs displays the package files with their download url -func PackageDownloadURLs(ctx *context.APIContext) { +func PackageDownloadURLs(ctx *context.Context) { pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) serveDownloadURLs( @@ -206,7 +206,7 @@ func PackageDownloadURLs(ctx *context.APIContext) { ) } -func serveDownloadURLs(ctx *context.APIContext, fileKey, downloadURL string) { +func serveDownloadURLs(ctx *context.Context, fileKey, downloadURL string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) @@ -242,7 +242,7 @@ func serveDownloadURLs(ctx *context.APIContext, fileKey, downloadURL string) { } // RecipeUploadURLs displays the upload urls for the provided recipe files -func RecipeUploadURLs(ctx *context.APIContext) { +func RecipeUploadURLs(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) serveUploadURLs( @@ -253,7 +253,7 @@ func RecipeUploadURLs(ctx *context.APIContext) { } // PackageUploadURLs displays the upload urls for the provided package files -func PackageUploadURLs(ctx *context.APIContext) { +func PackageUploadURLs(ctx *context.Context) { pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) serveUploadURLs( @@ -263,12 +263,12 @@ func PackageUploadURLs(ctx *context.APIContext) { ) } -func serveUploadURLs(ctx *context.APIContext, fileFilter stringSet, uploadURL string) { +func serveUploadURLs(ctx *context.Context, fileFilter stringSet, uploadURL string) { defer ctx.Req.Body.Close() var files map[string]int64 if err := json.NewDecoder(ctx.Req.Body).Decode(&files); err != nil { - ctx.Error(http.StatusBadRequest, "", err) + apiError(ctx, http.StatusBadRequest, err) return } @@ -283,20 +283,20 @@ func serveUploadURLs(ctx *context.APIContext, fileFilter stringSet, uploadURL st } // UploadRecipeFile handles the upload of a recipe file -func UploadRecipeFile(ctx *context.APIContext) { +func UploadRecipeFile(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) uploadFile(ctx, recipeFileList, rref.AsKey()) } // UploadPackageFile handles the upload of a package file -func UploadPackageFile(ctx *context.APIContext) { +func UploadPackageFile(ctx *context.Context) { pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) uploadFile(ctx, packageFileList, pref.AsKey()) } -func uploadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { +func uploadFile(ctx *context.Context, fileFilter stringSet, fileKey string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) @@ -425,20 +425,20 @@ func uploadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { } // DownloadRecipeFile serves the conent of the requested recipe file -func DownloadRecipeFile(ctx *context.APIContext) { +func DownloadRecipeFile(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) downloadFile(ctx, recipeFileList, rref.AsKey()) } // DownloadPackageFile serves the conent of the requested package file -func DownloadPackageFile(ctx *context.APIContext) { +func DownloadPackageFile(ctx *context.Context) { pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) downloadFile(ctx, packageFileList, pref.AsKey()) } -func downloadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) { +func downloadFile(ctx *context.Context, fileFilter stringSet, fileKey string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) filename := ctx.Params("filename") @@ -473,7 +473,7 @@ func downloadFile(ctx *context.APIContext, fileFilter stringSet, fileKey string) } // DeleteRecipeV1 deletes the requested recipe(s) -func DeleteRecipeV1(ctx *context.APIContext) { +func DeleteRecipeV1(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) if err := deleteRecipeOrPackage(ctx, rref, true, nil, false); err != nil { @@ -488,7 +488,7 @@ func DeleteRecipeV1(ctx *context.APIContext) { } // DeleteRecipeV2 deletes the requested recipe(s) respecting its revisions -func DeleteRecipeV2(ctx *context.APIContext) { +func DeleteRecipeV2(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) if err := deleteRecipeOrPackage(ctx, rref, rref.Revision == "", nil, false); err != nil { @@ -503,7 +503,7 @@ func DeleteRecipeV2(ctx *context.APIContext) { } // DeletePackageV1 deletes the requested package(s) -func DeletePackageV1(ctx *context.APIContext) { +func DeletePackageV1(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) type PackageReferences struct { @@ -552,7 +552,7 @@ func DeletePackageV1(ctx *context.APIContext) { } // DeletePackageV2 deletes the requested package(s) respecting its revisions -func DeletePackageV2(ctx *context.APIContext) { +func DeletePackageV2(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) @@ -595,7 +595,7 @@ func DeletePackageV2(ctx *context.APIContext) { ctx.Status(http.StatusOK) } -func deleteRecipeOrPackage(apictx *context.APIContext, rref *conan_module.RecipeReference, ignoreRecipeRevision bool, pref *conan_module.PackageReference, ignorePackageRevision bool) error { +func deleteRecipeOrPackage(apictx *context.Context, rref *conan_module.RecipeReference, ignoreRecipeRevision bool, pref *conan_module.PackageReference, ignorePackageRevision bool) error { ctx, committer, err := db.TxContext() if err != nil { return err @@ -679,7 +679,7 @@ func deleteRecipeOrPackage(apictx *context.APIContext, rref *conan_module.Recipe } // ListRecipeRevisions gets a list of all recipe revisions -func ListRecipeRevisions(ctx *context.APIContext) { +func ListRecipeRevisions(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) revisions, err := conan_model.GetRecipeRevisions(ctx, ctx.Package.Owner.ID, rref) @@ -692,7 +692,7 @@ func ListRecipeRevisions(ctx *context.APIContext) { } // ListPackageRevisions gets a list of all package revisions -func ListPackageRevisions(ctx *context.APIContext) { +func ListPackageRevisions(ctx *context.Context) { pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) revisions, err := conan_model.GetPackageRevisions(ctx, ctx.Package.Owner.ID, pref) @@ -709,7 +709,7 @@ type revisionInfo struct { Time time.Time `json:"time"` } -func listRevisions(ctx *context.APIContext, revisions []*conan_model.PropertyValue) { +func listRevisions(ctx *context.Context, revisions []*conan_model.PropertyValue) { if len(revisions) == 0 { apiError(ctx, http.StatusNotFound, conan_model.ErrRecipeReferenceNotExist) return @@ -728,7 +728,7 @@ func listRevisions(ctx *context.APIContext, revisions []*conan_model.PropertyVal } // LatestRecipeRevision gets the latest recipe revision -func LatestRecipeRevision(ctx *context.APIContext) { +func LatestRecipeRevision(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) revision, err := conan_model.GetLastRecipeRevision(ctx, ctx.Package.Owner.ID, rref) @@ -745,7 +745,7 @@ func LatestRecipeRevision(ctx *context.APIContext) { } // LatestPackageRevision gets the latest package revision -func LatestPackageRevision(ctx *context.APIContext) { +func LatestPackageRevision(ctx *context.Context) { pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) revision, err := conan_model.GetLastPackageRevision(ctx, ctx.Package.Owner.ID, pref) @@ -762,20 +762,20 @@ func LatestPackageRevision(ctx *context.APIContext) { } // ListRecipeRevisionFiles gets a list of all recipe revision files -func ListRecipeRevisionFiles(ctx *context.APIContext) { +func ListRecipeRevisionFiles(ctx *context.Context) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) listRevisionFiles(ctx, rref.AsKey()) } // ListPackageRevisionFiles gets a list of all package revision files -func ListPackageRevisionFiles(ctx *context.APIContext) { +func ListPackageRevisionFiles(ctx *context.Context) { pref := ctx.Data[packageReferenceKey].(*conan_module.PackageReference) listRevisionFiles(ctx, pref.AsKey()) } -func listRevisionFiles(ctx *context.APIContext, fileKey string) { +func listRevisionFiles(ctx *context.Context, fileKey string) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeConan, rref.Name, rref.Version) diff --git a/routers/api/v1/packages/conan/search.go b/routers/api/v1/packages/conan/search.go index ea8ed284be685..d3d8e8732b197 100644 --- a/routers/api/v1/packages/conan/search.go +++ b/routers/api/v1/packages/conan/search.go @@ -21,7 +21,7 @@ type SearchResult struct { } // SearchRecipes searches all recipes matching the query -func SearchRecipes(ctx *context.APIContext) { +func SearchRecipes(ctx *context.Context) { q := ctx.FormTrim("q") opts := parseQuery(ctx.Package.Owner, q) @@ -62,16 +62,16 @@ func parseQuery(owner *user_model.User, query string) *conan_model.RecipeSearchO } // SearchPackagesV1 searches all packages of a recipe (Conan v1 endpoint) -func SearchPackagesV1(ctx *context.APIContext) { +func SearchPackagesV1(ctx *context.Context) { searchPackages(ctx, true) } // SearchPackagesV2 searches all packages of a recipe (Conan v2 endpoint) -func SearchPackagesV2(ctx *context.APIContext) { +func SearchPackagesV2(ctx *context.Context) { searchPackages(ctx, false) } -func searchPackages(ctx *context.APIContext, searchAllRevisions bool) { +func searchPackages(ctx *context.Context, searchAllRevisions bool) { rref := ctx.Data[recipeReferenceKey].(*conan_module.RecipeReference) if !searchAllRevisions && rref.Revision == "" { diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/v1/packages/generic/generic.go index b8eba11b233f4..fb6116f1d101b 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/v1/packages/generic/generic.go @@ -24,14 +24,14 @@ var ( filenameRegex = packageNameRegex ) -func apiError(ctx *context.APIContext, status int, obj interface{}) { +func apiError(ctx *context.Context, status int, obj interface{}) { packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } // DownloadPackageFile serves the specific generic package. -func DownloadPackageFile(ctx *context.APIContext) { +func DownloadPackageFile(ctx *context.Context) { packageName, packageVersion, filename, err := sanitizeParameters(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -64,7 +64,7 @@ func DownloadPackageFile(ctx *context.APIContext) { // UploadPackage uploads the specific generic package. // Duplicated packages get rejected. -func UploadPackage(ctx *context.APIContext) { +func UploadPackage(ctx *context.Context) { packageName, packageVersion, filename, err := sanitizeParameters(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -120,7 +120,7 @@ func UploadPackage(ctx *context.APIContext) { } // DeletePackage deletes the specific generic package. -func DeletePackage(ctx *context.APIContext) { +func DeletePackage(ctx *context.Context) { packageName, packageVersion, _, err := sanitizeParameters(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -146,7 +146,7 @@ func DeletePackage(ctx *context.APIContext) { } } -func sanitizeParameters(ctx *context.APIContext) (string, string, string, error) { +func sanitizeParameters(ctx *context.Context) (string, string, string, error) { packageName := ctx.Params("packagename") filename := ctx.Params("filename") diff --git a/routers/api/v1/packages/helper.go b/routers/api/v1/packages/helper.go index e6133b1eeb806..5d27a2807c05e 100644 --- a/routers/api/v1/packages/helper.go +++ b/routers/api/v1/packages/helper.go @@ -15,7 +15,7 @@ import ( // LogAndProcessError logs an error and calls a custom callback with the processed error message. // If the error is an InternalServerError the message is stripped if the user is not an admin. -func LogAndProcessError(ctx *context.APIContext, status int, obj interface{}, cb func(string)) { +func LogAndProcessError(ctx *context.Context, status int, obj interface{}, cb func(string)) { var message string if err, ok := obj.(error); ok { message = err.Error() diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/v1/packages/maven/maven.go index ed9617e49ea45..daccec00e0c6a 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/v1/packages/maven/maven.go @@ -42,14 +42,14 @@ var ( illegalCharacters = regexp.MustCompile(`[\\/:"<>|?\*]`) ) -func apiError(ctx *context.APIContext, status int, obj interface{}) { +func apiError(ctx *context.Context, status int, obj interface{}) { packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } // DownloadPackageFile serves the content of a package -func DownloadPackageFile(ctx *context.APIContext) { +func DownloadPackageFile(ctx *context.Context) { params, err := extractPathParameters(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -63,7 +63,7 @@ func DownloadPackageFile(ctx *context.APIContext) { } } -func serveMavenMetadata(ctx *context.APIContext, params parameters) { +func serveMavenMetadata(ctx *context.Context, params parameters) { // /com/foo/project/maven-metadata.xml[.md5/.sha1/.sha256/.sha512] packageName := params.GroupID + "-" + params.ArtifactID @@ -114,7 +114,7 @@ func serveMavenMetadata(ctx *context.APIContext, params parameters) { ctx.PlainTextBytes(http.StatusOK, xmlMetadataWithHeader) } -func servePackageFile(ctx *context.APIContext, params parameters) { +func servePackageFile(ctx *context.Context, params parameters) { packageName := params.GroupID + "-" + params.ArtifactID pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeMaven, packageName, params.Version) @@ -182,7 +182,7 @@ func servePackageFile(ctx *context.APIContext, params parameters) { } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. -func UploadPackageFile(ctx *context.APIContext) { +func UploadPackageFile(ctx *context.Context) { params, err := extractPathParameters(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -335,7 +335,7 @@ type parameters struct { IsMeta bool } -func extractPathParameters(ctx *context.APIContext) (parameters, error) { +func extractPathParameters(ctx *context.Context) (parameters, error) { parts := strings.Split(ctx.Params("*"), "/") p := parameters{ diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/v1/packages/npm/npm.go index 69c1ff6e07978..8f4a665f74697 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/v1/packages/npm/npm.go @@ -28,7 +28,7 @@ import ( // errInvalidTagName indicates an invalid tag name var errInvalidTagName = errors.New("The tag name is invalid") -func apiError(ctx *context.APIContext, status int, obj interface{}) { +func apiError(ctx *context.Context, status int, obj interface{}) { packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.JSON(status, map[string]string{ "error": message, @@ -38,7 +38,7 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { // packageNameFromParams gets the package name from the url parameters // Variations: /name/, /@scope/name/, /@scope%2Fname/ -func packageNameFromParams(ctx *context.APIContext) (string, error) { +func packageNameFromParams(ctx *context.Context) (string, error) { scope := ctx.Params("scope") id := ctx.Params("id") if scope != "" { @@ -48,7 +48,7 @@ func packageNameFromParams(ctx *context.APIContext) (string, error) { } // PackageMetadata returns the metadata for a single package -func PackageMetadata(ctx *context.APIContext) { +func PackageMetadata(ctx *context.Context) { packageName, err := packageNameFromParams(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -72,7 +72,7 @@ func PackageMetadata(ctx *context.APIContext) { } resp := createPackageMetadataResponse( - setting.AppURL+"api/v1/packages/"+ctx.Package.Owner.Name+"/npm", + setting.AppURL+"api/packages/"+ctx.Package.Owner.Name+"/npm", pds, ) @@ -80,7 +80,7 @@ func PackageMetadata(ctx *context.APIContext) { } // DownloadPackageFile serves the content of a package -func DownloadPackageFile(ctx *context.APIContext) { +func DownloadPackageFile(ctx *context.Context) { packageName, err := packageNameFromParams(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -114,7 +114,7 @@ func DownloadPackageFile(ctx *context.APIContext) { } // UploadPackage creates a new package -func UploadPackage(ctx *context.APIContext) { +func UploadPackage(ctx *context.Context) { npmPackage, err := npm_module.ParsePackage(ctx.Req.Body) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -172,7 +172,7 @@ func UploadPackage(ctx *context.APIContext) { } // ListPackageTags returns all tags for a package -func ListPackageTags(ctx *context.APIContext) { +func ListPackageTags(ctx *context.Context) { packageName, err := packageNameFromParams(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -201,7 +201,7 @@ func ListPackageTags(ctx *context.APIContext) { } // AddPackageTag adds a tag to the package -func AddPackageTag(ctx *context.APIContext) { +func AddPackageTag(ctx *context.Context) { packageName, err := packageNameFromParams(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -215,7 +215,7 @@ func AddPackageTag(ctx *context.APIContext) { } version := strings.Trim(string(body), "\"") // is as "version" in the body - pv, err := packages.GetVersionByNameAndVersion(ctx, ctx.ContextUser.ID, packages.TypeNpm, packageName, version) + pv, err := packages.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages.TypeNpm, packageName, version) if err != nil { if err == packages.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) @@ -236,14 +236,14 @@ func AddPackageTag(ctx *context.APIContext) { } // DeletePackageTag deletes a package tag -func DeletePackageTag(ctx *context.APIContext) { +func DeletePackageTag(ctx *context.Context) { packageName, err := packageNameFromParams(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) return } - pvs, err := packages.GetVersionsByPackageName(ctx.ContextUser.ID, packages.TypeNpm, packageName) + pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNpm, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/v1/packages/nuget/nuget.go index 086ed42b8ee7c..c780c296e6fc8 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/v1/packages/nuget/nuget.go @@ -21,7 +21,7 @@ import ( packages_service "code.gitea.io/gitea/services/packages" ) -func apiError(ctx *context.APIContext, status int, obj interface{}) { +func apiError(ctx *context.Context, status int, obj interface{}) { packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.JSON(status, map[string]string{ "Message": message, @@ -30,14 +30,14 @@ func apiError(ctx *context.APIContext, status int, obj interface{}) { } // ServiceIndex https://docs.microsoft.com/en-us/nuget/api/service-index -func ServiceIndex(ctx *context.APIContext) { - resp := createServiceIndexResponse(setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/nuget") +func ServiceIndex(ctx *context.Context) { + resp := createServiceIndexResponse(setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/nuget") ctx.JSON(http.StatusOK, resp) } // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages -func SearchService(ctx *context.APIContext) { +func SearchService(ctx *context.Context) { pvs, count, err := packages.SearchVersions(&packages.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, Type: string(packages.TypeNuGet), @@ -59,7 +59,7 @@ func SearchService(ctx *context.APIContext) { } resp := createSearchResultResponse( - &linkBuilder{setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/nuget"}, + &linkBuilder{setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/nuget"}, count, pds, ) @@ -68,7 +68,7 @@ func SearchService(ctx *context.APIContext) { } // RegistrationIndex https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-index -func RegistrationIndex(ctx *context.APIContext) { +func RegistrationIndex(ctx *context.Context) { packageName := ctx.Params("id") pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNuGet, packageName) @@ -88,7 +88,7 @@ func RegistrationIndex(ctx *context.APIContext) { } resp := createRegistrationIndexResponse( - &linkBuilder{setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/nuget"}, + &linkBuilder{setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/nuget"}, pds, ) @@ -96,7 +96,7 @@ func RegistrationIndex(ctx *context.APIContext) { } // RegistrationLeaf https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf -func RegistrationLeaf(ctx *context.APIContext) { +func RegistrationLeaf(ctx *context.Context) { packageName := ctx.Params("id") packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") @@ -117,7 +117,7 @@ func RegistrationLeaf(ctx *context.APIContext) { } resp := createRegistrationLeafResponse( - &linkBuilder{setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/nuget"}, + &linkBuilder{setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/nuget"}, pd, ) @@ -125,7 +125,7 @@ func RegistrationLeaf(ctx *context.APIContext) { } // EnumeratePackageVersions https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#enumerate-package-versions -func EnumeratePackageVersions(ctx *context.APIContext) { +func EnumeratePackageVersions(ctx *context.Context) { packageName := ctx.Params("id") pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNuGet, packageName) @@ -150,7 +150,7 @@ func EnumeratePackageVersions(ctx *context.APIContext) { } // DownloadPackageFile https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg -func DownloadPackageFile(ctx *context.APIContext) { +func DownloadPackageFile(ctx *context.Context) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") filename := ctx.Params("filename") @@ -181,7 +181,7 @@ func DownloadPackageFile(ctx *context.APIContext) { // UploadPackage creates a new package with the metadata contained in the uploaded nupgk file // https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#push-a-package -func UploadPackage(ctx *context.APIContext) { +func UploadPackage(ctx *context.Context) { np, buf, closables := processUploadedFile(ctx, nuget_module.DependencyPackage) defer func() { for _, c := range closables { @@ -226,7 +226,7 @@ func UploadPackage(ctx *context.APIContext) { // UploadSymbolPackage adds a symbol package to an existing package // https://docs.microsoft.com/en-us/nuget/api/symbol-package-publish-resource -func UploadSymbolPackage(ctx *context.APIContext) { +func UploadSymbolPackage(ctx *context.Context) { np, buf, closables := processUploadedFile(ctx, nuget_module.SymbolsPackage) defer func() { for _, c := range closables { @@ -267,7 +267,7 @@ func UploadSymbolPackage(ctx *context.APIContext) { ctx.Status(http.StatusCreated) } -func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.PackageType) (*nuget_module.Package, *packages_module.HashedBuffer, []io.Closer) { +func processUploadedFile(ctx *context.Context, expectedType nuget_module.PackageType) (*nuget_module.Package, *packages_module.HashedBuffer, []io.Closer) { closables := make([]io.Closer, 0, 2) upload, close, err := ctx.UploadStream() @@ -309,7 +309,7 @@ func processUploadedFile(ctx *context.APIContext, expectedType nuget_module.Pack // DeletePackage hard deletes the package // https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#delete-a-package -func DeletePackage(ctx *context.APIContext) { +func DeletePackage(ctx *context.Context) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/v1/packages/pypi/pypi.go index a9e4b929dc577..b94930e11491e 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/v1/packages/pypi/pypi.go @@ -29,14 +29,14 @@ var nameMatcher = regexp.MustCompile(`\A[a-z0-9\.\-_]+\z`) // https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions var versionMatcher = regexp.MustCompile(`^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$`) -func apiError(ctx *context.APIContext, status int, obj interface{}) { +func apiError(ctx *context.Context, status int, obj interface{}) { packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } // PackageMetadata returns the metadata for a single package -func PackageMetadata(ctx *context.APIContext) { +func PackageMetadata(ctx *context.Context) { packageName := normalizer.Replace(ctx.Params("id")) pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypePyPI, packageName) @@ -55,7 +55,7 @@ func PackageMetadata(ctx *context.APIContext) { return } - ctx.Data["RegistryURL"] = setting.AppURL + "api/v1/packages/" + ctx.Package.Owner.Name + "/pypi" + ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi" ctx.Data["PackageDescriptor"] = pds[0] ctx.Data["PackageDescriptors"] = pds ctx.Render = templates.HTMLRenderer() @@ -63,7 +63,7 @@ func PackageMetadata(ctx *context.APIContext) { } // DownloadPackageFile serves the content of a package -func DownloadPackageFile(ctx *context.APIContext) { +func DownloadPackageFile(ctx *context.Context) { packageName := normalizer.Replace(ctx.Params("id")) packageVersion := ctx.Params("version") filename := ctx.Params("filename") @@ -93,7 +93,7 @@ func DownloadPackageFile(ctx *context.APIContext) { } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. -func UploadPackageFile(ctx *context.APIContext) { +func UploadPackageFile(ctx *context.Context) { file, fileHeader, err := ctx.Req.FormFile("content") if err != nil { apiError(ctx, http.StatusBadRequest, err) diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/v1/packages/rubygems/rubygems.go index e6c8eba1e5673..202d529b1dc8e 100644 --- a/routers/api/v1/packages/rubygems/rubygems.go +++ b/routers/api/v1/packages/rubygems/rubygems.go @@ -20,14 +20,14 @@ import ( packages_service "code.gitea.io/gitea/services/packages" ) -func apiError(ctx *context.APIContext, status int, obj interface{}) { +func apiError(ctx *context.Context, status int, obj interface{}) { packages_router.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } // EnumeratePackages serves the package list -func EnumeratePackages(ctx *context.APIContext) { +func EnumeratePackages(ctx *context.Context) { packages, err := packages.GetVersionsByPackageType(ctx.Package.Owner.ID, packages.TypeRubyGems) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -38,7 +38,7 @@ func EnumeratePackages(ctx *context.APIContext) { } // EnumeratePackagesLatest serves the list of the lastest version of every package -func EnumeratePackagesLatest(ctx *context.APIContext) { +func EnumeratePackagesLatest(ctx *context.Context) { pvs, _, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, Type: string(packages.TypeRubyGems), @@ -52,11 +52,11 @@ func EnumeratePackagesLatest(ctx *context.APIContext) { } // EnumeratePackagesPreRelease is not supported and serves an empty list -func EnumeratePackagesPreRelease(ctx *context.APIContext) { +func EnumeratePackagesPreRelease(ctx *context.Context) { enumeratePackages(ctx, "prerelease_specs.4.8", []*packages.PackageVersion{}) } -func enumeratePackages(ctx *context.APIContext, filename string, pvs []*packages.PackageVersion) { +func enumeratePackages(ctx *context.Context, filename string, pvs []*packages.PackageVersion) { pds, err := packages.GetPackageDescriptors(pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -88,7 +88,7 @@ func enumeratePackages(ctx *context.APIContext, filename string, pvs []*packages } // ServePackageSpecification serves the compressed Gemspec file of a package -func ServePackageSpecification(ctx *context.APIContext) { +func ServePackageSpecification(ctx *context.Context) { filename := ctx.Params("filename") if !strings.HasSuffix(filename, ".gemspec.rz") { @@ -155,7 +155,7 @@ func ServePackageSpecification(ctx *context.APIContext) { } // DownloadPackageFile serves the content of a package -func DownloadPackageFile(ctx *context.APIContext) { +func DownloadPackageFile(ctx *context.Context) { filename := ctx.Params("filename") pvs, err := packages.GetVersionsByFilename(ctx.Package.Owner.ID, packages.TypeRubyGems, filename) @@ -189,7 +189,7 @@ func DownloadPackageFile(ctx *context.APIContext) { } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. -func UploadPackageFile(ctx *context.APIContext) { +func UploadPackageFile(ctx *context.Context) { upload, close, err := ctx.UploadStream() if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -256,7 +256,7 @@ func UploadPackageFile(ctx *context.APIContext) { } // DeletePackage deletes a package -func DeletePackage(ctx *context.APIContext) { +func DeletePackage(ctx *context.Context) { // Go populates the form only for POST, PUT and PATCH requests if err := ctx.Req.ParseMultipartForm(32 << 20); err != nil { apiError(ctx, http.StatusInternalServerError, err) diff --git a/routers/init.go b/routers/init.go index 804dfd65335b6..6b71768fe65c1 100644 --- a/routers/init.go +++ b/routers/init.go @@ -32,6 +32,7 @@ import ( "code.gitea.io/gitea/modules/svg" "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/web" + packages_router "code.gitea.io/gitea/routers/api/packages" apiv1 "code.gitea.io/gitea/routers/api/v1" "code.gitea.io/gitea/routers/common" "code.gitea.io/gitea/routers/private" @@ -188,5 +189,6 @@ func NormalRoutes() *web.Route { r.Mount("/", web_routers.Routes(sessioner)) r.Mount("/api/v1", apiv1.Routes(sessioner)) r.Mount("/api/internal", private.Routes()) + r.Mount("/api/packages", packages_router.Routes()) return r } From 1b4854ca4f8f4b631c49ef45db176521f59ede87 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 15 Feb 2022 17:36:23 +0000 Subject: [PATCH 097/130] Moved files to new location. --- integrations/api_packages_composer_test.go | 2 +- integrations/api_packages_conan_test.go | 2 +- integrations/api_packages_nuget_test.go | 2 +- routers/api/packages/api.go | 16 ++++++++-------- routers/api/{v1 => }/packages/composer/api.go | 0 .../api/{v1 => }/packages/composer/composer.go | 4 ++-- routers/api/{v1 => }/packages/conan/auth.go | 0 routers/api/{v1 => }/packages/conan/conan.go | 4 ++-- routers/api/{v1 => }/packages/conan/search.go | 0 routers/api/{v1 => }/packages/generic/generic.go | 4 ++-- .../{v1/packages => packages/helper}/helper.go | 2 +- routers/api/{v1 => }/packages/maven/api.go | 0 routers/api/{v1 => }/packages/maven/maven.go | 4 ++-- routers/api/{v1 => }/packages/npm/api.go | 0 routers/api/{v1 => }/packages/npm/npm.go | 4 ++-- routers/api/{v1 => }/packages/nuget/api.go | 0 routers/api/{v1 => }/packages/nuget/links.go | 0 routers/api/{v1 => }/packages/nuget/nuget.go | 4 ++-- routers/api/{v1 => }/packages/pypi/pypi.go | 4 ++-- .../api/{v1 => }/packages/rubygems/rubygems.go | 4 ++-- 20 files changed, 28 insertions(+), 28 deletions(-) rename routers/api/{v1 => }/packages/composer/api.go (100%) rename routers/api/{v1 => }/packages/composer/composer.go (97%) rename routers/api/{v1 => }/packages/conan/auth.go (100%) rename routers/api/{v1 => }/packages/conan/conan.go (99%) rename routers/api/{v1 => }/packages/conan/search.go (100%) rename routers/api/{v1 => }/packages/generic/generic.go (96%) rename routers/api/{v1/packages => packages/helper}/helper.go (98%) rename routers/api/{v1 => }/packages/maven/api.go (100%) rename routers/api/{v1 => }/packages/maven/maven.go (98%) rename routers/api/{v1 => }/packages/npm/api.go (100%) rename routers/api/{v1 => }/packages/npm/npm.go (98%) rename routers/api/{v1 => }/packages/nuget/api.go (100%) rename routers/api/{v1 => }/packages/nuget/links.go (100%) rename routers/api/{v1 => }/packages/nuget/nuget.go (98%) rename routers/api/{v1 => }/packages/pypi/pypi.go (97%) rename routers/api/{v1 => }/packages/rubygems/rubygems.go (98%) diff --git a/integrations/api_packages_composer_test.go b/integrations/api_packages_composer_test.go index 4db7bab2577e5..d9987df962e90 100644 --- a/integrations/api_packages_composer_test.go +++ b/integrations/api_packages_composer_test.go @@ -17,7 +17,7 @@ import ( user_model "code.gitea.io/gitea/models/user" composer_module "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/routers/api/v1/packages/composer" + "code.gitea.io/gitea/routers/api/packages/composer" "github.com/stretchr/testify/assert" ) diff --git a/integrations/api_packages_conan_test.go b/integrations/api_packages_conan_test.go index 9dc0c2a3a396f..0293bcc9f8406 100644 --- a/integrations/api_packages_conan_test.go +++ b/integrations/api_packages_conan_test.go @@ -19,7 +19,7 @@ import ( user_model "code.gitea.io/gitea/models/user" conan_module "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/setting" - conan_router "code.gitea.io/gitea/routers/api/v1/packages/conan" + conan_router "code.gitea.io/gitea/routers/api/packages/conan" "github.com/stretchr/testify/assert" ) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index 9069ac6222fb6..e92ba5ca1c844 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -19,7 +19,7 @@ import ( user_model "code.gitea.io/gitea/models/user" nuget_module "code.gitea.io/gitea/modules/packages/nuget" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/routers/api/v1/packages/nuget" + "code.gitea.io/gitea/routers/api/packages/nuget" "github.com/stretchr/testify/assert" ) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 0c8e9a1416a45..aa8077ca97fc3 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -10,14 +10,14 @@ import ( "code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/web" - "code.gitea.io/gitea/routers/api/v1/packages/composer" - "code.gitea.io/gitea/routers/api/v1/packages/conan" - "code.gitea.io/gitea/routers/api/v1/packages/generic" - "code.gitea.io/gitea/routers/api/v1/packages/maven" - "code.gitea.io/gitea/routers/api/v1/packages/npm" - "code.gitea.io/gitea/routers/api/v1/packages/nuget" - "code.gitea.io/gitea/routers/api/v1/packages/pypi" - "code.gitea.io/gitea/routers/api/v1/packages/rubygems" + "code.gitea.io/gitea/routers/api/packages/composer" + "code.gitea.io/gitea/routers/api/packages/conan" + "code.gitea.io/gitea/routers/api/packages/generic" + "code.gitea.io/gitea/routers/api/packages/maven" + "code.gitea.io/gitea/routers/api/packages/npm" + "code.gitea.io/gitea/routers/api/packages/nuget" + "code.gitea.io/gitea/routers/api/packages/pypi" + "code.gitea.io/gitea/routers/api/packages/rubygems" "code.gitea.io/gitea/services/auth" ) diff --git a/routers/api/v1/packages/composer/api.go b/routers/api/packages/composer/api.go similarity index 100% rename from routers/api/v1/packages/composer/api.go rename to routers/api/packages/composer/api.go diff --git a/routers/api/v1/packages/composer/composer.go b/routers/api/packages/composer/composer.go similarity index 97% rename from routers/api/v1/packages/composer/composer.go rename to routers/api/packages/composer/composer.go index 183eea32c2e51..bcd81e4d8ddb7 100644 --- a/routers/api/v1/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -19,14 +19,14 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" composer_module "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/setting" - packages_router "code.gitea.io/gitea/routers/api/v1/packages" + "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" "github.com/hashicorp/go-version" ) func apiError(ctx *context.Context, status int, obj interface{}) { - packages_router.LogAndProcessError(ctx, status, obj, func(message string) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { type Error struct { Status int `json:"status"` Message string `json:"message"` diff --git a/routers/api/v1/packages/conan/auth.go b/routers/api/packages/conan/auth.go similarity index 100% rename from routers/api/v1/packages/conan/auth.go rename to routers/api/packages/conan/auth.go diff --git a/routers/api/v1/packages/conan/conan.go b/routers/api/packages/conan/conan.go similarity index 99% rename from routers/api/v1/packages/conan/conan.go rename to routers/api/packages/conan/conan.go index 76ecf4eb95223..9a96885ee6a97 100644 --- a/routers/api/v1/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -21,7 +21,7 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" conan_module "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/setting" - packages_router "code.gitea.io/gitea/routers/api/v1/packages" + "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" ) @@ -59,7 +59,7 @@ func jsonResponse(ctx *context.Context, status int, obj interface{}) { } func apiError(ctx *context.Context, status int, obj interface{}) { - packages_router.LogAndProcessError(ctx, status, obj, func(message string) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { jsonResponse(ctx, status, map[string]string{ "message": message, }) diff --git a/routers/api/v1/packages/conan/search.go b/routers/api/packages/conan/search.go similarity index 100% rename from routers/api/v1/packages/conan/search.go rename to routers/api/packages/conan/search.go diff --git a/routers/api/v1/packages/generic/generic.go b/routers/api/packages/generic/generic.go similarity index 96% rename from routers/api/v1/packages/generic/generic.go rename to routers/api/packages/generic/generic.go index fb6116f1d101b..ca61eddc64b3a 100644 --- a/routers/api/v1/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -13,7 +13,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" - packages_router "code.gitea.io/gitea/routers/api/v1/packages" + "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" "github.com/hashicorp/go-version" @@ -25,7 +25,7 @@ var ( ) func apiError(ctx *context.Context, status int, obj interface{}) { - packages_router.LogAndProcessError(ctx, status, obj, func(message string) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } diff --git a/routers/api/v1/packages/helper.go b/routers/api/packages/helper/helper.go similarity index 98% rename from routers/api/v1/packages/helper.go rename to routers/api/packages/helper/helper.go index 5d27a2807c05e..b36292c547aa3 100644 --- a/routers/api/v1/packages/helper.go +++ b/routers/api/packages/helper/helper.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package packages +package helper import ( "fmt" diff --git a/routers/api/v1/packages/maven/api.go b/routers/api/packages/maven/api.go similarity index 100% rename from routers/api/v1/packages/maven/api.go rename to routers/api/packages/maven/api.go diff --git a/routers/api/v1/packages/maven/maven.go b/routers/api/packages/maven/maven.go similarity index 98% rename from routers/api/v1/packages/maven/maven.go rename to routers/api/packages/maven/maven.go index daccec00e0c6a..ca1c116b975bc 100644 --- a/routers/api/v1/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -25,7 +25,7 @@ import ( "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" maven_module "code.gitea.io/gitea/modules/packages/maven" - packages_router "code.gitea.io/gitea/routers/api/v1/packages" + "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" ) @@ -43,7 +43,7 @@ var ( ) func apiError(ctx *context.Context, status int, obj interface{}) { - packages_router.LogAndProcessError(ctx, status, obj, func(message string) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } diff --git a/routers/api/v1/packages/npm/api.go b/routers/api/packages/npm/api.go similarity index 100% rename from routers/api/v1/packages/npm/api.go rename to routers/api/packages/npm/api.go diff --git a/routers/api/v1/packages/npm/npm.go b/routers/api/packages/npm/npm.go similarity index 98% rename from routers/api/v1/packages/npm/npm.go rename to routers/api/packages/npm/npm.go index 8f4a665f74697..1e6ecbb90d1bd 100644 --- a/routers/api/v1/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -19,7 +19,7 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" npm_module "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/setting" - packages_router "code.gitea.io/gitea/routers/api/v1/packages" + "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" "github.com/hashicorp/go-version" @@ -29,7 +29,7 @@ import ( var errInvalidTagName = errors.New("The tag name is invalid") func apiError(ctx *context.Context, status int, obj interface{}) { - packages_router.LogAndProcessError(ctx, status, obj, func(message string) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { ctx.JSON(status, map[string]string{ "error": message, }) diff --git a/routers/api/v1/packages/nuget/api.go b/routers/api/packages/nuget/api.go similarity index 100% rename from routers/api/v1/packages/nuget/api.go rename to routers/api/packages/nuget/api.go diff --git a/routers/api/v1/packages/nuget/links.go b/routers/api/packages/nuget/links.go similarity index 100% rename from routers/api/v1/packages/nuget/links.go rename to routers/api/packages/nuget/links.go diff --git a/routers/api/v1/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go similarity index 98% rename from routers/api/v1/packages/nuget/nuget.go rename to routers/api/packages/nuget/nuget.go index c780c296e6fc8..5cf7b4d5e1742 100644 --- a/routers/api/v1/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -17,12 +17,12 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" nuget_module "code.gitea.io/gitea/modules/packages/nuget" "code.gitea.io/gitea/modules/setting" - packages_router "code.gitea.io/gitea/routers/api/v1/packages" + "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" ) func apiError(ctx *context.Context, status int, obj interface{}) { - packages_router.LogAndProcessError(ctx, status, obj, func(message string) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { ctx.JSON(status, map[string]string{ "Message": message, }) diff --git a/routers/api/v1/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go similarity index 97% rename from routers/api/v1/packages/pypi/pypi.go rename to routers/api/packages/pypi/pypi.go index b94930e11491e..c86c932d19525 100644 --- a/routers/api/v1/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -18,7 +18,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/validation" - packages_router "code.gitea.io/gitea/routers/api/v1/packages" + "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" ) @@ -30,7 +30,7 @@ var nameMatcher = regexp.MustCompile(`\A[a-z0-9\.\-_]+\z`) var versionMatcher = regexp.MustCompile(`^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$`) func apiError(ctx *context.Context, status int, obj interface{}) { - packages_router.LogAndProcessError(ctx, status, obj, func(message string) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } diff --git a/routers/api/v1/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go similarity index 98% rename from routers/api/v1/packages/rubygems/rubygems.go rename to routers/api/packages/rubygems/rubygems.go index 202d529b1dc8e..40323624e2ccb 100644 --- a/routers/api/v1/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -16,12 +16,12 @@ import ( "code.gitea.io/gitea/modules/context" packages_module "code.gitea.io/gitea/modules/packages" rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" - packages_router "code.gitea.io/gitea/routers/api/v1/packages" + "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" ) func apiError(ctx *context.Context, status int, obj interface{}) { - packages_router.LogAndProcessError(ctx, status, obj, func(message string) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) }) } From c209e68d9eeeb156fc283fd0ec043229e009734b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 15 Feb 2022 17:38:56 +0000 Subject: [PATCH 098/130] Updated docs. --- docs/content/doc/packages/composer.en-us.md | 10 +++++----- docs/content/doc/packages/conan.en-us.md | 4 ++-- docs/content/doc/packages/generic.en-us.md | 8 ++++---- docs/content/doc/packages/maven.en-us.md | 6 +++--- docs/content/doc/packages/npm.en-us.md | 12 ++++++------ docs/content/doc/packages/nuget.en-us.md | 4 ++-- docs/content/doc/packages/pypi.en-us.md | 6 +++--- docs/content/doc/packages/rubygems.en-us.md | 14 +++++++------- templates/package/content/composer.tmpl | 2 +- templates/package/content/conan.tmpl | 2 +- templates/package/content/generic.tmpl | 2 +- templates/package/content/maven.tmpl | 8 ++++---- templates/package/content/npm.tmpl | 2 +- templates/package/content/nuget.tmpl | 2 +- templates/package/content/pypi.tmpl | 2 +- templates/package/content/rubygems.tmpl | 4 ++-- 16 files changed, 44 insertions(+), 44 deletions(-) diff --git a/docs/content/doc/packages/composer.en-us.md b/docs/content/doc/packages/composer.en-us.md index 382206a51efd7..4e02ed27ad737 100644 --- a/docs/content/doc/packages/composer.en-us.md +++ b/docs/content/doc/packages/composer.en-us.md @@ -31,7 +31,7 @@ The package content must be the zipped PHP project with the `composer.json` file You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. ``` -PUT https://gitea.example.com/api/v1/packages/{owner}/composer +PUT https://gitea.example.com/api/packages/{owner}/composer ``` | Parameter | Description | @@ -41,7 +41,7 @@ PUT https://gitea.example.com/api/v1/packages/{owner}/composer If the `composer.json` file does not contain a `version` property, you must provide it as a query parameter: ``` -PUT https://gitea.example.com/api/v1/packages/{owner}/composer?version={x.y.z} +PUT https://gitea.example.com/api/packages/{owner}/composer?version={x.y.z} ``` Example request using HTTP Basic authentication: @@ -49,7 +49,7 @@ Example request using HTTP Basic authentication: ```shell curl --user your_username:your_password_or_token \ --upload-file path/to/project.zip \ - https://gitea.example.com/api/v1/packages/testuser/composer + https://gitea.example.com/api/packages/testuser/composer ``` Or specify the package version as query parameter: @@ -57,7 +57,7 @@ Or specify the package version as query parameter: ```shell curl --user your_username:your_password_or_token \ --upload-file path/to/project.zip \ - https://gitea.example.com/api/v1/packages/testuser/composer?version=1.0.3 + https://gitea.example.com/api/packages/testuser/composer?version=1.0.3 ``` The server reponds with the following HTTP Status codes. @@ -75,7 +75,7 @@ To register the package registry you need to add it to the Composer `config.json { "repositories": [{ "type": "composer", - "url": "https://gitea.example.com/api/v1/packages/{owner}/composer" + "url": "https://gitea.example.com/api/packages/{owner}/composer" } ] } diff --git a/docs/content/doc/packages/conan.en-us.md b/docs/content/doc/packages/conan.en-us.md index fe24e87e78594..968ff77d9aa6f 100644 --- a/docs/content/doc/packages/conan.en-us.md +++ b/docs/content/doc/packages/conan.en-us.md @@ -29,7 +29,7 @@ To work with the Conan package registry, you need to use the [conan](https://con To register the package registry you need to configure a new Conan remote: ```shell -conan remote add {remote} https://gitea.example.com/api/v1/packages/{owner}/conan +conan remote add {remote} https://gitea.example.com/api/packages/{owner}/conan conan user --remote {remote} --password {password} {username} ``` @@ -43,7 +43,7 @@ conan user --remote {remote} --password {password} {username} For example: ```shell -conan remote add gitea https://gitea.example.com/api/v1/packages/testuser/conan +conan remote add gitea https://gitea.example.com/api/packages/testuser/conan conan user --remote gitea --password password123 testuser ``` diff --git a/docs/content/doc/packages/generic.en-us.md b/docs/content/doc/packages/generic.en-us.md index 15a714c9242c9..5e2f6a658cc21 100644 --- a/docs/content/doc/packages/generic.en-us.md +++ b/docs/content/doc/packages/generic.en-us.md @@ -30,7 +30,7 @@ To publish a generic package perform a HTTP PUT operation with the package conte You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. ``` -PUT https://gitea.example.com/api/v1/packages/{owner}/generic/{package_name}/{package_version}/{file_name} +PUT https://gitea.example.com/api/packages/{owner}/generic/{package_name}/{package_version}/{file_name} ``` | Parameter | Description | @@ -45,7 +45,7 @@ Example request using HTTP Basic authentication: ```shell curl --user your_username:your_password_or_token \ --upload-file path/to/file.bin \ - https://gitea.example.com/api/v1/packages/testuser/generic/test_package/1.0.0/file.bin + https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0/file.bin ``` The server reponds with the following HTTP Status codes. @@ -60,7 +60,7 @@ The server reponds with the following HTTP Status codes. To download a generic package perform a HTTP GET operation. ``` -GET https://gitea.example.com/api/v1/packages/{owner}/generic/{package_name}/{package_version}/{file_name} +GET https://gitea.example.com/api/packages/{owner}/generic/{package_name}/{package_version}/{file_name} ``` | Parameter | Description | @@ -76,5 +76,5 @@ Example request using HTTP Basic authentication: ```shell curl --user your_username:your_token_or_password \ - https://gitea.example.com/api/v1/packages/testuser/generic/test_package/1.0.0/file.bin + https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0/file.bin ``` diff --git a/docs/content/doc/packages/maven.en-us.md b/docs/content/doc/packages/maven.en-us.md index d55f4eb47397e..b95d58ce23a73 100644 --- a/docs/content/doc/packages/maven.en-us.md +++ b/docs/content/doc/packages/maven.en-us.md @@ -53,17 +53,17 @@ Afterwards add the following sections to your project `pom.xml` file: gitea - https://gitea.example.com/api/v1/packages/{owner}/maven + https://gitea.example.com/api/packages/{owner}/maven gitea - https://gitea.example.com/api/v1/packages/{owner}/maven + https://gitea.example.com/api/packages/{owner}/maven gitea - https://gitea.example.com/api/v1/packages/{owner}/maven + https://gitea.example.com/api/packages/{owner}/maven ``` diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index 818ded0fdb024..fccbd920ebc3e 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -33,8 +33,8 @@ The following examples use the `npm` tool with the scope `@test`. To register the package registry you need to configure a new package source. ```shell -npm config set {scope}:registry https://gitea.example.com/api/v1/packages/{owner}/npm/ -npm config set -- '//gitea.example.com/api/v1/packages/{owner}/npm/:_authToken' "{token}" +npm config set {scope}:registry https://gitea.example.com/api/packages/{owner}/npm/ +npm config set -- '//gitea.example.com/api/packages/{owner}/npm/:_authToken' "{token}" ``` | Parameter | Description | @@ -46,15 +46,15 @@ npm config set -- '//gitea.example.com/api/v1/packages/{owner}/npm/:_authToken' For example: ```shell -npm config set @test:registry https://gitea.example.com/api/v1/packages/testuser/npm/ -npm config set -- '//gitea.example.com/api/v1/packages/testuser/npm/:_authToken' "personal_access_token" +npm config set @test:registry https://gitea.example.com/api/packages/testuser/npm/ +npm config set -- '//gitea.example.com/api/packages/testuser/npm/:_authToken' "personal_access_token" ``` or without scope: ```shell -npm config set registry https://gitea.example.com/api/v1/packages/testuser/npm/ -npm config set -- '//gitea.example.com/api/v1/packages/testuser/npm/:_authToken' "personal_access_token" +npm config set registry https://gitea.example.com/api/packages/testuser/npm/ +npm config set -- '//gitea.example.com/api/packages/testuser/npm/:_authToken' "personal_access_token" ``` ## Publish a package diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md index b7974f06921d1..d41f7e8d45d4f 100644 --- a/docs/content/doc/packages/nuget.en-us.md +++ b/docs/content/doc/packages/nuget.en-us.md @@ -31,7 +31,7 @@ The following examples use the `dotnet nuget` tool. To register the package registry you need to configure a new NuGet feed source: ```shell -dotnet nuget add source --name {source_name} --username {username} --password {password} https://gitea.example.com/api/v1/packages/{owner}/nuget/index.json +dotnet nuget add source --name {source_name} --username {username} --password {password} https://gitea.example.com/api/packages/{owner}/nuget/index.json ``` | Parameter | Description | @@ -44,7 +44,7 @@ dotnet nuget add source --name {source_name} --username {username} --password {p For example: ```shell -dotnet nuget add source --name gitea --username testuser --password password123 https://gitea.example.com/api/v1/packages/testuser/nuget/index.json +dotnet nuget add source --name gitea --username testuser --password password123 https://gitea.example.com/api/packages/testuser/nuget/index.json ``` ## Publish a package diff --git a/docs/content/doc/packages/pypi.en-us.md b/docs/content/doc/packages/pypi.en-us.md index 4411bb1f760a5..1b88f27049d61 100644 --- a/docs/content/doc/packages/pypi.en-us.md +++ b/docs/content/doc/packages/pypi.en-us.md @@ -33,7 +33,7 @@ To register the package registry you need to edit your local `~/.pypirc` file. A index-servers = gitea [gitea] -repository = https://gitea.example.com/api/v1/packages/{owner}/pypi +repository = https://gitea.example.com/api/packages/{owner}/pypi username = {username} password = {password} ``` @@ -61,7 +61,7 @@ You cannot publish a package if a package of the same name and version already e To install a PyPI package from the package registry, execute the following command: ```shell -pip install --index-url https://{username}:{password}@gitea.example.com/api/v1/packages/{owner}/pypi/simple --no-deps {package_name} +pip install --index-url https://{username}:{password}@gitea.example.com/api/packages/{owner}/pypi/simple --no-deps {package_name} ``` | Parameter | Description | @@ -74,7 +74,7 @@ pip install --index-url https://{username}:{password}@gitea.example.com/api/v1/p For example: ```shell -pip install --index-url https://testuser:password123@gitea.example.com/api/v1/packages/testuser/pypi/simple --no-deps test_package +pip install --index-url https://testuser:password123@gitea.example.com/api/packages/testuser/pypi/simple --no-deps test_package ``` ## Supported commands diff --git a/docs/content/doc/packages/rubygems.en-us.md b/docs/content/doc/packages/rubygems.en-us.md index 8a43dda407d4a..3ce01ecf9851e 100644 --- a/docs/content/doc/packages/rubygems.en-us.md +++ b/docs/content/doc/packages/rubygems.en-us.md @@ -30,7 +30,7 @@ To register the package registry edit the `~/.gem/credentials` file and add: ```ini --- -https://gitea.example.com/api/v1/packages/{owner}/rubygems: Bearer {token} +https://gitea.example.com/api/packages/{owner}/rubygems: Bearer {token} ``` | Parameter | Description | @@ -42,7 +42,7 @@ For example: ``` --- -https://gitea.example.com/api/v1/packages/testuser/rubygems: Bearer 3bd626f84b01cd26b873931eace1e430a5773cc4 +https://gitea.example.com/api/packages/testuser/rubygems: Bearer 3bd626f84b01cd26b873931eace1e430a5773cc4 ``` ## Publish a package @@ -61,7 +61,7 @@ gem push --host {host} {package_file} For example: ```shell -gem push --host https://gitea.example.com/api/v1/packages/testuser/rubygems test_package-1.0.0.gem +gem push --host https://gitea.example.com/api/packages/testuser/rubygems test_package-1.0.0.gem ``` You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. @@ -75,7 +75,7 @@ To install a package from the package registry you can use [Bundler](https://bun Add a new `source` block to your `Gemfile`: ``` -source "https://gitea.example.com/api/v1/packages/{owner}/rubygems" do +source "https://gitea.example.com/api/packages/{owner}/rubygems" do gem "{package_name}" end ``` @@ -88,7 +88,7 @@ end For example: ``` -source "https://gitea.example.com/api/v1/packages/testuser/rubygems" do +source "https://gitea.example.com/api/packages/testuser/rubygems" do gem "test_package" end ``` @@ -104,7 +104,7 @@ bundle install Execute the following command: ```shell -gem install --host https://gitea.example.com/api/v1/packages/{owner}/rubygems {package_name} +gem install --host https://gitea.example.com/api/packages/{owner}/rubygems {package_name} ``` | Parameter | Description | @@ -115,7 +115,7 @@ gem install --host https://gitea.example.com/api/v1/packages/{owner}/rubygems {p For example: ```shell -gem install --host https://gitea.example.com/api/v1/packages/testuser/rubygems test_package +gem install --host https://gitea.example.com/api/packages/testuser/rubygems test_package ``` ## Supported commands diff --git a/templates/package/content/composer.tmpl b/templates/package/content/composer.tmpl index 5a10ec0759cf2..ceb3bda8f763d 100644 --- a/templates/package/content/composer.tmpl +++ b/templates/package/content/composer.tmpl @@ -7,7 +7,7 @@ diff --git a/templates/package/content/conan.tmpl b/templates/package/content/conan.tmpl index 25baada9e5396..a9882499530a5 100644 --- a/templates/package/content/conan.tmpl +++ b/templates/package/content/conan.tmpl @@ -4,7 +4,7 @@
    - +
    diff --git a/templates/package/content/generic.tmpl b/templates/package/content/generic.tmpl index bfaae85b507fb..4211da3ac1edb 100644 --- a/templates/package/content/generic.tmpl +++ b/templates/package/content/generic.tmpl @@ -4,7 +4,7 @@
    - +
    diff --git a/templates/package/content/maven.tmpl b/templates/package/content/maven.tmpl index 956a9ceb0729a..6856fa7618069 100644 --- a/templates/package/content/maven.tmpl +++ b/templates/package/content/maven.tmpl @@ -7,19 +7,19 @@
    @@ -37,7 +37,7 @@
    - +
    diff --git a/templates/package/content/npm.tmpl b/templates/package/content/npm.tmpl index fe33ae4dc97eb..9f75983a75016 100644 --- a/templates/package/content/npm.tmpl +++ b/templates/package/content/npm.tmpl @@ -4,7 +4,7 @@
    - +
    diff --git a/templates/package/content/nuget.tmpl b/templates/package/content/nuget.tmpl index 2df2222092e43..1abb4ed339c26 100644 --- a/templates/package/content/nuget.tmpl +++ b/templates/package/content/nuget.tmpl @@ -4,7 +4,7 @@
    - +
    diff --git a/templates/package/content/pypi.tmpl b/templates/package/content/pypi.tmpl index 70bcf0ad9cd70..73e5c81a1475e 100644 --- a/templates/package/content/pypi.tmpl +++ b/templates/package/content/pypi.tmpl @@ -4,7 +4,7 @@
    - +
    diff --git a/templates/package/content/rubygems.tmpl b/templates/package/content/rubygems.tmpl index 8f1d4e083b2a8..80c5ce77ccbf9 100644 --- a/templates/package/content/rubygems.tmpl +++ b/templates/package/content/rubygems.tmpl @@ -4,11 +4,11 @@
    - +
    -
    From 04b3aae81499dcaa206abeb547adcf157f9dd024 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 15 Feb 2022 22:23:57 +0000 Subject: [PATCH 099/130] Fixed JOIN order. --- models/packages/conan/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/packages/conan/search.go b/models/packages/conan/search.go index 36f3915b2762a..f9361174dd056 100644 --- a/models/packages/conan/search.go +++ b/models/packages/conan/search.go @@ -75,8 +75,8 @@ func SearchRecipes(ctx context.Context, opts *RecipeSearchOptions) ([]string, er query := builder. Select("package.name, package_version.version, package_file.id"). From("package_file"). - Join("INNER", "package", "package.id = package_version.package_id"). Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("INNER", "package", "package.id = package_version.package_id"). Where(cond) results := make([]struct { From ff51ca9947fb0a2b64a28b94a644c39447cb9c30 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 16 Feb 2022 07:44:00 +0100 Subject: [PATCH 100/130] Fixed GROUP BY statement. --- models/packages/conan/references.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/packages/conan/references.go b/models/packages/conan/references.go index 492d8b9e42187..be3d6d04d5bcb 100644 --- a/models/packages/conan/references.go +++ b/models/packages/conan/references.go @@ -69,7 +69,7 @@ func findPropertyValues(ctx context.Context, propertyName string, ownerID int64, Where(cond) query := builder. - Select("package_property.value, package_file.created_unix"). + Select("package_property.value, MAX(package_file.created_unix)"). From("package_property"). Join("INNER", "package_file", "package_file.id = package_property.ref_id"). Where(builder.Eq{"package_property.name": propertyName}.And(builder.In("package_property.ref_id", in2))). From f596b5818c875adb407abf8c02385ea5db344284 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 17 Feb 2022 22:15:27 +0000 Subject: [PATCH 101/130] Fixed GROUP BY #2. --- models/packages/conan/references.go | 4 ++-- routers/api/packages/conan/search.go | 27 ++++++++++++++------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/models/packages/conan/references.go b/models/packages/conan/references.go index be3d6d04d5bcb..cc0db250f8177 100644 --- a/models/packages/conan/references.go +++ b/models/packages/conan/references.go @@ -69,12 +69,12 @@ func findPropertyValues(ctx context.Context, propertyName string, ownerID int64, Where(cond) query := builder. - Select("package_property.value, MAX(package_file.created_unix)"). + Select("package_property.value, MAX(package_file.created_unix) AS created_unix"). From("package_property"). Join("INNER", "package_file", "package_file.id = package_property.ref_id"). Where(builder.Eq{"package_property.name": propertyName}.And(builder.In("package_property.ref_id", in2))). GroupBy("package_property.value"). - OrderBy("package_file.created_unix DESC") + OrderBy("created_unix DESC") var values []*PropertyValue return values, db.GetEngine(ctx).SQL(query).Find(&values) diff --git a/routers/api/packages/conan/search.go b/routers/api/packages/conan/search.go index d3d8e8732b197..39dd6362aa488 100644 --- a/routers/api/packages/conan/search.go +++ b/routers/api/packages/conan/search.go @@ -85,24 +85,25 @@ func searchPackages(ctx *context.Context, searchAllRevisions bool) { return } rref = rref.WithRevision(lastRevision.Value) - } - - has, err := conan_model.RecipeExists(ctx, ctx.Package.Owner.ID, rref) - if err != nil { - if err == conan_model.ErrRecipeReferenceNotExist { - apiError(ctx, http.StatusNotFound, err) - } else { - apiError(ctx, http.StatusInternalServerError, err) + } else { + has, err := conan_model.RecipeExists(ctx, ctx.Package.Owner.ID, rref) + if err != nil { + if err == conan_model.ErrRecipeReferenceNotExist { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + if !has { + apiError(ctx, http.StatusNotFound, nil) + return } - return - } - if !has { - apiError(ctx, http.StatusNotFound, nil) - return } recipeRevisions := []*conan_model.PropertyValue{{Value: rref.Revision}} if searchAllRevisions { + var err error recipeRevisions, err = conan_model.GetRecipeRevisions(ctx, ctx.Package.Owner.ID, rref) if err != nil { apiError(ctx, http.StatusInternalServerError, err) From 6bdf396c3f1b523539a908d508e9a40d7d144009 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 20 Feb 2022 12:48:43 +0000 Subject: [PATCH 102/130] Added symbol server support. --- docs/content/doc/packages/nuget.en-us.md | 21 +- integrations/api_packages_nuget_test.go | 76 +++++-- models/packages/package_file.go | 13 ++ modules/packages/nuget/metadata.go | 2 + modules/packages/nuget/symbol_extractor.go | 187 ++++++++++++++++++ .../packages/nuget/symbol_extractor_test.go | 82 ++++++++ routers/api/packages/api.go | 2 + routers/api/packages/nuget/nuget.go | 102 +++++++++- 8 files changed, 458 insertions(+), 27 deletions(-) create mode 100644 modules/packages/nuget/symbol_extractor.go create mode 100644 modules/packages/nuget/symbol_extractor_test.go diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md index d41f7e8d45d4f..3022427e79097 100644 --- a/docs/content/doc/packages/nuget.en-us.md +++ b/docs/content/doc/packages/nuget.en-us.md @@ -14,7 +14,7 @@ menu: # NuGet Packages Repository -Publish [NuGet](https://www.nuget.org/) packages for your user or organization. +Publish [NuGet](https://www.nuget.org/) packages for your user or organization. The package registry supports [NuGet Symbol Packages](https://docs.microsoft.com/en-us/nuget/create-packages/symbol-packages-snupkg) too. **Table of Contents** @@ -68,6 +68,25 @@ dotnet nuget push --source gitea test_package.1.0.0.nupkg You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. +### Symbol Packages + +The NuGet package registry has build support for a symbol server. The PDB files embedded in a symbol package (`.snupkg`) can get requested by clients. +To do so, register the NuGet package registry as symbol source: + +``` +https://gitea.example.com/api/packages/{owner}/nuget/symbols +``` + +| Parameter | Description | +| --------- | ----------- | +| `owner` | The owner of the package registry. | + +For example: + +```shell +https://gitea.example.com/api/packages/testuser/nuget/symbols +``` + ## Install a package To install a NuGet package from the package registry, execute the following command: diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index e92ba5ca1c844..bc848ea63a6cf 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -7,10 +7,10 @@ package integrations import ( "archive/zip" "bytes" + "encoding/base64" "fmt" "io" "net/http" - "strings" "testing" "code.gitea.io/gitea/models/db" @@ -32,6 +32,8 @@ func TestPackageNuGet(t *testing.T) { packageVersion := "1.0.3" packageAuthors := "KN4CK3R" packageDescription := "Gitea Test Package" + symbolFilename := "test.pdb" + symbolID := "d910bb6948bd4c6cb40155bcf52c3c94" var buf bytes.Buffer archive := zip.NewWriter(&buf) @@ -129,6 +131,7 @@ func TestPackageNuGet(t *testing.T) { createPackage := func(id, packageType string) io.Reader { var buf bytes.Buffer archive := zip.NewWriter(&buf) + w, _ := archive.Create("package.nuspec") w.Write([]byte(` @@ -140,6 +143,13 @@ func TestPackageNuGet(t *testing.T) { `)) + + w, _ = archive.Create(symbolFilename) + b, _ := base64.StdEncoding.DecodeString(`QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAABgB8AAAAWAAAACNQZGIAAAAA1AAAAAgBAAAj +fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB +AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) + w.Write(b) + archive.Close() return &buf } @@ -169,17 +179,32 @@ func TestPackageNuGet(t *testing.T) { pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) - assert.Len(t, pfs, 2) - i := 0 - if strings.HasSuffix(pfs[1].Name, ".snupkg") { - i = 1 + assert.Len(t, pfs, 3) + for _, pf := range pfs { + switch pf.Name { + case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion): + case fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion): + assert.False(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(616), pb.Size) + case symbolFilename: + assert.False(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(160), pb.Size) + + pps, err := packages.GetProperties(db.DefaultContext, packages.PropertyTypeFile, pf.ID) + assert.NoError(t, err) + assert.Len(t, pps, 1) + assert.Equal(t, nuget_module.PropertySymbolID, pps[0].Name) + assert.Equal(t, symbolID, pps[0].Value) + default: + assert.Fail(t, "unexpected file: %v", pf.Name) + } } - assert.Equal(t, fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion), pfs[i].Name) - assert.False(t, pfs[i].IsLead) - - pb, err := packages.GetBlobByID(db.DefaultContext, pfs[i].BlobID) - assert.NoError(t, err) - assert.Equal(t, int64(368), pb.Size) req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/symbolpackage", url), createPackage(packageName, "SymbolsPackage")) req = AddBasicAuthHeader(req, user.Name) @@ -190,25 +215,38 @@ func TestPackageNuGet(t *testing.T) { t.Run("Download", func(t *testing.T) { defer PrintCurrentTest(t)() + checkDownloadCount := func(count int64) { + pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + assert.Equal(t, count, pvs[0].DownloadCount) + } + + checkDownloadCount(0) + req := NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.nupkg", url, packageName, packageVersion, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) assert.Equal(t, content, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) - assert.NoError(t, err) - assert.Len(t, pvs, 1) - assert.Equal(t, int64(1), pvs[0].DownloadCount) + checkDownloadCount(1) req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.snupkg", url, packageName, packageVersion, packageName, packageVersion)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - pvs, err = packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) - assert.NoError(t, err) - assert.Len(t, pvs, 1) - assert.Equal(t, int64(1), pvs[0].DownloadCount) + checkDownloadCount(1) + + t.Run("Symbol", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/symbols/%s/%sFFFFFFFF/%s", url, symbolFilename, symbolID, symbolFilename)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusOK) + + checkDownloadCount(1) + }) }) t.Run("SearchService", func(t *testing.T) { diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 9404956a8223c..d2d8ac98a789b 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -117,6 +117,8 @@ func DeleteFileByID(ctx context.Context, fileID int64) error { // PackageFileSearchOptions are options for SearchXXX methods type PackageFileSearchOptions struct { + OwnerID int64 + PackageType string VersionID int64 Query string CompositeKey string @@ -129,6 +131,17 @@ func (opts *PackageFileSearchOptions) toConds() builder.Cond { if opts.VersionID != 0 { cond = cond.And(builder.Eq{"package_file.version_id": opts.VersionID}) + } else if opts.OwnerID != 0 && opts.PackageType != "" && opts.PackageType != "all" { + in := builder. + Select("package_version.id"). + From("package_version"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(builder.Eq{ + "package.owner_id": opts.OwnerID, + "package.type": opts.PackageType, + }) + + cond = cond.And(builder.In("package_file.version_id", in)) } if opts.CompositeKey != "" { cond = cond.And(builder.Eq{"package_file.composite_key": opts.CompositeKey}) diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 6f6e3fa18d64d..bb5ebe9c1eb1d 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -37,6 +37,8 @@ const ( DependencyPackage PackageType = iota + 1 // SymbolsPackage represents a symbol package (*.snupkg) SymbolsPackage + + PropertySymbolID = "nuget.symbol.id" ) var idmatch = regexp.MustCompile(`\A\w+(?:[.-]\w+)*\z`) diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go new file mode 100644 index 0000000000000..13641ca6ef94a --- /dev/null +++ b/modules/packages/nuget/symbol_extractor.go @@ -0,0 +1,187 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "path" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/packages" +) + +var ( + ErrMissingPdbFiles = errors.New("Package does not contain PDB files") + ErrInvalidFiles = errors.New("Package contains invalid files") + ErrInvalidPdbMagicNumber = errors.New("Invalid Portable PDB magic number") + ErrMissingPdbStream = errors.New("Missing PDB stream") +) + +type PortablePdb struct { + Name string + ID string + Content *packages.HashedBuffer +} + +type PortablePdbList []*PortablePdb + +func (l PortablePdbList) Close() { + for _, pdb := range l { + pdb.Content.Close() + } +} + +// ExtractPortablePdb extracts PDB files from a .snupkg file +func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + var pdbs PortablePdbList + + err = func() error { + for _, file := range archive.File { + if strings.HasSuffix(file.Name, "/") { + continue + } + ext := strings.ToLower(filepath.Ext(file.Name)) + + switch ext { + case ".nuspec", ".xml", ".psmdcp", ".rels", ".p7s": + continue + case ".pdb": + f, err := archive.Open(file.Name) + if err != nil { + return err + } + + buf, err := packages.CreateHashedBufferFromReader(f, 32*1024*1024) + + f.Close() + + if err != nil { + return err + } + + id, err := ParseDebugHeaderID(buf) + if err != nil { + buf.Close() + return fmt.Errorf("Invalid PDB file: %v", err) + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + buf.Close() + return err + } + + pdbs = append(pdbs, &PortablePdb{ + Name: path.Base(file.Name), + ID: id, + Content: buf, + }) + default: + return ErrInvalidFiles + } + } + return nil + }() + if err != nil { + pdbs.Close() + return nil, err + } + + if len(pdbs) == 0 { + return nil, ErrMissingPdbFiles + } + + return pdbs, nil +} + +// ParseDebugHeaderID TODO +func ParseDebugHeaderID(r io.ReadSeeker) (string, error) { + var magic uint32 + if err := binary.Read(r, binary.LittleEndian, &magic); err != nil { + return "", err + } + if magic != 0x424A5342 { + return "", ErrInvalidPdbMagicNumber + } + + if _, err := r.Seek(8, io.SeekCurrent); err != nil { + return "", err + } + + var versionStringSize int32 + if err := binary.Read(r, binary.LittleEndian, &versionStringSize); err != nil { + return "", err + } + if _, err := r.Seek(int64(versionStringSize), io.SeekCurrent); err != nil { + return "", err + } + if _, err := r.Seek(2, io.SeekCurrent); err != nil { + return "", err + } + + var streamCount int16 + if err := binary.Read(r, binary.LittleEndian, &streamCount); err != nil { + return "", err + } + + read4ByteAlignedString := func(r io.Reader) (string, error) { + b := make([]byte, 4) + var buf bytes.Buffer + for { + if _, err := r.Read(b); err != nil { + return "", err + } + if i := bytes.IndexByte(b, 0); i != -1 { + buf.Write(b[:i]) + return buf.String(), nil + } + buf.Write(b) + } + } + + for i := 0; i < int(streamCount); i++ { + var offset uint32 + if err := binary.Read(r, binary.LittleEndian, &offset); err != nil { + return "", err + } + if _, err := r.Seek(4, io.SeekCurrent); err != nil { + return "", err + } + name, err := read4ByteAlignedString(r) + if err != nil { + return "", err + } + + if name == "#Pdb" { + if _, err := r.Seek(int64(offset), io.SeekStart); err != nil { + return "", err + } + + b := make([]byte, 16) + if _, err := r.Read(b); err != nil { + return "", err + } + + data1 := binary.LittleEndian.Uint32(b[0:4]) + data2 := binary.LittleEndian.Uint16(b[4:6]) + data3 := binary.LittleEndian.Uint16(b[6:8]) + data4 := b[8:16] + + return fmt.Sprintf("%08x%04x%04x%04x%012x", data1, data2, data3, data4[:2], data4[2:]), nil + } + } + + return "", ErrMissingPdbStream +} diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go new file mode 100644 index 0000000000000..892d718caa57f --- /dev/null +++ b/modules/packages/nuget/symbol_extractor_test.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" +) + +const pdbContent = `QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAABgB8AAAAWAAAACNQZGIAAAAA1AAAAAgBAAAj +fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB +AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==` + +func TestExtractPortablePdb(t *testing.T) { + createArchive := func(name string, content []byte) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create(name) + w.Write(content) + archive.Close() + return buf.Bytes() + } + + t.Run("MissingPdbFiles", func(t *testing.T) { + var buf bytes.Buffer + zip.NewWriter(&buf).Close() + + pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + assert.ErrorIs(t, err, ErrMissingPdbFiles) + assert.Empty(t, pdbs) + }) + + t.Run("InvalidFiles", func(t *testing.T) { + data := createArchive("sub/test.bin", []byte{}) + + pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data))) + assert.ErrorIs(t, err, ErrInvalidFiles) + assert.Empty(t, pdbs) + }) + + t.Run("Valid", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(pdbContent) + data := createArchive("test.pdb", b) + + pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data))) + assert.NoError(t, err) + assert.Len(t, pdbs, 1) + assert.Equal(t, "test.pdb", pdbs[0].Name) + assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", pdbs[0].ID) + pdbs.Close() + }) +} + +func TestParseDebugHeaderID(t *testing.T) { + t.Run("InvalidPdbMagicNumber", func(t *testing.T) { + id, err := ParseDebugHeaderID(bytes.NewReader([]byte{0, 0, 0, 0})) + assert.ErrorIs(t, err, ErrInvalidPdbMagicNumber) + assert.Empty(t, id) + }) + + t.Run("MissingPdbStream", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(`QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAAAQB8AAAAWAAAACNVUwA=`) + + id, err := ParseDebugHeaderID(bytes.NewReader(b)) + assert.ErrorIs(t, err, ErrMissingPdbStream) + assert.Empty(t, id) + }) + + t.Run("Valid", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(pdbContent) + + id, err := ParseDebugHeaderID(bytes.NewReader(b)) + assert.NoError(t, err) + assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", id) + }) +} diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index aa8077ca97fc3..78169ccd4f9ac 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -24,6 +24,7 @@ import ( func reqPackageAccess(accessMode perm.AccessMode) func(ctx *context.Context) { return func(ctx *context.Context) { if ctx.Package.AccessMode < accessMode && !ctx.IsUserSiteAdmin() { + ctx.Resp.Header().Set("WWW-Authenticate", `Basic realm="Gitea Package API"`) ctx.Error(http.StatusUnauthorized, "reqPackageAccess", "user should have specific permission or be a site admin") return } @@ -167,6 +168,7 @@ func Routes() *web.Route { r.Put("/symbolpackage", nuget.UploadSymbolPackage) r.Delete("/{id}/{version}", nuget.DeletePackage) }, reqPackageAccess(perm.AccessModeWrite)) + r.Get("/symbols/{filename}/{guid:[0-9a-f]{32}}FFFFFFFF/{filename2}", nuget.DownloadSymbolFile) }) r.Group("/npm", func() { r.Group("/@{scope}/{id}", func() { diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index 5cf7b4d5e1742..2df9dfbf4a594 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -237,13 +237,27 @@ func UploadSymbolPackage(ctx *context.Context) { return } - _, _, err := packages_service.AddFileToExistingPackage( - &packages_service.PackageInfo{ - Owner: ctx.Package.Owner, - PackageType: packages.TypeNuGet, - Name: np.ID, - Version: np.Version, - }, + pdbs, err := nuget_module.ExtractPortablePdb(buf, buf.Size()) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + defer pdbs.Close() + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pi := &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages.TypeNuGet, + Name: np.ID, + Version: np.Version, + } + + _, _, err = packages_service.AddFileToExistingPackage( + pi, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ Filename: strings.ToLower(fmt.Sprintf("%s.%s.snupkg", np.ID, np.Version)), @@ -264,6 +278,32 @@ func UploadSymbolPackage(ctx *context.Context) { return } + for _, pdb := range pdbs { + _, _, err := packages_service.AddFileToExistingPackage( + pi, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(pdb.Name), + CompositeKey: strings.ToLower(pdb.ID), + }, + Data: pdb.Content, + IsLead: false, + Properties: map[string]string{ + nuget_module.PropertySymbolID: strings.ToLower(pdb.ID), + }, + }, + ) + if err != nil { + switch err { + case packages.ErrDuplicatePackageFile: + apiError(ctx, http.StatusBadRequest, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + } + ctx.Status(http.StatusCreated) } @@ -307,6 +347,54 @@ func processUploadedFile(ctx *context.Context, expectedType nuget_module.Package return np, buf, closables } +// DownloadSymbolFile https://github.com/dotnet/symstore/blob/main/docs/specs/Simple_Symbol_Query_Protocol.md#request +func DownloadSymbolFile(ctx *context.Context) { + filename := ctx.Params("filename") + guid := ctx.Params("guid") + filename2 := ctx.Params("filename2") + + if filename != filename2 { + apiError(ctx, http.StatusBadRequest, nil) + return + } + + pfs, _, err := packages.SearchFiles(ctx, &packages.PackageFileSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + PackageType: string(packages.TypeNuGet), + Query: filename, + Properties: map[string]string{ + nuget_module.PropertySymbolID: strings.ToLower(guid), + }, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pfs) != 1 { + apiError(ctx, http.StatusNotFound, nil) + return + } + + pv, err := packages.GetVersionByID(ctx, pfs[0].VersionID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + s, _, err := packages_service.GetPackageFileStream(pv, pfs[0]) + if err != nil { + if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeStream(s, pfs[0].Name) +} + // DeletePackage hard deletes the package // https://docs.microsoft.com/en-us/nuget/api/package-publish-resource#delete-a-package func DeletePackage(ctx *context.Context) { From e70ec8eacc7e54e19e76e69b1f6b6b546939aaf4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 20 Feb 2022 13:19:26 +0000 Subject: [PATCH 103/130] Added more tests. --- integrations/api_packages_nuget_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index bc848ea63a6cf..661500a77696c 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -241,7 +241,14 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) t.Run("Symbol", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("%s/symbols/%s/%sFFFFFFFF/%s", url, symbolFilename, symbolID, symbolFilename)) + req := NewRequest(t, "GET", fmt.Sprintf("%s/symbols/%s/%sFFFFFFFF/gitea.pdb", url, symbolFilename, symbolID)) + MakeRequest(t, req, http.StatusBadRequest) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/symbols/%s/%sFFFFFFFF/%s", url, symbolFilename, "00000000000000000000000000000000", symbolFilename)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/symbols/%s/%sFFFFFFFF/%s", url, symbolFilename, symbolID, symbolFilename)) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) From 0aa606963d17063730257d69110a0f73aa444680 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 24 Feb 2022 16:44:00 +0100 Subject: [PATCH 104/130] Set NOT NULL. --- models/migrations/v211.go | 32 +++++++++++++++--------------- models/packages/package.go | 12 +++++------ models/packages/package_blob.go | 4 ++-- models/packages/package_file.go | 14 ++++++------- models/packages/package_version.go | 10 +++++----- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/models/migrations/v211.go b/models/migrations/v211.go index 94b3ad708bf45..5ef8618a4db19 100644 --- a/models/migrations/v211.go +++ b/models/migrations/v211.go @@ -16,9 +16,9 @@ func addPackageTables(x *xorm.Engine) error { OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` RepoID int64 `xorm:"INDEX"` Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string + Name string `xorm:"NOT NULL"` LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - SemverCompatible bool + SemverCompatible bool `xorm:"NOT NULL DEFAULT false"` } if err := x.Sync2(new(Package)); err != nil { @@ -26,14 +26,14 @@ func addPackageTables(x *xorm.Engine) error { } type PackageVersion struct { - ID int64 `xorm:"pk autoincr"` - PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - CreatorID int64 - Version string + ID int64 `xorm:"pk autoincr"` + PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatorID int64 `xorm:"NOT NULL DEFAULT 0"` + Version string `xorm:"NOT NULL"` LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` MetadataJSON string `xorm:"TEXT metadata_json"` - DownloadCount int64 + DownloadCount int64 `xorm:"NOT NULL DEFAULT 0"` } if err := x.Sync2(new(PackageVersion)); err != nil { @@ -53,13 +53,13 @@ func addPackageTables(x *xorm.Engine) error { } type PackageFile struct { - ID int64 `xorm:"pk autoincr"` - VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - BlobID int64 `xorm:"INDEX NOT NULL"` - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - CompositeKey string `xorm:"UNIQUE(s) INDEX"` - IsLead bool + ID int64 `xorm:"pk autoincr"` + VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + BlobID int64 `xorm:"INDEX NOT NULL"` + Name string `xorm:"NOT NULL"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + CompositeKey string `xorm:"UNIQUE(s) INDEX"` + IsLead bool `xorm:"NOT NULL DEFAULT false"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` } @@ -68,8 +68,8 @@ func addPackageTables(x *xorm.Engine) error { } type PackageBlob struct { - ID int64 `xorm:"pk autoincr"` - Size int64 + ID int64 `xorm:"pk autoincr"` + Size int64 `xorm:"NOT NULL DEFAULT 0"` HashMD5 string `xorm:"hash_md5 char(32) UNIQUE(md5) INDEX NOT NULL"` HashSHA1 string `xorm:"hash_sha1 char(40) UNIQUE(sha1) INDEX NOT NULL"` HashSHA256 string `xorm:"hash_sha256 char(64) UNIQUE(sha256) INDEX NOT NULL"` diff --git a/models/packages/package.go b/models/packages/package.go index 2ef03953b5276..c9f46d1393370 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -87,13 +87,13 @@ func (pt Type) SVGName() string { // Package represents a package type Package struct { - ID int64 `xorm:"pk autoincr"` - OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - RepoID int64 `xorm:"INDEX"` - Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string + ID int64 `xorm:"pk autoincr"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + RepoID int64 `xorm:"INDEX"` + Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string `xorm:"NOT NULL"` LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - SemverCompatible bool + SemverCompatible bool `xorm:"NOT NULL DEFAULT false"` } // TryInsertPackage inserts a package. If a package exists already, ErrDuplicatePackage is returned diff --git a/models/packages/package_blob.go b/models/packages/package_blob.go index 233b4a26488b6..c0f904c06ab19 100644 --- a/models/packages/package_blob.go +++ b/models/packages/package_blob.go @@ -21,8 +21,8 @@ func init() { // PackageBlob represents a package blob type PackageBlob struct { - ID int64 `xorm:"pk autoincr"` - Size int64 + ID int64 `xorm:"pk autoincr"` + Size int64 `xorm:"NOT NULL DEFAULT 0"` HashMD5 string `xorm:"hash_md5 char(32) UNIQUE(md5) INDEX NOT NULL"` HashSHA1 string `xorm:"hash_sha1 char(40) UNIQUE(sha1) INDEX NOT NULL"` HashSHA256 string `xorm:"hash_sha256 char(64) UNIQUE(sha256) INDEX NOT NULL"` diff --git a/models/packages/package_file.go b/models/packages/package_file.go index d2d8ac98a789b..1eedcb25c00a7 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -32,13 +32,13 @@ const EmptyFileKey = "" // PackageFile represents a package file type PackageFile struct { - ID int64 `xorm:"pk autoincr"` - VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - BlobID int64 `xorm:"INDEX NOT NULL"` - Name string - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - CompositeKey string `xorm:"UNIQUE(s) INDEX"` - IsLead bool + ID int64 `xorm:"pk autoincr"` + VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + BlobID int64 `xorm:"INDEX NOT NULL"` + Name string `xorm:"NOT NULL"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + CompositeKey string `xorm:"UNIQUE(s) INDEX"` + IsLead bool `xorm:"NOT NULL DEFAULT false"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` } diff --git a/models/packages/package_version.go b/models/packages/package_version.go index d7dc01fda2bee..37a8832352506 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -29,14 +29,14 @@ func init() { // PackageVersion represents a package version type PackageVersion struct { - ID int64 `xorm:"pk autoincr"` - PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` - CreatorID int64 - Version string + ID int64 `xorm:"pk autoincr"` + PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` + CreatorID int64 `xorm:"NOT NULL DEFAULT 0"` + Version string `xorm:"NOT NULL"` LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` MetadataJSON string `xorm:"TEXT metadata_json"` - DownloadCount int64 + DownloadCount int64 `xorm:"NOT NULL DEFAULT 0"` } // GetOrInsertVersion inserts a version. If the same version exist already ErrDuplicatePackageVersion is returned From a5216a009f54347dc981bafcb92ba8f11d96d570 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 3 Mar 2022 15:16:33 +0000 Subject: [PATCH 105/130] Added setting to disable package registries. --- custom/conf/app.example.ini | 9 ++++++++ .../doc/advanced/config-cheat-sheet.en-us.md | 4 ++++ modules/setting/packages.go | 23 +++++++++++++++++++ modules/setting/repository.go | 4 ---- routers/api/packages/api.go | 5 ++++ 5 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 modules/setting/packages.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index fbae4b270c2e6..a1453781eea68 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2188,6 +2188,15 @@ PATH = ;; Enable/Disable federation capabilities ; ENABLED = true +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;[packages] +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Enable/Disable package registry capabilities +; ENABLED = true + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; default storage for attachments, lfs and avatars diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index a7e5a62282172..ba4e51591eb43 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -1064,6 +1064,10 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `ENABLED`: **true**: Enable/Disable federation capabilities +## Packages (`packages`) + +- `ENABLED`: **true**: Enable/Disable package registry capabilities + ## Mirror (`mirror`) - `ENABLED`: **true**: Enables the mirror functionality. Set to **false** to disable all mirrors. diff --git a/modules/setting/packages.go b/modules/setting/packages.go new file mode 100644 index 0000000000000..4e9d554d85e5d --- /dev/null +++ b/modules/setting/packages.go @@ -0,0 +1,23 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import "code.gitea.io/gitea/modules/log" + +// Package registry settings +var ( + Packages = struct { + Storage + Enabled bool + }{ + Enabled: true, + } +) + +func newPackagesService() { + if err := Cfg.Section("packages").MapTo(&Packages); err != nil { + log.Fatal("Failed to map Packages settings: %v", err) + } +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index a950c3ca771fa..becd354174e7d 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -256,10 +256,6 @@ var ( RepoArchive = struct { Storage }{} - - Packages = struct { - Storage - }{} ) func newRepository() { diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 78169ccd4f9ac..2ccebc8e65382 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/packages/composer" "code.gitea.io/gitea/routers/api/packages/conan" @@ -34,6 +35,10 @@ func reqPackageAccess(accessMode perm.AccessMode) func(ctx *context.Context) { func Routes() *web.Route { r := web.NewRoute() + if !setting.Packages.Enabled { + return r + } + r.Use(context.PackageContexter()) authGroup := auth.NewGroup(&auth.OAuth2{}, &auth.Basic{}, &conan.Auth{}) From 7ba602ca6efb11c3a7ea66d525c87b5dd171cff4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 13 Mar 2022 17:34:56 +0000 Subject: [PATCH 106/130] Moved auth into service. --- integrations/api_packages_generic_test.go | 8 +-- integrations/api_packages_test.go | 3 +- routers/api/packages/api.go | 2 +- routers/api/packages/conan/auth.go | 66 +++-------------------- routers/api/packages/conan/conan.go | 7 ++- services/packages/auth.go | 66 +++++++++++++++++++++++ 6 files changed, 88 insertions(+), 64 deletions(-) create mode 100644 services/packages/auth.go diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index 34ea48c3e1dc5..82ffdf13abd7a 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -21,20 +21,19 @@ import ( func TestPackageGeneric(t *testing.T) { defer prepareTestEnv(t)() user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User) - session := loginUser(t, user.Name) - token := getTokenForLoggedInUser(t, session) packageName := "te-st_pac.kage" packageVersion := "1.0.3" filename := "fi-le_na.me" content := []byte{1, 2, 3} - url := fmt.Sprintf("/api/packages/%s/generic/%s/%s/%s?token=%s", user.Name, packageName, packageVersion, filename, token) + url := fmt.Sprintf("/api/packages/%s/generic/%s/%s/%s", user.Name, packageName, packageVersion, filename) t.Run("Upload", func(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) + AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeGeneric) @@ -63,6 +62,7 @@ func TestPackageGeneric(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)) + AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusBadRequest) }) @@ -84,6 +84,7 @@ func TestPackageGeneric(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequest(t, "DELETE", url) + AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeGeneric) @@ -102,6 +103,7 @@ func TestPackageGeneric(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequest(t, "DELETE", url) + AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusNotFound) }) } diff --git a/integrations/api_packages_test.go b/integrations/api_packages_test.go index 8a9cf16edf356..e676971b7730c 100644 --- a/integrations/api_packages_test.go +++ b/integrations/api_packages_test.go @@ -28,8 +28,9 @@ func TestPackageAPI(t *testing.T) { packageVersion := "1.0.3" filename := "file.bin" - url := fmt.Sprintf("/api/packages/%s/generic/%s/%s/%s?token=%s", user.Name, packageName, packageVersion, filename, token) + url := fmt.Sprintf("/api/packages/%s/generic/%s/%s/%s", user.Name, packageName, packageVersion, filename) req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{})) + AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) var packageID int64 diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 2ccebc8e65382..cb5e7e0e523ca 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -41,7 +41,7 @@ func Routes() *web.Route { r.Use(context.PackageContexter()) - authGroup := auth.NewGroup(&auth.OAuth2{}, &auth.Basic{}, &conan.Auth{}) + authGroup := auth.NewGroup(&auth.Basic{}, &conan.Auth{}) r.Use(func(ctx *context.Context) { ctx.User = authGroup.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) }) diff --git a/routers/api/packages/conan/auth.go b/routers/api/packages/conan/auth.go index cc5fa341add59..a718053a27ae1 100644 --- a/routers/api/packages/conan/auth.go +++ b/routers/api/packages/conan/auth.go @@ -5,18 +5,12 @@ package conan import ( - "fmt" "net/http" - "strings" - "time" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/auth" - - "github.com/golang-jwt/jwt/v4" + "code.gitea.io/gitea/services/packages" ) type Auth struct{} @@ -26,65 +20,21 @@ func (a *Auth) Name() string { } func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) *user_model.User { - token, err := parseAuthorizationToken(req) + uid, err := packages.ParseAuthorizationToken(req) if err != nil { - log.Trace("parseAuthorizationToken: %v", err) + log.Trace("ParseAuthorizationToken: %v", err) return nil } - u, err := user_model.GetUserByID(token.UserID) - if err != nil { - log.Error("GetUserByID: %v", err) + if uid == 0 { return nil } - return u -} - -type conanClaims struct { - jwt.RegisteredClaims - UserID int64 -} - -func createAuthorizationToken(ctx *context.Context) (string, error) { - now := time.Now() - claims := conanClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - ExpiresAt: jwt.NewNumericDate(now.Add(24 * time.Hour)), - NotBefore: jwt.NewNumericDate(now), - }, - UserID: ctx.User.ID, - } - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - - tokenString, err := token.SignedString([]byte(setting.SecretKey)) - if err != nil { - return "", err - } - - return tokenString, nil -} - -func parseAuthorizationToken(req *http.Request) (*conanClaims, error) { - parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("no token") - } - - token, err := jwt.ParseWithClaims(parts[1], &conanClaims{}, func(t *jwt.Token) (interface{}, error) { - if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"]) - } - return []byte(setting.SecretKey), nil - }) + u, err := user_model.GetUserByID(uid) if err != nil { - return nil, err - } - - c, ok := token.Claims.(*conanClaims) - if !token.Valid || !ok { - return nil, fmt.Errorf("invalid token claim") + log.Error("GetUserByID: %v", err) + return nil } - return c, nil + return u } diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index 9a96885ee6a97..73e61489104a0 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -113,7 +113,12 @@ func Ping(ctx *context.Context) { // Authenticate creates an authentication token for the user func Authenticate(ctx *context.Context) { - token, err := createAuthorizationToken(ctx) + if ctx.User == nil { + apiError(ctx, http.StatusBadRequest, nil) + return + } + + token, err := packages_service.CreateAuthorizationToken(ctx.User) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/services/packages/auth.go b/services/packages/auth.go new file mode 100644 index 0000000000000..50212fccfd327 --- /dev/null +++ b/services/packages/auth.go @@ -0,0 +1,66 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "fmt" + "net/http" + "strings" + "time" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + + "github.com/golang-jwt/jwt/v4" +) + +type packageClaims struct { + jwt.RegisteredClaims + UserID int64 +} + +func CreateAuthorizationToken(u *user_model.User) (string, error) { + now := time.Now() + + claims := packageClaims{ + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(now.Add(24 * time.Hour)), + NotBefore: jwt.NewNumericDate(now), + }, + UserID: u.ID, + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + + tokenString, err := token.SignedString([]byte(setting.SecretKey)) + if err != nil { + return "", err + } + + return tokenString, nil +} + +func ParseAuthorizationToken(req *http.Request) (int64, error) { + parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2) + if len(parts) != 2 { + return 0, fmt.Errorf("no token") + } + + token, err := jwt.ParseWithClaims(parts[1], &packageClaims{}, func(t *jwt.Token) (interface{}, error) { + if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"]) + } + return []byte(setting.SecretKey), nil + }) + if err != nil { + return 0, err + } + + c, ok := token.Claims.(*packageClaims) + if !token.Valid || !ok { + return 0, fmt.Errorf("invalid token claim") + } + + return c.UserID, nil +} From 0293e7741ac02d237572b19146a240a3c61cc03b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 13 Mar 2022 18:03:36 +0000 Subject: [PATCH 107/130] refactor --- services/packages/packages.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/services/packages/packages.go b/services/packages/packages.go index 1b1625af02723..1c84ac29e15fe 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -174,7 +174,7 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) ( pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci) removeBlob := false defer func() { - if blobCreated && removeBlob { + if removeBlob { contentStore := packages_module.NewContentStore() if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { log.Error("Error deleting package blob from content store: %v", err) @@ -182,12 +182,12 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) ( } }() if err != nil { - removeBlob = true + removeBlob = blobCreated return nil, nil, err } if err := committer.Commit(); err != nil { - removeBlob = true + removeBlob = blobCreated return nil, nil, err } @@ -199,13 +199,11 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers hashMD5, hashSHA1, hashSHA256, hashSHA512 := pfci.Data.Sums() - blobKey := fmt.Sprintf("%x", hashSHA256) - pb := &packages_model.PackageBlob{ Size: pfci.Data.Size(), HashMD5: fmt.Sprintf("%x", hashMD5), HashSHA1: fmt.Sprintf("%x", hashSHA1), - HashSHA256: blobKey, + HashSHA256: fmt.Sprintf("%x", hashSHA256), HashSHA512: fmt.Sprintf("%x", hashSHA512), } pb, exists, err := packages_model.GetOrInsertBlob(ctx, pb) @@ -215,7 +213,7 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers } if !exists { contentStore := packages_module.NewContentStore() - if err := contentStore.Save(packages_module.BlobHash256Key(blobKey), pfci.Data, pfci.Data.Size()); err != nil { + if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), pfci.Data, pfci.Data.Size()); err != nil { log.Error("Error saving package blob in content store: %v", err) return nil, nil, false, err } @@ -412,7 +410,7 @@ func GetFileStreamByPackageVersion(pv *packages_model.PackageVersion, pfi *Packa return GetPackageFileStream(pv, pf) } -// GetPackageFileStream returns the cotent of the specific package file +// GetPackageFileStream returns the content of the specific package file func GetPackageFileStream(pv *packages_model.PackageVersion, pf *packages_model.PackageFile) (io.ReadCloser, *packages_model.PackageFile, error) { pb, err := packages_model.GetBlobByID(db.DefaultContext, pf.BlobID) if err != nil { From 35c04f01d8394fc259ff3c572c09ca026b84f94e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 15 Mar 2022 17:47:00 +0100 Subject: [PATCH 108/130] Use ctx everywhere. --- integrations/api_packages_composer_test.go | 8 +-- integrations/api_packages_conan_test.go | 6 +- integrations/api_packages_generic_test.go | 8 +-- integrations/api_packages_maven_test.go | 16 +++--- integrations/api_packages_npm_test.go | 6 +- integrations/api_packages_nuget_test.go | 12 ++-- integrations/api_packages_pypi_test.go | 10 ++-- integrations/api_packages_rubygems_test.go | 10 ++-- models/packages/descriptor.go | 64 ++++++++++++++-------- models/packages/package.go | 8 +-- models/packages/package_version.go | 28 +++++----- modules/context/package.go | 4 +- modules/packages/conan/metadata.go | 12 ++-- modules/packages/maven/metadata.go | 20 +++---- modules/packages/npm/metadata.go | 24 ++++---- modules/packages/nuget/metadata.go | 12 ++-- modules/packages/pypi/metadata.go | 14 ++--- modules/packages/rubygems/metadata.go | 20 +++---- routers/api/packages/composer/composer.go | 25 +++++---- routers/api/packages/conan/auth.go | 1 + routers/api/packages/conan/conan.go | 5 +- routers/api/packages/generic/generic.go | 15 ++--- routers/api/packages/maven/maven.go | 41 +++++++------- routers/api/packages/npm/npm.go | 35 ++++++------ routers/api/packages/nuget/nuget.go | 53 +++++++++--------- routers/api/packages/pypi/pypi.go | 15 ++--- routers/api/packages/rubygems/rubygems.go | 31 ++++++----- routers/api/v1/packages/package.go | 6 +- routers/web/admin/packages.go | 4 +- routers/web/repo/packages.go | 6 +- routers/web/user/package.go | 11 ++-- services/packages/packages.go | 30 +++++----- 32 files changed, 292 insertions(+), 268 deletions(-) diff --git a/integrations/api_packages_composer_test.go b/integrations/api_packages_composer_test.go index d9987df962e90..d85293fede21a 100644 --- a/integrations/api_packages_composer_test.go +++ b/integrations/api_packages_composer_test.go @@ -87,11 +87,11 @@ func TestPackageComposer(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeComposer) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeComposer) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.NotNil(t, pd.SemVer) assert.IsType(t, &composer_module.Metadata{}, pd.Metadata) @@ -117,7 +117,7 @@ func TestPackageComposer(t *testing.T) { t.Run("Download", func(t *testing.T) { defer PrintCurrentTest(t)() - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeComposer) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeComposer) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(0), pvs[0].DownloadCount) @@ -132,7 +132,7 @@ func TestPackageComposer(t *testing.T) { assert.Equal(t, content, resp.Body.Bytes()) - pvs, err = packages.GetVersionsByPackageType(user.ID, packages.TypeComposer) + pvs, err = packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeComposer) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) diff --git a/integrations/api_packages_conan_test.go b/integrations/api_packages_conan_test.go index 0293bcc9f8406..65d16801fc596 100644 --- a/integrations/api_packages_conan_test.go +++ b/integrations/api_packages_conan_test.go @@ -260,11 +260,11 @@ func TestPackageConan(t *testing.T) { t.Run("Validate", func(t *testing.T) { defer PrintCurrentTest(t)() - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeConan) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeConan) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.NotNil(t, pd.SemVer) assert.Equal(t, name, pd.Package.Name) @@ -518,7 +518,7 @@ func TestPackageConan(t *testing.T) { t.Run("Validate", func(t *testing.T) { defer PrintCurrentTest(t)() - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeConan) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeConan) assert.NoError(t, err) assert.Len(t, pvs, 2) }) diff --git a/integrations/api_packages_generic_test.go b/integrations/api_packages_generic_test.go index 82ffdf13abd7a..c507702eaafd1 100644 --- a/integrations/api_packages_generic_test.go +++ b/integrations/api_packages_generic_test.go @@ -36,11 +36,11 @@ func TestPackageGeneric(t *testing.T) { AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeGeneric) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeGeneric) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.NotNil(t, pd.SemVer) assert.Nil(t, pd.Metadata) @@ -74,7 +74,7 @@ func TestPackageGeneric(t *testing.T) { assert.Equal(t, content, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeGeneric) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeGeneric) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) @@ -87,7 +87,7 @@ func TestPackageGeneric(t *testing.T) { AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeGeneric) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeGeneric) assert.NoError(t, err) assert.Empty(t, pvs) }) diff --git a/integrations/api_packages_maven_test.go b/integrations/api_packages_maven_test.go index 63a64385b5788..c7c45426859e7 100644 --- a/integrations/api_packages_maven_test.go +++ b/integrations/api_packages_maven_test.go @@ -44,11 +44,11 @@ func TestPackageMaven(t *testing.T) { putFile(t, fmt.Sprintf("/%s/%s", packageVersion, filename), "test", http.StatusCreated) putFile(t, "/maven-metadata.xml", "test", http.StatusOK) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.Nil(t, pd.SemVer) assert.Nil(t, pd.Metadata) @@ -81,7 +81,7 @@ func TestPackageMaven(t *testing.T) { assert.Equal(t, []byte("test"), resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(0), pvs[0].DownloadCount) @@ -113,21 +113,21 @@ func TestPackageMaven(t *testing.T) { t.Run("UploadPOM", func(t *testing.T) { defer PrintCurrentTest(t)() - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.Nil(t, pd.Metadata) putFile(t, fmt.Sprintf("/%s/%s.pom", packageVersion, filename), pomContent, http.StatusCreated) - pvs, err = packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) + pvs, err = packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err = packages.GetPackageDescriptor(pvs[0]) + pd, err = packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.IsType(t, &maven.Metadata{}, pd.Metadata) assert.Equal(t, packageDescription, pd.Metadata.(*maven.Metadata).Description) @@ -152,7 +152,7 @@ func TestPackageMaven(t *testing.T) { assert.Equal(t, []byte(pomContent), resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeMaven) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index 8e6cbce4df62c..4d9eeb3aae575 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -73,11 +73,11 @@ func TestPackageNpm(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNpm) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNpm) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.NotNil(t, pd.SemVer) assert.IsType(t, &npm.Metadata{}, pd.Metadata) @@ -116,7 +116,7 @@ func TestPackageNpm(t *testing.T) { b, _ := base64.StdEncoding.DecodeString(data) assert.Equal(t, b, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNpm) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNpm) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) diff --git a/integrations/api_packages_nuget_test.go b/integrations/api_packages_nuget_test.go index 661500a77696c..e69dd0ff9b669 100644 --- a/integrations/api_packages_nuget_test.go +++ b/integrations/api_packages_nuget_test.go @@ -99,11 +99,11 @@ func TestPackageNuGet(t *testing.T) { req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.NotNil(t, pd.SemVer) assert.IsType(t, &nuget_module.Metadata{}, pd.Metadata) @@ -166,11 +166,11 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.NotNil(t, pd.SemVer) assert.IsType(t, &nuget_module.Metadata{}, pd.Metadata) @@ -216,7 +216,7 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) defer PrintCurrentTest(t)() checkDownloadCount := func(count int64) { - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, count, pvs[0].DownloadCount) @@ -354,7 +354,7 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeNuGet) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNuGet) assert.NoError(t, err) assert.Empty(t, pvs) }) diff --git a/integrations/api_packages_pypi_test.go b/integrations/api_packages_pypi_test.go index 170daa69456ee..5d610df39da92 100644 --- a/integrations/api_packages_pypi_test.go +++ b/integrations/api_packages_pypi_test.go @@ -65,11 +65,11 @@ func TestPackagePyPI(t *testing.T) { filename := "test.whl" uploadFile(t, filename, content, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypePyPI) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypePyPI) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.NotNil(t, pd.SemVer) assert.IsType(t, &pypi.Metadata{}, pd.Metadata) @@ -93,11 +93,11 @@ func TestPackagePyPI(t *testing.T) { filename := "test.tar.gz" uploadFile(t, filename, content, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypePyPI) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypePyPI) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.NotNil(t, pd.SemVer) assert.IsType(t, &pypi.Metadata{}, pd.Metadata) @@ -146,7 +146,7 @@ func TestPackagePyPI(t *testing.T) { downloadFile("test.whl") downloadFile("test.tar.gz") - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypePyPI) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypePyPI) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(2), pvs[0].DownloadCount) diff --git a/integrations/api_packages_rubygems_test.go b/integrations/api_packages_rubygems_test.go index 32a374cd3f48d..269bc953b4181 100644 --- a/integrations/api_packages_rubygems_test.go +++ b/integrations/api_packages_rubygems_test.go @@ -124,11 +124,11 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) uploadFile(t, http.StatusCreated) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Len(t, pvs, 1) - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) assert.NotNil(t, pd.SemVer) assert.IsType(t, &rubygems.Metadata{}, pd.Metadata) @@ -161,7 +161,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`) assert.Equal(t, gemContent, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) @@ -180,7 +180,7 @@ DGAWSKc7zFhPJamg0qRK99TcYphehZLU4hKInFhGSUlBsZW+PtgZepn5+iDxECRzDUDGcfh6hoA4 gAAAAP//MS06Gw==`) assert.Equal(t, b, resp.Body.Bytes()) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Len(t, pvs, 1) assert.Equal(t, int64(1), pvs[0].DownloadCount) @@ -219,7 +219,7 @@ gAAAAP//MS06Gw==`) req = AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusOK) - pvs, err := packages.GetVersionsByPackageType(user.ID, packages.TypeRubyGems) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeRubyGems) assert.NoError(t, err) assert.Empty(t, pvs) }) diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 12887e76b328c..3878e7c8e2fa4 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -8,7 +8,6 @@ import ( "context" "strconv" - "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" @@ -23,6 +22,19 @@ import ( "github.com/hashicorp/go-version" ) +// PackagePropertyList is a list of package properties +type PackagePropertyList []*PackageProperty + +// GetByName gets the first property value with the specific name +func (l PackagePropertyList) GetByName(name string) string { + for _, pp := range l { + if pp.Name == name { + return pp.Value + } + } + return "" +} + // PackageDescriptor describes a package type PackageDescriptor struct { Package *Package @@ -31,16 +43,16 @@ type PackageDescriptor struct { Version *PackageVersion SemVer *version.Version Creator *user_model.User - Properties []*PackageProperty + Properties PackagePropertyList Metadata interface{} - Files []PackageFileDescriptor + Files []*PackageFileDescriptor } // PackageFileDescriptor describes a package file type PackageFileDescriptor struct { File *PackageFile Blob *PackageBlob - Properties []*PackageProperty + Properties PackagePropertyList } // WebLink returns the package's web link @@ -58,12 +70,7 @@ func (pd *PackageDescriptor) CalculateBlobSize() int64 { } // GetPackageDescriptor gets the package description for a version -func GetPackageDescriptor(pv *PackageVersion) (*PackageDescriptor, error) { - return GetPackageDescriptorCtx(db.DefaultContext, pv) -} - -// GetPackageDescriptorCtx gets the package description for a version -func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageDescriptor, error) { +func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDescriptor, error) { p, err := GetPackageByID(ctx, pv.PackageID) if err != nil { return nil, err @@ -96,21 +103,13 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD return nil, err } - pfds := make([]PackageFileDescriptor, 0, len(pfs)) + pfds := make([]*PackageFileDescriptor, 0, len(pfs)) for _, pf := range pfs { - pb, err := GetBlobByID(ctx, pf.BlobID) - if err != nil { - return nil, err - } - pfps, err := GetProperties(ctx, PropertyTypeFile, pf.ID) + pfd, err := GetPackageFileDescriptor(ctx, pf) if err != nil { return nil, err } - pfds = append(pfds, PackageFileDescriptor{ - pf, - pb, - pfps, - }) + pfds = append(pfds, pfd) } var metadata interface{} @@ -143,17 +142,34 @@ func GetPackageDescriptorCtx(ctx context.Context, pv *PackageVersion) (*PackageD Version: pv, SemVer: semVer, Creator: creator, - Properties: pvps, + Properties: PackagePropertyList(pvps), Metadata: metadata, Files: pfds, }, nil } +// GetPackageFileDescriptor gets a package file descriptor for a package file +func GetPackageFileDescriptor(ctx context.Context, pf *PackageFile) (*PackageFileDescriptor, error) { + pb, err := GetBlobByID(ctx, pf.BlobID) + if err != nil { + return nil, err + } + pfps, err := GetProperties(ctx, PropertyTypeFile, pf.ID) + if err != nil { + return nil, err + } + return &PackageFileDescriptor{ + pf, + pb, + PackagePropertyList(pfps), + }, nil +} + // GetPackageDescriptors gets the package descriptions for the versions -func GetPackageDescriptors(pvs []*PackageVersion) ([]*PackageDescriptor, error) { +func GetPackageDescriptors(ctx context.Context, pvs []*PackageVersion) ([]*PackageDescriptor, error) { pds := make([]*PackageDescriptor, 0, len(pvs)) for _, pv := range pvs { - pd, err := GetPackageDescriptor(pv) + pd, err := GetPackageDescriptor(ctx, pv) if err != nil { return nil, err } diff --git a/models/packages/package.go b/models/packages/package.go index c9f46d1393370..8a5a10eb8d4b6 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -120,8 +120,8 @@ func TryInsertPackage(ctx context.Context, p *Package) (*Package, error) { } // SetRepositoryLink sets the linked repository -func SetRepositoryLink(packageID, repoID int64) error { - _, err := db.GetEngine(db.DefaultContext).ID(packageID).Cols("repo_id").Update(&Package{RepoID: repoID}) +func SetRepositoryLink(ctx context.Context, packageID, repoID int64) error { + _, err := db.GetEngine(ctx).ID(packageID).Cols("repo_id").Update(&Package{RepoID: repoID}) return err } @@ -181,6 +181,6 @@ func HasOwnerPackages(ctx context.Context, ownerID int64) (bool, error) { } // HasRepositoryPackages tests if a repository has packages -func HasRepositoryPackages(repositoryID int64) (bool, error) { - return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repositoryID).Exist(&Package{}) +func HasRepositoryPackages(ctx context.Context, repositoryID int64) (bool, error) { + return db.GetEngine(ctx).Where("repo_id = ?", repositoryID).Exist(&Package{}) } diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 37a8832352506..54538f5423dff 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -62,14 +62,14 @@ func GetOrInsertVersion(ctx context.Context, pv *PackageVersion) (*PackageVersio } // UpdateVersion updates a version -func UpdateVersion(pv *PackageVersion) error { - _, err := db.GetEngine(db.DefaultContext).ID(pv.ID).Update(pv) +func UpdateVersion(ctx context.Context, pv *PackageVersion) error { + _, err := db.GetEngine(ctx).ID(pv.ID).Update(pv) return err } // IncrementDownloadCounter increments the download counter of a version -func IncrementDownloadCounter(versionID int64) error { - _, err := db.GetEngine(db.DefaultContext).Exec("UPDATE `package_version` SET `download_count` = `download_count` + 1 WHERE `id` = ?", versionID) +func IncrementDownloadCounter(ctx context.Context, versionID int64) error { + _, err := db.GetEngine(ctx).Exec("UPDATE `package_version` SET `download_count` = `download_count` + 1 WHERE `id` = ?", versionID) return err } @@ -112,21 +112,21 @@ func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType } // GetVersionsByPackageType gets all versions of a specific type -func GetVersionsByPackageType(ownerID int64, packageType Type) ([]*PackageVersion, error) { +func GetVersionsByPackageType(ctx context.Context, ownerID int64, packageType Type) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ "package.owner_id": ownerID, "package.type": packageType, } pvs := make([]*PackageVersion, 0, 10) - return pvs, db.GetEngine(db.DefaultContext). + return pvs, db.GetEngine(ctx). Where(cond). Join("INNER", "package", "package.id = package_version.package_id"). Find(&pvs) } // GetVersionsByPackageName gets all versions of a specific package -func GetVersionsByPackageName(ownerID int64, packageType Type, name string) ([]*PackageVersion, error) { +func GetVersionsByPackageName(ctx context.Context, ownerID int64, packageType Type, name string) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ "package.owner_id": ownerID, "package.type": packageType, @@ -134,14 +134,14 @@ func GetVersionsByPackageName(ownerID int64, packageType Type, name string) ([]* } pvs := make([]*PackageVersion, 0, 10) - return pvs, db.GetEngine(db.DefaultContext). + return pvs, db.GetEngine(ctx). Where(cond). Join("INNER", "package", "package.id = package_version.package_id"). Find(&pvs) } // GetVersionsByFilename gets all versions which are linked to a filename -func GetVersionsByFilename(ownerID int64, packageType Type, filename string) ([]*PackageVersion, error) { +func GetVersionsByFilename(ctx context.Context, ownerID int64, packageType Type, filename string) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ "package.owner_id": ownerID, "package.type": packageType, @@ -149,7 +149,7 @@ func GetVersionsByFilename(ownerID int64, packageType Type, filename string) ([] } pvs := make([]*PackageVersion, 0, 10) - return pvs, db.GetEngine(db.DefaultContext). + return pvs, db.GetEngine(ctx). Where(cond). Join("INNER", "package_file", "package_file.version_id = package_version.id"). Join("INNER", "package", "package.id = package_version.package_id"). @@ -237,8 +237,8 @@ func (opts *PackageSearchOptions) configureOrderBy(e db.Engine) { } // SearchVersions gets all versions of packages matching the search options -func SearchVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, error) { - sess := db.GetEngine(db.DefaultContext). +func SearchVersions(ctx context.Context, opts *PackageSearchOptions) ([]*PackageVersion, int64, error) { + sess := db.GetEngine(ctx). Where(opts.toConds()). Table("package_version"). Join("INNER", "package", "package.id = package_version.package_id") @@ -255,11 +255,11 @@ func SearchVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, error } // SearchLatestVersions gets the latest version of every package matching the search options -func SearchLatestVersions(opts *PackageSearchOptions) ([]*PackageVersion, int64, error) { +func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*PackageVersion, int64, error) { cond := opts.toConds(). And(builder.Expr("pv2.id IS NULL")) - sess := db.GetEngine(db.DefaultContext). + sess := db.GetEngine(ctx). Table("package_version"). Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND package_version.created_unix < pv2.created_unix"). Join("INNER", "package", "package.id = package_version.package_id"). diff --git a/modules/context/package.go b/modules/context/package.go index eb1a72fbb4006..bc411ef0ab750 100644 --- a/modules/context/package.go +++ b/modules/context/package.go @@ -86,9 +86,9 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) { return } - ctx.Package.Descriptor, err = packages_model.GetPackageDescriptorCtx(ctx, pv) + ctx.Package.Descriptor, err = packages_model.GetPackageDescriptor(ctx, pv) if err != nil { - errCb(http.StatusInternalServerError, "GetPackageDescriptorCtx", err) + errCb(http.StatusInternalServerError, "GetPackageDescriptor", err) return } diff --git a/modules/packages/conan/metadata.go b/modules/packages/conan/metadata.go index fc82d87fe328c..a7d6a9df0b4c4 100644 --- a/modules/packages/conan/metadata.go +++ b/modules/packages/conan/metadata.go @@ -15,10 +15,10 @@ const ( // Metadata represents the metadata of a Conan package type Metadata struct { - Author string `json:"author"` - License string `json:"license"` - ProjectURL string `json:"project_url"` - RepositoryURL string `json:"repository_url"` - Description string `json:"description"` - Keywords []string `json:"keywords"` + Author string `json:"author,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Description string `json:"description,omitempty"` + Keywords []string `json:"keywords,omitempty"` } diff --git a/modules/packages/maven/metadata.go b/modules/packages/maven/metadata.go index 6e0dd88a6add8..6ee9d69687ced 100644 --- a/modules/packages/maven/metadata.go +++ b/modules/packages/maven/metadata.go @@ -13,20 +13,20 @@ import ( // Metadata represents the metadata of a Maven package type Metadata struct { - GroupID string `json:"group_id"` - ArtifactID string `json:"artifact_id"` - Name string `json:"name"` - Description string `json:"description"` - ProjectURL string `json:"project_url"` - Licenses []string `json:"licenses"` - Dependencies []*Dependency `json:"dependencies"` + GroupID string `json:"group_id,omitempty"` + ArtifactID string `json:"artifact_id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Licenses []string `json:"licenses,omitempty"` + Dependencies []*Dependency `json:"dependencies,omitempty"` } // Dependency represents a dependency of a Maven package type Dependency struct { - GroupID string `json:"group_id"` - ArtifactID string `json:"artifact_id"` - Version string `json:"version"` + GroupID string `json:"group_id,omitempty"` + ArtifactID string `json:"artifact_id,omitempty"` + Version string `json:"version,omitempty"` } type pomStruct struct { diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index 9543e94f9b646..643a4d344b7de 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -9,16 +9,16 @@ const TagProperty = "npm.tag" // Metadata represents the metadata of a npm package type Metadata struct { - Scope string `json:"scope"` - Name string `json:"name"` - Description string `json:"description"` - Author string `json:"author"` - License string `json:"license"` - ProjectURL string `json:"project_url"` - Keywords []string `json:"keywords"` - Dependencies map[string]string `json:"dependencies"` - DevelopmentDependencies map[string]string `json:"development_dependencies"` - PeerDependencies map[string]string `json:"peer_dependencies"` - OptionalDependencies map[string]string `json:"optional_dependencies"` - Readme string `json:"readme"` + Scope string `json:"scope,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Author string `json:"author,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevelopmentDependencies map[string]string `json:"development_dependencies,omitempty"` + PeerDependencies map[string]string `json:"peer_dependencies,omitempty"` + OptionalDependencies map[string]string `json:"optional_dependencies,omitempty"` + Readme string `json:"readme,omitempty"` } diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index bb5ebe9c1eb1d..797bff45ac632 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -55,12 +55,12 @@ type Package struct { // Metadata represents the metadata of a Nuget package type Metadata struct { - Description string `json:"description"` - ReleaseNotes string `json:"release_notes"` - Authors string `json:"authors"` - ProjectURL string `json:"project_url"` - RepositoryURL string `json:"repository_url"` - Dependencies map[string][]Dependency `json:"dependencies"` + Description string `json:"description,omitempty"` + ReleaseNotes string `json:"release_notes,omitempty"` + Authors string `json:"authors,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Dependencies map[string][]Dependency `json:"dependencies,omitempty"` } // Dependency represents a dependency of a Nuget package diff --git a/modules/packages/pypi/metadata.go b/modules/packages/pypi/metadata.go index 27fc585975644..df367d10e21e3 100644 --- a/modules/packages/pypi/metadata.go +++ b/modules/packages/pypi/metadata.go @@ -6,11 +6,11 @@ package pypi // Metadata represents the metadata of a PyPI package type Metadata struct { - Author string `json:"author"` - Description string `json:"description"` - LongDescription string `json:"long_description"` - Summary string `json:"summary"` - ProjectURL string `json:"project_url"` - License string `json:"license"` - RequiresPython string `json:"requires_python"` + Author string `json:"author,omitempty"` + Description string `json:"description,omitempty"` + LongDescription string `json:"long_description,omitempty"` + Summary string `json:"summary,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + License string `json:"license,omitempty"` + RequiresPython string `json:"requires_python,omitempty"` } diff --git a/modules/packages/rubygems/metadata.go b/modules/packages/rubygems/metadata.go index f337ba3530f2d..942f205fc3157 100644 --- a/modules/packages/rubygems/metadata.go +++ b/modules/packages/rubygems/metadata.go @@ -37,16 +37,16 @@ type Package struct { // Metadata represents the metadata of a RubyGems package type Metadata struct { - Platform string `json:"platform"` - Description string `json:"description"` - Summary string `json:"summary"` - Authors []string `json:"authors"` - Licenses []string `json:"licenses"` - RequiredRubyVersion []VersionRequirement `json:"required_ruby_version"` - RequiredRubygemsVersion []VersionRequirement `json:"required_rubygems_version"` - ProjectURL string `json:"project_url"` - RuntimeDependencies []Dependency `json:"runtime_dependencies"` - DevelopmentDependencies []Dependency `json:"development_dependencies"` + Platform string `json:"platform,omitempty"` + Description string `json:"description,omitempty"` + Summary string `json:"summary,omitempty"` + Authors []string `json:"authors,omitempty"` + Licenses []string `json:"licenses,omitempty"` + RequiredRubyVersion []VersionRequirement `json:"required_ruby_version,omitempty"` + RequiredRubygemsVersion []VersionRequirement `json:"required_rubygems_version,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RuntimeDependencies []Dependency `json:"runtime_dependencies,omitempty"` + DevelopmentDependencies []Dependency `json:"development_dependencies,omitempty"` } // VersionRequirement represents a version restriction diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index bcd81e4d8ddb7..c952bf75b42c7 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -13,7 +13,7 @@ import ( "strings" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" packages_module "code.gitea.io/gitea/modules/packages" @@ -61,9 +61,9 @@ func SearchPackages(ctx *context.Context) { PageSize: convert.ToCorrectPageSize(perPage), } - opts := &packages.PackageSearchOptions{ + opts := &packages_model.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, - Type: string(packages.TypeComposer), + Type: string(packages_model.TypeComposer), Query: ctx.FormTrim("q"), Paginator: &paginator, } @@ -73,7 +73,7 @@ func SearchPackages(ctx *context.Context) { } } - pvs, total, err := packages.SearchLatestVersions(opts) + pvs, total, err := packages_model.SearchLatestVersions(ctx, opts) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -98,7 +98,7 @@ func SearchPackages(ctx *context.Context) { nextLink = u.String() } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -112,7 +112,7 @@ func SearchPackages(ctx *context.Context) { // EnumeratePackages lists all package names // https://packagist.org/apidoc#list-packages func EnumeratePackages(ctx *context.Context) { - ps, err := packages.GetPackagesByType(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeComposer) + ps, err := packages_model.GetPackagesByType(db.DefaultContext, ctx.Package.Owner.ID, packages_model.TypeComposer) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -134,17 +134,17 @@ func PackageMetadata(ctx *context.Context) { vendorName := ctx.Params("vendorname") projectName := ctx.Params("projectname") - pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeComposer, vendorName+"/"+projectName) + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeComposer, vendorName+"/"+projectName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } if len(pvs) == 0 { - apiError(ctx, http.StatusNotFound, packages.ErrPackageNotExist) + apiError(ctx, http.StatusNotFound, packages_model.ErrPackageNotExist) return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -164,12 +164,13 @@ func DownloadPackageFile(ctx *context.Context) { fileID := ctx.ParamsInt64("fileid") s, pf, err := packages_service.GetFileStreamByPackageVersionAndFileID( + ctx, ctx.Package.Owner, versionID, fileID, ) if err != nil { - if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -214,7 +215,7 @@ func UploadPackage(ctx *context.Context) { &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeComposer, + PackageType: packages_model.TypeComposer, Name: cp.Name, Version: cp.Version, }, @@ -234,7 +235,7 @@ func UploadPackage(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrDuplicatePackageVersion { + if err == packages_model.ErrDuplicatePackageVersion { apiError(ctx, http.StatusBadRequest, err) return } diff --git a/routers/api/packages/conan/auth.go b/routers/api/packages/conan/auth.go index a718053a27ae1..00855a97a47a2 100644 --- a/routers/api/packages/conan/auth.go +++ b/routers/api/packages/conan/auth.go @@ -19,6 +19,7 @@ func (a *Auth) Name() string { return "conan" } +// Verify extracts the user from the Bearer token func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) *user_model.User { uid, err := packages.ParseAuthorizationToken(req) if err != nil { diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index 73e61489104a0..2ccb88066a458 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -385,7 +385,7 @@ func uploadFile(ctx *context.Context, fileFilter stringSet, fileKey string) { return } pv.MetadataJSON = string(raw) - if err := packages_model.UpdateVersion(pv); err != nil { + if err := packages_model.UpdateVersion(ctx, pv); err != nil { apiError(ctx, http.StatusInternalServerError, err) return } @@ -453,6 +453,7 @@ func downloadFile(ctx *context.Context, fileFilter stringSet, fileKey string) { } s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, PackageType: packages_model.TypeConan, @@ -612,7 +613,7 @@ func deleteRecipeOrPackage(apictx *context.Context, rref *conan_module.RecipeRef return err } - pd, err := packages_model.GetPackageDescriptorCtx(ctx, pv) + pd, err := packages_model.GetPackageDescriptor(ctx, pv) if err != nil { return err } diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index ca61eddc64b3a..e7b20e7da9726 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -9,7 +9,7 @@ import ( "net/http" "regexp" - "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" @@ -39,9 +39,10 @@ func DownloadPackageFile(ctx *context.Context) { } s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeGeneric, + PackageType: packages_model.TypeGeneric, Name: packageName, Version: packageVersion, }, @@ -50,7 +51,7 @@ func DownloadPackageFile(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -92,7 +93,7 @@ func UploadPackage(ctx *context.Context) { &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeGeneric, + PackageType: packages_model.TypeGeneric, Name: packageName, Version: packageVersion, }, @@ -108,7 +109,7 @@ func UploadPackage(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrDuplicatePackageVersion { + if err == packages_model.ErrDuplicatePackageVersion { apiError(ctx, http.StatusBadRequest, err) return } @@ -131,13 +132,13 @@ func DeletePackage(ctx *context.Context) { ctx.User, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeGeneric, + PackageType: packages_model.TypeGeneric, Name: packageName, Version: packageVersion, }, ) if err != nil { - if err == packages.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) return } diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index ca1c116b975bc..b023d18959335 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -18,8 +18,7 @@ import ( "regexp" "strings" - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" @@ -67,17 +66,17 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { // /com/foo/project/maven-metadata.xml[.md5/.sha1/.sha256/.sha512] packageName := params.GroupID + "-" + params.ArtifactID - pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeMaven, packageName) + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeMaven, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } if len(pvs) == 0 { - apiError(ctx, http.StatusNotFound, packages.ErrPackageNotExist) + apiError(ctx, http.StatusNotFound, packages_model.ErrPackageNotExist) return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -117,9 +116,9 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { func servePackageFile(ctx *context.Context, params parameters) { packageName := params.GroupID + "-" + params.ArtifactID - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeMaven, packageName, params.Version) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeMaven, packageName, params.Version) if err != nil { - if err == packages.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -134,9 +133,9 @@ func servePackageFile(ctx *context.Context, params parameters) { filename = filename[:len(filename)-len(ext)] } - pf, err := packages.GetFileForVersionByName(db.DefaultContext, pv.ID, filename, packages.EmptyFileKey) + pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, filename, packages_model.EmptyFileKey) if err != nil { - if err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -144,7 +143,7 @@ func servePackageFile(ctx *context.Context, params parameters) { return } - pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -173,7 +172,7 @@ func servePackageFile(ctx *context.Context, params parameters) { defer s.Close() if pf.IsLead { - if err := packages.IncrementDownloadCounter(pv.ID); err != nil { + if err := packages_model.IncrementDownloadCounter(ctx, pv.ID); err != nil { log.Error("Error incrementing download counter: %v", err) } } @@ -209,7 +208,7 @@ func UploadPackageFile(ctx *context.Context) { pvci := &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeMaven, + PackageType: packages_model.TypeMaven, Name: packageName, Version: params.Version, }, @@ -221,25 +220,25 @@ func UploadPackageFile(ctx *context.Context) { // Do not upload checksum files but compare the hashes. if isChecksumExtension(ext) { - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) if err != nil { - if err == packages.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) return } apiError(ctx, http.StatusInternalServerError, err) return } - pf, err := packages.GetFileForVersionByName(db.DefaultContext, pv.ID, params.Filename[:len(params.Filename)-len(ext)], packages.EmptyFileKey) + pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, params.Filename[:len(params.Filename)-len(ext)], packages_model.EmptyFileKey) if err != nil { - if err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) return } apiError(ctx, http.StatusInternalServerError, err) return } - pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -282,8 +281,8 @@ func UploadPackageFile(ctx *context.Context) { } if pvci.Metadata != nil { - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) - if err != nil && err != packages.ErrPackageNotExist { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvci.Owner.ID, pvci.PackageType, pvci.Name, pvci.Version) + if err != nil && err != packages_model.ErrPackageNotExist { apiError(ctx, http.StatusInternalServerError, err) return } @@ -294,7 +293,7 @@ func UploadPackageFile(ctx *context.Context) { return } pv.MetadataJSON = string(raw) - if err := packages.UpdateVersion(pv); err != nil { + if err := packages_model.UpdateVersion(ctx, pv); err != nil { apiError(ctx, http.StatusInternalServerError, err) return } @@ -312,7 +311,7 @@ func UploadPackageFile(ctx *context.Context) { pfci, ) if err != nil { - if err == packages.ErrDuplicatePackageFile { + if err == packages_model.ErrDuplicatePackageFile { apiError(ctx, http.StatusBadRequest, err) return } diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 1e6ecbb90d1bd..eb92d47eacb2c 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -14,7 +14,7 @@ import ( "strings" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" packages_module "code.gitea.io/gitea/modules/packages" npm_module "code.gitea.io/gitea/modules/packages/npm" @@ -55,7 +55,7 @@ func PackageMetadata(ctx *context.Context) { return } - pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNpm, packageName) + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -65,7 +65,7 @@ func PackageMetadata(ctx *context.Context) { return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -90,9 +90,10 @@ func DownloadPackageFile(ctx *context.Context) { filename := ctx.Params("filename") s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeNpm, + PackageType: packages_model.TypeNpm, Name: packageName, Version: packageVersion, }, @@ -101,7 +102,7 @@ func DownloadPackageFile(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -132,7 +133,7 @@ func UploadPackage(ctx *context.Context) { &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeNpm, + PackageType: packages_model.TypeNpm, Name: npmPackage.Name, Version: npmPackage.Version, }, @@ -149,7 +150,7 @@ func UploadPackage(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrDuplicatePackageVersion { + if err == packages_model.ErrDuplicatePackageVersion { apiError(ctx, http.StatusBadRequest, err) return } @@ -179,7 +180,7 @@ func ListPackageTags(ctx *context.Context) { return } - pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNpm, packageName) + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -187,7 +188,7 @@ func ListPackageTags(ctx *context.Context) { tags := make(map[string]string) for _, pv := range pvs { - pvps, err := packages.GetPropertiesByName(ctx, packages.PropertyTypeVersion, pv.ID, npm_module.TagProperty) + pvps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, npm_module.TagProperty) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -215,9 +216,9 @@ func AddPackageTag(ctx *context.Context) { } version := strings.Trim(string(body), "\"") // is as "version" in the body - pv, err := packages.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages.TypeNpm, packageName, version) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName, version) if err != nil { - if err == packages.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -243,7 +244,7 @@ func DeletePackageTag(ctx *context.Context) { return } - pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNpm, packageName) + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -261,7 +262,7 @@ func DeletePackageTag(ctx *context.Context) { } } -func setPackageTag(tag string, pv *packages.PackageVersion, deleteOnly bool) error { +func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly bool) error { if tag == "" { return errInvalidTagName } @@ -276,20 +277,20 @@ func setPackageTag(tag string, pv *packages.PackageVersion, deleteOnly bool) err } defer committer.Close() - pvs, err := packages.FindVersionsByPropertyNameAndValue(ctx, pv.PackageID, npm_module.TagProperty, tag) + pvs, err := packages_model.FindVersionsByPropertyNameAndValue(ctx, pv.PackageID, npm_module.TagProperty, tag) if err != nil { return err } if len(pvs) == 1 { - pvps, err := packages.GetPropertiesByName(ctx, packages.PropertyTypeVersion, pvs[0].ID, npm_module.TagProperty) + pvps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pvs[0].ID, npm_module.TagProperty) if err != nil { return err } for _, pvp := range pvps { if pvp.Value == tag { - if err := packages.DeletePropertyByID(ctx, pvp.ID); err != nil { + if err := packages_model.DeletePropertyByID(ctx, pvp.ID); err != nil { return err } break @@ -298,7 +299,7 @@ func setPackageTag(tag string, pv *packages.PackageVersion, deleteOnly bool) err } if !deleteOnly { - _, err = packages.InsertProperty(ctx, packages.PropertyTypeVersion, pv.ID, npm_module.TagProperty, tag) + _, err = packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, npm_module.TagProperty, tag) if err != nil { return err } diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index 2df9dfbf4a594..19e06c2bdff49 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -12,7 +12,7 @@ import ( "strings" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" packages_module "code.gitea.io/gitea/modules/packages" nuget_module "code.gitea.io/gitea/modules/packages/nuget" @@ -38,9 +38,9 @@ func ServiceIndex(ctx *context.Context) { // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages func SearchService(ctx *context.Context) { - pvs, count, err := packages.SearchVersions(&packages.PackageSearchOptions{ + pvs, count, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, - Type: string(packages.TypeNuGet), + Type: string(packages_model.TypeNuGet), Query: ctx.FormTrim("q"), Paginator: db.NewAbsoluteListOptions( ctx.FormInt("skip"), @@ -52,7 +52,7 @@ func SearchService(ctx *context.Context) { return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -71,7 +71,7 @@ func SearchService(ctx *context.Context) { func RegistrationIndex(ctx *context.Context) { packageName := ctx.Params("id") - pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNuGet, packageName) + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNuGet, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -81,7 +81,7 @@ func RegistrationIndex(ctx *context.Context) { return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -100,9 +100,9 @@ func RegistrationLeaf(ctx *context.Context) { packageName := ctx.Params("id") packageVersion := strings.TrimSuffix(ctx.Params("version"), ".json") - pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages.TypeNuGet, packageName, packageVersion) + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, ctx.Package.Owner.ID, packages_model.TypeNuGet, packageName, packageVersion) if err != nil { - if err == packages.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -110,7 +110,7 @@ func RegistrationLeaf(ctx *context.Context) { return } - pd, err := packages.GetPackageDescriptor(pv) + pd, err := packages_model.GetPackageDescriptor(ctx, pv) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -128,7 +128,7 @@ func RegistrationLeaf(ctx *context.Context) { func EnumeratePackageVersions(ctx *context.Context) { packageName := ctx.Params("id") - pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypeNuGet, packageName) + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNuGet, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -138,7 +138,7 @@ func EnumeratePackageVersions(ctx *context.Context) { return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -156,9 +156,10 @@ func DownloadPackageFile(ctx *context.Context) { filename := ctx.Params("filename") s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeNuGet, + PackageType: packages_model.TypeNuGet, Name: packageName, Version: packageVersion, }, @@ -167,7 +168,7 @@ func DownloadPackageFile(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -196,7 +197,7 @@ func UploadPackage(ctx *context.Context) { &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeNuGet, + PackageType: packages_model.TypeNuGet, Name: np.ID, Version: np.Version, }, @@ -213,7 +214,7 @@ func UploadPackage(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrDuplicatePackageVersion { + if err == packages_model.ErrDuplicatePackageVersion { apiError(ctx, http.StatusBadRequest, err) return } @@ -251,7 +252,7 @@ func UploadSymbolPackage(ctx *context.Context) { pi := &packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeNuGet, + PackageType: packages_model.TypeNuGet, Name: np.ID, Version: np.Version, } @@ -268,9 +269,9 @@ func UploadSymbolPackage(ctx *context.Context) { ) if err != nil { switch err { - case packages.ErrPackageNotExist: + case packages_model.ErrPackageNotExist: apiError(ctx, http.StatusNotFound, err) - case packages.ErrDuplicatePackageFile: + case packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusBadRequest, err) default: apiError(ctx, http.StatusInternalServerError, err) @@ -295,7 +296,7 @@ func UploadSymbolPackage(ctx *context.Context) { ) if err != nil { switch err { - case packages.ErrDuplicatePackageFile: + case packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusBadRequest, err) default: apiError(ctx, http.StatusInternalServerError, err) @@ -358,9 +359,9 @@ func DownloadSymbolFile(ctx *context.Context) { return } - pfs, _, err := packages.SearchFiles(ctx, &packages.PackageFileSearchOptions{ + pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{ OwnerID: ctx.Package.Owner.ID, - PackageType: string(packages.TypeNuGet), + PackageType: string(packages_model.TypeNuGet), Query: filename, Properties: map[string]string{ nuget_module.PropertySymbolID: strings.ToLower(guid), @@ -375,15 +376,15 @@ func DownloadSymbolFile(ctx *context.Context) { return } - pv, err := packages.GetVersionByID(ctx, pfs[0].VersionID) + pv, err := packages_model.GetVersionByID(ctx, pfs[0].VersionID) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return } - s, _, err := packages_service.GetPackageFileStream(pv, pfs[0]) + s, _, err := packages_service.GetPackageFileStream(ctx, pv, pfs[0]) if err != nil { - if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -405,13 +406,13 @@ func DeletePackage(ctx *context.Context) { ctx.User, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeNuGet, + PackageType: packages_model.TypeNuGet, Name: packageName, Version: packageVersion, }, ) if err != nil { - if err == packages.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) return } diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index c86c932d19525..590ec7c99ed73 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -11,7 +11,7 @@ import ( "regexp" "strings" - "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" packages_module "code.gitea.io/gitea/modules/packages" pypi_module "code.gitea.io/gitea/modules/packages/pypi" @@ -39,7 +39,7 @@ func apiError(ctx *context.Context, status int, obj interface{}) { func PackageMetadata(ctx *context.Context) { packageName := normalizer.Replace(ctx.Params("id")) - pvs, err := packages.GetVersionsByPackageName(ctx.Package.Owner.ID, packages.TypePyPI, packageName) + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -49,7 +49,7 @@ func PackageMetadata(ctx *context.Context) { return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -69,9 +69,10 @@ func DownloadPackageFile(ctx *context.Context) { filename := ctx.Params("filename") s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypePyPI, + PackageType: packages_model.TypePyPI, Name: packageName, Version: packageVersion, }, @@ -80,7 +81,7 @@ func DownloadPackageFile(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -136,7 +137,7 @@ func UploadPackageFile(ctx *context.Context) { &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypePyPI, + PackageType: packages_model.TypePyPI, Name: packageName, Version: packageVersion, }, @@ -161,7 +162,7 @@ func UploadPackageFile(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrDuplicatePackageFile { + if err == packages_model.ErrDuplicatePackageFile { apiError(ctx, http.StatusBadRequest, err) return } diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index 40323624e2ccb..e79122830b9c0 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -12,7 +12,7 @@ import ( "net/http" "strings" - "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/context" packages_module "code.gitea.io/gitea/modules/packages" rubygems_module "code.gitea.io/gitea/modules/packages/rubygems" @@ -28,7 +28,7 @@ func apiError(ctx *context.Context, status int, obj interface{}) { // EnumeratePackages serves the package list func EnumeratePackages(ctx *context.Context) { - packages, err := packages.GetVersionsByPackageType(ctx.Package.Owner.ID, packages.TypeRubyGems) + packages, err := packages_model.GetVersionsByPackageType(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -39,9 +39,9 @@ func EnumeratePackages(ctx *context.Context) { // EnumeratePackagesLatest serves the list of the lastest version of every package func EnumeratePackagesLatest(ctx *context.Context) { - pvs, _, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ + pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, - Type: string(packages.TypeRubyGems), + Type: string(packages_model.TypeRubyGems), }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -53,11 +53,11 @@ func EnumeratePackagesLatest(ctx *context.Context) { // EnumeratePackagesPreRelease is not supported and serves an empty list func EnumeratePackagesPreRelease(ctx *context.Context) { - enumeratePackages(ctx, "prerelease_specs.4.8", []*packages.PackageVersion{}) + enumeratePackages(ctx, "prerelease_specs.4.8", []*packages_model.PackageVersion{}) } -func enumeratePackages(ctx *context.Context, filename string, pvs []*packages.PackageVersion) { - pds, err := packages.GetPackageDescriptors(pvs) +func enumeratePackages(ctx *context.Context, filename string, pvs []*packages_model.PackageVersion) { + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -96,7 +96,7 @@ func ServePackageSpecification(ctx *context.Context) { return } - pvs, err := packages.GetVersionsByFilename(ctx.Package.Owner.ID, packages.TypeRubyGems, filename[:len(filename)-10]+"gem") + pvs, err := packages_model.GetVersionsByFilename(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems, filename[:len(filename)-10]+"gem") if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -107,7 +107,7 @@ func ServePackageSpecification(ctx *context.Context) { return } - pd, err := packages.GetPackageDescriptor(pvs[0]) + pd, err := packages_model.GetPackageDescriptor(ctx, pvs[0]) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -158,7 +158,7 @@ func ServePackageSpecification(ctx *context.Context) { func DownloadPackageFile(ctx *context.Context) { filename := ctx.Params("filename") - pvs, err := packages.GetVersionsByFilename(ctx.Package.Owner.ID, packages.TypeRubyGems, filename) + pvs, err := packages_model.GetVersionsByFilename(ctx, ctx.Package.Owner.ID, packages_model.TypeRubyGems, filename) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -170,13 +170,14 @@ func DownloadPackageFile(ctx *context.Context) { } s, pf, err := packages_service.GetFileStreamByPackageVersion( + ctx, pvs[0], &packages_service.PackageFileInfo{ Filename: filename, }, ) if err != nil { - if err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageFileNotExist { apiError(ctx, http.StatusNotFound, err) return } @@ -227,7 +228,7 @@ func UploadPackageFile(ctx *context.Context) { &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeRubyGems, + PackageType: packages_model.TypeRubyGems, Name: rp.Name, Version: rp.Version, }, @@ -244,7 +245,7 @@ func UploadPackageFile(ctx *context.Context) { }, ) if err != nil { - if err == packages.ErrDuplicatePackageVersion { + if err == packages_model.ErrDuplicatePackageVersion { apiError(ctx, http.StatusBadRequest, err) return } @@ -269,13 +270,13 @@ func DeletePackage(ctx *context.Context) { ctx.User, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, - PackageType: packages.TypeRubyGems, + PackageType: packages_model.TypeRubyGems, Name: packageName, Version: packageVersion, }, ) if err != nil { - if err == packages.ErrPackageNotExist { + if err == packages_model.ErrPackageNotExist { apiError(ctx, http.StatusNotFound, err) return } diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 88c35c82b0f96..1675378385547 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -54,7 +54,7 @@ func ListPackages(ctx *context.APIContext) { packageType := ctx.FormTrim("type") query := ctx.FormTrim("q") - pvs, count, err := packages.SearchVersions(&packages.PackageSearchOptions{ + pvs, count, err := packages.SearchVersions(ctx, &packages.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, Type: packageType, Query: query, @@ -65,7 +65,7 @@ func ListPackages(ctx *context.APIContext) { return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages.GetPackageDescriptors(ctx, pvs) if err != nil { ctx.Error(http.StatusInternalServerError, "GetPackageDescriptors", err) return @@ -167,7 +167,7 @@ func ListPackageFiles(ctx *context.APIContext) { apiPackageFiles := make([]*api.PackageFile, 0, len(ctx.Package.Descriptor.Files)) for _, pfd := range ctx.Package.Descriptor.Files { - apiPackageFiles = append(apiPackageFiles, convert.ToPackageFile(&pfd)) + apiPackageFiles = append(apiPackageFiles, convert.ToPackageFile(pfd)) } ctx.JSON(http.StatusOK, apiPackageFiles) diff --git a/routers/web/admin/packages.go b/routers/web/admin/packages.go index 86c26db62af43..41d6a968e3363 100644 --- a/routers/web/admin/packages.go +++ b/routers/web/admin/packages.go @@ -30,7 +30,7 @@ func Packages(ctx *context.Context) { packageType := ctx.FormTrim("type") sort := ctx.FormTrim("sort") - pvs, total, err := packages_model.SearchVersions(&packages_model.PackageSearchOptions{ + pvs, total, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ Query: query, Type: packageType, Sort: sort, @@ -44,7 +44,7 @@ func Packages(ctx *context.Context) { return } - pds, err := packages_model.GetPackageDescriptors(pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { ctx.ServerError("GetPackageDescriptors", err) return diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index 20abd827bfdb9..de251a102e1b4 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -27,7 +27,7 @@ func Packages(ctx *context.Context) { query := ctx.FormTrim("q") packageType := ctx.FormTrim("type") - pvs, total, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ + pvs, total, err := packages.SearchLatestVersions(ctx, &packages.PackageSearchOptions{ Paginator: &db.ListOptions{ PageSize: setting.UI.PackagesPagingNum, Page: page, @@ -42,13 +42,13 @@ func Packages(ctx *context.Context) { return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages.GetPackageDescriptors(ctx, pvs) if err != nil { ctx.ServerError("GetPackageDescriptors", err) return } - hasPackages, err := packages.HasRepositoryPackages(ctx.Repo.Repository.ID) + hasPackages, err := packages.HasRepositoryPackages(ctx, ctx.Repo.Repository.ID) if err != nil { ctx.ServerError("HasRepositoryPackages", err) return diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 1cc71c5a7c399..874e6692c4889 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -37,7 +37,7 @@ func Packages(ctx *context.Context) { query := ctx.FormTrim("q") packageType := ctx.FormTrim("type") - pvs, total, err := packages.SearchLatestVersions(&packages.PackageSearchOptions{ + pvs, total, err := packages.SearchLatestVersions(ctx, &packages.PackageSearchOptions{ Paginator: &db.ListOptions{ PageSize: setting.UI.PackagesPagingNum, Page: page, @@ -51,13 +51,13 @@ func Packages(ctx *context.Context) { return } - pds, err := packages.GetPackageDescriptors(pvs) + pds, err := packages.GetPackageDescriptors(ctx, pvs) if err != nil { ctx.ServerError("GetPackageDescriptors", err) return } - hasPackages, err := packages.HasOwnerPackages(db.DefaultContext, ctx.ContextUser.ID) + hasPackages, err := packages.HasOwnerPackages(ctx, ctx.ContextUser.ID) if err != nil { ctx.ServerError("HasOwnerPackages", err) return @@ -94,7 +94,7 @@ func ViewPackage(ctx *context.Context) { ctx.Data["ContextUser"] = ctx.ContextUser ctx.Data["PackageDescriptor"] = pd - otherVersions, err := packages.GetVersionsByPackageName(pd.Owner.ID, pd.Package.Type, pd.Package.LowerName) + otherVersions, err := packages.GetVersionsByPackageName(ctx, pd.Owner.ID, pd.Package.Type, pd.Package.LowerName) if err != nil { ctx.ServerError("GetVersionsByPackageName", err) return @@ -160,7 +160,7 @@ func PackageSettingsPost(ctx *context.Context) { repoID = repo.ID } - if err := packages.SetRepositoryLink(pd.Package.ID, repoID); err != nil { + if err := packages.SetRepositoryLink(ctx, pd.Package.ID, repoID); err != nil { log.Error("Error updating package: %v", err) return false } @@ -193,6 +193,7 @@ func PackageSettingsPost(ctx *context.Context) { // DownloadPackageFile serves the content of a package file func DownloadPackageFile(ctx *context.Context) { s, pf, err := packages_service.GetFileStreamByPackageVersionAndFileID( + ctx, ctx.ContextUser, ctx.ParamsInt64(":versionid"), ctx.ParamsInt64(":fileid"), diff --git a/services/packages/packages.go b/services/packages/packages.go index 1c84ac29e15fe..5f294589d63a1 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -94,7 +94,7 @@ func createPackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreatio } if created { - pd, err := packages_model.GetPackageDescriptor(pv) + pd, err := packages_model.GetPackageDescriptor(ctx, pv) if err != nil { return nil, nil, err } @@ -282,7 +282,7 @@ func DeletePackageVersion(doer *user_model.User, pv *packages_model.PackageVersi } defer committer.Close() - pd, err := packages_model.GetPackageDescriptorCtx(ctx, pv) + pd, err := packages_model.GetPackageDescriptor(ctx, pv) if err != nil { return err } @@ -353,10 +353,10 @@ func DeleteUnreferencedBlobs() error { } // GetFileStreamByPackageNameAndVersion returns the content of the specific package file -func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) { +func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey) - pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) + pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { if err == packages_model.ErrPackageNotExist { return nil, nil, err @@ -365,14 +365,14 @@ func GetFileStreamByPackageNameAndVersion(pvi *PackageInfo, pfi *PackageFileInfo return nil, nil, err } - return GetFileStreamByPackageVersion(pv, pfi) + return GetFileStreamByPackageVersion(ctx, pv, pfi) } // GetFileStreamByPackageVersionAndFileID returns the content of the specific package file -func GetFileStreamByPackageVersionAndFileID(owner *user_model.User, versionID, fileID int64) (io.ReadCloser, *packages_model.PackageFile, error) { +func GetFileStreamByPackageVersionAndFileID(ctx context.Context, owner *user_model.User, versionID, fileID int64) (io.ReadCloser, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %v", owner.ID, versionID, fileID) - pv, err := packages_model.GetVersionByID(db.DefaultContext, versionID) + pv, err := packages_model.GetVersionByID(ctx, versionID) if err != nil { if err == packages_model.ErrPackageVersionNotExist { return nil, nil, packages_model.ErrPackageNotExist @@ -381,7 +381,7 @@ func GetFileStreamByPackageVersionAndFileID(owner *user_model.User, versionID, f return nil, nil, err } - p, err := packages_model.GetPackageByID(db.DefaultContext, pv.PackageID) + p, err := packages_model.GetPackageByID(ctx, pv.PackageID) if err != nil { log.Error("Error getting package: %v", err) return nil, nil, err @@ -391,28 +391,28 @@ func GetFileStreamByPackageVersionAndFileID(owner *user_model.User, versionID, f return nil, nil, packages_model.ErrPackageNotExist } - pf, err := packages_model.GetFileForVersionByID(db.DefaultContext, versionID, fileID) + pf, err := packages_model.GetFileForVersionByID(ctx, versionID, fileID) if err != nil { log.Error("Error getting file: %v", err) return nil, nil, err } - return GetPackageFileStream(pv, pf) + return GetPackageFileStream(ctx, pv, pf) } // GetFileStreamByPackageVersion returns the content of the specific package file -func GetFileStreamByPackageVersion(pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) { +func GetFileStreamByPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadCloser, *packages_model.PackageFile, error) { pf, err := packages_model.GetFileForVersionByName(db.DefaultContext, pv.ID, pfi.Filename, pfi.CompositeKey) if err != nil { return nil, nil, err } - return GetPackageFileStream(pv, pf) + return GetPackageFileStream(ctx, pv, pf) } // GetPackageFileStream returns the content of the specific package file -func GetPackageFileStream(pv *packages_model.PackageVersion, pf *packages_model.PackageFile) (io.ReadCloser, *packages_model.PackageFile, error) { - pb, err := packages_model.GetBlobByID(db.DefaultContext, pf.BlobID) +func GetPackageFileStream(ctx context.Context, pv *packages_model.PackageVersion, pf *packages_model.PackageFile) (io.ReadCloser, *packages_model.PackageFile, error) { + pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) if err != nil { return nil, nil, err } @@ -420,7 +420,7 @@ func GetPackageFileStream(pv *packages_model.PackageVersion, pf *packages_model. s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256)) if err == nil { if pf.IsLead { - if err := packages_model.IncrementDownloadCounter(pv.ID); err != nil { + if err := packages_model.IncrementDownloadCounter(ctx, pv.ID); err != nil { log.Error("Error incrementing download counter: %v", err) } } From 118770f9ad3749f825d06a9b14017cb4a94a2c84 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 17 Mar 2022 15:13:32 +0000 Subject: [PATCH 109/130] Added package cleanup task. --- custom/conf/app.example.ini | 18 ++++++++++++++++++ models/packages/package_blob.go | 7 ++++--- options/locale/locale_en-US.ini | 1 + routers/api/packages/conan/conan.go | 2 +- services/cron/tasks_basic.go | 19 +++++++++++++++++++ services/packages/packages.go | 9 +++++---- 6 files changed, 48 insertions(+), 8 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 5c3f382a9d80d..531a2803b8a3d 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1882,6 +1882,24 @@ PATH = ;; If CLEANUP_TYPE is set to PerWebhook, this is number of hook_task records to keep for a webhook (i.e. keep the most recent x deliveries). ;NUMBER_TO_KEEP = 10 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Cleanup expired packages +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;[cron.cleanup_packages] +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Whether to enable the job +;ENABLED = true +;; Whether to always run at least once at start up time (if ENABLED) +;RUN_AT_START = true +;; Notice if not success +;NO_SUCCESS_NOTICE = true +;; Time interval for job to run +;SCHEDULE = @midnight +;; Unreferenced blobs created more than OLDER_THAN ago are subject to deletion +;OLDER_THAN = 24h + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/models/packages/package_blob.go b/models/packages/package_blob.go index c0f904c06ab19..efe91e711a28f 100644 --- a/models/packages/package_blob.go +++ b/models/packages/package_blob.go @@ -7,6 +7,7 @@ package packages import ( "context" "errors" + "time" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" @@ -61,13 +62,13 @@ func GetBlobByID(ctx context.Context, blobID int64) (*PackageBlob, error) { return pb, nil } -// GetUnreferencedBlobs gets all blobs without associated files -func GetUnreferencedBlobs(ctx context.Context) ([]*PackageBlob, error) { +// GetUnreferencedBlobsOlderThan gets all blobs without associated files older than the specific duration +func GetUnreferencedBlobsOlderThan(ctx context.Context, olderThan time.Duration) ([]*PackageBlob, error) { pbs := make([]*PackageBlob, 0, 10) return pbs, db.GetEngine(ctx). Table("package_blob"). Join("LEFT OUTER", "package_file", "package_file.blob_id = package_blob.id"). - Where("package_file.id IS NULL"). + Where("package_file.id IS NULL AND package_blob.created_unix < ?", time.Now().Add(-olderThan).Unix()). Find(&pbs) } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 4540e34d012a1..1709bcf7664a9 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2435,6 +2435,7 @@ dashboard.resync_all_hooks = Resynchronize pre-receive, update and post-receive dashboard.reinit_missing_repos = Reinitialize all missing Git repositories for which records exist dashboard.sync_external_users = Synchronize external user data dashboard.cleanup_hook_task_table = Cleanup hook_task table +dashboard.cleanup_packages = Cleanup expired packages dashboard.server_uptime = Server Uptime dashboard.current_goroutine = Current Goroutines dashboard.current_memory_usage = Current Memory Usage diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index 2ccb88066a458..87400c7cbc960 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -681,7 +681,7 @@ func deleteRecipeOrPackage(apictx *context.Context, rref *conan_module.RecipeRef notification.NotifyPackageDelete(apictx.User, pd) } - return packages_service.DeleteUnreferencedBlobs() + return nil } // ListRecipeRevisions gets a list of all recipe revisions diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index fdf8550c31fbf..9c5bd5ddcb49e 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/migrations" mirror_service "code.gitea.io/gitea/services/mirror" + packages_service "code.gitea.io/gitea/services/packages" repo_service "code.gitea.io/gitea/services/repository" archiver_service "code.gitea.io/gitea/services/repository/archiver" ) @@ -140,6 +141,21 @@ func registerCleanupHookTaskTable() { }) } +func registerCleanupPackages() { + RegisterTaskFatal("cleanup_packages", &OlderThanConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@midnight", + NoSuccessNotice: true, + }, + OlderThan: 24 * time.Hour, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + realConfig := config.(*OlderThanConfig) + return packages_service.Cleanup(ctx, realConfig.OlderThan) + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -151,4 +167,7 @@ func initBasicTasks() { registerUpdateMigrationPosterID() } registerCleanupHookTaskTable() + if setting.Packages.Enabled { + registerCleanupPackages() + } } diff --git a/services/packages/packages.go b/services/packages/packages.go index 5f294589d63a1..91b83216a12c4 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "strings" + "time" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -316,18 +317,18 @@ func DeletePackageVersion(doer *user_model.User, pv *packages_model.PackageVersi notification.NotifyPackageDelete(doer, pd) - return DeleteUnreferencedBlobs() + return nil } -// DeleteUnreferencedBlobs deletes all unreferenced package blobs -func DeleteUnreferencedBlobs() error { +// Cleanup removes old unreferenced package blobs +func Cleanup(unused context.Context, olderThan time.Duration) error { ctx, committer, err := db.TxContext() if err != nil { return err } defer committer.Close() - pbs, err := packages_model.GetUnreferencedBlobs(ctx) + pbs, err := packages_model.GetUnreferencedBlobsOlderThan(ctx, olderThan) if err != nil { return err } From abf3a1e1c01c5f8add93becbfc1424258b735ae6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 18 Mar 2022 19:04:21 +0000 Subject: [PATCH 110/130] Changed packages path. --- integrations/api_packages_composer_test.go | 3 +- integrations/api_packages_test.go | 16 +++---- models/packages/descriptor.go | 14 ++++-- modules/context/package.go | 24 +++------- modules/setting/packages.go | 2 +- modules/setting/setting.go | 2 + routers/api/packages/api.go | 2 +- routers/api/packages/composer/api.go | 5 +- routers/api/packages/composer/composer.go | 17 ++++--- routers/api/v1/api.go | 2 +- routers/api/v1/packages/package.go | 56 ++++++++++++++++------ routers/web/admin/packages.go | 2 +- routers/web/user/package.go | 35 ++++++++------ routers/web/web.go | 2 +- services/cron/tasks_basic.go | 6 +-- templates/admin/packages/list.tmpl | 2 +- templates/package/settings.tmpl | 2 +- templates/package/shared/list.tmpl | 2 +- templates/package/view.tmpl | 2 +- 19 files changed, 113 insertions(+), 83 deletions(-) diff --git a/integrations/api_packages_composer_test.go b/integrations/api_packages_composer_test.go index d85293fede21a..59b975408dfb1 100644 --- a/integrations/api_packages_composer_test.go +++ b/integrations/api_packages_composer_test.go @@ -9,6 +9,7 @@ import ( "bytes" "fmt" "net/http" + neturl "net/url" "testing" "code.gitea.io/gitea/models/db" @@ -126,7 +127,7 @@ func TestPackageComposer(t *testing.T) { assert.NoError(t, err) assert.Len(t, pfs, 1) - req := NewRequest(t, "GET", fmt.Sprintf("%s/files/%d/%d", url, pvs[0].ID, pfs[0].ID)) + req := NewRequest(t, "GET", fmt.Sprintf("%s/files/%s/%s/%s", url, neturl.PathEscape(packageName), neturl.PathEscape(pvs[0].LowerVersion), neturl.PathEscape(pfs[0].LowerName))) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) diff --git a/integrations/api_packages_test.go b/integrations/api_packages_test.go index e676971b7730c..263e7cea53fac 100644 --- a/integrations/api_packages_test.go +++ b/integrations/api_packages_test.go @@ -33,8 +33,6 @@ func TestPackageAPI(t *testing.T) { AddBasicAuthHeader(req, user.Name) MakeRequest(t, req, http.StatusCreated) - var packageID int64 - t.Run("ListPackages", func(t *testing.T) { defer PrintCurrentTest(t)() @@ -50,17 +48,15 @@ func TestPackageAPI(t *testing.T) { assert.Equal(t, packageVersion, apiPackages[0].Version) assert.NotNil(t, apiPackages[0].Creator) assert.Equal(t, user.Name, apiPackages[0].Creator.UserName) - - packageID = apiPackages[0].ID }) t.Run("GetPackage", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/%d?token=%s", user.Name, 123456, token)) + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/dummy/%s/%s?token=%s", user.Name, packageName, packageVersion, token)) MakeRequest(t, req, http.StatusNotFound) - req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/%d?token=%s", user.Name, packageID, token)) + req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/generic/%s/%s?token=%s", user.Name, packageName, packageVersion, token)) resp := MakeRequest(t, req, http.StatusOK) var p *api.Package @@ -76,10 +72,10 @@ func TestPackageAPI(t *testing.T) { t.Run("ListPackageFiles", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/%d/files?token=%s", user.Name, 123456, token)) + req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/dummy/%s/%s/files?token=%s", user.Name, packageName, packageVersion, token)) MakeRequest(t, req, http.StatusNotFound) - req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/%d/files?token=%s", user.Name, packageID, token)) + req = NewRequest(t, "GET", fmt.Sprintf("/api/v1/packages/%s/generic/%s/%s/files?token=%s", user.Name, packageName, packageVersion, token)) resp := MakeRequest(t, req, http.StatusOK) var files []*api.PackageFile @@ -97,10 +93,10 @@ func TestPackageAPI(t *testing.T) { t.Run("DeletePackage", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/packages/%s/%d?token=%s", user.Name, 123456, token)) + req := NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/packages/%s/dummy/%s/%s?token=%s", user.Name, packageName, packageVersion, token)) MakeRequest(t, req, http.StatusNotFound) - req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/packages/%s/%d?token=%s", user.Name, packageID, token)) + req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/packages/%s/generic/%s/%s?token=%s", user.Name, packageName, packageVersion, token)) MakeRequest(t, req, http.StatusNoContent) }) } diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 3878e7c8e2fa4..c20bd8b1f2ad9 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -6,7 +6,8 @@ package packages import ( "context" - "strconv" + "fmt" + "net/url" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" @@ -55,9 +56,14 @@ type PackageFileDescriptor struct { Properties PackagePropertyList } -// WebLink returns the package's web link -func (pd *PackageDescriptor) WebLink() string { - return pd.Owner.HTMLURL() + "/-/packages/" + strconv.FormatInt(pd.Version.ID, 10) +// PackageWebLink returns the package web link +func (pd *PackageDescriptor) PackageWebLink() string { + return fmt.Sprintf("%s/-/packages/%s/%s", pd.Owner.HTMLURL(), string(pd.Package.Type), url.PathEscape(pd.Package.LowerName)) +} + +// FullWebLink returns the package version web link +func (pd *PackageDescriptor) FullWebLink() string { + return fmt.Sprintf("%s/%s", pd.PackageWebLink(), url.PathEscape(pd.Version.LowerVersion)) } // CalculateBlobSize returns the total blobs size in bytes diff --git a/modules/context/package.go b/modules/context/package.go index bc411ef0ab750..0d5459a52bd8e 100644 --- a/modules/context/package.go +++ b/modules/context/package.go @@ -7,7 +7,6 @@ package context import ( "fmt" "net/http" - "strconv" "code.gitea.io/gitea/models" packages_model "code.gitea.io/gitea/models/packages" @@ -68,20 +67,16 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) { } } - versionID := ctx.Params(":versionid") - if versionID != "" { - id, err := strconv.ParseInt(versionID, 10, 64) - if err != nil { - errCb(http.StatusInternalServerError, "ParseInt", err) - return - } - - pv, err := packages_model.GetVersionByID(ctx, id) + packageType := ctx.Params("type") + name := ctx.Params("name") + version := ctx.Params("version") + if packageType != "" && name != "" && version != "" { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.Type(packageType), name, version) if err != nil { if err == packages_model.ErrPackageNotExist { - errCb(http.StatusNotFound, "GetVersionByID", err) + errCb(http.StatusNotFound, "GetVersionByNameAndVersion", err) } else { - errCb(http.StatusInternalServerError, "GetVersionByID", err) + errCb(http.StatusInternalServerError, "GetVersionByNameAndVersion", err) } return } @@ -91,11 +86,6 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) { errCb(http.StatusInternalServerError, "GetPackageDescriptor", err) return } - - if ctx.Package.Descriptor.Owner.ID != ctx.Package.Owner.ID { - errCb(http.StatusNotFound, "Package owner does not match", "Package owner does not match") - return - } } } diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 4e9d554d85e5d..9a89fa9a3e3b9 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -16,7 +16,7 @@ var ( } ) -func newPackagesService() { +func newPackages() { if err := Cfg.Section("packages").MapTo(&Packages); err != nil { log.Fatal("Failed to map Packages settings: %v", err) } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0dd70752c2849..f82aec94ce8bb 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1008,6 +1008,8 @@ func loadFromConf(allowEmpty bool, extraConfig string) { newPictureService() + newPackages() + if err = Cfg.Section("ui").MapTo(&UI); err != nil { log.Fatal("Failed to map UI settings: %v", err) } else if err = Cfg.Section("markdown").MapTo(&Markdown); err != nil { diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index cb5e7e0e523ca..9b854cbd015bb 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -53,7 +53,7 @@ func Routes() *web.Route { r.Get("/list.json", composer.EnumeratePackages) r.Get("/p2/{vendorname}/{projectname}~dev.json", composer.PackageMetadata) r.Get("/p2/{vendorname}/{projectname}.json", composer.PackageMetadata) - r.Get("/files/{versionid}/{fileid}", composer.DownloadPackageFile) + r.Get("/files/{package}/{version}/{filename}", composer.DownloadPackageFile) r.Put("", reqPackageAccess(perm.AccessModeWrite), composer.UploadPackage) }) r.Group("/conan", func() { diff --git a/routers/api/packages/composer/api.go b/routers/api/packages/composer/api.go index 70aa336993a0d..d8f67d130cd09 100644 --- a/routers/api/packages/composer/api.go +++ b/routers/api/packages/composer/api.go @@ -5,7 +5,8 @@ package composer import ( - "strconv" + "fmt" + "net/url" "time" packages_model "code.gitea.io/gitea/models/packages" @@ -102,7 +103,7 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_model.Pac Metadata: pd.Metadata.(*composer_module.Metadata), Dist: Dist{ Type: "zip", - URL: registryURL + "/files/" + strconv.FormatInt(pd.Version.ID, 10) + "/" + strconv.FormatInt(pd.Files[0].File.ID, 10), + URL: fmt.Sprintf("%s/files/%s/%s/%s", registryURL, url.PathEscape(pd.Package.LowerName), url.PathEscape(pd.Version.LowerVersion), url.PathEscape(pd.Files[0].File.LowerName)), Checksum: pd.Files[0].Blob.HashSHA1, }, }) diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index c952bf75b42c7..aabb7dadb7f6d 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -160,14 +160,17 @@ func PackageMetadata(ctx *context.Context) { // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.Context) { - versionID := ctx.ParamsInt64("versionid") - fileID := ctx.ParamsInt64("fileid") - - s, pf, err := packages_service.GetFileStreamByPackageVersionAndFileID( + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( ctx, - ctx.Package.Owner, - versionID, - fileID, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeComposer, + Name: ctx.Params("package"), + Version: ctx.Params("version"), + }, + &packages_service.PackageFileInfo{ + Filename: ctx.Params("filename"), + }, ) if err != nil { if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 0f7a4d86c4271..edc93b1918914 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1015,7 +1015,7 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { }) m.Group("/packages/{username}", func() { - m.Group("/{versionid}", func() { + m.Group("/{type}/{name}/{version}", func() { m.Get("", packages.GetPackage) m.Delete("", reqPackageAccess(perm.AccessModeWrite), packages.DeletePackage) m.Get("/files", packages.ListPackageFiles) diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 1675378385547..bb300bc655e9e 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -83,7 +83,7 @@ func ListPackages(ctx *context.APIContext) { // GetPackage gets a package func GetPackage(ctx *context.APIContext) { - // swagger:operation GET /packages/{owner}/{versionid} package getPackage + // swagger:operation GET /packages/{owner}/{type}/{name}/{version} package getPackage // --- // summary: Gets a package // produces: @@ -94,11 +94,20 @@ func GetPackage(ctx *context.APIContext) { // description: owner of the package // type: string // required: true - // - name: versionid + // - name: type // in: path - // description: id of the package version - // type: integer - // format: int64 + // description: type of the package + // type: string + // required: true + // - name: name + // in: path + // description: name of the package + // type: string + // required: true + // - name: version + // in: path + // description: version of the package + // type: string // required: true // responses: // "200": @@ -111,7 +120,7 @@ func GetPackage(ctx *context.APIContext) { // DeletePackage deletes a package func DeletePackage(ctx *context.APIContext) { - // swagger:operation DELETE /packages/{owner}/{versionid} package deletePackage + // swagger:operation DELETE /packages/{owner}/{type}/{name}/{version} package deletePackage // --- // summary: Delete a package // parameters: @@ -120,12 +129,20 @@ func DeletePackage(ctx *context.APIContext) { // description: owner of the package // type: string // required: true - // - name: versionid + // - name: type // in: path - // description: id of the package version - // type: integer - // format: int64 + // description: type of the package + // type: string + // required: true + // - name: name + // in: path + // description: name of the package + // type: string // required: true + // - name: version + // in: path + // description: version of the package + // type: string // responses: // "204": // "$ref": "#/responses/empty" @@ -142,7 +159,7 @@ func DeletePackage(ctx *context.APIContext) { // ListPackageFiles gets all files of a package func ListPackageFiles(ctx *context.APIContext) { - // swagger:operation GET /packages/{owner}/{versionid}/files package listPackageFiles + // swagger:operation GET /packages/{owner}/{type}/{name}/{version}/files package listPackageFiles // --- // summary: Gets all files of a package // produces: @@ -153,11 +170,20 @@ func ListPackageFiles(ctx *context.APIContext) { // description: owner of the package // type: string // required: true - // - name: versionid + // - name: type // in: path - // description: id of the package version - // type: integer - // format: int64 + // description: type of the package + // type: string + // required: true + // - name: name + // in: path + // description: name of the package + // type: string + // required: true + // - name: version + // in: path + // description: version of the package + // type: string // required: true // responses: // "200": diff --git a/routers/web/admin/packages.go b/routers/web/admin/packages.go index 41d6a968e3363..aad92ff12e595 100644 --- a/routers/web/admin/packages.go +++ b/routers/web/admin/packages.go @@ -40,7 +40,7 @@ func Packages(ctx *context.Context) { }, }) if err != nil { - ctx.ServerError("SearchLatestVersions", err) + ctx.ServerError("SearchVersions", err) return } diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 874e6692c4889..ce28c91552964 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -10,7 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/packages" + packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/base" @@ -37,7 +37,7 @@ func Packages(ctx *context.Context) { query := ctx.FormTrim("q") packageType := ctx.FormTrim("type") - pvs, total, err := packages.SearchLatestVersions(ctx, &packages.PackageSearchOptions{ + pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ Paginator: &db.ListOptions{ PageSize: setting.UI.PackagesPagingNum, Page: page, @@ -51,13 +51,13 @@ func Packages(ctx *context.Context) { return } - pds, err := packages.GetPackageDescriptors(ctx, pvs) + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { ctx.ServerError("GetPackageDescriptors", err) return } - hasPackages, err := packages.HasOwnerPackages(ctx, ctx.ContextUser.ID) + hasPackages, err := packages_model.HasOwnerPackages(ctx, ctx.ContextUser.ID) if err != nil { ctx.ServerError("HasOwnerPackages", err) return @@ -94,7 +94,7 @@ func ViewPackage(ctx *context.Context) { ctx.Data["ContextUser"] = ctx.ContextUser ctx.Data["PackageDescriptor"] = pd - otherVersions, err := packages.GetVersionsByPackageName(ctx, pd.Owner.ID, pd.Package.Type, pd.Package.LowerName) + otherVersions, err := packages_model.GetVersionsByPackageName(ctx, pd.Owner.ID, pd.Package.Type, pd.Package.LowerName) if err != nil { ctx.ServerError("GetVersionsByPackageName", err) return @@ -160,7 +160,7 @@ func PackageSettingsPost(ctx *context.Context) { repoID = repo.ID } - if err := packages.SetRepositoryLink(ctx, pd.Package.ID, repoID); err != nil { + if err := packages_model.SetRepositoryLink(ctx, pd.Package.ID, repoID); err != nil { log.Error("Error updating package: %v", err) return false } @@ -192,18 +192,23 @@ func PackageSettingsPost(ctx *context.Context) { // DownloadPackageFile serves the content of a package file func DownloadPackageFile(ctx *context.Context) { - s, pf, err := packages_service.GetFileStreamByPackageVersionAndFileID( - ctx, - ctx.ContextUser, - ctx.ParamsInt64(":versionid"), - ctx.ParamsInt64(":fileid"), - ) + pf, err := packages_model.GetFileForVersionByID(ctx, ctx.Package.Descriptor.Version.ID, ctx.ParamsInt64(":fileid")) if err != nil { - if err == packages.ErrPackageNotExist || err == packages.ErrPackageFileNotExist { + if err == packages_model.ErrPackageFileNotExist { ctx.NotFound("", err) - return + } else { + ctx.ServerError("GetFileForVersionByID", err) } - ctx.ServerError("GetFileStreamByPackageVersionAndFileID", err) + return + } + + s, _, err := packages_service.GetPackageFileStream( + ctx, + ctx.Package.Descriptor.Version, + pf, + ) + if err != nil { + ctx.ServerError("GetPackageFileStream", err) return } defer s.Close() diff --git a/routers/web/web.go b/routers/web/web.go index f606bec55a40f..fb17a6f8e6ba2 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -629,7 +629,7 @@ func RegisterRoutes(m *web.Route) { m.Group("/{username}/-", func() { m.Group("/packages", func() { m.Get("", user.Packages) - m.Group("/{versionid}", func() { + m.Group("/{type}/{name}/{version}", func() { m.Get("", user.ViewPackage) m.Get("/files/{fileid}", user.DownloadPackageFile) m.Group("/settings", func() { diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index 9c5bd5ddcb49e..505145c643eb5 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -144,9 +144,9 @@ func registerCleanupHookTaskTable() { func registerCleanupPackages() { RegisterTaskFatal("cleanup_packages", &OlderThanConfig{ BaseConfig: BaseConfig{ - Enabled: true, - RunAtStart: true, - Schedule: "@midnight", + Enabled: true, + RunAtStart: true, + Schedule: "@midnight", NoSuccessNotice: true, }, OlderThan: 24 * time.Hour, diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index 65181b67176e0..a7143c70bee62 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -63,7 +63,7 @@
    - + - - + +
    ID{{SortArrow "oldest" "newest" "$.SortType" false}}ID {{.i18n.Tr "admin.packages.owner"}} {{.i18n.Tr "admin.packages.type"}} {{.i18n.Tr "admin.packages.name"}} - {{SortArrow "alphabetically" "reversealphabetically" "$.SortType" false}} + {{SortArrow "alphabetically" "reversealphabetically" .SortType false}} + + {{.i18n.Tr "admin.packages.version"}} + {{SortArrow "highestversion" "lowestversion" .SortType false}} {{.i18n.Tr "admin.packages.version"}} {{.i18n.Tr "admin.packages.creator"}} {{.i18n.Tr "admin.packages.repository"}} - {{.i18n.Tr "admin.packages.size"}} - {{SortArrow "size" "reversesize" "$.SortType" false}} + {{.i18n.Tr "admin.packages.size"}} + {{.i18n.Tr "admin.packages.published"}} + {{SortArrow "oldest" "newest" .SortType true}} {{.i18n.Tr "admin.packages.published"}} {{.i18n.Tr "admin.notices.op"}}
    {{.Package.Type.Name}} {{.Package.Name}}{{.Version.Version}}{{.Version.Version}} {{.Creator.Name}} {{if .Repository}} diff --git a/templates/package/settings.tmpl b/templates/package/settings.tmpl index 97159112edff6..cc50cb4c5b91c 100644 --- a/templates/package/settings.tmpl +++ b/templates/package/settings.tmpl @@ -3,7 +3,7 @@ {{template "user/overview/header" .}}
    {{template "base/alert" .}} -

    {{.PackageDescriptor.Package.Name}} ({{.PackageDescriptor.Version.Version}}) / {{.i18n.Tr "repo.settings"}}

    +

    {{.PackageDescriptor.Package.Name}} ({{.PackageDescriptor.Version.Version}}) / {{.i18n.Tr "repo.settings"}}

    {{.i18n.Tr "packages.settings.link"}}

    diff --git a/templates/package/shared/list.tmpl b/templates/package/shared/list.tmpl index 45b05f2bec515..cdfa624ab1923 100644 --- a/templates/package/shared/list.tmpl +++ b/templates/package/shared/list.tmpl @@ -23,7 +23,7 @@
  • - {{.Package.Name}} + {{.Package.Name}} {{svg .Package.Type.SVGName 16}} {{.Package.Type.Name}}
    diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 86573a2db5120..ceb0c46494a4d 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -63,7 +63,7 @@
    {{range .OtherVersions}}
    - {{.Version}} + {{.Version}} {{$.i18n.Tr "packages.versions.on"}} {{.CreatedUnix.FormatDate}}
    {{end}} From 5a4128afa466a753f7ae271a37a10fa3974605f4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 20 Mar 2022 12:55:59 +0000 Subject: [PATCH 111/130] Added container registry. --- custom/conf/app.example.ini | 5 +- docs/content/doc/packages/container.en-us.md | 91 +++ docs/content/doc/packages/generic.en-us.md | 2 +- docs/content/doc/packages/maven.en-us.md | 2 +- docs/content/doc/packages/npm.en-us.md | 2 +- docs/content/doc/packages/nuget.en-us.md | 2 +- docs/content/doc/packages/overview.en-us.md | 1 + docs/content/doc/packages/pypi.en-us.md | 2 +- docs/content/doc/packages/rubygems.en-us.md | 2 +- integrations/api_packages_container_test.go | 527 +++++++++++++++ models/migrations/v211.go | 15 +- models/packages/conan/references.go | 1 + models/packages/conan/search.go | 13 +- models/packages/container/const.go | 10 + models/packages/container/search.go | 222 +++++++ models/packages/descriptor.go | 3 + models/packages/package.go | 70 +- models/packages/package_blob.go | 4 +- models/packages/package_blob_upload.go | 81 +++ models/packages/package_file.go | 11 +- models/packages/package_version.go | 73 ++- models/user/user.go | 1 + modules/packages/container/helm/helm.go | 56 ++ modules/packages/container/metadata.go | 157 +++++ modules/packages/container/metadata_test.go | 62 ++ modules/packages/container/oci/digest.go | 27 + modules/packages/container/oci/mediatype.go | 36 ++ modules/packages/container/oci/oci.go | 191 ++++++ modules/packages/container/oci/reference.go | 17 + modules/packages/hashed_buffer.go | 36 +- modules/packages/multi_hasher.go | 123 ++++ modules/packages/multi_hasher_test.go | 54 ++ modules/setting/packages.go | 28 +- options/locale/locale_en-US.ini | 14 + routers/api/packages/api.go | 165 ++++- routers/api/packages/composer/composer.go | 2 +- routers/api/packages/conan/conan.go | 4 - routers/api/packages/container/auth.go | 45 ++ routers/api/packages/container/blob.go | 136 ++++ routers/api/packages/container/container.go | 603 ++++++++++++++++++ routers/api/packages/container/errors.go | 53 ++ routers/api/packages/container/manifest.go | 430 +++++++++++++ routers/api/packages/generic/generic.go | 2 +- routers/api/packages/nuget/nuget.go | 8 +- routers/api/packages/rubygems/rubygems.go | 2 +- routers/api/v1/packages/package.go | 6 +- routers/init.go | 5 +- routers/web/admin/packages.go | 10 +- routers/web/repo/packages.go | 8 +- routers/web/user/package.go | 188 +++++- routers/web/web.go | 20 +- services/auth/auth.go | 5 + services/auth/basic.go | 2 +- services/packages/container/blob_uploader.go | 134 ++++ services/packages/container/cleanup.go | 75 +++ services/packages/packages.go | 84 ++- templates/admin/packages/list.tmpl | 5 +- templates/package/content/container.tmpl | 78 +++ templates/package/metadata/container.tmpl | 9 + templates/package/settings.tmpl | 1 - templates/package/shared/list.tmpl | 1 + templates/package/shared/versionlist.tmpl | 32 + templates/package/view.tmpl | 29 +- templates/user/overview/package_versions.tmpl | 6 + 64 files changed, 3884 insertions(+), 205 deletions(-) create mode 100644 docs/content/doc/packages/container.en-us.md create mode 100644 integrations/api_packages_container_test.go create mode 100644 models/packages/container/const.go create mode 100644 models/packages/container/search.go create mode 100644 models/packages/package_blob_upload.go create mode 100644 modules/packages/container/helm/helm.go create mode 100644 modules/packages/container/metadata.go create mode 100644 modules/packages/container/metadata_test.go create mode 100644 modules/packages/container/oci/digest.go create mode 100644 modules/packages/container/oci/mediatype.go create mode 100644 modules/packages/container/oci/oci.go create mode 100644 modules/packages/container/oci/reference.go create mode 100644 modules/packages/multi_hasher.go create mode 100644 modules/packages/multi_hasher_test.go create mode 100644 routers/api/packages/container/auth.go create mode 100644 routers/api/packages/container/blob.go create mode 100644 routers/api/packages/container/container.go create mode 100644 routers/api/packages/container/errors.go create mode 100644 routers/api/packages/container/manifest.go create mode 100644 services/packages/container/blob_uploader.go create mode 100644 services/packages/container/cleanup.go create mode 100644 templates/package/content/container.tmpl create mode 100644 templates/package/metadata/container.tmpl create mode 100644 templates/package/shared/versionlist.tmpl create mode 100644 templates/user/overview/package_versions.tmpl diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 531a2803b8a3d..23290859f204b 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2215,7 +2215,10 @@ PATH = ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; ;; Enable/Disable package registry capabilities -; ENABLED = true +;ENABLED = true +;; +;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload` +;CHUNKED_UPLOAD_PATH = tmp/package-upload ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/doc/packages/container.en-us.md b/docs/content/doc/packages/container.en-us.md new file mode 100644 index 0000000000000..28559eb22b8d5 --- /dev/null +++ b/docs/content/doc/packages/container.en-us.md @@ -0,0 +1,91 @@ +--- +date: "2021-07-20T00:00:00+00:00" +title: "Container Registry" +slug: "packages/container" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Container Registry" + weight: 30 + identifier: "container" +--- + +# Container Registry + +Publish [Open Container Initiative](https://opencontainers.org/) compliant images for your user or organization. +The container registry follows the OCI specs and supports all compatible images like [Docker](https://www.docker.com/) and [Helm Charts](https://helm.sh/). + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Container registry, you can use the tools for your specific image type. +The following examples use the `docker` client. + +## Login to the container registry + +To push an image or if the image is in a private registry, you have to authenticate: + +```shell +docker login gitea.example.com +``` + +## Image naming convention + +Images must follow this naming convention: + +`{registry}/{owner}/{image}` + +For example, these are all valid image names for the owner `testuser`: + +`gitea.example.com/testuser/myimage` + +`gitea.example.com/testuser/my-image` + +`gitea.example.com/testuser/my/image` + +**NOTE:** The registry only supports case-insensitive tag names. So `image:tag` and `image:Tag` get treated as the same image and tag. + +## Push an image + +Push an image by executing the following command: + +```shell +docker push gitea.example.com/{owner}/{image}:{tag} +``` + +| Parameter | Description | +| ----------| ----------- | +| `owner` | The owner of the image. | +| `image` | The name of the image. | +| `tag` | The tag of the image. | + +For example: + +```shell +docker push gitea.example.com/testuser/myimage:latest +``` + +## Pull an image + +Pull an image by executing the following command: + +```shell +docker pull gitea.example.com/{owner}/{image}:{tag} +``` + +| Parameter | Description | +| ----------| ----------- | +| `owner` | The owner of the image. | +| `image` | The name of the image. | +| `tag` | The tag of the image. | + +For example: + +```shell +docker pull gitea.example.com/testuser/myimage:latest +``` diff --git a/docs/content/doc/packages/generic.en-us.md b/docs/content/doc/packages/generic.en-us.md index 5e2f6a658cc21..afef323938d4f 100644 --- a/docs/content/doc/packages/generic.en-us.md +++ b/docs/content/doc/packages/generic.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "Generic" - weight: 30 + weight: 40 identifier: "generic" --- diff --git a/docs/content/doc/packages/maven.en-us.md b/docs/content/doc/packages/maven.en-us.md index b95d58ce23a73..78288a9e42faf 100644 --- a/docs/content/doc/packages/maven.en-us.md +++ b/docs/content/doc/packages/maven.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "Maven" - weight: 40 + weight: 50 identifier: "maven" --- diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index fccbd920ebc3e..083c2799837f0 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "npm" - weight: 50 + weight: 60 identifier: "npm" --- diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md index 3022427e79097..88c7a857d64ec 100644 --- a/docs/content/doc/packages/nuget.en-us.md +++ b/docs/content/doc/packages/nuget.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "NuGet" - weight: 60 + weight: 70 identifier: "nuget" --- diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 53ccddf130a58..0a7415e92bb5c 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -28,6 +28,7 @@ The following package managers are currently supported: | ---- | -------- | -------------- | | [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` | | [Conan]({{< relref "doc/packages/conan.en-us.md" >}}) | C++ | `conan` | +| [Container]({{< relref "doc/packages/container.en-us.md" >}}) | - | any OCI compliant client | | [Generic]({{< relref "doc/packages/generic.en-us.md" >}}) | - | any HTTP client | | [Maven]({{< relref "doc/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` | | [npm]({{< relref "doc/packages/npm.en-us.md" >}}) | JavaScript | `npm` | diff --git a/docs/content/doc/packages/pypi.en-us.md b/docs/content/doc/packages/pypi.en-us.md index 1b88f27049d61..1d7a8f22e86de 100644 --- a/docs/content/doc/packages/pypi.en-us.md +++ b/docs/content/doc/packages/pypi.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "PyPI" - weight: 70 + weight: 80 identifier: "pypi" --- diff --git a/docs/content/doc/packages/rubygems.en-us.md b/docs/content/doc/packages/rubygems.en-us.md index 3ce01ecf9851e..603e925e32afd 100644 --- a/docs/content/doc/packages/rubygems.en-us.md +++ b/docs/content/doc/packages/rubygems.en-us.md @@ -8,7 +8,7 @@ menu: sidebar: parent: "packages" name: "RubyGems" - weight: 80 + weight: 90 identifier: "rubygems" --- diff --git a/integrations/api_packages_container_test.go b/integrations/api_packages_container_test.go new file mode 100644 index 0000000000000..ec031a56b0f54 --- /dev/null +++ b/integrations/api_packages_container_test.go @@ -0,0 +1,527 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "bytes" + "encoding/base64" + "fmt" + "net/http" + "strings" + "testing" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + container_model "code.gitea.io/gitea/models/packages/container" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/packages/container/oci" + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestPackageContainer(t *testing.T) { + defer prepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User) + + has := func(l packages_model.PackagePropertyList, name string) bool { + for _, pp := range l { + if pp.Name == name { + return true + } + } + return false + } + + images := []string{"test", "te/st"} + tags := []string{"latest", "main"} + multiTag := "multi" + + unknownDigest := "sha256:0000000000000000000000000000000000000000000000000000000000000000" + + blobDigest := "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" + blobContent, _ := base64.StdEncoding.DecodeString(`H4sIAAAJbogA/2IYBaNgFIxYAAgAAP//Lq+17wAEAAA=`) + + configDigest := "sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d" + configContent := `{"architecture":"amd64","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/true"],"ArgsEscaped":true,"Image":"sha256:9bd8b88dc68b80cffe126cc820e4b52c6e558eb3b37680bfee8e5f3ed7b8c257"},"container":"b89fe92a887d55c0961f02bdfbfd8ac3ddf66167db374770d2d9e9fab3311510","container_config":{"Hostname":"b89fe92a887d","Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/true\"]"],"ArgsEscaped":true,"Image":"sha256:9bd8b88dc68b80cffe126cc820e4b52c6e558eb3b37680bfee8e5f3ed7b8c257"},"created":"2022-01-01T00:00:00.000000000Z","docker_version":"20.10.12","history":[{"created":"2022-01-01T00:00:00.000000000Z","created_by":"/bin/sh -c #(nop) COPY file:0e7589b0c800daaf6fa460d2677101e4676dd9491980210cb345480e513f3602 in /true "},{"created":"2022-01-01T00:00:00.000000001Z","created_by":"/bin/sh -c #(nop) CMD [\"/true\"]","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:0ff3b91bdf21ecdf2f2f3d4372c2098a14dbe06cd678e8f0a85fd4902d00e2e2"]}}` + + manifestDigest := "sha256:4f10484d1c1bb13e3956b4de1cd42db8e0f14a75be1617b60f2de3cd59c803c6" + manifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeDockerManifest + `","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d","size":1069},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4","size":32}]}` + + untaggedManifestDigest := "sha256:4305f5f5572b9a426b88909b036e52ee3cf3d7b9c1b01fac840e90747f56623d" + untaggedManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageManifest + `","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:4607e093bec406eaadb6f3a340f63400c9d3a7038680744c406903766b938f0d","size":1069},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4","size":32}]}` + + indexManifestDigest := "sha256:bab112d6efb9e7f221995caaaa880352feb5bd8b1faf52fae8d12c113aa123ec" + indexManifestContent := `{"schemaVersion":2,"mediaType":"` + oci.MediaTypeImageIndex + `","manifests":[{"mediaType":"` + oci.MediaTypeDockerManifest + `","digest":"` + manifestDigest + `","platform":{"os":"linux","architecture":"arm","variant":"v7"}},{"mediaType":"` + oci.MediaTypeImageManifest + `","digest":"` + untaggedManifestDigest + `","platform":{"os":"linux","architecture":"arm64","variant":"v8"}}]}` + + anonymousToken := "" + userToken := "" + + t.Run("Authenticate", func(t *testing.T) { + type TokenResponse struct { + Token string `json:"token"` + } + + authenticate := []string{ + `Bearer realm="` + setting.AppURL + `v2/token"`, + `Basic`, + } + + t.Run("Anonymous", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL)) + resp := MakeRequest(t, req, http.StatusUnauthorized) + + assert.ElementsMatch(t, authenticate, resp.Header().Values("WWW-Authenticate")) + + req = NewRequest(t, "GET", fmt.Sprintf("%sv2/token", setting.AppURL)) + resp = MakeRequest(t, req, http.StatusOK) + + tokenResponse := &TokenResponse{} + DecodeJSON(t, resp, &tokenResponse) + + assert.NotEmpty(t, tokenResponse.Token) + + anonymousToken = fmt.Sprintf("Bearer %s", tokenResponse.Token) + + req = NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL)) + addTokenAuthHeader(req, anonymousToken) + resp = MakeRequest(t, req, http.StatusOK) + }) + + t.Run("User", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL)) + resp := MakeRequest(t, req, http.StatusUnauthorized) + + assert.ElementsMatch(t, authenticate, resp.Header().Values("WWW-Authenticate")) + + req = NewRequest(t, "GET", fmt.Sprintf("%sv2/token", setting.AppURL)) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + tokenResponse := &TokenResponse{} + DecodeJSON(t, resp, &tokenResponse) + + assert.NotEmpty(t, tokenResponse.Token) + + userToken = fmt.Sprintf("Bearer %s", tokenResponse.Token) + + req = NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL)) + addTokenAuthHeader(req, userToken) + resp = MakeRequest(t, req, http.StatusOK) + }) + }) + + t.Run("DetermineSupport", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusOK) + assert.Equal(t, "registry/2.0", resp.Header().Get("Docker-Distribution-Api-Version")) + }) + + for _, image := range images { + t.Run(fmt.Sprintf("[Image:%s]", image), func(t *testing.T) { + url := fmt.Sprintf("%sv2/%s/%s", setting.AppURL, user.Name, image) + + t.Run("UploadBlob/Monolithic", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads", url)) + addTokenAuthHeader(req, anonymousToken) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, unknownDigest), bytes.NewReader(blobContent)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusBadRequest) + + req = NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, blobDigest), bytes.NewReader(blobContent)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusCreated) + + assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location")) + assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest")) + + pv, err := packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeContainer, image, container_model.UploadVersion) + assert.NoError(t, err) + + pfs, err := packages_model.GetFilesByVersionID(db.DefaultContext, pv.ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + + pb, err := packages_model.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.EqualValues(t, len(blobContent), pb.Size) + }) + + t.Run("UploadBlob/Chunked", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads", url)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusAccepted) + + uuid := resp.Header().Get("Docker-Upload-Uuid") + assert.NotEmpty(t, uuid) + + pbu, err := packages_model.GetBlobUploadByID(db.DefaultContext, uuid) + assert.NoError(t, err) + assert.EqualValues(t, 0, pbu.BytesReceived) + + uploadURL := resp.Header().Get("Location") + assert.NotEmpty(t, uploadURL) + + req = NewRequestWithBody(t, "PATCH", setting.AppURL+uploadURL[1:]+"000", bytes.NewReader(blobContent)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequestWithBody(t, "PATCH", setting.AppURL+uploadURL[1:], bytes.NewReader(blobContent)) + addTokenAuthHeader(req, userToken) + + req.Header.Set("Content-Range", "1-10") + MakeRequest(t, req, http.StatusRequestedRangeNotSatisfiable) + + contentRange := fmt.Sprintf("0-%d", len(blobContent)-1) + req.Header.Set("Content-Range", contentRange) + resp = MakeRequest(t, req, http.StatusAccepted) + + assert.Equal(t, uuid, resp.Header().Get("Docker-Upload-Uuid")) + assert.Equal(t, contentRange, resp.Header().Get("Range")) + + pbu, err = packages_model.GetBlobUploadByID(db.DefaultContext, uuid) + assert.NoError(t, err) + assert.EqualValues(t, len(blobContent), pbu.BytesReceived) + + uploadURL = resp.Header().Get("Location") + + req = NewRequest(t, "PUT", fmt.Sprintf("%s?digest=%s", setting.AppURL+uploadURL[1:], blobDigest)) + addTokenAuthHeader(req, userToken) + resp = MakeRequest(t, req, http.StatusCreated) + + assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location")) + assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest")) + }) + + for _, tag := range tags { + t.Run(fmt.Sprintf("[Tag:%s]", tag), func(t *testing.T) { + t.Run("UploadManifest", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, configDigest), strings.NewReader(configContent)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, tag), strings.NewReader(manifestContent)) + addTokenAuthHeader(req, anonymousToken) + req.Header.Set("Content-Type", oci.MediaTypeDockerManifest) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, tag), strings.NewReader(manifestContent)) + addTokenAuthHeader(req, userToken) + req.Header.Set("Content-Type", oci.MediaTypeDockerManifest) + resp := MakeRequest(t, req, http.StatusCreated) + + assert.Equal(t, manifestDigest, resp.Header().Get("Docker-Content-Digest")) + + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeContainer, image, tag) + assert.NoError(t, err) + + pd, err := packages_model.GetPackageDescriptor(db.DefaultContext, pv) + assert.NoError(t, err) + assert.Nil(t, pd.SemVer) + assert.Equal(t, image, pd.Package.Name) + assert.Equal(t, tag, pd.Version.Version) + assert.True(t, has(pd.Properties, container_module.PropertyManifestTagged)) + + assert.IsType(t, &container_module.Metadata{}, pd.Metadata) + metadata := pd.Metadata.(*container_module.Metadata) + assert.Equal(t, container_module.TypeOCI, metadata.Type) + assert.Len(t, metadata.ImageLayers, 2) + assert.Empty(t, metadata.MultiArch) + + assert.Len(t, pd.Files, 3) + for _, pfd := range pd.Files { + switch pfd.File.Name { + case container_model.ManifestFilename: + assert.True(t, pfd.File.IsLead) + assert.Equal(t, oci.MediaTypeDockerManifest, pfd.Properties.GetByName(container_module.PropertyMediaType)) + assert.Equal(t, manifestDigest, pfd.Properties.GetByName(container_module.PropertyDigest)) + case strings.Replace(configDigest, ":", "_", 1): + assert.False(t, pfd.File.IsLead) + assert.Equal(t, "application/vnd.docker.container.image.v1+json", pfd.Properties.GetByName(container_module.PropertyMediaType)) + assert.Equal(t, configDigest, pfd.Properties.GetByName(container_module.PropertyDigest)) + case strings.Replace(blobDigest, ":", "_", 1): + assert.False(t, pfd.File.IsLead) + assert.Equal(t, "application/vnd.docker.image.rootfs.diff.tar.gzip", pfd.Properties.GetByName(container_module.PropertyMediaType)) + assert.Equal(t, blobDigest, pfd.Properties.GetByName(container_module.PropertyDigest)) + default: + assert.Fail(t, "unknown file: %s", pfd.File.Name) + } + } + + // Overwrite existing tag + req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, tag), strings.NewReader(manifestContent)) + addTokenAuthHeader(req, userToken) + req.Header.Set("Content-Type", oci.MediaTypeDockerManifest) + MakeRequest(t, req, http.StatusCreated) + }) + + t.Run("HeadManifest", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "HEAD", fmt.Sprintf("%s/manifests/unknown-tag", url)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "HEAD", fmt.Sprintf("%s/manifests/%s", url, tag)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, fmt.Sprintf("%d", len(manifestContent)), resp.Header().Get("Content-Length")) + assert.Equal(t, manifestDigest, resp.Header().Get("Docker-Content-Digest")) + }) + + t.Run("GetManifest", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/manifests/unknown-tag", url)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/manifests/%s", url, tag)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, fmt.Sprintf("%d", len(manifestContent)), resp.Header().Get("Content-Length")) + assert.Equal(t, oci.MediaTypeDockerManifest, resp.Header().Get("Content-Type")) + assert.Equal(t, manifestDigest, resp.Header().Get("Docker-Content-Digest")) + assert.Equal(t, manifestContent, resp.Body.String()) + }) + }) + } + + t.Run("UploadUntaggedManifest", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, untaggedManifestDigest), strings.NewReader(untaggedManifestContent)) + addTokenAuthHeader(req, userToken) + req.Header.Set("Content-Type", oci.MediaTypeImageManifest) + resp := MakeRequest(t, req, http.StatusCreated) + + assert.Equal(t, untaggedManifestDigest, resp.Header().Get("Docker-Content-Digest")) + + req = NewRequest(t, "HEAD", fmt.Sprintf("%s/manifests/%s", url, untaggedManifestDigest)) + addTokenAuthHeader(req, userToken) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, fmt.Sprintf("%d", len(untaggedManifestContent)), resp.Header().Get("Content-Length")) + assert.Equal(t, untaggedManifestDigest, resp.Header().Get("Docker-Content-Digest")) + + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeContainer, image, untaggedManifestDigest) + assert.NoError(t, err) + + pd, err := packages_model.GetPackageDescriptor(db.DefaultContext, pv) + assert.NoError(t, err) + assert.Nil(t, pd.SemVer) + assert.Equal(t, image, pd.Package.Name) + assert.Equal(t, untaggedManifestDigest, pd.Version.Version) + assert.False(t, has(pd.Properties, container_module.PropertyManifestTagged)) + + assert.IsType(t, &container_module.Metadata{}, pd.Metadata) + + assert.Len(t, pd.Files, 3) + for _, pfd := range pd.Files { + if pfd.File.Name == container_model.ManifestFilename { + assert.True(t, pfd.File.IsLead) + assert.Equal(t, oci.MediaTypeImageManifest, pfd.Properties.GetByName(container_module.PropertyMediaType)) + assert.Equal(t, untaggedManifestDigest, pfd.Properties.GetByName(container_module.PropertyDigest)) + } + } + }) + + t.Run("UploadIndexManifest", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/%s", url, multiTag), strings.NewReader(indexManifestContent)) + addTokenAuthHeader(req, userToken) + req.Header.Set("Content-Type", oci.MediaTypeImageIndex) + resp := MakeRequest(t, req, http.StatusCreated) + + assert.Equal(t, indexManifestDigest, resp.Header().Get("Docker-Content-Digest")) + + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeContainer, image, multiTag) + assert.NoError(t, err) + + pd, err := packages_model.GetPackageDescriptor(db.DefaultContext, pv) + assert.NoError(t, err) + assert.Nil(t, pd.SemVer) + assert.Equal(t, image, pd.Package.Name) + assert.Equal(t, multiTag, pd.Version.Version) + assert.True(t, has(pd.Properties, container_module.PropertyManifestTagged)) + + getAllByName := func(l packages_model.PackagePropertyList, name string) []string { + values := make([]string, 0, len(l)) + for _, pp := range l { + if pp.Name == name { + values = append(values, pp.Value) + } + } + return values + } + assert.ElementsMatch(t, []string{manifestDigest, untaggedManifestDigest}, getAllByName(pd.Properties, container_module.PropertyManifestReference)) + + assert.IsType(t, &container_module.Metadata{}, pd.Metadata) + metadata := pd.Metadata.(*container_module.Metadata) + assert.Equal(t, container_module.TypeOCI, metadata.Type) + assert.Contains(t, metadata.MultiArch, "linux/arm/v7") + assert.Equal(t, manifestDigest, metadata.MultiArch["linux/arm/v7"]) + assert.Contains(t, metadata.MultiArch, "linux/arm64/v8") + assert.Equal(t, untaggedManifestDigest, metadata.MultiArch["linux/arm64/v8"]) + + assert.Len(t, pd.Files, 1) + assert.True(t, pd.Files[0].File.IsLead) + assert.Equal(t, oci.MediaTypeImageIndex, pd.Files[0].Properties.GetByName(container_module.PropertyMediaType)) + assert.Equal(t, indexManifestDigest, pd.Files[0].Properties.GetByName(container_module.PropertyDigest)) + }) + + t.Run("UploadBlob/Mount", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, unknownDigest)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusAccepted) + + req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, blobDigest)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusCreated) + + assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location")) + assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest")) + }) + + t.Run("HeadBlob", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "HEAD", fmt.Sprintf("%s/blobs/%s", url, unknownDigest)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "HEAD", fmt.Sprintf("%s/blobs/%s", url, blobDigest)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, fmt.Sprintf("%d", len(blobContent)), resp.Header().Get("Content-Length")) + assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest")) + }) + + t.Run("GetBlob", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/blobs/%s", url, unknownDigest)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/blobs/%s", url, blobDigest)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, fmt.Sprintf("%d", len(blobContent)), resp.Header().Get("Content-Length")) + assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest")) + assert.Equal(t, blobContent, resp.Body.Bytes()) + }) + + t.Run("GetTagList", func(t *testing.T) { + defer PrintCurrentTest(t)() + + cases := []struct { + URL string + Expected []string + }{ + { + URL: fmt.Sprintf("%s/tags/list", url), + Expected: []string{"latest", "main", "multi"}, + }, + { + URL: fmt.Sprintf("%s/tags/list?n=0", url), + Expected: []string{}, + }, + { + URL: fmt.Sprintf("%s/tags/list?n=2", url), + Expected: []string{"latest", "main"}, + }, + { + URL: fmt.Sprintf("%s/tags/list?last=main", url), + Expected: []string{"multi"}, + }, + { + URL: fmt.Sprintf("%s/tags/list?n=1&last=latest", url), + Expected: []string{"main"}, + }, + } + + for _, c := range cases { + req := NewRequest(t, "GET", c.URL) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusOK) + + type TagList struct { + Name string `json:"name"` + Tags []string `json:"tags"` + } + + tagList := &TagList{} + DecodeJSON(t, resp, &tagList) + + assert.Equal(t, user.Name+"/"+image, tagList.Name) + assert.Equal(t, c.Expected, tagList.Tags) + } + }) + + t.Run("Delete", func(t *testing.T) { + t.Run("Blob", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/blobs/%s", url, blobDigest)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusAccepted) + + req = NewRequest(t, "HEAD", fmt.Sprintf("%s/blobs/%s", url, blobDigest)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusNotFound) + }) + + t.Run("ManifestByDigest", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/manifests/%s", url, untaggedManifestDigest)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusAccepted) + + req = NewRequest(t, "HEAD", fmt.Sprintf("%s/manifests/%s", url, untaggedManifestDigest)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusNotFound) + }) + + t.Run("ManifestByTag", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/manifests/%s", url, multiTag)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusAccepted) + + req = NewRequest(t, "HEAD", fmt.Sprintf("%s/manifests/%s", url, multiTag)) + addTokenAuthHeader(req, userToken) + MakeRequest(t, req, http.StatusNotFound) + }) + }) + }) + } +} diff --git a/models/migrations/v211.go b/models/migrations/v211.go index 5ef8618a4db19..30318278fad45 100644 --- a/models/migrations/v211.go +++ b/models/migrations/v211.go @@ -32,6 +32,7 @@ func addPackageTables(x *xorm.Engine) error { Version string `xorm:"NOT NULL"` LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` + IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"` MetadataJSON string `xorm:"TEXT metadata_json"` DownloadCount int64 `xorm:"NOT NULL DEFAULT 0"` } @@ -77,5 +78,17 @@ func addPackageTables(x *xorm.Engine) error { CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` } - return x.Sync2(new(PackageBlob)) + if err := x.Sync2(new(PackageBlob)); err != nil { + return err + } + + type PackageBlobUpload struct { + ID string `xorm:"pk"` + BytesReceived int64 `xorm:"NOT NULL DEFAULT 0"` + HashStateBytes []byte `xorm:"BLOB"` + CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated INDEX NOT NULL"` + } + + return x.Sync2(new(PackageBlobUpload)) } diff --git a/models/packages/conan/references.go b/models/packages/conan/references.go index cc0db250f8177..4b7b2014304ff 100644 --- a/models/packages/conan/references.go +++ b/models/packages/conan/references.go @@ -58,6 +58,7 @@ func findPropertyValues(ctx context.Context, propertyName string, ownerID int64, "package.owner_id": ownerID, "package.lower_name": strings.ToLower(name), "package_version.lower_version": strings.ToLower(version), + "package_version.is_internal": false, strconv.Itoa(len(propertyFilter)): builder.Select("COUNT(*)").Where(propsCond).From("package_property"), } diff --git a/models/packages/conan/search.go b/models/packages/conan/search.go index f9361174dd056..6d86b0eae41ec 100644 --- a/models/packages/conan/search.go +++ b/models/packages/conan/search.go @@ -35,7 +35,12 @@ type RecipeSearchOptions struct { // SearchRecipes gets all recipes matching the search options func SearchRecipes(ctx context.Context, opts *RecipeSearchOptions) ([]string, error) { - cond := builder.NewCond() + var cond builder.Cond = builder.Eq{ + "package_file.is_lead": true, + "package.type": packages.TypeConan, + "package.owner_id": opts.OwnerID, + "package_version.is_internal": false, + } if opts.Name != "" { cond = cond.And(buildCondition("package.lower_name", strings.ToLower(opts.Name))) @@ -66,12 +71,6 @@ func SearchRecipes(ctx context.Context, opts *RecipeSearchOptions) ([]string, er }) } - cond = cond.And(builder.Eq{ - "package_file.is_lead": true, - "package.type": packages.TypeConan, - "package.owner_id": opts.OwnerID, - }) - query := builder. Select("package.name, package_version.version, package_file.id"). From("package_file"). diff --git a/models/packages/container/const.go b/models/packages/container/const.go new file mode 100644 index 0000000000000..9d3ed64a6e695 --- /dev/null +++ b/models/packages/container/const.go @@ -0,0 +1,10 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +const ( + ManifestFilename = "manifest.json" + UploadVersion = "_upload" +) diff --git a/models/packages/container/search.go b/models/packages/container/search.go new file mode 100644 index 0000000000000..638363c14d3b4 --- /dev/null +++ b/models/packages/container/search.go @@ -0,0 +1,222 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "errors" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + container_module "code.gitea.io/gitea/modules/packages/container" + + "xorm.io/builder" +) + +var ErrContainerBlobNotExist = errors.New("Container blob does not exist") + +type BlobSearchOptions struct { + OwnerID int64 + Image string + Digest string + Tag string + IsManifest bool +} + +func (opts *BlobSearchOptions) toConds() builder.Cond { + var cond builder.Cond = builder.Eq{ + "package.type": packages.TypeContainer, + } + + if opts.OwnerID != 0 { + cond = cond.And(builder.Eq{"package.owner_id": opts.OwnerID}) + } + if opts.Image != "" { + cond = cond.And(builder.Eq{"package.lower_name": strings.ToLower(opts.Image)}) + } + if opts.Tag != "" { + cond = cond.And(builder.Eq{"package_version.lower_version": strings.ToLower(opts.Tag)}) + } + if opts.IsManifest { + cond = cond.And(builder.Eq{"package_file.lower_name": ManifestFilename}) + } + if opts.Digest != "" { + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeFile, + "package_property.name": container_module.PropertyDigest, + "package_property.value": opts.Digest, + } + + cond = cond.And(builder.In("package_file.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property"))) + } + + return cond +} + +// GetContainerBlob gets the container blob matching the blob search options +// If multiple matching blobs are found (manifests with the same digest) the first (according to the database) is selected. +func GetContainerBlob(ctx context.Context, opts *BlobSearchOptions) (*packages.PackageFileDescriptor, error) { + pfds, err := getContainerBlobsLimit(ctx, opts, 1) + if err != nil { + return nil, err + } + if len(pfds) != 1 { + return nil, ErrContainerBlobNotExist + } + + return pfds[0], nil +} + +// GetContainerBlobs gets the container blobs matching the blob search options +func GetContainerBlobs(ctx context.Context, opts *BlobSearchOptions) ([]*packages.PackageFileDescriptor, error) { + return getContainerBlobsLimit(ctx, opts, 0) +} + +func getContainerBlobsLimit(ctx context.Context, opts *BlobSearchOptions, limit int) ([]*packages.PackageFileDescriptor, error) { + pfs := make([]*packages.PackageFile, 0, limit) + sess := db.GetEngine(ctx). + Join("INNER", "package", "package.id = package_version.package_id"). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Where(opts.toConds()) + + if limit > 0 { + sess = sess.Limit(limit) + } + + if err := sess.Find(&pfs); err != nil { + return nil, err + } + + pfds := make([]*packages.PackageFileDescriptor, 0, len(pfs)) + for _, pf := range pfs { + pfd, err := packages.GetPackageFileDescriptor(ctx, pf) + if err != nil { + return nil, err + } + pfds = append(pfds, pfd) + } + + return pfds, nil +} + +// GetManifestVersions gets all package versions representing the matching manifest +func GetManifestVersions(ctx context.Context, opts *BlobSearchOptions) ([]*packages.PackageVersion, error) { + cond := opts.toConds().And(builder.Eq{"package_version.is_internal": false}) + + pvs := make([]*packages.PackageVersion, 0, 10) + return pvs, db.GetEngine(ctx). + Join("INNER", "package", "package.id = package_version.package_id"). + Join("INNER", "package_file", "package_file.version_id = package_version.id"). + Where(cond). + Find(&pvs) +} + +// GetImageTags gets a sorted list of the tags of an image +// The result is suitable for the api call. +func GetImageTags(ctx context.Context, ownerID int64, image string, n *int, last string) ([]string, error) { + var cond builder.Cond = builder.Eq{ + "package.type": packages.TypeContainer, + "package.owner_id": ownerID, + "package.lower_name": strings.ToLower(image), + "package_version.is_internal": false, + } + + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeVersion, + "package_property.name": container_module.PropertyManifestTagged, + } + + cond = cond.And(builder.In("package_version.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property"))) + + if last != "" { + cond = cond.And(builder.Gt{"package_version.lower_version": strings.ToLower(last)}) + } + + sess := db.GetEngine(ctx). + Table("package_version"). + Select("package_version.lower_version"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond). + Asc("package_version.lower_version") + + var tags []string + if n != nil && *n >= 0 { + sess = sess.Limit(*n) + + tags = make([]string, 0, *n) + } else { + tags = make([]string, 0, 10) + } + + return tags, sess.Find(&tags) +} + +type ImageTagsSearchOptions struct { + PackageID int64 + Query string + IsTagged bool + db.Paginator +} + +func (opts *ImageTagsSearchOptions) toConds() builder.Cond { + var cond builder.Cond = builder.Eq{ + "package.type": packages.TypeContainer, + "package.id": opts.PackageID, + "package_version.is_internal": false, + } + + if opts.Query != "" { + cond = cond.And(builder.Like{"package_version.lower_version", strings.ToLower(opts.Query)}) + } + + var propsCond builder.Cond = builder.Eq{ + "package_property.ref_type": packages.PropertyTypeVersion, + "package_property.name": container_module.PropertyManifestTagged, + } + + in := builder.In("package_version.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")) + + if opts.IsTagged { + cond = cond.And(in) + } else { + cond = cond.And(builder.Not{in}) + } + + return cond +} + +// SearchImageTags gets a sorted list of the tags of an image +func SearchImageTags(ctx context.Context, opts *ImageTagsSearchOptions) ([]*packages.PackageVersion, int64, error) { + sess := db.GetEngine(ctx). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(opts.toConds()). + Desc("package_version.created_unix") + + if opts.Paginator != nil { + sess = db.SetSessionPagination(sess, opts) + } + + pvs := make([]*packages.PackageVersion, 0, 10) + count, err := sess.FindAndCount(&pvs) + return pvs, count, err +} + +func SearchExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) ([]*packages.PackageFile, error) { + var cond builder.Cond = builder.Eq{ + "package_version.is_internal": true, + "package_version.lower_version": UploadVersion, + "package.type": packages.TypeContainer, + } + cond = cond.And(builder.Lt{"package_file.created_unix": time.Now().Add(-olderThan).Unix()}) + + var pfs []*packages.PackageFile + return pfs, db.GetEngine(ctx). + Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(cond). + Find(&pfs) +} diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index c20bd8b1f2ad9..5a622fa32fc5c 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/packages/conan" + "code.gitea.io/gitea/modules/packages/container" "code.gitea.io/gitea/modules/packages/maven" "code.gitea.io/gitea/modules/packages/npm" "code.gitea.io/gitea/modules/packages/nuget" @@ -124,6 +125,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &composer.Metadata{} case TypeConan: metadata = &conan.Metadata{} + case TypeContainer: + metadata = &container.Metadata{} case TypeNuGet: metadata = &nuget.Metadata{} case TypeNpm: diff --git a/models/packages/package.go b/models/packages/package.go index 8a5a10eb8d4b6..9b4a98f845152 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -7,6 +7,7 @@ package packages import ( "context" "errors" + "strings" "code.gitea.io/gitea/models/db" @@ -29,14 +30,15 @@ type Type string // List of supported packages const ( - TypeComposer Type = "composer" - TypeConan Type = "conan" - TypeGeneric Type = "generic" - TypeNuGet Type = "nuget" - TypeNpm Type = "npm" - TypeMaven Type = "maven" - TypePyPI Type = "pypi" - TypeRubyGems Type = "rubygems" + TypeComposer Type = "composer" + TypeConan Type = "conan" + TypeContainer Type = "container" + TypeGeneric Type = "generic" + TypeNuGet Type = "nuget" + TypeNpm Type = "npm" + TypeMaven Type = "maven" + TypePyPI Type = "pypi" + TypeRubyGems Type = "rubygems" ) // Name gets the name of the package type @@ -46,6 +48,8 @@ func (pt Type) Name() string { return "Composer" case TypeConan: return "Conan" + case TypeContainer: + return "Container" case TypeGeneric: return "Generic" case TypeNuGet: @@ -69,6 +73,8 @@ func (pt Type) SVGName() string { return "gitea-composer" case TypeConan: return "gitea-conan" + case TypeContainer: + return "octicon-container" case TypeGeneric: return "octicon-package" case TypeNuGet: @@ -145,6 +151,28 @@ func GetPackageByID(ctx context.Context, packageID int64) (*Package, error) { return p, nil } +// GetPackageByName gets a package by name +func GetPackageByName(ctx context.Context, ownerID int64, packageType Type, name string) (*Package, error) { + var cond builder.Cond = builder.Eq{ + "package.owner_id": ownerID, + "package.type": packageType, + "package.lower_name": strings.ToLower(name), + } + + p := &Package{} + + has, err := db.GetEngine(ctx). + Where(cond). + Get(p) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageNotExist + } + return p, nil +} + // GetPackagesByType gets all packages of a specific type func GetPackagesByType(ctx context.Context, ownerID int64, packageType Type) ([]*Package, error) { var cond builder.Cond = builder.Eq{ @@ -158,21 +186,19 @@ func GetPackagesByType(ctx context.Context, ownerID int64, packageType Type) ([] Find(&ps) } -// DeletePackageByIDIfUnreferenced deletes a package if there are no associated versions -func DeletePackageByIDIfUnreferenced(ctx context.Context, packageID int64) error { - e := db.GetEngine(ctx) +// DeletePackagesIfUnreferenced deletes a package if there are no associated versions +func DeletePackagesIfUnreferenced(ctx context.Context) error { + in := builder. + Select("package_version.package_id"). + From("package"). + Join("LEFT", "package_version", "package_version.package_id = package.id"). + Where(builder.Expr("package_version.id IS NULL")) - count, err := e.Count(&PackageVersion{ - PackageID: packageID, - }) - if err != nil { - return err - } - if count == 0 { - _, err := e.ID(packageID).Delete(&Package{}) - return err - } - return nil + _, err := db.GetEngine(ctx). + Where(builder.In("package.id", in)). + Delete(&Package{}) + + return err } // HasOwnerPackages tests if a user/org has packages diff --git a/models/packages/package_blob.go b/models/packages/package_blob.go index efe91e711a28f..d9a8314c8801b 100644 --- a/models/packages/package_blob.go +++ b/models/packages/package_blob.go @@ -62,8 +62,8 @@ func GetBlobByID(ctx context.Context, blobID int64) (*PackageBlob, error) { return pb, nil } -// GetUnreferencedBlobsOlderThan gets all blobs without associated files older than the specific duration -func GetUnreferencedBlobsOlderThan(ctx context.Context, olderThan time.Duration) ([]*PackageBlob, error) { +// FindExpiredUnreferencedBlobs gets all blobs without associated files older than the specific duration +func FindExpiredUnreferencedBlobs(ctx context.Context, olderThan time.Duration) ([]*PackageBlob, error) { pbs := make([]*PackageBlob, 0, 10) return pbs, db.GetEngine(ctx). Table("package_blob"). diff --git a/models/packages/package_blob_upload.go b/models/packages/package_blob_upload.go new file mode 100644 index 0000000000000..635068f1d865d --- /dev/null +++ b/models/packages/package_blob_upload.go @@ -0,0 +1,81 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + "errors" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +// ErrPackageBlobUploadNotExist indicates a package blob upload not exist error +var ErrPackageBlobUploadNotExist = errors.New("Package blob upload does not exist") + +func init() { + db.RegisterModel(new(PackageBlobUpload)) +} + +// PackageBlobUpload represents a package blob upload +type PackageBlobUpload struct { + ID string `xorm:"pk"` + BytesReceived int64 `xorm:"NOT NULL DEFAULT 0"` + HashStateBytes []byte `xorm:"BLOB"` + CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated INDEX NOT NULL"` +} + +// CreateBlobUpload inserts a blob upload +func CreateBlobUpload(ctx context.Context) (*PackageBlobUpload, error) { + id, err := util.CryptoRandomString(25) + if err != nil { + return nil, err + } + + pbu := &PackageBlobUpload{ + ID: strings.ToLower(id), + } + + _, err = db.GetEngine(ctx).Insert(pbu) + return pbu, err +} + +// GetBlobUploadByID gets a blob upload by id +func GetBlobUploadByID(ctx context.Context, id string) (*PackageBlobUpload, error) { + pbu := &PackageBlobUpload{} + + has, err := db.GetEngine(ctx).ID(id).Get(pbu) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageBlobUploadNotExist + } + return pbu, nil +} + +// UpdateBlobUpload updates the blob upload +func UpdateBlobUpload(ctx context.Context, pbu *PackageBlobUpload) error { + _, err := db.GetEngine(ctx).ID(pbu.ID).Update(pbu) + return err +} + +// DeleteBlobUploadByID deletes the blob upload +func DeleteBlobUploadByID(ctx context.Context, id string) error { + _, err := db.GetEngine(ctx).ID(id).Delete(&PackageBlobUpload{}) + return err +} + +// FindExpiredBlobUploads gets all expired blob uploads +func FindExpiredBlobUploads(ctx context.Context, olderThan time.Duration) ([]*PackageBlobUpload, error) { + pbus := make([]*PackageBlobUpload, 0, 10) + return pbus, db.GetEngine(ctx). + Where("updated_unix < ?", time.Now().Add(-olderThan).Unix()). + Find(&pbus) +} diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 1eedcb25c00a7..1c70324aa98d9 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -9,6 +9,7 @@ import ( "errors" "strconv" "strings" + "time" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" @@ -123,6 +124,7 @@ type PackageFileSearchOptions struct { Query string CompositeKey string Properties map[string]string + OlderThan time.Duration db.Paginator } @@ -137,8 +139,9 @@ func (opts *PackageFileSearchOptions) toConds() builder.Cond { From("package_version"). Join("INNER", "package", "package.id = package_version.package_id"). Where(builder.Eq{ - "package.owner_id": opts.OwnerID, - "package.type": opts.PackageType, + "package.owner_id": opts.OwnerID, + "package.type": opts.PackageType, + "package_version.is_internal": false, }) cond = cond.And(builder.In("package_file.version_id", in)) @@ -170,6 +173,10 @@ func (opts *PackageFileSearchOptions) toConds() builder.Cond { }) } + if opts.OlderThan != 0 { + cond = cond.And(builder.Lt{"package_file.created_unix": time.Now().Add(-opts.OlderThan).Unix()}) + } + return cond } diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 54538f5423dff..083c8be0dd8e1 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -35,6 +35,7 @@ type PackageVersion struct { Version string `xorm:"NOT NULL"` LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` + IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"` MetadataJSON string `xorm:"TEXT metadata_json"` DownloadCount int64 `xorm:"NOT NULL DEFAULT 0"` } @@ -89,10 +90,20 @@ func GetVersionByID(ctx context.Context, versionID int64) (*PackageVersion, erro // GetVersionByNameAndVersion gets a version by name and version number func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType Type, name, version string) (*PackageVersion, error) { + return getVersionByNameAndVersion(ctx, ownerID, packageType, name, version, false) +} + +// GetInternalVersionByNameAndVersion gets a version by name and version number +func GetInternalVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType Type, name, version string) (*PackageVersion, error) { + return getVersionByNameAndVersion(ctx, ownerID, packageType, name, version, true) +} + +func getVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType Type, name, version string, isInternal bool) (*PackageVersion, error) { var cond builder.Cond = builder.Eq{ - "package.owner_id": ownerID, - "package.type": packageType, - "package.lower_name": strings.ToLower(name), + "package.owner_id": ownerID, + "package.type": packageType, + "package.lower_name": strings.ToLower(name), + "package_version.is_internal": isInternal, } pv := &PackageVersion{ LowerVersion: strings.ToLower(version), @@ -114,8 +125,9 @@ func GetVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType // GetVersionsByPackageType gets all versions of a specific type func GetVersionsByPackageType(ctx context.Context, ownerID int64, packageType Type) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ - "package.owner_id": ownerID, - "package.type": packageType, + "package.owner_id": ownerID, + "package.type": packageType, + "package_version.is_internal": false, } pvs := make([]*PackageVersion, 0, 10) @@ -128,9 +140,10 @@ func GetVersionsByPackageType(ctx context.Context, ownerID int64, packageType Ty // GetVersionsByPackageName gets all versions of a specific package func GetVersionsByPackageName(ctx context.Context, ownerID int64, packageType Type, name string) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ - "package.owner_id": ownerID, - "package.type": packageType, - "package.lower_name": strings.ToLower(name), + "package.owner_id": ownerID, + "package.type": packageType, + "package.lower_name": strings.ToLower(name), + "package_version.is_internal": false, } pvs := make([]*PackageVersion, 0, 10) @@ -143,9 +156,10 @@ func GetVersionsByPackageName(ctx context.Context, ownerID int64, packageType Ty // GetVersionsByFilename gets all versions which are linked to a filename func GetVersionsByFilename(ctx context.Context, ownerID int64, packageType Type, filename string) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ - "package.owner_id": ownerID, - "package.type": packageType, - "package_file.lower_name": strings.ToLower(filename), + "package.owner_id": ownerID, + "package.type": packageType, + "package_file.lower_name": strings.ToLower(filename), + "package_version.is_internal": false, } pvs := make([]*PackageVersion, 0, 10) @@ -171,17 +185,19 @@ func HasVersionFileReferences(ctx context.Context, versionID int64) (bool, error // PackageSearchOptions are options for SearchXXX methods type PackageSearchOptions struct { - OwnerID int64 - RepoID int64 - Type string - Query string - Properties map[string]string - Sort string + OwnerID int64 + RepoID int64 + Type string + PackageID int64 + QueryName string + QueryVersion string + Properties map[string]string + Sort string db.Paginator } func (opts *PackageSearchOptions) toConds() builder.Cond { - cond := builder.NewCond() + var cond builder.Cond = builder.Eq{"package_version.is_internal": false} if opts.OwnerID != 0 { cond = cond.And(builder.Eq{"package.owner_id": opts.OwnerID}) @@ -192,8 +208,14 @@ func (opts *PackageSearchOptions) toConds() builder.Cond { if opts.Type != "" && opts.Type != "all" { cond = cond.And(builder.Eq{"package.type": opts.Type}) } - if opts.Query != "" { - cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.Query)}) + if opts.PackageID != 0 { + cond = cond.And(builder.Eq{"package.id": opts.PackageID}) + } + if opts.QueryName != "" { + cond = cond.And(builder.Like{"package.lower_name", strings.ToLower(opts.QueryName)}) + } + if opts.QueryVersion != "" { + cond = cond.And(builder.Like{"package_version.lower_version", strings.ToLower(opts.QueryVersion)}) } if len(opts.Properties) != 0 { @@ -261,7 +283,7 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P sess := db.GetEngine(ctx). Table("package_version"). - Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND package_version.created_unix < pv2.created_unix"). + Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))"). Join("INNER", "package", "package.id = package_version.package_id"). Where(cond) @@ -279,10 +301,11 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P // FindVersionsByPropertyNameAndValue gets all package versions which are associated with a specific property + value func FindVersionsByPropertyNameAndValue(ctx context.Context, packageID int64, name, value string) ([]*PackageVersion, error) { var cond builder.Cond = builder.Eq{ - "package_property.ref_type": PropertyTypeVersion, - "package_property.name": name, - "package_property.value": value, - "package_version.package_id": packageID, + "package_property.ref_type": PropertyTypeVersion, + "package_property.name": name, + "package_property.value": value, + "package_version.package_id": packageID, + "package_version.is_internal": false, } pvs := make([]*PackageVersion, 0, 5) diff --git a/models/user/user.go b/models/user/user.go index a3094a13ceb16..fd7bc30c854f5 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -605,6 +605,7 @@ var ( "stars", "template", "user", + "v2", } reservedUserPatterns = []string{"*.keys", "*.gpg", "*.rss", "*.atom"} diff --git a/modules/packages/container/helm/helm.go b/modules/packages/container/helm/helm.go new file mode 100644 index 0000000000000..98d3824a85e47 --- /dev/null +++ b/modules/packages/container/helm/helm.go @@ -0,0 +1,56 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package helm + +// https://github.com/helm/helm/blob/main/pkg/chart/ + +const ConfigMediaType = "application/vnd.cncf.helm.config.v1+json" + +// Maintainer describes a Chart maintainer. +type Maintainer struct { + // Name is a user name or organization name + Name string `json:"name,omitempty"` + // Email is an optional email address to contact the named maintainer + Email string `json:"email,omitempty"` + // URL is an optional URL to an address for the named maintainer + URL string `json:"url,omitempty"` +} + +// Metadata for a Chart file. This models the structure of a Chart.yaml file. +type Metadata struct { + // The name of the chart. Required. + Name string `json:"name,omitempty"` + // The URL to a relevant project page, git repo, or contact person + Home string `json:"home,omitempty"` + // Source is the URL to the source code of this chart + Sources []string `json:"sources,omitempty"` + // A SemVer 2 conformant version string of the chart. Required. + Version string `json:"version,omitempty"` + // A one-sentence description of the chart + Description string `json:"description,omitempty"` + // A list of string keywords + Keywords []string `json:"keywords,omitempty"` + // A list of name and URL/email address combinations for the maintainer(s) + Maintainers []*Maintainer `json:"maintainers,omitempty"` + // The URL to an icon file. + Icon string `json:"icon,omitempty"` + // The API Version of this chart. Required. + APIVersion string `json:"apiVersion,omitempty"` + // The condition to check to enable chart + Condition string `json:"condition,omitempty"` + // The tags to check to enable chart + Tags string `json:"tags,omitempty"` + // The version of the application enclosed inside of this chart. + AppVersion string `json:"appVersion,omitempty"` + // Whether or not this chart is deprecated + Deprecated bool `json:"deprecated,omitempty"` + // Annotations are additional mappings uninterpreted by Helm, + // made available for inspection by other applications. + Annotations map[string]string `json:"annotations,omitempty"` + // KubeVersion is a SemVer constraint specifying the version of Kubernetes required. + KubeVersion string `json:"kubeVersion,omitempty"` + // Specifies the chart type: application or library + Type string `json:"type,omitempty"` +} diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go new file mode 100644 index 0000000000000..087d38e5bd142 --- /dev/null +++ b/modules/packages/container/metadata.go @@ -0,0 +1,157 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "fmt" + "io" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/container/helm" + "code.gitea.io/gitea/modules/packages/container/oci" + "code.gitea.io/gitea/modules/validation" +) + +const ( + PropertyDigest = "container.digest" + PropertyMediaType = "container.mediatype" + PropertyManifestTagged = "container.manifest.tagged" + PropertyManifestReference = "container.manifest.reference" + + DefaultPlatform = "linux/amd64" + + labelLicenses = "org.opencontainers.image.licenses" + labelURL = "org.opencontainers.image.url" + labelSource = "org.opencontainers.image.source" + labelDocumentation = "org.opencontainers.image.documentation" + labelDescription = "org.opencontainers.image.description" + labelAuthors = "org.opencontainers.image.authors" +) + +type ImageType string + +const ( + TypeOCI ImageType = "oci" + TypeHelm ImageType = "helm" +) + +// Name gets the name of the image type +func (it ImageType) Name() string { + switch it { + case TypeHelm: + return "Helm Chart" + default: + return "OCI / Docker" + } +} + +// Metadata represents the metadata of a Container package +type Metadata struct { + Type ImageType `json:"type"` + IsTagged bool `json:"is_tagged"` + Platform string `json:"platform,omitempty"` + Description string `json:"description,omitempty"` + Authors []string `json:"authors,omitempty"` + Licenses string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ImageLayers []string `json:"layer_creation,omitempty"` + MultiArch map[string]string `json:"multiarch,omitempty"` +} + +// ParseImageConfig parses the metadata of an image config +func ParseImageConfig(mediaType oci.MediaType, r io.Reader) (*Metadata, error) { + if strings.EqualFold(string(mediaType), helm.ConfigMediaType) { + return parseHelmConfig(r) + } + + // fallback to OCI Image Config + return parseOCIImageConfig(r) +} + +func parseOCIImageConfig(r io.Reader) (*Metadata, error) { + var image oci.Image + if err := json.NewDecoder(r).Decode(&image); err != nil { + return nil, err + } + + platform := DefaultPlatform + if image.OS != "" && image.Architecture != "" { + platform = fmt.Sprintf("%s/%s", image.OS, image.Architecture) + if image.Variant != "" { + platform = fmt.Sprintf("%s/%s", platform, image.Variant) + } + } + + imageLayers := make([]string, 0, len(image.History)) + for _, history := range image.History { + cmd := history.CreatedBy + if i := strings.Index(cmd, "#(nop) "); i != -1 { + cmd = strings.TrimSpace(cmd[i+7:]) + } + imageLayers = append(imageLayers, cmd) + } + + metadata := &Metadata{ + Type: TypeOCI, + Platform: platform, + Licenses: image.Config.Labels[labelLicenses], + ProjectURL: image.Config.Labels[labelURL], + RepositoryURL: image.Config.Labels[labelSource], + DocumentationURL: image.Config.Labels[labelDocumentation], + Description: image.Config.Labels[labelDescription], + Labels: image.Config.Labels, + ImageLayers: imageLayers, + } + + if authors, ok := image.Config.Labels[labelAuthors]; ok { + metadata.Authors = []string{authors} + } + + if !validation.IsValidURL(metadata.ProjectURL) { + metadata.ProjectURL = "" + } + if !validation.IsValidURL(metadata.RepositoryURL) { + metadata.RepositoryURL = "" + } + if !validation.IsValidURL(metadata.DocumentationURL) { + metadata.DocumentationURL = "" + } + + return metadata, nil +} + +func parseHelmConfig(r io.Reader) (*Metadata, error) { + var config helm.Metadata + if err := json.NewDecoder(r).Decode(&config); err != nil { + return nil, err + } + + metadata := &Metadata{ + Type: TypeHelm, + Description: config.Description, + ProjectURL: config.Home, + } + + if len(config.Maintainers) > 0 { + authors := make([]string, 0, len(config.Maintainers)) + for _, maintainer := range config.Maintainers { + authors = append(authors, maintainer.Name) + } + metadata.Authors = authors + } + + if len(config.Sources) > 0 && validation.IsValidURL(config.Sources[0]) { + metadata.RepositoryURL = config.Sources[0] + } + if !validation.IsValidURL(metadata.ProjectURL) { + metadata.ProjectURL = "" + } + + return metadata, nil +} diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go new file mode 100644 index 0000000000000..9400cf695443d --- /dev/null +++ b/modules/packages/container/metadata_test.go @@ -0,0 +1,62 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "strings" + "testing" + + "code.gitea.io/gitea/modules/packages/container/helm" + "code.gitea.io/gitea/modules/packages/container/oci" + + "github.com/stretchr/testify/assert" +) + +func TestParseImageConfig(t *testing.T) { + description := "Image Description" + author := "Gitea" + license := "MIT" + projectURL := "https://gitea.io" + repositoryURL := "https://gitea.com/gitea" + documentationURL := "https://docs.gitea.io" + + configOCI := `{"config": {"labels": {"` + labelAuthors + `": "` + author + `", "` + labelLicenses + `": "` + license + `", "` + labelURL + `": "` + projectURL + `", "` + labelSource + `": "` + repositoryURL + `", "` + labelDocumentation + `": "` + documentationURL + `", "` + labelDescription + `": "` + description + `"}}, "history": [{"created_by": "do it 1"}, {"created_by": "dummy #(nop) do it 2"}]}` + + metadata, err := ParseImageConfig(oci.MediaType(oci.MediaTypeImageManifest), strings.NewReader(configOCI)) + assert.NoError(t, err) + + assert.Equal(t, TypeOCI, metadata.Type) + assert.Equal(t, description, metadata.Description) + assert.ElementsMatch(t, []string{author}, metadata.Authors) + assert.Equal(t, license, metadata.Licenses) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) + assert.Equal(t, documentationURL, metadata.DocumentationURL) + assert.Equal(t, []string{"do it 1", "do it 2"}, metadata.ImageLayers) + assert.Equal( + t, + map[string]string{ + labelAuthors: author, + labelLicenses: license, + labelURL: projectURL, + labelSource: repositoryURL, + labelDocumentation: documentationURL, + labelDescription: description, + }, + metadata.Labels, + ) + assert.Empty(t, metadata.MultiArch) + + configHelm := `{"description":"` + description + `", "home": "` + projectURL + `", "sources": ["` + repositoryURL + `"], "maintainers":[{"name":"` + author + `"}]}` + + metadata, err = ParseImageConfig(oci.MediaType(helm.ConfigMediaType), strings.NewReader(configHelm)) + assert.NoError(t, err) + + assert.Equal(t, TypeHelm, metadata.Type) + assert.Equal(t, description, metadata.Description) + assert.ElementsMatch(t, []string{author}, metadata.Authors) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) +} diff --git a/modules/packages/container/oci/digest.go b/modules/packages/container/oci/digest.go new file mode 100644 index 0000000000000..5234814cfe485 --- /dev/null +++ b/modules/packages/container/oci/digest.go @@ -0,0 +1,27 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package oci + +import ( + "regexp" + "strings" +) + +var digestPattern = regexp.MustCompile(`\Asha256:[a-f0-9]{64}\z`) + +type Digest string + +// Validate checks if the digest has a valid SHA256 signature +func (d Digest) Validate() bool { + return digestPattern.MatchString(string(d)) +} + +func (d Digest) Hash() string { + p := strings.SplitN(string(d), ":", 2) + if len(p) != 2 { + return "" + } + return p[1] +} diff --git a/modules/packages/container/oci/mediatype.go b/modules/packages/container/oci/mediatype.go new file mode 100644 index 0000000000000..2636fbe2880fc --- /dev/null +++ b/modules/packages/container/oci/mediatype.go @@ -0,0 +1,36 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package oci + +import ( + "strings" +) + +const ( + MediaTypeImageManifest = "application/vnd.oci.image.manifest.v1+json" + MediaTypeImageIndex = "application/vnd.oci.image.index.v1+json" + MediaTypeDockerManifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeDockerManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" +) + +type MediaType string + +// IsValid tests if the media type is in the OCI or Docker namespace +func (m MediaType) IsValid() bool { + s := string(m) + return strings.HasPrefix(s, "application/vnd.docker.") || strings.HasPrefix(s, "application/vnd.oci.") +} + +// IsImageManifest tests if the media type is an image manifest +func (m MediaType) IsImageManifest() bool { + s := string(m) + return strings.EqualFold(s, MediaTypeDockerManifest) || strings.EqualFold(s, MediaTypeImageManifest) +} + +// IsImageIndex tests if the media type is an image index +func (m MediaType) IsImageIndex() bool { + s := string(m) + return strings.EqualFold(s, MediaTypeDockerManifestList) || strings.EqualFold(s, MediaTypeImageIndex) +} diff --git a/modules/packages/container/oci/oci.go b/modules/packages/container/oci/oci.go new file mode 100644 index 0000000000000..01cca8fe69865 --- /dev/null +++ b/modules/packages/container/oci/oci.go @@ -0,0 +1,191 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package oci + +import ( + "time" +) + +// https://github.com/opencontainers/image-spec/tree/main/specs-go/v1 + +// ImageConfig defines the execution parameters which should be used as a base when running a container using an image. +type ImageConfig struct { + // User defines the username or UID which the process in the container should run as. + User string `json:"User,omitempty"` + + // ExposedPorts a set of ports to expose from a container running this image. + ExposedPorts map[string]struct{} `json:"ExposedPorts,omitempty"` + + // Env is a list of environment variables to be used in a container. + Env []string `json:"Env,omitempty"` + + // Entrypoint defines a list of arguments to use as the command to execute when the container starts. + Entrypoint []string `json:"Entrypoint,omitempty"` + + // Cmd defines the default arguments to the entrypoint of the container. + Cmd []string `json:"Cmd,omitempty"` + + // Volumes is a set of directories describing where the process is likely write data specific to a container instance. + Volumes map[string]struct{} `json:"Volumes,omitempty"` + + // WorkingDir sets the current working directory of the entrypoint process in the container. + WorkingDir string `json:"WorkingDir,omitempty"` + + // Labels contains arbitrary metadata for the container. + Labels map[string]string `json:"Labels,omitempty"` + + // StopSignal contains the system call signal that will be sent to the container to exit. + StopSignal string `json:"StopSignal,omitempty"` +} + +// RootFS describes a layer content addresses +type RootFS struct { + // Type is the type of the rootfs. + Type string `json:"type"` + + // DiffIDs is an array of layer content hashes, in order from bottom-most to top-most. + DiffIDs []string `json:"diff_ids"` +} + +// History describes the history of a layer. +type History struct { + // Created is the combined date and time at which the layer was created, formatted as defined by RFC 3339, section 5.6. + Created *time.Time `json:"created,omitempty"` + + // CreatedBy is the command which created the layer. + CreatedBy string `json:"created_by,omitempty"` + + // Author is the author of the build point. + Author string `json:"author,omitempty"` + + // Comment is a custom message set when creating the layer. + Comment string `json:"comment,omitempty"` + + // EmptyLayer is used to mark if the history item created a filesystem diff. + EmptyLayer bool `json:"empty_layer,omitempty"` +} + +// Image is the JSON structure which describes some basic information about the image. +// This provides the `application/vnd.oci.image.config.v1+json` mediatype when marshalled to JSON. +type Image struct { + // Created is the combined date and time at which the image was created, formatted as defined by RFC 3339, section 5.6. + Created *time.Time `json:"created,omitempty"` + + // Author defines the name and/or email address of the person or entity which created and is responsible for maintaining the image. + Author string `json:"author,omitempty"` + + // Architecture is the CPU architecture which the binaries in this image are built to run on. + Architecture string `json:"architecture"` + + // Variant is the variant of the specified CPU architecture which image binaries are intended to run on. + Variant string `json:"variant,omitempty"` + + // OS is the name of the operating system which the image is built to run on. + OS string `json:"os"` + + // OSVersion is an optional field specifying the operating system + // version, for example on Windows `10.0.14393.1066`. + OSVersion string `json:"os.version,omitempty"` + + // OSFeatures is an optional field specifying an array of strings, + // each listing a required OS feature (for example on Windows `win32k`). + OSFeatures []string `json:"os.features,omitempty"` + + // Config defines the execution parameters which should be used as a base when running a container using the image. + Config ImageConfig `json:"config,omitempty"` + + // RootFS references the layer content addresses used by the image. + RootFS RootFS `json:"rootfs"` + + // History describes the history of each layer. + History []History `json:"history,omitempty"` +} + +// Descriptor describes the disposition of targeted content. +// This structure provides `application/vnd.oci.descriptor.v1+json` mediatype +// when marshalled to JSON. +type Descriptor struct { + // MediaType is the media type of the object this schema refers to. + MediaType MediaType `json:"mediaType,omitempty"` + + // Digest is the digest of the targeted content. + Digest Digest `json:"digest"` + + // Size specifies the size in bytes of the blob. + Size int64 `json:"size"` + + // URLs specifies a list of URLs from which this object MAY be downloaded + URLs []string `json:"urls,omitempty"` + + // Annotations contains arbitrary metadata relating to the targeted content. + Annotations map[string]string `json:"annotations,omitempty"` + + // Data is an embedding of the targeted content. This is encoded as a base64 + // string when marshalled to JSON (automatically, by encoding/json). If + // present, Data can be used directly to avoid fetching the targeted content. + Data []byte `json:"data,omitempty"` + + // Platform describes the platform which the image in the manifest runs on. + // + // This should only be used when referring to a manifest. + Platform *Platform `json:"platform,omitempty"` +} + +// Platform describes the platform which the image in the manifest runs on. +type Platform struct { + // Architecture field specifies the CPU architecture, for example + // `amd64` or `ppc64`. + Architecture string `json:"architecture"` + + // OS specifies the operating system, for example `linux` or `windows`. + OS string `json:"os"` + + // OSVersion is an optional field specifying the operating system + // version, for example on Windows `10.0.14393.1066`. + OSVersion string `json:"os.version,omitempty"` + + // OSFeatures is an optional field specifying an array of strings, + // each listing a required OS feature (for example on Windows `win32k`). + OSFeatures []string `json:"os.features,omitempty"` + + // Variant is an optional field specifying a variant of the CPU, for + // example `v7` to specify ARMv7 when architecture is `arm`. + Variant string `json:"variant,omitempty"` +} + +type SchemaMediaBase struct { + // SchemaVersion is the image manifest schema that this image follows + SchemaVersion int `json:"schemaVersion"` + + // MediaType specifies the type of this document data structure e.g. `application/vnd.oci.image.manifest.v1+json` + MediaType MediaType `json:"mediaType,omitempty"` +} + +// Manifest provides `application/vnd.oci.image.manifest.v1+json` mediatype structure when marshalled to JSON. +type Manifest struct { + SchemaMediaBase + + // Config references a configuration object for a container, by digest. + // The referenced configuration object is a JSON blob that the runtime uses to set up the container. + Config Descriptor `json:"config"` + + // Layers is an indexed list of layers referenced by the manifest. + Layers []Descriptor `json:"layers"` + + // Annotations contains arbitrary metadata for the image manifest. + Annotations map[string]string `json:"annotations,omitempty"` +} + +// Index references manifests for various platforms. +// This structure provides `application/vnd.oci.image.index.v1+json` mediatype when marshalled to JSON. +type Index struct { + SchemaMediaBase + + // Manifests references platform specific manifests. + Manifests []Descriptor `json:"manifests"` + + // Annotations contains arbitrary metadata for the image index. + Annotations map[string]string `json:"annotations,omitempty"` +} diff --git a/modules/packages/container/oci/reference.go b/modules/packages/container/oci/reference.go new file mode 100644 index 0000000000000..120ff122d4e68 --- /dev/null +++ b/modules/packages/container/oci/reference.go @@ -0,0 +1,17 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package oci + +import ( + "regexp" +) + +var referencePattern = regexp.MustCompile(`\A[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}\z`) + +type Reference string + +func (r Reference) Validate() bool { + return referencePattern.MatchString(string(r)) +} diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go index 5f2d7132c5ed0..3f8cafcfb536d 100644 --- a/modules/packages/hashed_buffer.go +++ b/modules/packages/hashed_buffer.go @@ -5,24 +5,23 @@ package packages import ( - "crypto/md5" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" - "hash" "io" "code.gitea.io/gitea/modules/util/filebuffer" ) +// HashedSizeReader provide methods to read, sum hashes and a Size method +type HashedSizeReader interface { + io.Reader + HashSummer + Size() int64 +} + // HashedBuffer is buffer which calculates multiple checksums type HashedBuffer struct { *filebuffer.FileBackedBuffer - md5 hash.Hash - sha1 hash.Hash - sha256 hash.Hash - sha512 hash.Hash + hash *MultiHasher combinedWriter io.Writer } @@ -33,19 +32,14 @@ func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) { if err != nil { return nil, err } - md5 := md5.New() - sha1 := sha1.New() - sha256 := sha256.New() - sha512 := sha512.New() - combinedWriter := io.MultiWriter(b, md5, sha1, sha256, sha512) + hash := NewMultiHasher() + + combinedWriter := io.MultiWriter(b, hash) return &HashedBuffer{ b, - md5, - sha1, - sha256, - sha512, + hash, combinedWriter, }, nil } @@ -72,9 +66,5 @@ func (b *HashedBuffer) Write(p []byte) (int, error) { // Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data func (b *HashedBuffer) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) { - hashMD5 = b.md5.Sum(nil) - hashSHA1 = b.sha1.Sum(nil) - hashSHA256 = b.sha256.Sum(nil) - hashSHA512 = b.sha512.Sum(nil) - return + return b.hash.Sums() } diff --git a/modules/packages/multi_hasher.go b/modules/packages/multi_hasher.go new file mode 100644 index 0000000000000..0659a18d2a020 --- /dev/null +++ b/modules/packages/multi_hasher.go @@ -0,0 +1,123 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding" + "errors" + "hash" + "io" +) + +const ( + marshaledSizeMD5 = 92 + marshaledSizeSHA1 = 96 + marshaledSizeSHA256 = 108 + marshaledSizeSHA512 = 204 + + marshaledSize = marshaledSizeMD5 + marshaledSizeSHA1 + marshaledSizeSHA256 + marshaledSizeSHA512 +) + +// HashSummer provide a Sums method +type HashSummer interface { + Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) +} + +// MultiHasher calculates multiple checksums +type MultiHasher struct { + md5 hash.Hash + sha1 hash.Hash + sha256 hash.Hash + sha512 hash.Hash + + combinedWriter io.Writer +} + +// NewMultiHasher creates a multi hasher +func NewMultiHasher() *MultiHasher { + md5 := md5.New() + sha1 := sha1.New() + sha256 := sha256.New() + sha512 := sha512.New() + + combinedWriter := io.MultiWriter(md5, sha1, sha256, sha512) + + return &MultiHasher{ + md5, + sha1, + sha256, + sha512, + combinedWriter, + } +} + +// MarshalBinary implements encoding.BinaryMarshaler +func (h *MultiHasher) MarshalBinary() ([]byte, error) { + md5Bytes, err := h.md5.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha1Bytes, err := h.sha1.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha256Bytes, err := h.sha256.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha512Bytes, err := h.sha512.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + + b := make([]byte, 0, marshaledSize) + b = append(b, md5Bytes...) + b = append(b, sha1Bytes...) + b = append(b, sha256Bytes...) + b = append(b, sha512Bytes...) + return b, nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler +func (h *MultiHasher) UnmarshalBinary(b []byte) error { + if len(b) != marshaledSize { + return errors.New("invalid hash state size") + } + + if err := h.md5.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeMD5]); err != nil { + return err + } + + b = b[marshaledSizeMD5:] + if err := h.sha1.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA1]); err != nil { + return err + } + + b = b[marshaledSizeSHA1:] + if err := h.sha256.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA256]); err != nil { + return err + } + + b = b[marshaledSizeSHA256:] + return h.sha512.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA512]) +} + +// Write implements io.Writer +func (h *MultiHasher) Write(p []byte) (int, error) { + return h.combinedWriter.Write(p) +} + +// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data +func (h *MultiHasher) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) { + hashMD5 = h.md5.Sum(nil) + hashSHA1 = h.sha1.Sum(nil) + hashSHA256 = h.sha256.Sum(nil) + hashSHA512 = h.sha512.Sum(nil) + return +} diff --git a/modules/packages/multi_hasher_test.go b/modules/packages/multi_hasher_test.go new file mode 100644 index 0000000000000..6c895ce120b50 --- /dev/null +++ b/modules/packages/multi_hasher_test.go @@ -0,0 +1,54 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + expectedMD5 = "e3bef03c5f3b7f6b3ab3e3053ed71e9c" + expectedSHA1 = "060b3b99f88e96085b4a68e095bc9e3d1d91e1bc" + expectedSHA256 = "6ccce4863b70f258d691f59609d31b4502e1ba5199942d3bc5d35d17a4ce771d" + expectedSHA512 = "7f70e439ba8c52025c1f06cdf6ae443c4b8ed2e90059cdb9bbbf8adf80846f185a24acca9245b128b226d61753b0d7ed46580a69c8999eeff3bc13a4d0bd816c" +) + +func TestMultiHasherSums(t *testing.T) { + t.Run("Sums", func(t *testing.T) { + h := NewMultiHasher() + h.Write([]byte("gitea")) + + hashMD5, hashSHA1, hashSHA256, hashSHA512 := h.Sums() + + assert.Equal(t, expectedMD5, fmt.Sprintf("%x", hashMD5)) + assert.Equal(t, expectedSHA1, fmt.Sprintf("%x", hashSHA1)) + assert.Equal(t, expectedSHA256, fmt.Sprintf("%x", hashSHA256)) + assert.Equal(t, expectedSHA512, fmt.Sprintf("%x", hashSHA512)) + }) + + t.Run("State", func(t *testing.T) { + h := NewMultiHasher() + h.Write([]byte("git")) + + state, err := h.MarshalBinary() + assert.NoError(t, err) + + h2 := NewMultiHasher() + err = h2.UnmarshalBinary(state) + assert.NoError(t, err) + + h2.Write([]byte("ea")) + + hashMD5, hashSHA1, hashSHA256, hashSHA512 := h2.Sums() + + assert.Equal(t, expectedMD5, fmt.Sprintf("%x", hashMD5)) + assert.Equal(t, expectedSHA1, fmt.Sprintf("%x", hashSHA1)) + assert.Equal(t, expectedSHA256, fmt.Sprintf("%x", hashSHA256)) + assert.Equal(t, expectedSHA512, fmt.Sprintf("%x", hashSHA512)) + }) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 9a89fa9a3e3b9..d3d67215404a9 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -4,20 +4,42 @@ package setting -import "code.gitea.io/gitea/modules/log" +import ( + "os" + "path/filepath" + + "code.gitea.io/gitea/modules/log" +) // Package registry settings var ( Packages = struct { Storage - Enabled bool + Enabled bool + ChunkedUploadPath string + RegistryHost string }{ Enabled: true, } ) func newPackages() { - if err := Cfg.Section("packages").MapTo(&Packages); err != nil { + sec := Cfg.Section("packages") + if err := sec.MapTo(&Packages); err != nil { log.Fatal("Failed to map Packages settings: %v", err) } + + Packages.RegistryHost = Domain + if (Protocol == HTTP && HTTPPort != "80") || (Protocol == HTTPS && HTTPPort != "443") { + Packages.RegistryHost += ":" + HTTPPort + } + + Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload")) + if !filepath.IsAbs(Packages.ChunkedUploadPath) { + Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath)) + } + + if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { + log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) + } } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 1709bcf7664a9..ed5be8f31256c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3006,6 +3006,8 @@ empty.documentation = For more information on the package registry, see %[3]s published_by_in = Published %[1]s by %[3]s in %[5]s installation = Installation @@ -3020,6 +3022,7 @@ details.license = License assets = Assets versions = Versions versions.on = on +versions.view_all = View all dependency.id = Id dependency.version = Version composer.registry = Setup this registry in your ~/.composer/config.json file: @@ -3031,6 +3034,17 @@ conan.details.repository = Repository conan.registry = Setup this registry from the command line: conan.install = To install the package using Conan, run the following command: conan.documentation = For more information on the Conan registry, see the documentation. +container.details.type = Image Type +container.details.platform = Platform +container.details.repository_site = Repository Site +container.details.documentation_site = Documentation Site +container.pull = Pull the image from the command line: +container.documentation = For more information on the Container registry, see the documentation. +container.multi_arch = OS / Arch +container.layers = Image Layers +container.labels = Labels +container.labels.key = Key +container.labels.value = Value generic.download = Download package from the command line: generic.documentation = For more information on the generic registry, see the documentation. maven.registry = Setup this registry in your project pom.xml file: diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 9b854cbd015bb..fd4b147f19421 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -6,13 +6,15 @@ package packages import ( "net/http" + "regexp" + "strings" "code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/packages/composer" "code.gitea.io/gitea/routers/api/packages/conan" + "code.gitea.io/gitea/routers/api/packages/container" "code.gitea.io/gitea/routers/api/packages/generic" "code.gitea.io/gitea/routers/api/packages/maven" "code.gitea.io/gitea/routers/api/packages/npm" @@ -35,10 +37,6 @@ func reqPackageAccess(accessMode perm.AccessMode) func(ctx *context.Context) { func Routes() *web.Route { r := web.NewRoute() - if !setting.Packages.Enabled { - return r - } - r.Use(context.PackageContexter()) authGroup := auth.NewGroup(&auth.Basic{}, &conan.Auth{}) @@ -221,3 +219,160 @@ func Routes() *web.Route { return r } + +func ContainerRoutes() *web.Route { + r := web.NewRoute() + + r.Use(context.PackageContexter()) + + authGroup := auth.NewGroup(&auth.Basic{}, &container.Auth{}) + r.Use(func(ctx *context.Context) { + ctx.User = authGroup.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) + }) + + r.Get("", container.ReqContainerAccess, container.DetermineSupport) + r.Get("/token", container.Authenticate) + r.Group("/{username}", func() { + r.Group("/{image}", func() { + r.Group("/blobs/uploads", func() { + r.Post("", container.InitiateUploadBlob) + r.Group("/{uuid}", func() { + r.Patch("", container.UploadBlob) + r.Put("", container.EndUploadBlob) + }) + }, reqPackageAccess(perm.AccessModeWrite)) + r.Group("/blobs/{digest}", func() { + r.Head("", container.HeadBlob) + r.Get("", container.GetBlob) + r.Delete("", reqPackageAccess(perm.AccessModeWrite), container.DeleteBlob) + }) + r.Group("/manifests/{reference}", func() { + r.Put("", reqPackageAccess(perm.AccessModeWrite), container.UploadManifest) + r.Head("", container.HeadManifest) + r.Get("", container.GetManifest) + r.Delete("", reqPackageAccess(perm.AccessModeWrite), container.DeleteManifest) + }) + r.Get("/tags/list", container.GetTagList) + }, container.VerifyImageName) + + var ( + blobsUploadsPattern = regexp.MustCompile(`\A(.+)/blobs/uploads/([a-zA-Z0-9-_.=]+)\z`) + blobsPattern = regexp.MustCompile(`\A(.+)/blobs/([^/]+)\z`) + manifestsPattern = regexp.MustCompile(`\A(.+)/manifests/([^/]+)\z`) + ) + + // Manual mapping of routes because {image} can contain slashes which chi does not support + r.Route("/*", "HEAD,GET,POST,PUT,PATCH,DELETE", func(ctx *context.Context) { + path := ctx.Params("*") + isHead := ctx.Req.Method == "HEAD" + isGet := ctx.Req.Method == "GET" + isPost := ctx.Req.Method == "POST" + isPut := ctx.Req.Method == "PUT" + isPatch := ctx.Req.Method == "PATCH" + isDelete := ctx.Req.Method == "DELETE" + + if isPost && strings.HasSuffix(path, "/blobs/uploads") { + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("image", path[:len(path)-14]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + container.InitiateUploadBlob(ctx) + return + } + if isGet && strings.HasSuffix(path, "/tags/list") { + ctx.SetParams("image", path[:len(path)-10]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + container.GetTagList(ctx) + return + } + + m := blobsUploadsPattern.FindStringSubmatch(path) + if len(m) == 3 && (isPut || isPatch) { + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("image", m[1]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("uuid", m[2]) + + if isPatch { + container.UploadBlob(ctx) + } else { + container.EndUploadBlob(ctx) + } + return + } + m = blobsPattern.FindStringSubmatch(path) + if len(m) == 3 && (isHead || isGet || isDelete) { + ctx.SetParams("image", m[1]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("digest", m[2]) + + if isHead { + container.HeadBlob(ctx) + } else if isGet { + container.GetBlob(ctx) + } else { + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + container.DeleteBlob(ctx) + } + return + } + m = manifestsPattern.FindStringSubmatch(path) + if len(m) == 3 && (isHead || isGet || isPut || isDelete) { + ctx.SetParams("image", m[1]) + container.VerifyImageName(ctx) + if ctx.Written() { + return + } + + ctx.SetParams("reference", m[2]) + + if isHead { + container.HeadManifest(ctx) + } else if isGet { + container.GetManifest(ctx) + } else { + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + if isPut { + container.UploadManifest(ctx) + } else { + container.DeleteManifest(ctx) + } + } + return + } + + ctx.Status(http.StatusNotFound) + }) + }, container.ReqContainerAccess, context.UserAssignment(), context.PackageAssignment(), reqPackageAccess(perm.AccessModeRead)) + + return r +} diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index aabb7dadb7f6d..b99db40f81ce6 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -64,7 +64,7 @@ func SearchPackages(ctx *context.Context) { opts := &packages_model.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, Type: string(packages_model.TypeComposer), - Query: ctx.FormTrim("q"), + QueryName: ctx.FormTrim("q"), Paginator: &paginator, } if ctx.FormTrim("type") != "" { diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index 87400c7cbc960..d6201778b6ea6 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -667,10 +667,6 @@ func deleteRecipeOrPackage(apictx *context.Context, rref *conan_module.RecipeRef if err := packages_model.DeleteVersionByID(ctx, pv.ID); err != nil { return err } - - if err := packages_model.DeletePackageByIDIfUnreferenced(ctx, pv.PackageID); err != nil { - return err - } } if err := committer.Commit(); err != nil { diff --git a/routers/api/packages/container/auth.go b/routers/api/packages/container/auth.go new file mode 100644 index 0000000000000..770068a3bfb7b --- /dev/null +++ b/routers/api/packages/container/auth.go @@ -0,0 +1,45 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "net/http" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/services/auth" + "code.gitea.io/gitea/services/packages" +) + +type Auth struct{} + +func (a *Auth) Name() string { + return "container" +} + +// Verify extracts the user from the Bearer token +// If it's an anonymous session a ghost user is returned +func (a *Auth) Verify(req *http.Request, w http.ResponseWriter, store auth.DataStore, sess auth.SessionStore) *user_model.User { + uid, err := packages.ParseAuthorizationToken(req) + if err != nil { + log.Trace("ParseAuthorizationToken: %v", err) + return nil + } + + if uid == 0 { + return nil + } + if uid == -1 { + return user_model.NewGhostUser() + } + + u, err := user_model.GetUserByID(uid) + if err != nil { + log.Error("GetUserByID: %v", err) + return nil + } + + return u +} diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go new file mode 100644 index 0000000000000..8f6254f583213 --- /dev/null +++ b/routers/api/packages/container/blob.go @@ -0,0 +1,136 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "encoding/hex" + "fmt" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + container_model "code.gitea.io/gitea/models/packages/container" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + container_module "code.gitea.io/gitea/modules/packages/container" + packages_service "code.gitea.io/gitea/services/packages" +) + +// saveAsPackageBlob creates a package blob from an upload +// The uploaded blob gets stored in a special upload version to link them to the package/image +func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) { + pb := packages_service.NewPackageBlob(hsr) + + exists := false + + contentStore := packages_module.NewContentStore() + + err := db.WithTx(func(ctx context.Context) error { + p := &packages_model.Package{ + OwnerID: pi.Owner.ID, + Type: packages_model.TypeContainer, + Name: strings.ToLower(pi.Name), + LowerName: strings.ToLower(pi.Name), + } + var err error + if p, err = packages_model.TryInsertPackage(ctx, p); err != nil { + if err != packages_model.ErrDuplicatePackage { + log.Error("Error inserting package: %v", err) + return err + } + } + + pv := &packages_model.PackageVersion{ + PackageID: p.ID, + CreatorID: pi.Owner.ID, + Version: container_model.UploadVersion, + LowerVersion: container_model.UploadVersion, + IsInternal: true, + MetadataJSON: "null", + } + if pv, err = packages_model.GetOrInsertVersion(ctx, pv); err != nil { + if err != packages_model.ErrDuplicatePackageVersion { + log.Error("Error inserting package: %v", err) + return err + } + } + + pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb) + if err != nil { + log.Error("Error inserting package blob: %v", err) + return err + } + if !exists { + if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil { + log.Error("Error saving package blob in content store: %v", err) + return err + } + } + + filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256)) + + pf := &packages_model.PackageFile{ + VersionID: pv.ID, + BlobID: pb.ID, + Name: filename, + LowerName: filename, + CompositeKey: packages_model.EmptyFileKey, + } + if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil { + if err == packages_model.ErrDuplicatePackageFile { + return nil + } + log.Error("Error inserting package file: %v", err) + return err + } + + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, container_module.PropertyDigest, digestFromPackageBlob(pb)); err != nil { + log.Error("Error setting package file property: %v", err) + return err + } + + return nil + }) + if err != nil { + if !exists { + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob from content store: %v", err) + } + } + return nil, err + } + + return pb, nil +} + +func deleteBlob(ownerID int64, image, digest string) error { + return db.WithTx(func(ctx context.Context) error { + pfds, err := container_model.GetContainerBlobs(ctx, &container_model.BlobSearchOptions{ + OwnerID: ownerID, + Image: image, + Digest: digest, + }) + if err != nil { + return err + } + + for _, file := range pfds { + if err := packages_service.DeletePackageFile(ctx, file.File); err != nil { + return err + } + } + return nil + }) +} + +func digestFromHashSummer(h packages_module.HashSummer) string { + _, _, hashSHA256, _ := h.Sums() + return "sha256:" + hex.EncodeToString(hashSHA256) +} + +func digestFromPackageBlob(pb *packages_model.PackageBlob) string { + return "sha256:" + pb.HashSHA256 +} diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go new file mode 100644 index 0000000000000..b1705a60bde59 --- /dev/null +++ b/routers/api/packages/container/container.go @@ -0,0 +1,603 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "errors" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + container_model "code.gitea.io/gitea/models/packages/container" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/packages/container/oci" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + container_service "code.gitea.io/gitea/services/packages/container" +) + +// maximum size of a container manifest +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests +const maxManifestSize = 10 * 1024 * 1024 + +var imageNamePattern = regexp.MustCompile(`\A[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*\z`) + +type containerHeaders struct { + Status int + ContentDigest string + UploadUUID string + Range string + Location string + ContentType string + ContentLength int64 +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#legacy-docker-support-http-headers +func setResponseHeaders(resp http.ResponseWriter, h *containerHeaders) { + if h.Location != "" { + resp.Header().Set("Location", h.Location) + } + if h.Range != "" { + resp.Header().Set("Range", h.Range) + } + if h.ContentType != "" { + resp.Header().Set("Content-Type", h.ContentType) + } + if h.ContentLength != 0 { + resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10)) + } + if h.UploadUUID != "" { + resp.Header().Set("Docker-Upload-Uuid", h.UploadUUID) + } + if h.ContentDigest != "" { + resp.Header().Set("Docker-Content-Digest", h.ContentDigest) + resp.Header().Set("ETag", fmt.Sprintf(`"%s"`, h.ContentDigest)) + } + resp.Header().Set("Docker-Distribution-Api-Version", "registry/2.0") + resp.WriteHeader(h.Status) +} + +func jsonResponse(ctx *context.Context, status int, obj interface{}) { + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: status, + ContentType: "application/json", + }) + if err := json.NewEncoder(ctx.Resp).Encode(obj); err != nil { + log.Error("JSON encode: %v", err) + } +} + +func apiError(ctx *context.Context, status int, err error) { + helper.LogAndProcessError(ctx, status, err, func(message string) { + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: status, + }) + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#error-codes +func apiErrorDefined(ctx *context.Context, err *namedError) { + type ContainerError struct { + Code string `json:"code"` + Message string `json:"message"` + } + + type ContainerErrors struct { + Errors []ContainerError `json:"errors"` + } + + jsonResponse(ctx, err.StatusCode, ContainerErrors{ + Errors: []ContainerError{ + { + Code: err.Code, + Message: err.Message, + }, + }, + }) +} + +// ReqContainerAccess is a middleware which checks the current user valid (real user or ghost for anonymous access) +func ReqContainerAccess(ctx *context.Context) { + if ctx.User == nil { + ctx.Resp.Header().Add("WWW-Authenticate", `Bearer realm="`+setting.AppURL+`v2/token"`) + ctx.Resp.Header().Add("WWW-Authenticate", `Basic`) + apiErrorDefined(ctx, errUnauthorized) + } +} + +// VerifyImageName is a middleware which checks if the image name is allowed +func VerifyImageName(ctx *context.Context) { + if !imageNamePattern.MatchString(ctx.Params("image")) { + apiErrorDefined(ctx, errNameInvalid) + } +} + +// DetermineSupport is used to test if the registry supports OCI +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#determining-support +func DetermineSupport(ctx *context.Context) { + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: http.StatusOK, + }) +} + +// Authenticate creates a token for the current user +// If the current user is anonymous, the ghost user is used +func Authenticate(ctx *context.Context) { + u := ctx.User + if u == nil { + u = user_model.NewGhostUser() + } + + token, err := packages_service.CreateAuthorizationToken(u) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.JSON(http.StatusOK, map[string]string{ + "token": token, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#mounting-a-blob-from-another-repository +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#single-post +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks +func InitiateUploadBlob(ctx *context.Context) { + image := ctx.Params("image") + + mount := ctx.FormTrim("mount") + from := ctx.FormTrim("from") + if mount != "" { + blob, _ := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + Image: from, + Digest: mount, + }) + if blob != nil { + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, mount), + ContentDigest: mount, + Status: http.StatusCreated, + }) + return + } + } + + digest := ctx.FormTrim("digest") + if digest != "" { + buf, err := packages_module.CreateHashedBufferFromReader(ctx.Req.Body, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + if digest != digestFromHashSummer(buf) { + apiErrorDefined(ctx, errDigestInvalid) + return + } + + if _, err := saveAsPackageBlob(buf, &packages_service.PackageInfo{Owner: ctx.Package.Owner, Name: image}); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, digest), + ContentDigest: digest, + Status: http.StatusCreated, + }) + return + } + + upload, err := packages_model.CreateBlobUpload(ctx) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, upload.ID), + Range: "0-0", + UploadUUID: upload.ID, + Status: http.StatusAccepted, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks +func UploadBlob(ctx *context.Context) { + image := ctx.Params("image") + + uploader, err := container_service.NewBlobUploader(ctx, ctx.Params("uuid")) + if err != nil { + if err == packages_model.ErrPackageBlobUploadNotExist { + apiErrorDefined(ctx, errBlobUploadUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + defer uploader.Close() + + contentRange := ctx.Req.Header.Get("Content-Range") + if contentRange != "" { + start, end := 0, 0 + if _, err := fmt.Sscanf(contentRange, "%d-%d", &start, &end); err != nil { + apiErrorDefined(ctx, errBlobUploadInvalid) + return + } + + if int64(start) != uploader.Size() { + apiErrorDefined(ctx, errBlobUploadInvalid.WithStatusCode(http.StatusRequestedRangeNotSatisfiable)) + return + } + } else if uploader.Size() != 0 { + apiErrorDefined(ctx, errBlobUploadInvalid.WithMessage("Stream uploads after first write are not allowed")) + return + } + + if err := uploader.Append(ctx, ctx.Req.Body); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/uploads/%s", ctx.Package.Owner.LowerName, image, uploader.ID), + Range: fmt.Sprintf("0-%d", uploader.Size()-1), + UploadUUID: uploader.ID, + Status: http.StatusAccepted, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks +func EndUploadBlob(ctx *context.Context) { + image := ctx.Params("image") + + digest := ctx.FormTrim("digest") + if digest == "" { + apiErrorDefined(ctx, errDigestInvalid) + return + } + + uploader, err := container_service.NewBlobUploader(ctx, ctx.Params("uuid")) + if err != nil { + if err == packages_model.ErrPackageBlobUploadNotExist { + apiErrorDefined(ctx, errBlobUploadUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + close := true + defer func() { + if close { + uploader.Close() + } + }() + + if ctx.Req.Body != nil { + if err := uploader.Append(ctx, ctx.Req.Body); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + if digest != digestFromHashSummer(uploader) { + apiErrorDefined(ctx, errDigestInvalid) + return + } + + if _, err := saveAsPackageBlob(uploader, &packages_service.PackageInfo{Owner: ctx.Package.Owner, Name: image}); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if err := uploader.Close(); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + close = false + + if err := container_service.RemoveBlobUploadByID(ctx, uploader.ID); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, digest), + ContentDigest: digest, + Status: http.StatusCreated, + }) +} + +func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) { + digest := ctx.Params("digest") + + if !oci.Digest(digest).Validate() { + return nil, container_model.ErrContainerBlobNotExist + } + + return container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Image: ctx.Params("image"), + Digest: digest, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry +func HeadBlob(ctx *context.Context) { + blob, err := getBlobFromContext(ctx) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + apiErrorDefined(ctx, errBlobUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + ContentDigest: blob.Properties.GetByName(container_module.PropertyDigest), + ContentLength: blob.Blob.Size, + Status: http.StatusOK, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-blobs +func GetBlob(ctx *context.Context) { + blob, err := getBlobFromContext(ctx) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + apiErrorDefined(ctx, errBlobUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(blob.Blob.HashSHA256)) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + setResponseHeaders(ctx.Resp, &containerHeaders{ + ContentDigest: blob.Properties.GetByName(container_module.PropertyDigest), + ContentType: blob.Properties.GetByName(container_module.PropertyMediaType), + ContentLength: blob.Blob.Size, + Status: http.StatusOK, + }) + if _, err := io.Copy(ctx.Resp, s); err != nil { + log.Error("Error whilst copying content to response: %v", err) + } +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-blobs +func DeleteBlob(ctx *context.Context) { + digest := ctx.Params("digest") + + if !oci.Digest(digest).Validate() { + apiErrorDefined(ctx, errBlobUnknown) + return + } + + if err := deleteBlob(ctx.Package.Owner.ID, ctx.Params("image"), digest); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: http.StatusAccepted, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests +func UploadManifest(ctx *context.Context) { + reference := ctx.Params("reference") + + mci := &manifestCreationInfo{ + MediaType: oci.MediaType(ctx.Req.Header.Get("Content-Type")), + Owner: ctx.Package.Owner, + Creator: ctx.User, + Image: ctx.Params("image"), + Reference: reference, + IsTagged: !oci.Digest(reference).Validate(), + } + + if mci.IsTagged && !oci.Reference(reference).Validate() { + apiErrorDefined(ctx, errManifestInvalid.WithMessage("Tag is invalid")) + return + } + + maxSize := maxManifestSize + 1 + buf, err := packages_module.CreateHashedBufferFromReader(&io.LimitedReader{R: ctx.Req.Body, N: int64(maxSize)}, maxSize) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + if buf.Size() > maxManifestSize { + apiErrorDefined(ctx, errManifestInvalid.WithMessage("Manifest exceeds maximum size").WithStatusCode(http.StatusRequestEntityTooLarge)) + return + } + + digest, err := processManifest(mci, buf) + if err != nil { + var namedError *namedError + if errors.As(err, &namedError) { + apiErrorDefined(ctx, namedError) + } else if errors.Is(err, container_model.ErrContainerBlobNotExist) { + apiErrorDefined(ctx, errBlobUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Location: fmt.Sprintf("/v2/%s/%s/manifests/%s", ctx.Package.Owner.LowerName, mci.Image, reference), + ContentDigest: digest, + Status: http.StatusCreated, + }) +} + +func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) { + reference := ctx.Params("reference") + + opts := &container_model.BlobSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Image: ctx.Params("image"), + IsManifest: true, + } + if oci.Digest(reference).Validate() { + opts.Digest = reference + } else if oci.Reference(reference).Validate() { + opts.Tag = reference + } else { + return nil, container_model.ErrContainerBlobNotExist + } + + return container_model.GetContainerBlob(ctx, opts) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry +func HeadManifest(ctx *context.Context) { + manifest, err := getManifestFromContext(ctx) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + apiErrorDefined(ctx, errManifestUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + ContentDigest: manifest.Properties.GetByName(container_module.PropertyDigest), + ContentType: manifest.Properties.GetByName(container_module.PropertyMediaType), + ContentLength: manifest.Blob.Size, + Status: http.StatusOK, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests +func GetManifest(ctx *context.Context) { + manifest, err := getManifestFromContext(ctx) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + apiErrorDefined(ctx, errManifestUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(manifest.Blob.HashSHA256)) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + setResponseHeaders(ctx.Resp, &containerHeaders{ + ContentDigest: manifest.Properties.GetByName(container_module.PropertyDigest), + ContentType: manifest.Properties.GetByName(container_module.PropertyMediaType), + ContentLength: manifest.Blob.Size, + Status: http.StatusOK, + }) + if _, err := io.Copy(ctx.Resp, s); err != nil { + log.Error("Error whilst copying content to response: %v", err) + } +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-tags +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-manifests +func DeleteManifest(ctx *context.Context) { + reference := ctx.Params("reference") + + opts := &container_model.BlobSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Image: ctx.Params("image"), + IsManifest: true, + } + if oci.Digest(reference).Validate() { + opts.Digest = reference + } else if oci.Reference(reference).Validate() { + opts.Tag = reference + } else { + apiErrorDefined(ctx, errManifestUnknown) + return + } + + pvs, err := container_model.GetManifestVersions(ctx, opts) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if len(pvs) == 0 { + apiErrorDefined(ctx, errManifestUnknown) + return + } + + for _, pv := range pvs { + if err := packages_service.RemovePackageVersion(ctx.User, pv); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + + setResponseHeaders(ctx.Resp, &containerHeaders{ + Status: http.StatusAccepted, + }) +} + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#content-discovery +func GetTagList(ctx *context.Context) { + image := ctx.Params("image") + + if _, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.TypeContainer, image); err != nil { + if err == packages_model.ErrPackageNotExist { + apiErrorDefined(ctx, errNameUnknown) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + var n *int + if ctx.FormTrim("n") != "" { + val := ctx.FormInt("n") + n = &val + } + last := ctx.FormTrim("last") + + tags, err := container_model.GetImageTags(ctx, ctx.Package.Owner.ID, image, n, last) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + type TagList struct { + Name string `json:"name"` + Tags []string `json:"tags"` + } + + jsonResponse(ctx, http.StatusOK, TagList{ + Name: strings.ToLower(ctx.Package.Owner.LowerName + "/" + image), + Tags: tags, + }) +} diff --git a/routers/api/packages/container/errors.go b/routers/api/packages/container/errors.go new file mode 100644 index 0000000000000..0efbb081ca2ff --- /dev/null +++ b/routers/api/packages/container/errors.go @@ -0,0 +1,53 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "net/http" +) + +// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#error-codes +var ( + errBlobUnknown = &namedError{Code: "BLOB_UNKNOWN", StatusCode: http.StatusNotFound} + errBlobUploadInvalid = &namedError{Code: "BLOB_UPLOAD_INVALID", StatusCode: http.StatusBadRequest} + errBlobUploadUnknown = &namedError{Code: "BLOB_UPLOAD_UNKNOWN", StatusCode: http.StatusNotFound} + errDigestInvalid = &namedError{Code: "DIGEST_INVALID", StatusCode: http.StatusBadRequest} + errManifestBlobUnknown = &namedError{Code: "MANIFEST_BLOB_UNKNOWN", StatusCode: http.StatusNotFound} + errManifestInvalid = &namedError{Code: "MANIFEST_INVALID", StatusCode: http.StatusBadRequest} + errManifestUnknown = &namedError{Code: "MANIFEST_UNKNOWN", StatusCode: http.StatusNotFound} + errNameInvalid = &namedError{Code: "NAME_INVALID", StatusCode: http.StatusBadRequest} + errNameUnknown = &namedError{Code: "NAME_UNKNOWN", StatusCode: http.StatusNotFound} + errSizeInvalid = &namedError{Code: "SIZE_INVALID", StatusCode: http.StatusBadRequest} + errUnauthorized = &namedError{Code: "UNAUTHORIZED", StatusCode: http.StatusUnauthorized} + errUnsupported = &namedError{Code: "UNSUPPORTED", StatusCode: http.StatusNotImplemented} +) + +type namedError struct { + Code string + StatusCode int + Message string +} + +func (e *namedError) Error() string { + return e.Message +} + +// WithMessage creates a new instance of the error with a different message +func (e *namedError) WithMessage(message string) *namedError { + return &namedError{ + Code: e.Code, + StatusCode: e.StatusCode, + Message: message, + } +} + +// WithStatusCode creates a new instance of the error with a different status code +func (e *namedError) WithStatusCode(statusCode int) *namedError { + return &namedError{ + Code: e.Code, + StatusCode: statusCode, + Message: e.Message, + } +} diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go new file mode 100644 index 0000000000000..7c614f314b24c --- /dev/null +++ b/routers/api/packages/container/manifest.go @@ -0,0 +1,430 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "fmt" + "io" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + container_model "code.gitea.io/gitea/models/packages/container" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/packages/container/oci" + packages_service "code.gitea.io/gitea/services/packages" +) + +// manifestCreationInfo describes a manifest to create +type manifestCreationInfo struct { + MediaType oci.MediaType + Owner *user_model.User + Creator *user_model.User + Image string + Reference string + IsTagged bool + Properties map[string]string +} + +func processManifest(mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) { + var schema oci.SchemaMediaBase + if err := json.NewDecoder(buf).Decode(&schema); err != nil { + return "", err + } + + if schema.SchemaVersion != 2 { + return "", errUnsupported.WithMessage("Schema version is not supported") + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + return "", err + } + + if !mci.MediaType.IsValid() { + mci.MediaType = schema.MediaType + if !mci.MediaType.IsValid() { + return "", errManifestInvalid.WithMessage("MediaType not recognized") + } + } + + if mci.MediaType.IsImageManifest() { + d, err := processImageManifest(mci, buf) + return d, err + } else if mci.MediaType.IsImageIndex() { + d, err := processImageManifestIndex(mci, buf) + return d, err + } + return "", errManifestInvalid +} + +func processImageManifest(mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) { + manifestDigest := "" + + err := func() error { + var manifest oci.Manifest + if err := json.NewDecoder(buf).Decode(&manifest); err != nil { + return err + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + return err + } + + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + configDescriptor, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + OwnerID: mci.Owner.ID, + Image: mci.Image, + Digest: string(manifest.Config.Digest), + }) + if err != nil { + return err + } + + configReader, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(configDescriptor.Blob.HashSHA256)) + if err != nil { + return err + } + defer configReader.Close() + + metadata, err := container_module.ParseImageConfig(manifest.Config.MediaType, configReader) + if err != nil { + return err + } + + blobReferences := make([]*blobReference, 0, 1+len(manifest.Layers)) + + blobReferences = append(blobReferences, &blobReference{ + Digest: manifest.Config.Digest, + MediaType: manifest.Config.MediaType, + File: configDescriptor, + ExpectedSize: manifest.Config.Size, + }) + + for _, layer := range manifest.Layers { + pfd, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + OwnerID: mci.Owner.ID, + Image: mci.Image, + Digest: string(layer.Digest), + }) + if err != nil { + return err + } + + blobReferences = append(blobReferences, &blobReference{ + Digest: layer.Digest, + MediaType: layer.MediaType, + File: pfd, + ExpectedSize: layer.Size, + }) + } + + pv, err := createPackageAndVersion(ctx, mci, metadata) + if err != nil { + return err + } + + uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_model.UploadVersion) + if err != nil && err != packages_model.ErrPackageNotExist { + return err + } + + for _, ref := range blobReferences { + if err := createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil { + return err + } + } + + pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf) + removeBlob := false + defer func() { + if removeBlob { + contentStore := packages_module.NewContentStore() + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob from content store: %v", err) + } + } + }() + if err != nil { + removeBlob = created + return err + } + + if err := committer.Commit(); err != nil { + removeBlob = created + return err + } + + manifestDigest = digest + + return nil + }() + if err != nil { + return "", err + } + + return manifestDigest, nil +} + +func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) { + manifestDigest := "" + + err := func() error { + var index oci.Index + if err := json.NewDecoder(buf).Decode(&index); err != nil { + return err + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + return err + } + + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + metadata := &container_module.Metadata{ + Type: container_module.TypeOCI, + MultiArch: make(map[string]string), + } + + for _, manifest := range index.Manifests { + if !manifest.MediaType.IsImageManifest() { + return errManifestInvalid + } + + platform := container_module.DefaultPlatform + if manifest.Platform != nil { + platform = fmt.Sprintf("%s/%s", manifest.Platform.OS, manifest.Platform.Architecture) + if manifest.Platform.Variant != "" { + platform = fmt.Sprintf("%s/%s", platform, manifest.Platform.Variant) + } + } + + _, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + OwnerID: mci.Owner.ID, + Image: mci.Image, + Digest: string(manifest.Digest), + IsManifest: true, + }) + if err != nil { + if err == container_model.ErrContainerBlobNotExist { + return errManifestBlobUnknown + } + return err + } + + metadata.MultiArch[platform] = string(manifest.Digest) + } + + pv, err := createPackageAndVersion(ctx, mci, metadata) + if err != nil { + return err + } + + pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf) + removeBlob := false + defer func() { + if removeBlob { + contentStore := packages_module.NewContentStore() + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob from content store: %v", err) + } + } + }() + if err != nil { + removeBlob = created + return err + } + + if err := committer.Commit(); err != nil { + removeBlob = created + return err + } + + manifestDigest = digest + + return nil + }() + if err != nil { + return "", err + } + + return manifestDigest, nil +} + +func deleteExistingVersionReferences(ctx context.Context, pv *packages_model.PackageVersion) error { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { + return err + } + + pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) + if err != nil { + return err + } + + for _, pf := range pfs { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { + return err + } + } + + return nil +} + +func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, metadata *container_module.Metadata) (*packages_model.PackageVersion, error) { + p := &packages_model.Package{ + OwnerID: mci.Owner.ID, + Type: packages_model.TypeContainer, + Name: strings.ToLower(mci.Image), + LowerName: strings.ToLower(mci.Image), + } + var err error + if p, err = packages_model.TryInsertPackage(ctx, p); err != nil { + if err != packages_model.ErrDuplicatePackage { + log.Error("Error inserting package: %v", err) + return nil, err + } + } + + metadata.IsTagged = mci.IsTagged + + metadataJSON, err := json.Marshal(metadata) + if err != nil { + return nil, err + } + + _pv := &packages_model.PackageVersion{ + PackageID: p.ID, + CreatorID: mci.Creator.ID, + Version: strings.ToLower(mci.Reference), + LowerVersion: strings.ToLower(mci.Reference), + MetadataJSON: string(metadataJSON), + } + var pv *packages_model.PackageVersion + if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil { + if err == packages_model.ErrDuplicatePackageVersion { + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + return nil, err + } + + if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil { + log.Error("Error inserting package: %v", err) + return nil, err + } + } else { + log.Error("Error inserting package: %v", err) + return nil, err + } + } + + if mci.IsTagged { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil { + log.Error("Error setting package version property: %v", err) + return nil, err + } + } + for _, digest := range metadata.MultiArch { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, digest); err != nil { + log.Error("Error setting package version property: %v", err) + return nil, err + } + } + + return pv, nil +} + +type blobReference struct { + Digest oci.Digest + MediaType oci.MediaType + Name string + File *packages_model.PackageFileDescriptor + ExpectedSize int64 + IsLead bool +} + +func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) error { + if ref.File.Blob.Size != ref.ExpectedSize { + return errSizeInvalid + } + + if ref.Name == "" { + ref.Name = strings.ToLower(fmt.Sprintf("sha256_%s", ref.File.Blob.HashSHA256)) + } + + pf := &packages_model.PackageFile{ + VersionID: pv.ID, + BlobID: ref.File.Blob.ID, + Name: ref.Name, + LowerName: ref.Name, + IsLead: ref.IsLead, + } + var err error + if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil { + log.Error("Error inserting package file: %v", err) + return err + } + + props := map[string]string{ + container_module.PropertyMediaType: string(ref.MediaType), + container_module.PropertyDigest: string(ref.Digest), + } + for name, value := range props { + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil { + log.Error("Error setting package file property: %v", err) + return err + } + } + + // Remove the file from the blob upload version + if uploadVersion != nil && ref.File.File != nil && uploadVersion.ID == ref.File.File.VersionID { + if err := packages_service.DeletePackageFile(ctx, ref.File.File); err != nil { + return err + } + } + + return nil +} + +func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *packages_model.PackageVersion, buf *packages_module.HashedBuffer) (*packages_model.PackageBlob, bool, string, error) { + pb, exists, err := packages_model.GetOrInsertBlob(ctx, packages_service.NewPackageBlob(buf)) + if err != nil { + log.Error("Error inserting package blob: %v", err) + return nil, false, "", err + } + if !exists { + contentStore := packages_module.NewContentStore() + if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil { + log.Error("Error saving package blob in content store: %v", err) + return nil, false, "", err + } + } + + manifestDigest := digestFromHashSummer(buf) + err = createFileFromBlobReference(ctx, pv, nil, &blobReference{ + Digest: oci.Digest(manifestDigest), + MediaType: mci.MediaType, + Name: container_model.ManifestFilename, + File: &packages_model.PackageFileDescriptor{Blob: pb}, + ExpectedSize: pb.Size, + IsLead: true, + }) + + return pb, !exists, manifestDigest, err +} diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index e7b20e7da9726..ceacdaa8bfdc4 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -128,7 +128,7 @@ func DeletePackage(ctx *context.Context) { return } - err = packages_service.DeletePackageVersionByNameAndVersion( + err = packages_service.RemovePackageVersionByNameAndVersion( ctx.User, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index 19e06c2bdff49..6cf9119c48a1e 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -39,9 +39,9 @@ func ServiceIndex(ctx *context.Context) { // SearchService https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages func SearchService(ctx *context.Context) { pvs, count, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ - OwnerID: ctx.Package.Owner.ID, - Type: string(packages_model.TypeNuGet), - Query: ctx.FormTrim("q"), + OwnerID: ctx.Package.Owner.ID, + Type: string(packages_model.TypeNuGet), + QueryName: ctx.FormTrim("q"), Paginator: db.NewAbsoluteListOptions( ctx.FormInt("skip"), ctx.FormInt("take"), @@ -402,7 +402,7 @@ func DeletePackage(ctx *context.Context) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") - err := packages_service.DeletePackageVersionByNameAndVersion( + err := packages_service.RemovePackageVersionByNameAndVersion( ctx.User, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index e79122830b9c0..6cff0b837291e 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -266,7 +266,7 @@ func DeletePackage(ctx *context.Context) { packageName := ctx.FormString("gem_name") packageVersion := ctx.FormString("version") - err := packages_service.DeletePackageVersionByNameAndVersion( + err := packages_service.RemovePackageVersionByNameAndVersion( ctx.User, &packages_service.PackageInfo{ Owner: ctx.Package.Owner, diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index bb300bc655e9e..0d9ffd66ee665 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -57,7 +57,7 @@ func ListPackages(ctx *context.APIContext) { pvs, count, err := packages.SearchVersions(ctx, &packages.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, Type: packageType, - Query: query, + QueryName: query, Paginator: &listOptions, }) if err != nil { @@ -149,9 +149,9 @@ func DeletePackage(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - err := packages_service.DeletePackageVersion(ctx.User, ctx.Package.Descriptor.Version) + err := packages_service.RemovePackageVersion(ctx.User, ctx.Package.Descriptor.Version) if err != nil { - ctx.Error(http.StatusInternalServerError, "DeletePackageVersion", err) + ctx.Error(http.StatusInternalServerError, "RemovePackageVersion", err) return } ctx.Status(http.StatusNoContent) diff --git a/routers/init.go b/routers/init.go index 6b71768fe65c1..489994789734b 100644 --- a/routers/init.go +++ b/routers/init.go @@ -189,6 +189,9 @@ func NormalRoutes() *web.Route { r.Mount("/", web_routers.Routes(sessioner)) r.Mount("/api/v1", apiv1.Routes(sessioner)) r.Mount("/api/internal", private.Routes()) - r.Mount("/api/packages", packages_router.Routes()) + if setting.Packages.Enabled { + r.Mount("/api/packages", packages_router.Routes()) + r.Mount("/v2", packages_router.ContainerRoutes()) + } return r } diff --git a/routers/web/admin/packages.go b/routers/web/admin/packages.go index aad92ff12e595..db2f050eb0d40 100644 --- a/routers/web/admin/packages.go +++ b/routers/web/admin/packages.go @@ -31,9 +31,9 @@ func Packages(ctx *context.Context) { sort := ctx.FormTrim("sort") pvs, total, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ - Query: query, - Type: packageType, - Sort: sort, + QueryName: query, + Type: packageType, + Sort: sort, Paginator: &db.ListOptions{ PageSize: setting.UI.PackagesPagingNum, Page: page, @@ -83,8 +83,8 @@ func DeletePackageVersion(ctx *context.Context) { return } - if err := packages_service.DeletePackageVersion(ctx.User, pv); err != nil { - ctx.ServerError("DeletePackageVersion", err) + if err := packages_service.RemovePackageVersion(ctx.User, pv); err != nil { + ctx.ServerError("RemovePackageVersion", err) return } diff --git a/routers/web/repo/packages.go b/routers/web/repo/packages.go index de251a102e1b4..f796bb0de5c70 100644 --- a/routers/web/repo/packages.go +++ b/routers/web/repo/packages.go @@ -32,10 +32,10 @@ func Packages(ctx *context.Context) { PageSize: setting.UI.PackagesPagingNum, Page: page, }, - OwnerID: ctx.ContextUser.ID, - RepoID: ctx.Repo.Repository.ID, - Query: query, - Type: packageType, + OwnerID: ctx.ContextUser.ID, + RepoID: ctx.Repo.Repository.ID, + QueryName: query, + Type: packageType, }) if err != nil { ctx.ServerError("SearchLatestVersions", err) diff --git a/routers/web/user/package.go b/routers/web/user/package.go index ce28c91552964..40a9dbbc6f401 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -6,11 +6,11 @@ package user import ( "net/http" - "sort" "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" + container_model "code.gitea.io/gitea/models/packages/container" "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/base" @@ -23,13 +23,14 @@ import ( ) const ( - tplPackagesList base.TplName = "user/overview/packages" - tplPackagesView base.TplName = "package/view" - tplPackagesSettings base.TplName = "package/settings" + tplPackagesList base.TplName = "user/overview/packages" + tplPackagesView base.TplName = "package/view" + tplPackageVersionList base.TplName = "user/overview/package_versions" + tplPackagesSettings base.TplName = "package/settings" ) -// Packages displays a list of all packages of the context user -func Packages(ctx *context.Context) { +// ListPackages displays a list of all packages of the context user +func ListPackages(ctx *context.Context) { page := ctx.FormInt("page") if page <= 1 { page = 1 @@ -42,9 +43,9 @@ func Packages(ctx *context.Context) { PageSize: setting.UI.PackagesPagingNum, Page: page, }, - OwnerID: ctx.ContextUser.ID, - Query: query, - Type: packageType, + OwnerID: ctx.ContextUser.ID, + Type: packageType, + QueryName: query, }) if err != nil { ctx.ServerError("SearchLatestVersions", err) @@ -80,44 +81,172 @@ func Packages(ctx *context.Context) { ctx.HTML(http.StatusOK, tplPackagesList) } -// ViewPackage displays a single package -func ViewPackage(ctx *context.Context) { - pd := ctx.Package.Descriptor +// RedirectToLastVersion redirects to the latest package version +func RedirectToLastVersion(ctx *context.Context) { + p, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.Type(ctx.Params("type")), ctx.Params("name")) + if err != nil { + if err == packages_model.ErrPackageNotExist { + ctx.NotFound("GetPackageByName", err) + } else { + ctx.ServerError("GetPackageByName", err) + } + return + } + + pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + }) + if err != nil { + ctx.ServerError("GetPackageByName", err) + return + } + if len(pvs) == 0 { + ctx.NotFound("", err) + } - if pd == nil { - ctx.NotFound("Package does not exist", nil) + pd, err := packages_model.GetPackageDescriptor(ctx, pvs[0]) + if err != nil { + ctx.ServerError("GetPackageDescriptor", err) return } + ctx.Redirect(pd.FullWebLink()) +} + +// ViewPackageVersion displays a single package version +func ViewPackageVersion(ctx *context.Context) { + pd := ctx.Package.Descriptor + ctx.Data["Title"] = pd.Package.Name ctx.Data["IsPackagesPage"] = true ctx.Data["ContextUser"] = ctx.ContextUser ctx.Data["PackageDescriptor"] = pd - otherVersions, err := packages_model.GetVersionsByPackageName(ctx, pd.Owner.ID, pd.Package.Type, pd.Package.LowerName) + var ( + total int64 + pvs []*packages_model.PackageVersion + err error + ) + switch pd.Package.Type { + case packages_model.TypeContainer: + ctx.Data["RegistryHost"] = setting.Packages.RegistryHost + + pvs, total, err = container_model.SearchImageTags(ctx, &container_model.ImageTagsSearchOptions{ + Paginator: db.NewAbsoluteListOptions(0, 5), + PackageID: pd.Package.ID, + IsTagged: true, + }) + default: + pvs, total, err = packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + Paginator: db.NewAbsoluteListOptions(0, 5), + PackageID: pd.Package.ID, + }) + if err != nil { + ctx.ServerError("SearchVersions", err) + } + } if err != nil { - ctx.ServerError("GetVersionsByPackageName", err) - return + ctx.ServerError("", err) } - sort.Slice(otherVersions, func(i, j int) bool { - return otherVersions[i].CreatedUnix > otherVersions[j].CreatedUnix - }) - ctx.Data["OtherVersions"] = otherVersions + + ctx.Data["LatestVersions"] = pvs + ctx.Data["TotalVersionCount"] = total ctx.Data["CanWritePackages"] = ctx.Package.AccessMode >= perm.AccessModeWrite || ctx.IsUserSiteAdmin() ctx.HTML(http.StatusOK, tplPackagesView) } -// PackageSettings displays the package settings page -func PackageSettings(ctx *context.Context) { - pd := ctx.Package.Descriptor +// ListPackageVersions lists all versions of a package +func ListPackageVersions(ctx *context.Context) { + p, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.Type(ctx.Params("type")), ctx.Params("name")) + if err != nil { + if err == packages_model.ErrPackageNotExist { + ctx.NotFound("GetPackageByName", err) + } else { + ctx.ServerError("GetPackageByName", err) + } + return + } + + page := ctx.FormInt("page") + if page <= 1 { + page = 1 + } + pagination := &db.ListOptions{ + PageSize: setting.UI.PackagesPagingNum, + Page: page, + } + + query := ctx.FormTrim("q") + + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["IsPackagesPage"] = true + ctx.Data["ContextUser"] = ctx.ContextUser + ctx.Data["PackageDescriptor"] = &packages_model.PackageDescriptor{ + Package: p, + Owner: ctx.Package.Owner, + } + ctx.Data["Query"] = query + + pagerParams := map[string]string{ + "q": query, + } + + var ( + total int64 + pvs []*packages_model.PackageVersion + ) + switch p.Type { + case packages_model.TypeContainer: + tagged := ctx.FormTrim("tagged") + + pagerParams["tagged"] = tagged + ctx.Data["Tagged"] = tagged + + pvs, total, err = container_model.SearchImageTags(ctx, &container_model.ImageTagsSearchOptions{ + Paginator: pagination, + PackageID: p.ID, + Query: query, + IsTagged: tagged == "" || tagged == "tagged", + }) + if err != nil { + ctx.ServerError("SearchImageTags", err) + return + } + default: + pvs, total, err = packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + Paginator: pagination, + PackageID: p.ID, + QueryVersion: query, + }) + if err != nil { + ctx.ServerError("SearchVersions", err) + return + } + } - if pd == nil { - ctx.NotFound("Package does not exist", nil) + ctx.Data["PackageDescriptors"], err = packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + ctx.ServerError("GetPackageDescriptors", err) return } + ctx.Data["Total"] = total + + pager := context.NewPagination(int(total), setting.UI.PackagesPagingNum, page, 5) + for k, v := range pagerParams { + pager.AddParamString(k, v) + } + ctx.Data["Page"] = pager + + ctx.HTML(http.StatusOK, tplPackageVersionList) +} + +// PackageSettings displays the package settings page +func PackageSettings(ctx *context.Context) { + pd := ctx.Package.Descriptor + ctx.Data["Title"] = pd.Package.Name ctx.Data["IsPackagesPage"] = true ctx.Data["ContextUser"] = ctx.ContextUser @@ -136,11 +265,6 @@ func PackageSettings(ctx *context.Context) { func PackageSettingsPost(ctx *context.Context) { pd := ctx.Package.Descriptor - if pd == nil { - ctx.NotFound("Package does not exist", nil) - return - } - form := web.GetForm(ctx).(*forms.PackageSettingForm) switch form.Action { case "link": @@ -177,7 +301,7 @@ func PackageSettingsPost(ctx *context.Context) { ctx.Redirect(ctx.Link) return case "delete": - err := packages_service.DeletePackageVersion(ctx.User, ctx.Package.Descriptor.Version) + err := packages_service.RemovePackageVersion(ctx.User, ctx.Package.Descriptor.Version) if err != nil { log.Error("Error deleting package: %v", err) ctx.Flash.Error(ctx.Tr("packages.settings.delete.error")) diff --git a/routers/web/web.go b/routers/web/web.go index fb17a6f8e6ba2..d6ff86e2a03f1 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -628,14 +628,18 @@ func RegisterRoutes(m *web.Route) { m.Group("/{username}/-", func() { m.Group("/packages", func() { - m.Get("", user.Packages) - m.Group("/{type}/{name}/{version}", func() { - m.Get("", user.ViewPackage) - m.Get("/files/{fileid}", user.DownloadPackageFile) - m.Group("/settings", func() { - m.Get("", user.PackageSettings) - m.Post("", bindIgnErr(forms.PackageSettingForm{}), user.PackageSettingsPost) - }, reqPackageAccess(perm.AccessModeWrite)) + m.Get("", user.ListPackages) + m.Group("/{type}/{name}", func() { + m.Get("", user.RedirectToLastVersion) + m.Get("/versions", user.ListPackageVersions) + m.Group("/{version}", func() { + m.Get("", user.ViewPackageVersion) + m.Get("/files/{fileid}", user.DownloadPackageFile) + m.Group("/settings", func() { + m.Get("", user.PackageSettings) + m.Post("", bindIgnErr(forms.PackageSettingForm{}), user.PackageSettingsPost) + }, reqPackageAccess(perm.AccessModeWrite)) + }) }) }, context.PackageAssignment(), reqPackageAccess(perm.AccessModeRead)) }, context.UserAssignment()) diff --git a/services/auth/auth.go b/services/auth/auth.go index bdff777f506e4..8dff078743eea 100644 --- a/services/auth/auth.go +++ b/services/auth/auth.go @@ -95,6 +95,11 @@ func isAttachmentDownload(req *http.Request) bool { return strings.HasPrefix(req.URL.Path, "/attachments/") && req.Method == "GET" } +// isContainerPath checks if the request targets the container endpoint +func isContainerPath(req *http.Request) bool { + return strings.HasPrefix(req.URL.Path, "/v2/") +} + var ( gitRawReleasePathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/(?:(?:git-(?:(?:upload)|(?:receive))-pack$)|(?:info/refs$)|(?:HEAD$)|(?:objects/)|(?:raw/)|(?:releases/download/))`) lfsPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/info/lfs/`) diff --git a/services/auth/basic.go b/services/auth/basic.go index d8667c65d5593..1869662e9277d 100644 --- a/services/auth/basic.go +++ b/services/auth/basic.go @@ -43,7 +43,7 @@ func (b *Basic) Name() string { // Returns nil if header is empty or validation fails. func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) *user_model.User { // Basic authentication should only fire on API, Download or on Git or LFSPaths - if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isGitRawReleaseOrLFSPath(req) { + if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawReleaseOrLFSPath(req) { return nil } diff --git a/services/packages/container/blob_uploader.go b/services/packages/container/blob_uploader.go new file mode 100644 index 0000000000000..c4421005ffca9 --- /dev/null +++ b/services/packages/container/blob_uploader.go @@ -0,0 +1,134 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "errors" + "io" + "os" + "path" + + packages_model "code.gitea.io/gitea/models/packages" + packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/setting" +) + +var ( + // errWriteAfterRead occurs if Write is called after a read operation + errWriteAfterRead = errors.New("write is unsupported after a read operation") + // errOffsetMissmatch occurs if the file offset is different than the model + errOffsetMissmatch = errors.New("offset mismatch between file and model") +) + +// BlobUploader handles chunked blob uploads +type BlobUploader struct { + *packages_model.PackageBlobUpload + *packages_module.MultiHasher + file *os.File + reading bool +} + +func buildFilePath(id string) string { + return path.Join(setting.Packages.ChunkedUploadPath, id) +} + +// NewBlobUploader creates a new blob uploader for the given id +func NewBlobUploader(ctx context.Context, id string) (*BlobUploader, error) { + model, err := packages_model.GetBlobUploadByID(ctx, id) + if err != nil { + return nil, err + } + + hash := packages_module.NewMultiHasher() + if len(model.HashStateBytes) != 0 { + if err := hash.UnmarshalBinary(model.HashStateBytes); err != nil { + return nil, err + } + } + + f, err := os.OpenFile(buildFilePath(model.ID), os.O_RDWR|os.O_CREATE, 0o666) + if err != nil { + return nil, err + } + + return &BlobUploader{ + model, + hash, + f, + false, + }, nil +} + +// Close implements io.Closer +func (u *BlobUploader) Close() error { + return u.file.Close() +} + +// Append appends a chunk of data and updates the model +func (u *BlobUploader) Append(ctx context.Context, r io.Reader) error { + if u.reading { + return errWriteAfterRead + } + + offset, err := u.file.Seek(0, io.SeekEnd) + if err != nil { + return err + } + if offset != u.BytesReceived { + return errOffsetMissmatch + } + + n, err := io.Copy(io.MultiWriter(u.file, u.MultiHasher), r) + if err != nil { + return err + } + + // fast path if nothing was written + if n == 0 { + return nil + } + + u.BytesReceived += n + + u.HashStateBytes, err = u.MultiHasher.MarshalBinary() + if err != nil { + return err + } + + return packages_model.UpdateBlobUpload(ctx, u.PackageBlobUpload) +} + +func (u *BlobUploader) Size() int64 { + return u.BytesReceived +} + +// Read implements io.Reader +func (u *BlobUploader) Read(p []byte) (int, error) { + if !u.reading { + _, err := u.file.Seek(0, io.SeekStart) + if err != nil { + return 0, err + } + + u.reading = true + } + + return u.file.Read(p) +} + +// Remove deletes the data and the model of a blob upload +func RemoveBlobUploadByID(ctx context.Context, id string) error { + if err := packages_model.DeleteBlobUploadByID(ctx, id); err != nil { + return err + } + + err := os.Remove(buildFilePath(id)) + if err != nil && !os.IsNotExist(err) { + return err + } + + return nil +} diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go new file mode 100644 index 0000000000000..91992a4d7f674 --- /dev/null +++ b/services/packages/container/cleanup.go @@ -0,0 +1,75 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "time" + + packages_model "code.gitea.io/gitea/models/packages" + container_model "code.gitea.io/gitea/models/packages/container" +) + +// Cleanup removes expired container data +func Cleanup(ctx context.Context, olderThan time.Duration) error { + if err := cleanupExpiredBlobUploads(ctx, olderThan); err != nil { + return err + } + return cleanupExpiredUploadedBlobs(ctx, olderThan) +} + +// cleanupExpiredBlobUploads removes expired blob uploads +func cleanupExpiredBlobUploads(ctx context.Context, olderThan time.Duration) error { + pbus, err := packages_model.FindExpiredBlobUploads(ctx, olderThan) + if err != nil { + return err + } + + for _, pbu := range pbus { + if err := RemoveBlobUploadByID(ctx, pbu.ID); err != nil { + return err + } + } + + return nil +} + +// cleanupExpiredUploadedBlobs removes expired uploaded blobs not referenced by a manifest +func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) error { + pfs, err := container_model.SearchExpiredUploadedBlobs(ctx, olderThan) + if err != nil { + return err + } + + versions := make(map[int64]struct{}) + for _, pf := range pfs { + versions[pf.VersionID] = struct{}{} + + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { + return err + } + } + + for versionID := range versions { + has, err := packages_model.HasVersionFileReferences(ctx, versionID) + if err != nil { + return err + } + if !has { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, versionID); err != nil { + return err + } + + if err := packages_model.DeleteVersionByID(ctx, versionID); err != nil { + return err + } + } + } + + return nil +} diff --git a/services/packages/packages.go b/services/packages/packages.go index 91b83216a12c4..b26e60c71185a 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" packages_module "code.gitea.io/gitea/modules/packages" + container_service "code.gitea.io/gitea/services/packages/container" ) // PackageInfo describes a package @@ -46,7 +47,7 @@ type PackageFileInfo struct { // PackageFileCreationInfo describes a package file to create type PackageFileCreationInfo struct { PackageFileInfo - Data *packages_module.HashedBuffer + Data packages_module.HashedSizeReader IsLead bool Properties map[string]string OverwriteExisting bool @@ -195,19 +196,23 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) ( return pv, pf, nil } -func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { - log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) - - hashMD5, hashSHA1, hashSHA256, hashSHA512 := pfci.Data.Sums() +// NewPackageBlob creates a package blob instance +func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.PackageBlob { + hashMD5, hashSHA1, hashSHA256, hashSHA512 := hsr.Sums() - pb := &packages_model.PackageBlob{ - Size: pfci.Data.Size(), + return &packages_model.PackageBlob{ + Size: hsr.Size(), HashMD5: fmt.Sprintf("%x", hashMD5), HashSHA1: fmt.Sprintf("%x", hashSHA1), HashSHA256: fmt.Sprintf("%x", hashSHA256), HashSHA512: fmt.Sprintf("%x", hashSHA512), } - pb, exists, err := packages_model.GetOrInsertBlob(ctx, pb) +} + +func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { + log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) + + pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data)) if err != nil { log.Error("Error inserting package blob: %v", err) return nil, nil, false, err @@ -265,18 +270,18 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers return pf, pb, !exists, nil } -// DeletePackageVersionByNameAndVersion deletes a package version and all associated files -func DeletePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error { +// RemovePackageVersionByNameAndVersion deletes a package version and all associated files +func RemovePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error { pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) if err != nil { return err } - return DeletePackageVersion(doer, pv) + return RemovePackageVersion(doer, pv) } -// DeletePackageVersion deletes the package version and all associated files -func DeletePackageVersion(doer *user_model.User, pv *packages_model.PackageVersion) error { +// RemovePackageVersion deletes the package version and all associated files +func RemovePackageVersion(doer *user_model.User, pv *packages_model.PackageVersion) error { ctx, committer, err := db.TxContext() if err != nil { return err @@ -290,34 +295,45 @@ func DeletePackageVersion(doer *user_model.User, pv *packages_model.PackageVersi log.Trace("Deleting package: %v", pv.ID) - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { + if err := DeletePackageVersionAndReferences(ctx, pv); err != nil { return err } - for _, file := range pd.Files { - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, file.File.ID); err != nil { - return err - } - if err := packages_model.DeleteFileByID(ctx, file.File.ID); err != nil { - return err - } + if err := committer.Commit(); err != nil { + return err } - if err := packages_model.DeleteVersionByID(ctx, pv.ID); err != nil { + notification.NotifyPackageDelete(doer, pd) + + return nil +} + +// DeletePackageVersionAndReferences deletes the package version and its properties and files +func DeletePackageVersionAndReferences(ctx context.Context, pv *packages_model.PackageVersion) error { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { return err } - if err := packages_model.DeletePackageByIDIfUnreferenced(ctx, pv.PackageID); err != nil { + pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) + if err != nil { return err } - if err := committer.Commit(); err != nil { - return err + for _, pf := range pfs { + if err := DeletePackageFile(ctx, pf); err != nil { + return err + } } - notification.NotifyPackageDelete(doer, pd) + return packages_model.DeleteVersionByID(ctx, pv.ID) +} - return nil +// DeletePackageFile deletes the package file and its properties +func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) error { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { + return err + } + return packages_model.DeleteFileByID(ctx, pf.ID) } // Cleanup removes old unreferenced package blobs @@ -328,13 +344,25 @@ func Cleanup(unused context.Context, olderThan time.Duration) error { } defer committer.Close() - pbs, err := packages_model.GetUnreferencedBlobsOlderThan(ctx, olderThan) + if err := container_service.Cleanup(ctx, olderThan); err != nil { + log.Error("hier") + return err + } + + if err := packages_model.DeletePackagesIfUnreferenced(ctx); err != nil { + log.Error("hier2") + return err + } + + pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan) if err != nil { + log.Error("hier3") return err } for _, pb := range pbs { if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil { + log.Error("hier4") return err } } diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index a7143c70bee62..114a108feb9f4 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -15,6 +15,7 @@ + @@ -62,8 +63,8 @@ {{end}}
    		• {{.Package.Type.Name}}{{.Package.Name}}{{.Version.Version}} {{.Creator.Name}} {{if .Repository}} diff --git a/templates/package/content/container.tmpl b/templates/package/content/container.tmpl new file mode 100644 index 0000000000000..c7d8a91edc6ef --- /dev/null +++ b/templates/package/content/container.tmpl @@ -0,0 +1,78 @@ +{{if eq .PackageDescriptor.Package.Type "container"}} +

    {{.i18n.Tr "packages.installation"}}

    +
    +
    +
    + + {{if eq .PackageDescriptor.Metadata.Type "helm"}} + + {{else}} + {{$seperator := ":"}} + {{if not .PackageDescriptor.Metadata.IsTagged}} + {{$seperator = "@"}} + {{end}} + + {{end}} +
    +
    + +
    +
    +
    + {{if .PackageDescriptor.Metadata.MultiArch}} +

    {{.i18n.Tr "packages.container.multi_arch"}}

    +
    +
    + {{range $arch, $digest := .PackageDescriptor.Metadata.MultiArch}} +
    + + {{if eq $.PackageDescriptor.Metadata.Type "oci"}} + + {{end}} +
    + {{end}} +
    +
    + {{end}} + {{if .PackageDescriptor.Metadata.Description}} +

    {{.i18n.Tr "packages.about"}}

    +
    + {{.PackageDescriptor.Metadata.Description}} +
    + {{end}} + {{if .PackageDescriptor.Metadata.ImageLayers}} +

    {{.i18n.Tr "packages.container.layers"}}

    +
    + + + {{range .PackageDescriptor.Metadata.ImageLayers}} + + + + {{end}} + +
    {{.}}
    +
    + {{end}} + {{if .PackageDescriptor.Metadata.Labels}} +

    {{.i18n.Tr "packages.container.labels"}}

    +
    + + + + + + + + + {{range $key, $value := .PackageDescriptor.Metadata.Labels}} + + + + + {{end}} + +
    {{.i18n.Tr "packages.container.labels.key"}}{{.i18n.Tr "packages.container.labels.value"}}
    {{$key}}{{$value}}
    +
    + {{end}} +{{end}} diff --git a/templates/package/metadata/container.tmpl b/templates/package/metadata/container.tmpl new file mode 100644 index 0000000000000..117d7e46a29f8 --- /dev/null +++ b/templates/package/metadata/container.tmpl @@ -0,0 +1,9 @@ +{{if eq .PackageDescriptor.Package.Type "container"}} +
    {{svg "octicon-package" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Type.Name}}
    + {{if .PackageDescriptor.Metadata.Platform}}
    {{svg "octicon-cpu" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Platform}}
    {{end}} + {{range .PackageDescriptor.Metadata.Authors}}
    {{svg "octicon-person" 16 "mr-3"}} {{.}}
    {{end}} + {{if .PackageDescriptor.Metadata.Licenses}}
    {{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Licenses}}
    {{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.details.project_site"}}
    {{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.container.details.repository_site"}}
    {{end}} + {{if .PackageDescriptor.Metadata.DocumentationURL}}
    {{svg "octicon-link-external" 16 "mr-3"}} {{.i18n.Tr "packages.container.details.documentation_site"}}
    {{end}} +{{end}} diff --git a/templates/package/settings.tmpl b/templates/package/settings.tmpl index cc50cb4c5b91c..bf2d1d4912dff 100644 --- a/templates/package/settings.tmpl +++ b/templates/package/settings.tmpl @@ -34,7 +34,6 @@ -

    {{.i18n.Tr "repo.settings.danger_zone"}}

    diff --git a/templates/package/shared/list.tmpl b/templates/package/shared/list.tmpl index cdfa624ab1923..9216e6b9de08d 100644 --- a/templates/package/shared/list.tmpl +++ b/templates/package/shared/list.tmpl @@ -8,6 +8,7 @@ + diff --git a/templates/package/shared/versionlist.tmpl b/templates/package/shared/versionlist.tmpl new file mode 100644 index 0000000000000..c5589d8b2163a --- /dev/null +++ b/templates/package/shared/versionlist.tmpl @@ -0,0 +1,32 @@ +
    +

    {{.PackageDescriptor.Package.Name}} / {{.i18n.Tr "packages.versions"}}

    +
    +
    + + {{if eq .PackageDescriptor.Package.Type "container"}} + + {{end}} + +
    +
    +
    + {{range .PackageDescriptors}} +
  • +
    +
    + {{.Version.LowerVersion}} +
    +
    + {{$.i18n.Tr "packages.published_by" (TimeSinceUnix .Version.CreatedUnix $.i18n.Lang) .Creator.HomeLink (.Creator.GetDisplayName | Escape) | Safe}} +
    +
    +
    • + {{else}} +

    {{.i18n.Tr "packages.filter.no_result"}}

    + {{end}} + {{template "base/paginate" .}} +
    +
    diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index ceb0c46494a4d..1b1c5d50c6553 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -21,6 +21,7 @@
    {{template "package/content/composer" .}} {{template "package/content/conan" .}} + {{template "package/content/container" .}} {{template "package/content/generic" .}} {{template "package/content/nuget" .}} {{template "package/content/npm" .}} @@ -30,7 +31,7 @@
    - {{.i18n.Tr "packages.details"}} + {{.i18n.Tr "packages.details"}}
    {{svg .PackageDescriptor.Package.Type.SVGName 16 "mr-3"}} {{.PackageDescriptor.Package.Type.Name}}
    {{if .PackageDescriptor.Repository}} @@ -40,6 +41,7 @@
    {{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
    {{template "package/metadata/composer" .}} {{template "package/metadata/conan" .}} + {{template "package/metadata/container" .}} {{template "package/metadata/generic" .}} {{template "package/metadata/nuget" .}} {{template "package/metadata/npm" .}} @@ -47,21 +49,24 @@ {{template "package/metadata/pypi" .}} {{template "package/metadata/rubygems" .}}
    -
    - {{.i18n.Tr "packages.assets"}} ({{len .PackageDescriptor.Files}}) -
    - {{range .PackageDescriptor.Files}} -
    - {{.File.Name}} - {{FileSize .Blob.Size}} + {{if not (eq .PackageDescriptor.Package.Type "container")}} +
    + {{.i18n.Tr "packages.assets"}} ({{len .PackageDescriptor.Files}}) +
    + {{range .PackageDescriptor.Files}} +
    + {{.File.Name}} + {{FileSize .Blob.Size}} +
    + {{end}}
    {{end}} -
    - {{if .OtherVersions}} + {{if .LatestVersions}}
    - {{.i18n.Tr "packages.versions"}} + {{.i18n.Tr "packages.versions"}} ({{.TotalVersionCount}}) + {{.i18n.Tr "packages.versions.view_all"}}
    - {{range .OtherVersions}} + {{range .LatestVersions}}
    {{.Version}} {{$.i18n.Tr "packages.versions.on"}} {{.CreatedUnix.FormatDate}} diff --git a/templates/user/overview/package_versions.tmpl b/templates/user/overview/package_versions.tmpl new file mode 100644 index 0000000000000..c647d5a71cd29 --- /dev/null +++ b/templates/user/overview/package_versions.tmpl @@ -0,0 +1,6 @@ +{{template "base/head" .}} +
    + {{template "user/overview/header" .}} + {{template "package/shared/versionlist" .}} +
    +{{template "base/footer" .}} From f41cbf09bd670af916b4d47e5daf26590c9440e4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 20 Mar 2022 15:09:30 +0000 Subject: [PATCH 112/130] Refactoring --- routers/api/packages/container/manifest.go | 22 -------- routers/api/v1/api.go | 4 +- routers/api/v1/swagger/package.go | 30 ++++++++++ routers/api/v1/swagger/repo.go | 21 ------- routers/web/web.go | 2 +- templates/swagger/v1_json.tmpl | 66 +++++++++++++++++----- 6 files changed, 84 insertions(+), 61 deletions(-) create mode 100644 routers/api/v1/swagger/package.go diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go index 7c614f314b24c..b327538e6f2c2 100644 --- a/routers/api/packages/container/manifest.go +++ b/routers/api/packages/container/manifest.go @@ -266,28 +266,6 @@ func processImageManifestIndex(mci *manifestCreationInfo, buf *packages_module.H return manifestDigest, nil } -func deleteExistingVersionReferences(ctx context.Context, pv *packages_model.PackageVersion) error { - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil { - return err - } - - pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) - if err != nil { - return err - } - - for _, pf := range pfs { - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil { - return err - } - if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil { - return err - } - } - - return nil -} - func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, metadata *container_module.Metadata) (*packages_model.PackageVersion, error) { p := &packages_model.Package{ OwnerID: mci.Owner.ID, diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index edc93b1918914..384838c93d5a2 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -197,8 +197,7 @@ func repoAssignment() func(ctx *context.APIContext) { func reqPackageAccess(accessMode perm.AccessMode) func(ctx *context.APIContext) { return func(ctx *context.APIContext) { if ctx.Package.AccessMode < accessMode && !ctx.IsUserSiteAdmin() { - ctx.Resp.Header().Set("WWW-Authenticate", `Basic realm="Gitea Package API"`) - ctx.Error(http.StatusUnauthorized, "reqPackageAccess", "user should have specific permission or be a site admin") + ctx.Error(http.StatusForbidden, "reqPackageAccess", "user should have specific permission or be a site admin") return } } @@ -236,7 +235,6 @@ func reqBasicOrRevProxyAuth() func(ctx *context.APIContext) { return } if !ctx.Context.IsBasicAuth { - ctx.Resp.Header().Set("WWW-Authenticate", `Basic realm="Gitea API"`) ctx.Error(http.StatusUnauthorized, "reqBasicOrRevProxyAuth", "auth required") return } diff --git a/routers/api/v1/swagger/package.go b/routers/api/v1/swagger/package.go new file mode 100644 index 0000000000000..2a1f057314775 --- /dev/null +++ b/routers/api/v1/swagger/package.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package swagger + +import ( + api "code.gitea.io/gitea/modules/structs" +) + +// Package +// swagger:response Package +type swaggerResponsePackage struct { + // in:body + Body api.Package `json:"body"` +} + +// PackageList +// swagger:response PackageList +type swaggerResponsePackageList struct { + // in:body + Body []api.Package `json:"body"` +} + +// PackageFileList +// swagger:response PackageFileList +type swaggerResponsePackageFileList struct { + // in:body + Body []api.PackageFile `json:"body"` +} diff --git a/routers/api/v1/swagger/repo.go b/routers/api/v1/swagger/repo.go index 995a06ddfbc4d..40aeca677de83 100644 --- a/routers/api/v1/swagger/repo.go +++ b/routers/api/v1/swagger/repo.go @@ -324,27 +324,6 @@ type swaggerCombinedStatus struct { Body api.CombinedStatus `json:"body"` } -// Package -// swagger:response Package -type swaggerResponsePackage struct { - // in:body - Body api.Package `json:"body"` -} - -// PackageList -// swagger:response PackageList -type swaggerResponsePackageList struct { - // in:body - Body []api.Package `json:"body"` -} - -// PackageFileList -// swagger:response PackageFileList -type swaggerResponsePackageFileList struct { - // in:body - Body []api.PackageFile `json:"body"` -} - // WikiPageList // swagger:response WikiPageList type swaggerWikiPageList struct { diff --git a/routers/web/web.go b/routers/web/web.go index 9fd4b64faf206..9c5c8cb58a222 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -529,7 +529,7 @@ func RegisterRoutes(m *web.Route) { reqPackageAccess := func(accessMode perm.AccessMode) func(ctx *context.Context) { return func(ctx *context.Context) { if ctx.Package.AccessMode < accessMode && !ctx.IsUserSiteAdmin() { - ctx.Error(http.StatusUnauthorized, "reqPackageAccess", "user should have specific permission or be a site admin") + ctx.Error(http.StatusForbidden, "reqPackageAccess", "user should have specific permission or be a site admin") } } } diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 67aa6fd7c0db5..436f445c63e02 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -1930,7 +1930,7 @@ } } }, - "/packages/{owner}/{versionid}": { + "/packages/{owner}/{type}/{name}/{version}": { "get": { "produces": [ "application/json" @@ -1949,10 +1949,23 @@ "required": true }, { - "type": "integer", - "format": "int64", - "description": "id of the package version", - "name": "versionid", + "type": "string", + "description": "type of the package", + "name": "type", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the package", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of the package", + "name": "version", "in": "path", "required": true } @@ -1981,12 +1994,24 @@ "required": true }, { - "type": "integer", - "format": "int64", - "description": "id of the package version", - "name": "versionid", + "type": "string", + "description": "type of the package", + "name": "type", "in": "path", "required": true + }, + { + "type": "string", + "description": "name of the package", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of the package", + "name": "version", + "in": "path" } ], "responses": { @@ -1999,7 +2024,7 @@ } } }, - "/packages/{owner}/{versionid}/files": { + "/packages/{owner}/{type}/{name}/{version}/files": { "get": { "produces": [ "application/json" @@ -2018,10 +2043,23 @@ "required": true }, { - "type": "integer", - "format": "int64", - "description": "id of the package version", - "name": "versionid", + "type": "string", + "description": "type of the package", + "name": "type", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the package", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of the package", + "name": "version", "in": "path", "required": true } From 17b949e21a7e98725e299741bf0e2cac20923598 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 20 Mar 2022 15:14:18 +0000 Subject: [PATCH 113/130] Updated comparison. --- docs/content/doc/features/comparison.en-us.md | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/content/doc/features/comparison.en-us.md b/docs/content/doc/features/comparison.en-us.md index 51ec6bf129c42..36180e3f5b4bd 100644 --- a/docs/content/doc/features/comparison.en-us.md +++ b/docs/content/doc/features/comparison.en-us.md @@ -33,26 +33,26 @@ _Symbols used in table:_ ## General Features -| Feature | Gitea | Gogs | GitHub EE | GitLab CE | GitLab EE | BitBucket | RhodeCode CE | -| ----------------------------------- | ---------------------------------------------------- | ---- | --------- | --------- | --------- | -------------- | ------------ | -| Open source and free | ✓ | ✓ | ✘ | ✓ | ✘ | ✘ | ✓ | -| Low resource usage (RAM/CPU) | ✓ | ✓ | ✘ | ✘ | ✘ | ✘ | ✘ | -| Multiple database support | ✓ | ✓ | ✘ | ⁄ | ⁄ | ✓ | ✓ | -| Multiple OS support | ✓ | ✓ | ✘ | ✘ | ✘ | ✘ | ✓ | -| Easy upgrade process | ✓ | ✓ | ✘ | ✓ | ✓ | ✘ | ✓ | -| Markdown support | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Orgmode support | ✓ | ✘ | ✓ | ✘ | ✘ | ✘ | ? | -| CSV support | ✓ | ✘ | ✓ | ✘ | ✘ | ✓ | ? | -| Third-party render tool support | ✓ | ✘ | ✘ | ✘ | ✘ | ✓ | ? | -| Static Git-powered pages | [✘](https://github.com/go-gitea/gitea/issues/302) | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | -| Integrated Git-powered wiki | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ (cloud only) | ✘ | -| Deploy Tokens | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Repository Tokens with write rights | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Built-in Package/Container Registry | ✓/[✘](https://github.com/go-gitea/gitea/issues/2316) | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | -| External git mirroring | ✓ | ✓ | ✘ | ✘ | ✓ | ✓ | ✓ | -| WebAuthn (2FA) | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ? | -| Built-in CI/CD | ✘ | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | -| Subgroups: groups within groups | ✘ | ✘ | ✘ | ✓ | ✓ | ✘ | ✓ | +| Feature | Gitea | Gogs | GitHub EE | GitLab CE | GitLab EE | BitBucket | RhodeCode CE | +| ----------------------------------- | ---------------------------------------------------| ---- | --------- | --------- | --------- | -------------- | ------------ | +| Open source and free | ✓ | ✓ | ✘ | ✓ | ✘ | ✘ | ✓ | +| Low resource usage (RAM/CPU) | ✓ | ✓ | ✘ | ✘ | ✘ | ✘ | ✘ | +| Multiple database support | ✓ | ✓ | ✘ | ⁄ | ⁄ | ✓ | ✓ | +| Multiple OS support | ✓ | ✓ | ✘ | ✘ | ✘ | ✘ | ✓ | +| Easy upgrade process | ✓ | ✓ | ✘ | ✓ | ✓ | ✘ | ✓ | +| Markdown support | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Orgmode support | ✓ | ✘ | ✓ | ✘ | ✘ | ✘ | ? | +| CSV support | ✓ | ✘ | ✓ | ✘ | ✘ | ✓ | ? | +| Third-party render tool support | ✓ | ✘ | ✘ | ✘ | ✘ | ✓ | ? | +| Static Git-powered pages | [✘](https://github.com/go-gitea/gitea/issues/302) | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | +| Integrated Git-powered wiki | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ (cloud only) | ✘ | +| Deploy Tokens | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Repository Tokens with write rights | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Built-in Package/Container Registry | ✓ | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | +| External git mirroring | ✓ | ✓ | ✘ | ✘ | ✓ | ✓ | ✓ | +| WebAuthn (2FA) | ✓ | ✘ | ✓ | ✓ | ✓ | ✓ | ? | +| Built-in CI/CD | ✘ | ✘ | ✓ | ✓ | ✓ | ✘ | ✘ | +| Subgroups: groups within groups | ✘ | ✘ | ✘ | ✓ | ✓ | ✘ | ✓ | ## Code management From dd953f2f243c716fcc0b2870e0894c6b91ace87e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 20 Mar 2022 15:29:25 +0000 Subject: [PATCH 114/130] Fix swagger. --- routers/api/v1/packages/package.go | 1 + templates/swagger/v1_json.tmpl | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 0d9ffd66ee665..27f888af7714d 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -143,6 +143,7 @@ func DeletePackage(ctx *context.APIContext) { // in: path // description: version of the package // type: string + // required: true // responses: // "204": // "$ref": "#/responses/empty" diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 436f445c63e02..41c9ed795e3f6 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -2011,7 +2011,8 @@ "type": "string", "description": "version of the package", "name": "version", - "in": "path" + "in": "path", + "required": true } ], "responses": { From 6804517fcd8a12e3f8cfc0279f3e4345202a712e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 20 Mar 2022 16:01:56 +0000 Subject: [PATCH 115/130] Fixed table order. --- models/packages/container/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/packages/container/search.go b/models/packages/container/search.go index 638363c14d3b4..40a42411677c2 100644 --- a/models/packages/container/search.go +++ b/models/packages/container/search.go @@ -79,8 +79,8 @@ func GetContainerBlobs(ctx context.Context, opts *BlobSearchOptions) ([]*package func getContainerBlobsLimit(ctx context.Context, opts *BlobSearchOptions, limit int) ([]*packages.PackageFileDescriptor, error) { pfs := make([]*packages.PackageFile, 0, limit) sess := db.GetEngine(ctx). - Join("INNER", "package", "package.id = package_version.package_id"). Join("INNER", "package_version", "package_version.id = package_file.version_id"). + Join("INNER", "package", "package.id = package_version.package_id"). Where(opts.toConds()) if limit > 0 { From 15246c291b8846a0dace76329adead33fbe50d68 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 22 Mar 2022 12:03:00 +0100 Subject: [PATCH 116/130] Use token auth for npm routes. --- integrations/api_packages_npm_test.go | 18 ++++++++++-------- routers/api/packages/api.go | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index 4d9eeb3aae575..c7d494f17b811 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -26,6 +26,8 @@ func TestPackageNpm(t *testing.T) { defer prepareTestEnv(t)() user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User) + token := fmt.Sprintf("Bearer %s", getTokenForLoggedInUser(t, loginUser(t, user.Name))) + packageName := "@scope/test-package" packageVersion := "1.0.1-pre" packageTag := "latest" @@ -70,7 +72,7 @@ func TestPackageNpm(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequestWithBody(t, "PUT", root, strings.NewReader(upload)) - req = AddBasicAuthHeader(req, user.Name) + req = addTokenAuthHeader(req, token) MakeRequest(t, req, http.StatusCreated) pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNpm) @@ -102,7 +104,7 @@ func TestPackageNpm(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequestWithBody(t, "PUT", root, strings.NewReader(upload)) - req = AddBasicAuthHeader(req, user.Name) + req = addTokenAuthHeader(req, token) MakeRequest(t, req, http.StatusBadRequest) }) @@ -110,7 +112,7 @@ func TestPackageNpm(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequest(t, "GET", fmt.Sprintf("%s/-/%s/%s", root, packageVersion, filename)) - req = AddBasicAuthHeader(req, user.Name) + req = addTokenAuthHeader(req, token) resp := MakeRequest(t, req, http.StatusOK) b, _ := base64.StdEncoding.DecodeString(data) @@ -126,7 +128,7 @@ func TestPackageNpm(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequest(t, "GET", root) - req = AddBasicAuthHeader(req, user.Name) + req = addTokenAuthHeader(req, token) resp := MakeRequest(t, req, http.StatusOK) var result npm.PackageMetadata @@ -154,7 +156,7 @@ func TestPackageNpm(t *testing.T) { test := func(t *testing.T, status int, tag, version string) { req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/%s", tagsRoot, tag), strings.NewReader(`"`+version+`"`)) - req = AddBasicAuthHeader(req, user.Name) + req = addTokenAuthHeader(req, token) MakeRequest(t, req, status) } @@ -169,7 +171,7 @@ func TestPackageNpm(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequest(t, "GET", tagsRoot) - req = AddBasicAuthHeader(req, user.Name) + req = addTokenAuthHeader(req, token) resp := MakeRequest(t, req, http.StatusOK) var result map[string]string @@ -186,7 +188,7 @@ func TestPackageNpm(t *testing.T) { defer PrintCurrentTest(t)() req := NewRequest(t, "GET", root) - req = AddBasicAuthHeader(req, user.Name) + req = addTokenAuthHeader(req, token) resp := MakeRequest(t, req, http.StatusOK) var result npm.PackageMetadata @@ -204,7 +206,7 @@ func TestPackageNpm(t *testing.T) { test := func(t *testing.T, status int, tag string) { req := NewRequest(t, "DELETE", fmt.Sprintf("%s/%s", tagsRoot, tag)) - req = AddBasicAuthHeader(req, user.Name) + req = addTokenAuthHeader(req, token) MakeRequest(t, req, status) } diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index fd4b147f19421..3ee74d0d67a64 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -39,7 +39,7 @@ func Routes() *web.Route { r.Use(context.PackageContexter()) - authGroup := auth.NewGroup(&auth.Basic{}, &conan.Auth{}) + authGroup := auth.NewGroup(&auth.OAuth2{}, &auth.Basic{}, &conan.Auth{}) r.Use(func(ctx *context.Context) { ctx.User = authGroup.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) }) From 250443d5363e59929219277fedb79aea9bdb1db4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 22 Mar 2022 11:29:25 +0000 Subject: [PATCH 117/130] Enabled ReverseProxy auth. --- routers/api/packages/api.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 3ee74d0d67a64..98b98863704ce 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/packages/composer" "code.gitea.io/gitea/routers/api/packages/conan" @@ -39,7 +40,16 @@ func Routes() *web.Route { r.Use(context.PackageContexter()) - authGroup := auth.NewGroup(&auth.OAuth2{}, &auth.Basic{}, &conan.Auth{}) + authMethods := []auth.Method{ + &auth.OAuth2{}, + &auth.Basic{}, + &conan.Auth{}, + } + if setting.Service.EnableReverseProxyAuth { + authMethods = append(authMethods, &auth.ReverseProxy{}) + } + + authGroup := auth.NewGroup(authMethods...) r.Use(func(ctx *context.Context) { ctx.User = authGroup.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) }) @@ -225,7 +235,15 @@ func ContainerRoutes() *web.Route { r.Use(context.PackageContexter()) - authGroup := auth.NewGroup(&auth.Basic{}, &container.Auth{}) + authMethods := []auth.Method{ + &auth.Basic{}, + &container.Auth{}, + } + if setting.Service.EnableReverseProxyAuth { + authMethods = append(authMethods, &auth.ReverseProxy{}) + } + + authGroup := auth.NewGroup(authMethods...) r.Use(func(ctx *context.Context) { ctx.User = authGroup.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) }) From 5f8e6b2c661f9a6bbd86b408521aa0abac5df37d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 23 Mar 2022 15:16:21 +0000 Subject: [PATCH 118/130] Added packages link for orgs. --- templates/org/menu.tmpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/org/menu.tmpl b/templates/org/menu.tmpl index 9c7c2bbfb23f4..1f7b1216a6269 100644 --- a/templates/org/menu.tmpl +++ b/templates/org/menu.tmpl @@ -3,6 +3,9 @@ {{svg "octicon-repo"}} {{.i18n.Tr "user.repositories"}} + + {{svg "octicon-package"}} {{.i18n.Tr "packages.title"}} + {{if .IsOrganizationMember}} {{svg "octicon-organization"}} {{$.i18n.Tr "org.people"}} From d8b1c69a7bd53b22a010bdf848eb4d08aade5cbd Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 25 Mar 2022 21:41:26 +0000 Subject: [PATCH 119/130] Fixed anonymous org access. --- modules/context/package.go | 17 ++++++++++------- routers/web/web.go | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/modules/context/package.go b/modules/context/package.go index 30eca2d8091c8..ac922a2969103 100644 --- a/modules/context/package.go +++ b/modules/context/package.go @@ -14,7 +14,7 @@ import ( user_model "code.gitea.io/gitea/models/user" ) -// Package contains owner, access mode and optional the package +// Package contains owner, access mode and optional the package descriptor type Package struct { Owner *user_model.User AccessMode perm.AccessMode @@ -54,12 +54,15 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) { ctx.Package.AccessMode = perm.AccessModeOwner } else { if ctx.Package.Owner.IsOrganization() { - if ctx.Doer != nil && models.HasOrgOrUserVisible(ctx.Package.Owner, ctx.Doer) { - var err error - ctx.Package.AccessMode, err = models.OrgFromUser(ctx.Package.Owner).GetOrgUserMaxAuthorizeLevel(ctx.Doer.ID) - if err != nil { - errCb(http.StatusInternalServerError, "GetOrgUserMaxAuthorizeLevel", err) - return + if models.HasOrgOrUserVisible(ctx.Package.Owner, ctx.Doer) { + ctx.Package.AccessMode = perm.AccessModeRead + if ctx.Doer != nil { + var err error + ctx.Package.AccessMode, err = models.OrgFromUser(ctx.Package.Owner).GetOrgUserMaxAuthorizeLevel(ctx.Doer.ID) + if err != nil { + errCb(http.StatusInternalServerError, "GetOrgUserMaxAuthorizeLevel", err) + return + } } } } else { diff --git a/routers/web/web.go b/routers/web/web.go index 9c5c8cb58a222..234f8e8ad78d5 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -529,7 +529,7 @@ func RegisterRoutes(m *web.Route) { reqPackageAccess := func(accessMode perm.AccessMode) func(ctx *context.Context) { return func(ctx *context.Context) { if ctx.Package.AccessMode < accessMode && !ctx.IsUserSiteAdmin() { - ctx.Error(http.StatusForbidden, "reqPackageAccess", "user should have specific permission or be a site admin") + ctx.NotFound("", nil) } } } From cb59cbd958071caf973a2cee5893af584e55bc77 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 25 Mar 2022 22:08:09 +0000 Subject: [PATCH 120/130] Enable copy button for setup instructions. --- templates/package/content/composer.tmpl | 10 ++--- templates/package/content/conan.tmpl | 8 ++-- templates/package/content/container.tmpl | 10 ++--- templates/package/content/generic.tmpl | 4 +- templates/package/content/maven.tmpl | 54 ++++++++++++------------ templates/package/content/npm.tmpl | 12 +++--- templates/package/content/nuget.tmpl | 8 ++-- templates/package/content/pypi.tmpl | 4 +- templates/package/content/rubygems.tmpl | 10 ++--- 9 files changed, 60 insertions(+), 60 deletions(-) diff --git a/templates/package/content/composer.tmpl b/templates/package/content/composer.tmpl index ceb3bda8f763d..29162f97b02a6 100644 --- a/templates/package/content/composer.tmpl +++ b/templates/package/content/composer.tmpl @@ -3,18 +3,18 @@
    - - +}
    - - + + 
    composer require {{.PackageDescriptor.Package.Name}}:{{.PackageDescriptor.Version.Version}}
    diff --git a/templates/package/content/conan.tmpl b/templates/package/content/conan.tmpl index a9882499530a5..3c1c35c00fac3 100644 --- a/templates/package/content/conan.tmpl +++ b/templates/package/content/conan.tmpl @@ -3,12 +3,12 @@
    - - + + 
    conan remote add gitea {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/conan
    - - + + 
    conan install --remote=gitea {{.PackageDescriptor.Package.Name}}/{{.PackageDescriptor.Version.Version}}
    diff --git a/templates/package/content/container.tmpl b/templates/package/content/container.tmpl index c7d8a91edc6ef..9fc8fc35b21f6 100644 --- a/templates/package/content/container.tmpl +++ b/templates/package/content/container.tmpl @@ -3,15 +3,15 @@
    - + {{if eq .PackageDescriptor.Metadata.Type "helm"}} - + 
    helm pull oci://{{.RegistryHost}}/{{.PackageDescriptor.Owner.LowerName}}/{{.PackageDescriptor.Package.LowerName}} --version {{.PackageDescriptor.Version.LowerVersion}}
    {{else}} {{$seperator := ":"}} {{if not .PackageDescriptor.Metadata.IsTagged}} {{$seperator = "@"}} {{end}} - + 
    docker pull {{.RegistryHost}}/{{.PackageDescriptor.Owner.LowerName}}/{{.PackageDescriptor.Package.LowerName}}{{$seperator}}{{.PackageDescriptor.Version.LowerVersion}}
    {{end}}
    @@ -25,9 +25,9 @@
    {{range $arch, $digest := .PackageDescriptor.Metadata.MultiArch}}
    - + {{if eq $.PackageDescriptor.Metadata.Type "oci"}} - + 
    docker pull {{$.RegistryHost}}/{{$.PackageDescriptor.Owner.LowerName}}/{{$.PackageDescriptor.Package.LowerName}}@{{$digest}}
    {{end}}
    {{end}} diff --git a/templates/package/content/generic.tmpl b/templates/package/content/generic.tmpl index 4211da3ac1edb..05a47b3ef4dcf 100644 --- a/templates/package/content/generic.tmpl +++ b/templates/package/content/generic.tmpl @@ -3,8 +3,8 @@
    - - + + 
    curl {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/generic/{{.PackageDescriptor.Package.Name}}/{{.PackageDescriptor.Version.Version}}/{{(index .PackageDescriptor.Files 0).File.Name}}
    diff --git a/templates/package/content/maven.tmpl b/templates/package/content/maven.tmpl index 6856fa7618069..32b89616cbfd2 100644 --- a/templates/package/content/maven.tmpl +++ b/templates/package/content/maven.tmpl @@ -3,41 +3,41 @@
    - - + <snapshotRepository> + <id>gitea</id> + <url>{{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/maven</url> + </snapshotRepository> +</distributionManagement>
    - - + + 
    <dependency>
+	<groupId>{{.PackageDescriptor.Metadata.GroupID}}</groupId>
+	<artifactId>{{.PackageDescriptor.Metadata.ArtifactID}}</artifactId>
+	<version>{{.PackageDescriptor.Version.Version}}</version>
+</dependency>
    - - + + 
    mvn install
    - - + + 
    mvn dependency:get -DremoteRepositories={{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/maven -Dartifact={{.PackageDescriptor.Metadata.GroupID}}:{{.PackageDescriptor.Metadata.ArtifactID}}:{{.PackageDescriptor.Version.Version}}
    diff --git a/templates/package/content/npm.tmpl b/templates/package/content/npm.tmpl index 9f75983a75016..16347d1b6edfc 100644 --- a/templates/package/content/npm.tmpl +++ b/templates/package/content/npm.tmpl @@ -3,16 +3,16 @@
    - - + + 
    {{if .PackageDescriptor.Metadata.Scope}}{{.PackageDescriptor.Metadata.Scope}}:{{end}}registry={{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/npm/
    - - + + 
    npm install {{.PackageDescriptor.Package.Name}}@{{.PackageDescriptor.Version.Version}}
    - - + + 
    "{{.PackageDescriptor.Package.Name}}": "{{.PackageDescriptor.Version.Version}}"
    diff --git a/templates/package/content/nuget.tmpl b/templates/package/content/nuget.tmpl index 1abb4ed339c26..879d7d017670c 100644 --- a/templates/package/content/nuget.tmpl +++ b/templates/package/content/nuget.tmpl @@ -3,12 +3,12 @@
    - - + + 
    dotnet nuget add source --name Gitea --username your_username --password your_token {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/nuget/index.json
    - - + + 
    dotnet add package --source Gitea --version {{.PackageDescriptor.Version.Version}} {{.PackageDescriptor.Package.Name}}
    diff --git a/templates/package/content/pypi.tmpl b/templates/package/content/pypi.tmpl index 73e5c81a1475e..352f4f617fad9 100644 --- a/templates/package/content/pypi.tmpl +++ b/templates/package/content/pypi.tmpl @@ -3,8 +3,8 @@
    - - + + 
    pip install --extra-index-url {{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/pypi/simple {{.PackageDescriptor.Package.Name}}
    diff --git a/templates/package/content/rubygems.tmpl b/templates/package/content/rubygems.tmpl index 80c5ce77ccbf9..6e22d7fbea256 100644 --- a/templates/package/content/rubygems.tmpl +++ b/templates/package/content/rubygems.tmpl @@ -3,14 +3,14 @@
    - - + + 
    gem install {{.PackageDescriptor.Package.Name}} --version "{{.PackageDescriptor.Version.Version}}" --source "{{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/rubygems"
    - - +end
    From 70c382598d1f048ea0a9baac6d554c4162cc994a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 27 Mar 2022 09:00:54 +0000 Subject: [PATCH 121/130] Merge error --- modules/context/repo.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/context/repo.go b/modules/context/repo.go index 634301ef034b9..ccdc810fb6906 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -441,7 +441,6 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { ctx.Repo.Owner = owner ctx.ContextUser = owner ctx.Data["Username"] = ctx.Repo.Owner.Name - ctx.ContextUser = owner // redirect link to wiki if strings.HasSuffix(repoName, ".wiki") { From af95307fef845bb19ea651a05bb191ad56676a49 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 27 Mar 2022 11:43:11 +0000 Subject: [PATCH 122/130] Added suggestions. --- integrations/api_packages_container_test.go | 33 +++++++++++++-------- models/migrations/v212.go | 2 +- models/packages/conan/search.go | 2 +- models/packages/container/search.go | 13 +++++--- models/packages/descriptor.go | 2 ++ models/packages/package.go | 5 ++-- models/packages/package_version.go | 2 +- modules/context/context.go | 2 +- routers/api/packages/container/container.go | 16 ++++++++-- routers/web/user/package.go | 3 ++ 10 files changed, 54 insertions(+), 26 deletions(-) diff --git a/integrations/api_packages_container_test.go b/integrations/api_packages_container_test.go index ec031a56b0f54..a8f49423e2a0c 100644 --- a/integrations/api_packages_container_test.go +++ b/integrations/api_packages_container_test.go @@ -442,28 +442,34 @@ func TestPackageContainer(t *testing.T) { defer PrintCurrentTest(t)() cases := []struct { - URL string - Expected []string + URL string + ExpectedTags []string + ExpectedLink string }{ { - URL: fmt.Sprintf("%s/tags/list", url), - Expected: []string{"latest", "main", "multi"}, + URL: fmt.Sprintf("%s/tags/list", url), + ExpectedTags: []string{"latest", "main", "multi"}, + ExpectedLink: fmt.Sprintf(`; rel="next"`, user.Name, image), }, { - URL: fmt.Sprintf("%s/tags/list?n=0", url), - Expected: []string{}, + URL: fmt.Sprintf("%s/tags/list?n=0", url), + ExpectedTags: []string{}, + ExpectedLink: "", }, { - URL: fmt.Sprintf("%s/tags/list?n=2", url), - Expected: []string{"latest", "main"}, + URL: fmt.Sprintf("%s/tags/list?n=2", url), + ExpectedTags: []string{"latest", "main"}, + ExpectedLink: fmt.Sprintf(`; rel="next"`, user.Name, image), }, { - URL: fmt.Sprintf("%s/tags/list?last=main", url), - Expected: []string{"multi"}, + URL: fmt.Sprintf("%s/tags/list?last=main", url), + ExpectedTags: []string{"multi"}, + ExpectedLink: fmt.Sprintf(`; rel="next"`, user.Name, image), }, { - URL: fmt.Sprintf("%s/tags/list?n=1&last=latest", url), - Expected: []string{"main"}, + URL: fmt.Sprintf("%s/tags/list?n=1&last=latest", url), + ExpectedTags: []string{"main"}, + ExpectedLink: fmt.Sprintf(`; rel="next"`, user.Name, image), }, } @@ -481,7 +487,8 @@ func TestPackageContainer(t *testing.T) { DecodeJSON(t, resp, &tagList) assert.Equal(t, user.Name+"/"+image, tagList.Name) - assert.Equal(t, c.Expected, tagList.Tags) + assert.Equal(t, c.ExpectedTags, tagList.Tags) + assert.Equal(t, c.ExpectedLink, resp.Header().Get("Link")) } }) diff --git a/models/migrations/v212.go b/models/migrations/v212.go index da6175f8ad26c..9d16f0556c8cb 100644 --- a/models/migrations/v212.go +++ b/models/migrations/v212.go @@ -33,7 +33,7 @@ func addPackageTables(x *xorm.Engine) error { LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"` - MetadataJSON string `xorm:"TEXT metadata_json"` + MetadataJSON string `xorm:"metadata_json TEXT"` DownloadCount int64 `xorm:"NOT NULL DEFAULT 0"` } diff --git a/models/packages/conan/search.go b/models/packages/conan/search.go index 6d86b0eae41ec..c274a7ce02504 100644 --- a/models/packages/conan/search.go +++ b/models/packages/conan/search.go @@ -20,7 +20,7 @@ import ( // buildCondition creates a Like condition if a wildcard is present. Otherwise Eq is used. func buildCondition(name, value string) builder.Cond { if strings.Contains(value, "*") { - return builder.Like{name, strings.ReplaceAll(value, "*", "%")} + return builder.Like{name, strings.ReplaceAll(strings.ReplaceAll(value, "_", "\\_"), "*", "%")} } return builder.Eq{name: value} } diff --git a/models/packages/container/search.go b/models/packages/container/search.go index 40a42411677c2..972cac9528f86 100644 --- a/models/packages/container/search.go +++ b/models/packages/container/search.go @@ -117,7 +117,12 @@ func GetManifestVersions(ctx context.Context, opts *BlobSearchOptions) ([]*packa // GetImageTags gets a sorted list of the tags of an image // The result is suitable for the api call. -func GetImageTags(ctx context.Context, ownerID int64, image string, n *int, last string) ([]string, error) { +func GetImageTags(ctx context.Context, ownerID int64, image string, n int, last string) ([]string, error) { + // Short circuit: n == 0 should return an empty list + if n == 0 { + return []string{}, nil + } + var cond builder.Cond = builder.Eq{ "package.type": packages.TypeContainer, "package.owner_id": ownerID, @@ -144,10 +149,10 @@ func GetImageTags(ctx context.Context, ownerID int64, image string, n *int, last Asc("package_version.lower_version") var tags []string - if n != nil && *n >= 0 { - sess = sess.Limit(*n) + if n > 0 { + sess = sess.Limit(n) - tags = make([]string, 0, *n) + tags = make([]string, 0, n) } else { tags = make([]string, 0, 10) } diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 5a622fa32fc5c..2efdafff966f5 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -137,6 +137,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &pypi.Metadata{} case TypeRubyGems: metadata = &rubygems.Metadata{} + default: + panic(fmt.Sprintf("unknown package type: %s", string(p.Type))) } if metadata != nil { if err := json.Unmarshal([]byte(pv.MetadataJSON), &metadata); err != nil { diff --git a/models/packages/package.go b/models/packages/package.go index 9b4a98f845152..05170ab3f430c 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -7,6 +7,7 @@ package packages import ( "context" "errors" + "fmt" "strings" "code.gitea.io/gitea/models/db" @@ -63,7 +64,7 @@ func (pt Type) Name() string { case TypeRubyGems: return "RubyGems" } - return "" + panic(fmt.Sprintf("unknown package type: %s", string(pt))) } // SVGName gets the name of the package type svg image @@ -88,7 +89,7 @@ func (pt Type) SVGName() string { case TypeRubyGems: return "gitea-rubygems" } - return "" + panic(fmt.Sprintf("unknown package type: %s", string(pt))) } // Package represents a package diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 083c8be0dd8e1..f7c6d4dc586e7 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -36,7 +36,7 @@ type PackageVersion struct { LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"` IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"` - MetadataJSON string `xorm:"TEXT metadata_json"` + MetadataJSON string `xorm:"metadata_json TEXT"` DownloadCount int64 `xorm:"NOT NULL DEFAULT 0"` } diff --git a/modules/context/context.go b/modules/context/context.go index 599c4440172f6..4905e1cb80e2f 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -380,7 +380,7 @@ func (ctx *Context) ServeStream(rd io.Reader, name string) { // UploadStream returns the request body or the first form file // Only form files need to get closed. -func (ctx *Context) UploadStream() (io.ReadCloser, bool, error) { +func (ctx *Context) UploadStream() (rd io.ReadCloser, needToClose bool, err error) { contentType := strings.ToLower(ctx.Req.Header.Get("Content-Type")) if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") || strings.HasPrefix(contentType, "multipart/form-data") { if err := ctx.Req.ParseMultipartForm(32 << 20); err != nil { diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index fd8cc9e365a5a..f0b1fafd26d35 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "net/url" "regexp" "strconv" "strings" @@ -578,10 +579,9 @@ func GetTagList(ctx *context.Context) { return } - var n *int + n := -1 if ctx.FormTrim("n") != "" { - val := ctx.FormInt("n") - n = &val + n = ctx.FormInt("n") } last := ctx.FormTrim("last") @@ -596,6 +596,16 @@ func GetTagList(ctx *context.Context) { Tags []string `json:"tags"` } + if len(tags) > 0 { + v := url.Values{} + if n > 0 { + v.Add("n", strconv.Itoa(n)) + } + v.Add("last", tags[len(tags)-1]) + + ctx.Resp.Header().Set("Link", fmt.Sprintf(`; rel="next"`, ctx.Package.Owner.LowerName, image, v.Encode())) + } + jsonResponse(ctx, http.StatusOK, TagList{ Name: strings.ToLower(ctx.Package.Owner.LowerName + "/" + image), Tags: tags, diff --git a/routers/web/user/package.go b/routers/web/user/package.go index d111c5073c6d2..8a5294dce1ec5 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -102,6 +102,7 @@ func RedirectToLastVersion(ctx *context.Context) { } if len(pvs) == 0 { ctx.NotFound("", err) + return } pd, err := packages_model.GetPackageDescriptor(ctx, pvs[0]) @@ -143,10 +144,12 @@ func ViewPackageVersion(ctx *context.Context) { }) if err != nil { ctx.ServerError("SearchVersions", err) + return } } if err != nil { ctx.ServerError("", err) + return } ctx.Data["LatestVersions"] = pvs From 1bf5b5f14e4ef6c0fe11ac4eb1d374cf9255e3b9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 27 Mar 2022 12:14:28 +0000 Subject: [PATCH 123/130] Fixed merge. --- services/cron/tasks_basic.go | 1 - 1 file changed, 1 deletion(-) diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index be395f55ed6f6..04cf93d78b12f 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -146,7 +146,6 @@ func registerCleanupPackages() { Enabled: true, RunAtStart: true, Schedule: "@midnight", - NoSuccessNotice: true, }, OlderThan: 24 * time.Hour, }, func(ctx context.Context, _ *user_model.User, config Config) error { From be74e4dcb878840d679d59c40426374e8febd0ef Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 28 Mar 2022 14:11:53 +0000 Subject: [PATCH 124/130] Handle "generic". --- models/packages/descriptor.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 2efdafff966f5..3249260f80a6f 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -127,6 +127,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &conan.Metadata{} case TypeContainer: metadata = &container.Metadata{} + case TypeGeneric: + // generic packages have no metadata case TypeNuGet: metadata = &nuget.Metadata{} case TypeNpm: From 3cb84ef5a9d83ef1d9aad6fa095d5c5912c611cf Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 28 Mar 2022 14:20:31 +0000 Subject: [PATCH 125/130] Added link for TODO. --- modules/notification/webhook/webhook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/notification/webhook/webhook.go b/modules/notification/webhook/webhook.go index 23cc22668445c..94d4d180be17c 100644 --- a/modules/notification/webhook/webhook.go +++ b/modules/notification/webhook/webhook.go @@ -867,7 +867,7 @@ func (m *webhookNotifier) NotifyPackageDelete(doer *user_model.User, pd *package func notifyPackage(sender *user_model.User, pd *packages_model.PackageDescriptor, action api.HookPackageAction) { if pd.Repository == nil { - // TODO + // TODO https://github.com/go-gitea/gitea/pull/17940 return } From 20facab13c9bccc2bdc0e9c18379f7117de12a9b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 28 Mar 2022 15:25:55 +0000 Subject: [PATCH 126/130] Added suggestions. --- integrations/api_packages_npm_test.go | 6 +++- routers/api/packages/generic/generic.go | 2 ++ routers/api/packages/npm/api.go | 2 +- routers/api/packages/npm/npm.go | 37 +++++--------------- services/packages/container/blob_uploader.go | 4 ++- 5 files changed, 19 insertions(+), 32 deletions(-) diff --git a/integrations/api_packages_npm_test.go b/integrations/api_packages_npm_test.go index c7d494f17b811..28a3711939828 100644 --- a/integrations/api_packages_npm_test.go +++ b/integrations/api_packages_npm_test.go @@ -127,7 +127,11 @@ func TestPackageNpm(t *testing.T) { t.Run("PackageMetadata", func(t *testing.T) { defer PrintCurrentTest(t)() - req := NewRequest(t, "GET", root) + req := NewRequest(t, "GET", fmt.Sprintf("/api/packages/%s/npm/%s", user.Name, "does-not-exist")) + req = addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", root) req = addTokenAuthHeader(req, token) resp := MakeRequest(t, req, http.StatusOK) diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index 925ec6cb35ce9..d862f77259ace 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -145,6 +145,8 @@ func DeletePackage(ctx *context.Context) { apiError(ctx, http.StatusInternalServerError, err) return } + + ctx.Status(http.StatusOK) } func sanitizeParameters(ctx *context.Context) (string, string, string, error) { diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index d168048ef5ebb..56c897704398d 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -67,7 +67,7 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package Dist: npm_module.PackageDistribution{ Shasum: pd.Files[0].Blob.HashSHA1, Integrity: "sha512-" + base64.StdEncoding.EncodeToString(hashBytes), - Tarball: fmt.Sprintf("%s/%s/-/%s/%s", registryURL, url.QueryEscape(pd.Package.Name), pd.Version.Version, pd.Files[0].File.LowerName), + Tarball: fmt.Sprintf("%s/%s/-/%s/%s", registryURL, url.QueryEscape(pd.Package.Name), url.PathEscape(pd.Version.Version), url.PathEscape(pd.Files[0].File.LowerName)), }, } } diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 214b4afb1d958..50151ee5ea6f8 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "net/http" - "net/url" "strings" "code.gitea.io/gitea/models/db" @@ -38,22 +37,18 @@ func apiError(ctx *context.Context, status int, obj interface{}) { // packageNameFromParams gets the package name from the url parameters // Variations: /name/, /@scope/name/, /@scope%2Fname/ -func packageNameFromParams(ctx *context.Context) (string, error) { +func packageNameFromParams(ctx *context.Context) string { scope := ctx.Params("scope") id := ctx.Params("id") if scope != "" { - return fmt.Sprintf("@%s/%s", scope, id), nil + return fmt.Sprintf("@%s/%s", scope, id) } - return url.QueryUnescape(id) + return id } // PackageMetadata returns the metadata for a single package func PackageMetadata(ctx *context.Context) { - packageName, err := packageNameFromParams(ctx) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } + packageName := packageNameFromParams(ctx) pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) if err != nil { @@ -81,11 +76,7 @@ func PackageMetadata(ctx *context.Context) { // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.Context) { - packageName, err := packageNameFromParams(ctx) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } + packageName := packageNameFromParams(ctx) packageVersion := ctx.Params("version") filename := ctx.Params("filename") @@ -174,11 +165,7 @@ func UploadPackage(ctx *context.Context) { // ListPackageTags returns all tags for a package func ListPackageTags(ctx *context.Context) { - packageName, err := packageNameFromParams(ctx) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } + packageName := packageNameFromParams(ctx) pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) if err != nil { @@ -203,11 +190,7 @@ func ListPackageTags(ctx *context.Context) { // AddPackageTag adds a tag to the package func AddPackageTag(ctx *context.Context) { - packageName, err := packageNameFromParams(ctx) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } + packageName := packageNameFromParams(ctx) body, err := io.ReadAll(ctx.Req.Body) if err != nil { @@ -238,11 +221,7 @@ func AddPackageTag(ctx *context.Context) { // DeletePackageTag deletes a package tag func DeletePackageTag(ctx *context.Context) { - packageName, err := packageNameFromParams(ctx) - if err != nil { - apiError(ctx, http.StatusBadRequest, err) - return - } + packageName := packageNameFromParams(ctx) pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) if err != nil { diff --git a/services/packages/container/blob_uploader.go b/services/packages/container/blob_uploader.go index c4421005ffca9..762f9e52598f8 100644 --- a/services/packages/container/blob_uploader.go +++ b/services/packages/container/blob_uploader.go @@ -10,6 +10,8 @@ import ( "io" "os" "path" + "path/filepath" + "strings" packages_model "code.gitea.io/gitea/models/packages" packages_module "code.gitea.io/gitea/modules/packages" @@ -32,7 +34,7 @@ type BlobUploader struct { } func buildFilePath(id string) string { - return path.Join(setting.Packages.ChunkedUploadPath, id) + return filepath.Join(setting.Packages.ChunkedUploadPath, path.Clean("/" + strings.ReplaceAll(id, "\\", "/"))[1:]) } // NewBlobUploader creates a new blob uploader for the given id From 10c5f03c52851849b2298a500dc5767ab4db3f83 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 28 Mar 2022 17:36:27 +0000 Subject: [PATCH 127/130] Changed temporary buffer filename. --- modules/util/filebuffer/file_backed_buffer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index ac3990219b737..128030b4c5c1b 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -75,7 +75,7 @@ func (b *FileBackedBuffer) Write(p []byte) (int, error) { n, err = b.file.Write(p) } else { if b.size+int64(len(p)) > b.maxMemorySize { - b.file, err = os.CreateTemp("", "buffer-") + b.file, err = os.CreateTemp("", "gitea-buffer-") if err != nil { return 0, err } From dbbbfa47e6e2e225996d1c12c53a5a539bc56154 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 29 Mar 2022 14:32:16 +0000 Subject: [PATCH 128/130] Added suggestions. --- custom/conf/app.example.ini | 4 ++-- docs/content/doc/advanced/config-cheat-sheet.en-us.md | 11 ++++++++++- docs/content/doc/packages/conan.en-us.md | 2 +- docs/content/doc/packages/npm.en-us.md | 4 ++-- docs/content/doc/packages/overview.en-us.md | 9 ++++++--- models/error.go | 2 +- options/locale/locale_en-US.ini | 8 ++++---- routers/api/packages/helper/helper.go | 2 +- templates/package/content/container.tmpl | 6 +++--- templates/package/shared/versionlist.tmpl | 5 +++-- 10 files changed, 33 insertions(+), 20 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 87d7db9dd3685..f6a3df51b4ea6 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1894,8 +1894,8 @@ PATH = ;ENABLED = true ;; Whether to always run at least once at start up time (if ENABLED) ;RUN_AT_START = true -;; Notice if not success -;NO_SUCCESS_NOTICE = true +;; Whether to emit notice on successful execution too +;NOTICE_ON_SUCCESS = false ;; Time interval for job to run ;SCHEDULE = @midnight ;; Unreferenced blobs created more than OLDER_THAN ago are subject to deletion diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 84ada648a1e81..3cc852a150d56 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -849,7 +849,7 @@ Default templates for project boards: - `RUN_AT_START`: **true**: Run repository statistics check at start time. - `SCHEDULE`: **@midnight**: Cron syntax for scheduling repository statistics check. -### Cron - Cleanup hook_task Table (`cron.cleanup_hook_task_table`) +#### Cron - Cleanup hook_task Table (`cron.cleanup_hook_task_table`) - `ENABLED`: **true**: Enable cleanup hook_task job. - `RUN_AT_START`: **false**: Run cleanup hook_task at start time (if ENABLED). @@ -858,6 +858,14 @@ Default templates for project boards: - `OLDER_THAN`: **168h**: If CLEANUP_TYPE is set to OlderThan, then any delivered hook_task records older than this expression will be deleted. - `NUMBER_TO_KEEP`: **10**: If CLEANUP_TYPE is set to PerWebhook, this is number of hook_task records to keep for a webhook (i.e. keep the most recent x deliveries). +#### Cron - Cleanup expired packages (`cron.cleanup_packages`) + +- `ENABLED`: **true**: Enable cleanup expired packages job. +- `RUN_AT_START`: **true**: Run job at start time (if ENABLED). +- `NOTICE_ON_SUCCESS`: **false**: Notify every time this job runs. +- `SCHEDULE`: **@midnight**: Cron syntax for the job. +- `OLDER_THAN`: **24h**: Unreferenced package data created more than OLDER_THAN ago is subject to deletion. + #### Cron - Update Migration Poster ID (`cron.update_migration_poster_id`) - `SCHEDULE`: **@midnight** : Interval as a duration between each synchronization, it will always attempt synchronization when the instance starts. @@ -1073,6 +1081,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf ## Packages (`packages`) - `ENABLED`: **true**: Enable/Disable package registry capabilities +- `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload` ## Mirror (`mirror`) diff --git a/docs/content/doc/packages/conan.en-us.md b/docs/content/doc/packages/conan.en-us.md index 968ff77d9aa6f..c650e9d7ea491 100644 --- a/docs/content/doc/packages/conan.en-us.md +++ b/docs/content/doc/packages/conan.en-us.md @@ -79,7 +79,7 @@ conan install --remote={remote} {recipe} | Parameter | Description | | ----------| ----------- | | `remote` | The remote name. | -| `recipe` | The recipe to upload. | +| `recipe` | The recipe to download. | For example: diff --git a/docs/content/doc/packages/npm.en-us.md b/docs/content/doc/packages/npm.en-us.md index 083c2799837f0..28b7cb8827029 100644 --- a/docs/content/doc/packages/npm.en-us.md +++ b/docs/content/doc/packages/npm.en-us.md @@ -22,9 +22,9 @@ Publish [npm](https://www.npmjs.com/) packages for your user or organization. ## Requirements -To work with the npm package registry, you need [Node.js and npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm/) or an other tool like [Yarn](https://classic.yarnpkg.com/en/docs/install). +To work with the npm package registry, you need [Node.js](https://nodejs.org/en/download/) coupled with a package manager such as [Yarn](https://classic.yarnpkg.com/en/docs/install) or [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm/) itself. -The registry supprots [scoped](https://docs.npmjs.com/misc/scope/) and unscoped packages. +The registry supports [scoped](https://docs.npmjs.com/misc/scope/) and unscoped packages. The following examples use the `npm` tool with the scope `@test`. diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 0a7415e92bb5c..f7809fc8a37bc 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -31,15 +31,18 @@ The following package managers are currently supported: | [Container]({{< relref "doc/packages/container.en-us.md" >}}) | - | any OCI compliant client | | [Generic]({{< relref "doc/packages/generic.en-us.md" >}}) | - | any HTTP client | | [Maven]({{< relref "doc/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` | -| [npm]({{< relref "doc/packages/npm.en-us.md" >}}) | JavaScript | `npm` | +| [npm]({{< relref "doc/packages/npm.en-us.md" >}}) | JavaScript | `npm`, `yarn` | | [NuGet]({{< relref "doc/packages/nuget.en-us.md" >}}) | .NET | `nuget` | -| [RubyGems]({{< relref "doc/packages/rubygems.en-us.md" >}}) | Ruby | `gem` | +| [PyPI]({{< relref "doc/packages/pypi.en-us.md" >}}) | Python | `pip`, `twine` | +| [RubyGems]({{< relref "doc/packages/rubygems.en-us.md" >}}) | Ruby | `gem`, `Bundler` | + +**The following paragraphs only apply if Packages are not globally disabled!** ## View packages You can view the packages of a repository on the repository page. -1. Go to the repoistory. +1. Go to the repository. 1. Go to **Packages** in the navigation bar. To view more details about a package, select the name of the package. diff --git a/models/error.go b/models/error.go index 99a8f31b7a712..c8d79c7f71738 100644 --- a/models/error.go +++ b/models/error.go @@ -58,7 +58,7 @@ func (err ErrUserHasOrgs) Error() string { return fmt.Sprintf("user still has membership of organizations [uid: %d]", err.UID) } -// ErrUserOwnPackages represents a "UserOwnPackages" kind of error. +// ErrUserOwnPackages notifies that the user (still) owns the packages. type ErrUserOwnPackages struct { UID int64 } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 6273e9f67fcba..de61f4e4f33fa 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -488,9 +488,9 @@ auth_failed = Authentication failed: %v still_own_repo = "Your account owns one or more repositories; delete or transfer them first." still_has_org = "Your account is a member of one or more organizations; leave them first." -still_own_packages = Your account owns one or more packages; delete them first. +still_own_packages = "Your account owns one or more packages; delete them first." org_still_own_repo = "This organization still owns one or more repositories; delete or transfer them first." -org_still_own_packages = This organization still owns one or more packages; delete them first. +org_still_own_packages = "This organization still owns one or more packages; delete them first." target_branch_not_exist = Target branch does not exist. @@ -2554,7 +2554,7 @@ repos.issues = Issues repos.size = Size packages.package_manage_panel = Package Management -packages.total_size = Total Blob Size: %s +packages.total_size = Total Size: %s packages.owner = Owner packages.creator = Creator packages.name = Name @@ -3025,7 +3025,7 @@ assets = Assets versions = Versions versions.on = on versions.view_all = View all -dependency.id = Id +dependency.id = ID dependency.version = Version composer.registry = Setup this registry in your ~/.composer/config.json file: composer.install = To install the package using Composer, run the following command: diff --git a/routers/api/packages/helper/helper.go b/routers/api/packages/helper/helper.go index a5135e3c45548..8cde84023f099 100644 --- a/routers/api/packages/helper/helper.go +++ b/routers/api/packages/helper/helper.go @@ -25,7 +25,7 @@ func LogAndProcessError(ctx *context.Context, status int, obj interface{}, cb fu if status == http.StatusInternalServerError { log.ErrorWithSkip(1, message) - if setting.IsProd && !(ctx.Doer != nil && ctx.Doer.IsAdmin) { + if setting.IsProd && (ctx.Doer == nil || !ctx.Doer.IsAdmin) { message = "" } } else { diff --git a/templates/package/content/container.tmpl b/templates/package/content/container.tmpl index 9fc8fc35b21f6..14d4a56398544 100644 --- a/templates/package/content/container.tmpl +++ b/templates/package/content/container.tmpl @@ -7,11 +7,11 @@ {{if eq .PackageDescriptor.Metadata.Type "helm"}} 
    helm pull oci://{{.RegistryHost}}/{{.PackageDescriptor.Owner.LowerName}}/{{.PackageDescriptor.Package.LowerName}} --version {{.PackageDescriptor.Version.LowerVersion}}
    {{else}} - {{$seperator := ":"}} + {{$separator := ":"}} {{if not .PackageDescriptor.Metadata.IsTagged}} - {{$seperator = "@"}} + {{$separator = "@"}} {{end}} - 
    docker pull {{.RegistryHost}}/{{.PackageDescriptor.Owner.LowerName}}/{{.PackageDescriptor.Package.LowerName}}{{$seperator}}{{.PackageDescriptor.Version.LowerVersion}}
    + 
    docker pull {{.RegistryHost}}/{{.PackageDescriptor.Owner.LowerName}}/{{.PackageDescriptor.Package.LowerName}}{{$separator}}{{.PackageDescriptor.Version.LowerVersion}}
    {{end}}
    diff --git a/templates/package/shared/versionlist.tmpl b/templates/package/shared/versionlist.tmpl index c5589d8b2163a..e2aa19cc8cb66 100644 --- a/templates/package/shared/versionlist.tmpl +++ b/templates/package/shared/versionlist.tmpl @@ -5,8 +5,9 @@ {{if eq .PackageDescriptor.Package.Type "container"}} {{end}} From 1d4ce1abff13fa2920fbf71480e99558c796f958 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 30 Mar 2022 14:02:24 +0800 Subject: [PATCH 129/130] Apply suggestions from code review Co-authored-by: Thomas Boerger --- docs/content/doc/packages/composer.en-us.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/doc/packages/composer.en-us.md b/docs/content/doc/packages/composer.en-us.md index 4e02ed27ad737..2502ee45b509b 100644 --- a/docs/content/doc/packages/composer.en-us.md +++ b/docs/content/doc/packages/composer.en-us.md @@ -22,7 +22,7 @@ Publish [Composer](https://getcomposer.org/) packages for your user or organizat ## Requirements -To work with the Composer package registry, you can use [Composer](hhttps://getcomposer.org/download/) to consume and a HTTP upload client like `curl` to publish packages. +To work with the Composer package registry, you can use [Composer](https://getcomposer.org/download/) to consume and a HTTP upload client like `curl` to publish packages. ## Publish a package @@ -60,7 +60,7 @@ curl --user your_username:your_password_or_token \ https://gitea.example.com/api/packages/testuser/composer?version=1.0.3 ``` -The server reponds with the following HTTP Status codes. +The server responds with the following HTTP Status codes. | HTTP Status Code | Meaning | | ----------------- | ------- | From 84ff7b0d2bb04d9660e508c715ef9e10062187bc Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 30 Mar 2022 14:08:42 +0800 Subject: [PATCH 130/130] Update docs/content/doc/packages/nuget.en-us.md --- docs/content/doc/packages/nuget.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/packages/nuget.en-us.md b/docs/content/doc/packages/nuget.en-us.md index 88c7a857d64ec..5565bf5b895d6 100644 --- a/docs/content/doc/packages/nuget.en-us.md +++ b/docs/content/doc/packages/nuget.en-us.md @@ -83,7 +83,7 @@ https://gitea.example.com/api/packages/{owner}/nuget/symbols For example: -```shell +``` https://gitea.example.com/api/packages/testuser/nuget/symbols ```