Add Debian package registry (#22854)

Co-authored-by: @awkwardbunny

This PR adds a Debian package registry. You can follow [this
tutorial](https://www.baeldung.com/linux/create-debian-package) to build
a *.deb package for testing. Source packages are not supported at the
moment and I did not find documentation of the architecture "all" and
how these packages should be treated.

---------

Co-authored-by: Brian Hong <[email protected]>
Co-authored-by: techknowlogick <[email protected]>

Author: 3 people

assets/go-licenses.json

@@ -25,7 +25,7 @@ func TestMigratePackages(t *testing.T) {
 	creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
 
 	content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n"
-	buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)
+	buf, err := packages_module.CreateHashedBufferFromReaderWithSize(strings.NewReader(content), 1024)
 	assert.NoError(t, err)
 	defer buf.Close()
 

cmd/migrate_storage_test.go

@@ -2501,6 +2501,8 @@ ROUTER = console
 ;LIMIT_SIZE_CONDA = -1
 ;; Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 ;LIMIT_SIZE_CONTAINER = -1
+;; Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_DEBIAN = -1
 ;; Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 ;LIMIT_SIZE_GENERIC = -1
 ;; Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

custom/conf/app.example.ini

@@ -1252,6 +1252,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 - `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_CONTAINER`: **-1**: Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_DEBIAN`: **-1**: Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_GENERIC`: **-1**: Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_HELM`: **-1**: Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_MAVEN`: **-1**: Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

docs/content/doc/administration/config-cheat-sheet.en-us.md

@@ -0,0 +1,134 @@
+---
+date: "2023-01-07T00:00:00+00:00"
+title: "Debian Packages Repository"
+slug: "packages/debian"
+draft: false
+toc: false
+menu:
+  sidebar:
+    parent: "packages"
+    name: "Debian"
+    weight: 35
+    identifier: "debian"
+---
+
+# Debian Packages Repository
+
+Publish [Debian](https://www.debian.org/distrib/packages) packages for your user or organization.
+
+**Table of Contents**
+
+{{< toc >}}
+
+## Requirements
+
+To work with the Debian registry, you need to use a HTTP client like `curl` to upload and a package manager like `apt` to consume packages.
+
+The following examples use `apt`.
+
+## Configuring the package registry
+
+To register the Debian registry add the url to the list of known apt sources:
+
+```shell
+echo "deb https://gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list
+```
+
+| Placeholder    | Description |
+| -------------- | ----------- |
+| `owner`        | The owner of the package. |
+| `distribution` | The distribution to use. |
+| `component`    | The component to use. |
+
+If the registry is private, provide credentials in the url. You can use a password or a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}):
+
+```shell
+echo "deb https://{username}:{your_password_or_token}@gitea.example.com/api/packages/{owner}/debian {distribution} {component}" | sudo tee -a /etc/apt/sources.list.d/gitea.list
+```
+
+The Debian registry files are signed with a PGP key which must be known to apt:
+
+```shell
+sudo curl https://gitea.example.com/api/packages/{owner}/debian/repository.key -o /etc/apt/trusted.gpg.d/gitea-{owner}.asc
+```
+
+Afterwards update the local package index:
+
+```shell
+apt update
+```
+
+## Publish a package
+
+To publish a Debian package (`*.deb`), perform a HTTP PUT operation with the package content in the request body.
+
+```
+PUT https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/upload
+```
+
+| Parameter      | Description |
+| -------------- | ----------- |
+| `owner`        | The owner of the package. |
+| `distribution` | The distribution may match the release name of the OS, ex: `bionic`. |
+| `component`    | The component can be used to group packages or just `main` or similar. |
+
+Example request using HTTP Basic authentication:
+
+```shell
+curl --user your_username:your_password_or_token \
+     --upload-file path/to/file.deb \
+     https://gitea.example.com/api/packages/testuser/debian/pool/bionic/main/upload
+```
+
+If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password.
+You cannot publish a file with the same name twice to a package. You must delete the existing package version first.
+
+The server reponds with the following HTTP Status codes.
+
+| HTTP Status Code  | Meaning |
+| ----------------- | ------- |
+| `201 Created`     | The package has been published. |
+| `400 Bad Request` | The package name, version, distribution, component or architecture are invalid. |
+| `409 Conflict`    | A package file with the same combination of parameters exist already in the package. |
+
+## Delete a package
+
+To delete a Debian package perform a HTTP DELETE operation. This will delete the package version too if there is no file left.
+
+```
+DELETE https://gitea.example.com/api/packages/{owner}/debian/pool/{distribution}/{component}/{package_name}/{package_version}/{architecture}
+```
+
+| Parameter         | Description |
+| ----------------- | ----------- |
+| `owner`           | The owner of the package. |
+| `package_name`    | The package name. |
+| `package_version` | The package version. |
+| `distribution`    | The package distribution. |
+| `component`       | The package component. |
+| `architecture`    | The package architecture. |
+
+Example request using HTTP Basic authentication:
+
+```shell
+curl --user your_username:your_token_or_password -X DELETE \
+     https://gitea.example.com/api/packages/testuser/debian/pools/bionic/main/test-package/1.0.0/amd64
+```
+
+The server reponds with the following HTTP Status codes.
+
+| HTTP Status Code  | Meaning |
+| ----------------- | ------- |
+| `204 No Content`  | Success |
+| `404 Not Found`   | The package or file was not found. |
+
+## Install a package
+
+To install a package from the Debian registry, execute the following commands:
+
+```shell
+# use latest version
+apt install {package_name}
+# use specific version
+apt install {package_name}={package_version}
+```

docs/content/doc/usage/packages/debian.en-us.md

@@ -33,6 +33,7 @@ The following package managers are currently supported:
 | [Conan]({{< relref "doc/usage/packages/conan.en-us.md" >}}) | C++ | `conan` |
 | [Conda]({{< relref "doc/usage/packages/conda.en-us.md" >}}) | - | `conda` |
 | [Container]({{< relref "doc/usage/packages/container.en-us.md" >}}) | - | any OCI compliant client |
+| [Debian]({{< relref "doc/usage/packages/debian.en-us.md" >}}) | - | `apt` |
 | [Generic]({{< relref "doc/usage/packages/generic.en-us.md" >}}) | - | any HTTP client |
 | [Helm]({{< relref "doc/usage/packages/helm.en-us.md" >}}) | - | any HTTP client, `cm-push` |
 | [Maven]({{< relref "doc/usage/packages/maven.en-us.md" >}}) | Java | `mvn`, `gradle` |

docs/content/doc/usage/packages/overview.en-us.md

@@ -17,6 +17,7 @@ require (
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/PuerkitoBio/goquery v1.8.0
 	github.com/alecthomas/chroma/v2 v2.5.0
+	github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
 	github.com/blevesearch/bleve/v2 v2.3.6
 	github.com/bufbuild/connect-go v1.3.1
 	github.com/buildkite/terminal-to-html/v3 v3.7.0
@@ -96,6 +97,7 @@ require (
 	github.com/stretchr/testify v1.8.1
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tstranex/u2f v1.0.0
+	github.com/ulikunitz/xz v0.5.11
 	github.com/urfave/cli v1.22.12
 	github.com/xanzy/go-gitlab v0.80.2
 	github.com/xeipuuv/gojsonschema v1.2.0
@@ -260,7 +262,6 @@ require (
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/toqueteos/webbrowser v1.2.0 // indirect
-	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/unknwon/com v1.0.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.44.0 // indirect

go.mod

@@ -162,6 +162,8 @@ github.com/bits-and-blooms/bitset v1.1.10/go.mod h1:w0XsmFg8qg6cmpTtJ0z3pKgjTDBM
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8=
 github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
+github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
 github.com/blevesearch/bleve/v2 v2.0.5/go.mod h1:ZjWibgnbRX33c+vBRgla9QhPb4QOjD6fdVJ+R1Bk8LM=
 github.com/blevesearch/bleve/v2 v2.3.6 h1:NlntUHcV5CSWIhpugx4d/BRMGCiaoI8ZZXrXlahzNq4=
 github.com/blevesearch/bleve/v2 v2.3.6/go.mod h1:JM2legf1cKVkdV8Ehu7msKIOKC0McSw0Q16Fmv9vsW4=

go.sum

@@ -489,6 +489,8 @@ var migrations = []Migration{
 	NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable),
 	// v255 -> v256
 	NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository),
+	// v256 -> v257
+	NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/migrations.go

@@ -0,0 +1,23 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_20 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+func AddIsInternalColumnToPackage(x *xorm.Engine) error {
+	type Package struct {
+		ID               int64  `xorm:"pk autoincr"`
+		OwnerID          int64  `xorm:"UNIQUE(s) INDEX NOT NULL"`
+		RepoID           int64  `xorm:"INDEX"`
+		Type             string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+		Name             string `xorm:"NOT NULL"`
+		LowerName        string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+		SemverCompatible bool   `xorm:"NOT NULL DEFAULT false"`
+		IsInternal       bool   `xorm:"INDEX NOT NULL DEFAULT false"`
+	}
+
+	return x.Sync2(new(Package))
+}