Add Go package registry (#24687)

Fixes #7608

This PR adds a Go package registry usable with the Go proxy protocol.

![grafik](https://github.com/go-gitea/gitea/assets/1666336/328feb5c-3df2-4f9d-8eae-fe3126d14c37)

Author: KN4CK3R

custom/conf/app.example.ini

@@ -2463,6 +2463,8 @@ ROUTER = console
 ;LIMIT_SIZE_DEBIAN = -1
 ;; Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 ;LIMIT_SIZE_GENERIC = -1
+;; Maximum size of a Go upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+;LIMIT_SIZE_GO = -1
 ;; Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 ;LIMIT_SIZE_HELM = -1
 ;; Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

docs/content/doc/administration/config-cheat-sheet.en-us.md

@@ -1223,6 +1223,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 - `LIMIT_SIZE_CONTAINER`: **-1**: Maximum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_DEBIAN`: **-1**: Maximum size of a Debian upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_GENERIC`: **-1**: Maximum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
+- `LIMIT_SIZE_GO`: **-1**: Maximum size of a Go upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_HELM`: **-1**: Maximum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_MAVEN`: **-1**: Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
 - `LIMIT_SIZE_NPM`: **-1**: Maximum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

docs/content/doc/usage/packages/debian.en-us.md

@@ -83,7 +83,7 @@ curl --user your_username:your_password_or_token \
 If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password.
 You cannot publish a file with the same name twice to a package. You must delete the existing package version first.
 
-The server reponds with the following HTTP Status codes.
+The server responds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |
 | ----------------- | ------- |
@@ -115,7 +115,7 @@ curl --user your_username:your_token_or_password -X DELETE \
      https://gitea.example.com/api/packages/testuser/debian/pools/bionic/main/test-package/1.0.0/amd64
 ```
 
-The server reponds with the following HTTP Status codes.
+The server responds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |
 | ----------------- | ------- |

docs/content/doc/usage/packages/generic.en-us.md

@@ -51,7 +51,7 @@ curl --user your_username:your_password_or_token \
 
 If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password.
 
-The server reponds with the following HTTP Status codes.
+The server responds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |
 | ----------------- | ------- |
@@ -83,7 +83,7 @@ curl --user your_username:your_token_or_password \
      https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0/file.bin
 ```
 
-The server reponds with the following HTTP Status codes.
+The server responds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |
 | ----------------- | ------- |
@@ -111,7 +111,7 @@ curl --user your_username:your_token_or_password -X DELETE \
      https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0
 ```
 
-The server reponds with the following HTTP Status codes.
+The server responds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |
 | ----------------- | ------- |
@@ -140,7 +140,7 @@ curl --user your_username:your_token_or_password -X DELETE \
      https://gitea.example.com/api/packages/testuser/generic/test_package/1.0.0/file.bin
 ```
 
-The server reponds with the following HTTP Status codes.
+The server responds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |
 | ----------------- | ------- |

docs/content/doc/usage/packages/go.en-us.md

@@ -0,0 +1,77 @@
+---
+date: "2023-05-10T00:00:00+00:00"
+title: "Go Packages Repository"
+slug: "go"
+weight: 45
+draft: false
+toc: false
+menu:
+  sidebar:
+    parent: "packages"
+    name: "Go"
+    weight: 45
+    identifier: "go"
+---
+
+# Go Packages Repository
+
+Publish Go packages for your user or organization.
+
+**Table of Contents**
+
+{{< toc >}}
+
+## Publish a package
+
+To publish a Go package perform a HTTP `PUT` operation with the package content in the request body.
+You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first.
+The package must follow the [documented structure](https://go.dev/ref/mod#zip-files).
+
+```
+PUT https://gitea.example.com/api/packages/{owner}/go/upload
+```
+
+| Parameter | Description |
+| --------- | ----------- |
+| `owner`   | The owner of the package. |
+
+To authenticate to the package registry, you need to provide [custom HTTP headers or use HTTP Basic authentication]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}):
+
+```shell
+curl --user your_username:your_password_or_token \
+     --upload-file path/to/file.zip \
+     https://gitea.example.com/api/packages/testuser/go/upload
+```
+
+If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password.
+
+The server responds with the following HTTP Status codes.
+
+| HTTP Status Code  | Meaning |
+| ----------------- | ------- |
+| `201 Created`     | The package has been published. |
+| `400 Bad Request` | The package is invalid. |
+| `409 Conflict`    | A package with the same name exist already. |
+
+## Install a package
+
+To install a Go package instruct Go to use the package registry as proxy:
+
+```shell
+# use latest version
+GOPROXY=https://gitea.example.com/api/packages/{owner}/go go install {package_name}
+# or
+GOPROXY=https://gitea.example.com/api/packages/{owner}/go go install {package_name}@latest
+# use specific version
+GOPROXY=https://gitea.example.com/api/packages/{owner}/go go install {package_name}@{package_version}
+```
+
+| Parameter         | Description |
+| ----------------- | ----------- |
+| `owner`           | The owner of the package. |
+| `package_name`    | The package name. |
+| `package_version` | The package version. |
+
+If the owner of the packages is private you need to [provide credentials](https://go.dev/ref/mod#private-module-proxy-auth).
+
+More information about the `GOPROXY` environment variable and how to protect against data leaks can be found in [the documentation](https://go.dev/ref/mod#private-modules).

docs/content/doc/usage/packages/rpm.en-us.md

@@ -69,7 +69,7 @@ curl --user your_username:your_password_or_token \
 If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password.
 You cannot publish a file with the same name twice to a package. You must delete the existing package version first.
 
-The server reponds with the following HTTP Status codes.
+The server responds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |
 | ----------------- | ------- |
@@ -99,7 +99,7 @@ curl --user your_username:your_token_or_password -X DELETE \
      https://gitea.example.com/api/packages/testuser/rpm/test-package/1.0.0/x86_64
 ```
 
-The server reponds with the following HTTP Status codes.
+The server responds with the following HTTP Status codes.
 
 | HTTP Status Code  | Meaning |
 | ----------------- | ------- |

models/packages/descriptor.go

@@ -155,6 +155,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
 		metadata = &debian.Metadata{}
 	case TypeGeneric:
 		// generic packages have no metadata
+	case TypeGo:
+		// go packages have no metadata
 	case TypeHelm:
 		metadata = &helm.Metadata{}
 	case TypeNuGet:

models/packages/package.go

@@ -39,6 +39,7 @@ const (
 	TypeContainer Type = "container"
 	TypeDebian    Type = "debian"
 	TypeGeneric   Type = "generic"
+	TypeGo        Type = "go"
 	TypeHelm      Type = "helm"
 	TypeMaven     Type = "maven"
 	TypeNpm       Type = "npm"
@@ -61,6 +62,7 @@ var TypeList = []Type{
 	TypeContainer,
 	TypeDebian,
 	TypeGeneric,
+	TypeGo,
 	TypeHelm,
 	TypeMaven,
 	TypeNpm,
@@ -94,6 +96,8 @@ func (pt Type) Name() string {
 		return "Debian"
 	case TypeGeneric:
 		return "Generic"
+	case TypeGo:
+		return "Go"
 	case TypeHelm:
 		return "Helm"
 	case TypeMaven:
@@ -139,6 +143,8 @@ func (pt Type) SVGName() string {
 		return "gitea-debian"
 	case TypeGeneric:
 		return "octicon-package"
+	case TypeGo:
+		return "gitea-go"
 	case TypeHelm:
 		return "gitea-helm"
 	case TypeMaven:

modules/packages/goproxy/metadata.go

@@ -0,0 +1,94 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package goproxy
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+const (
+	PropertyGoMod = "go.mod"
+
+	maxGoModFileSize = 16 * 1024 * 1024 // https://go.dev/ref/mod#zip-path-size-constraints
+)
+
+var (
+	ErrInvalidStructure  = util.NewInvalidArgumentErrorf("package has invalid structure")
+	ErrGoModFileTooLarge = util.NewInvalidArgumentErrorf("go.mod file is too large")
+)
+
+type Package struct {
+	Name    string
+	Version string
+	GoMod   string
+}
+
+// ParsePackage parses the Go package file
+// https://go.dev/ref/mod#zip-files
+func ParsePackage(r io.ReaderAt, size int64) (*Package, error) {
+	archive, err := zip.NewReader(r, size)
+	if err != nil {
+		return nil, err
+	}
+
+	var p *Package
+
+	for _, file := range archive.File {
+		nameAndVersion := path.Dir(file.Name)
+
+		parts := strings.SplitN(nameAndVersion, "@", 2)
+		if len(parts) != 2 {
+			continue
+		}
+
+		versionParts := strings.SplitN(parts[1], "/", 2)
+
+		if p == nil {
+			p = &Package{
+				Name:    strings.TrimSuffix(nameAndVersion, "@"+parts[1]),
+				Version: versionParts[0],
+			}
+		}
+
+		if len(versionParts) > 1 {
+			// files are expected in the "root" folder
+			continue
+		}
+
+		if path.Base(file.Name) == "go.mod" {
+			if file.UncompressedSize64 > maxGoModFileSize {
+				return nil, ErrGoModFileTooLarge
+			}
+
+			f, err := archive.Open(file.Name)
+			if err != nil {
+				return nil, err
+			}
+			defer f.Close()
+
+			bytes, err := io.ReadAll(&io.LimitedReader{R: f, N: maxGoModFileSize})
+			if err != nil {
+				return nil, err
+			}
+
+			p.GoMod = string(bytes)
+
+			return p, nil
+		}
+	}
+
+	if p == nil {
+		return nil, ErrInvalidStructure
+	}
+
+	p.GoMod = fmt.Sprintf("module %s", p.Name)
+
+	return p, nil
+}

modules/packages/goproxy/metadata_test.go

@@ -0,0 +1,75 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package goproxy
+
+import (
+	"archive/zip"
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	packageName    = "gitea.com/go-gitea/gitea"
+	packageVersion = "v0.0.1"
+)
+
+func TestParsePackage(t *testing.T) {
+	createArchive := func(files map[string][]byte) *bytes.Reader {
+		var buf bytes.Buffer
+		zw := zip.NewWriter(&buf)
+		for name, content := range files {
+			w, _ := zw.Create(name)
+			w.Write(content)
+		}
+		zw.Close()
+		return bytes.NewReader(buf.Bytes())
+	}
+
+	t.Run("EmptyPackage", func(t *testing.T) {
+		data := createArchive(nil)
+
+		p, err := ParsePackage(data, int64(data.Len()))
+		assert.Nil(t, p)
+		assert.ErrorIs(t, err, ErrInvalidStructure)
+	})
+
+	t.Run("InvalidNameOrVersionStructure", func(t *testing.T) {
+		data := createArchive(map[string][]byte{
+			packageName + "/" + packageVersion + "/go.mod": {},
+		})
+
+		p, err := ParsePackage(data, int64(data.Len()))
+		assert.Nil(t, p)
+		assert.ErrorIs(t, err, ErrInvalidStructure)
+	})
+
+	t.Run("GoModFileInWrongDirectory", func(t *testing.T) {
+		data := createArchive(map[string][]byte{
+			packageName + "@" + packageVersion + "/subdir/go.mod": {},
+		})
+
+		p, err := ParsePackage(data, int64(data.Len()))
+		assert.NotNil(t, p)
+		assert.NoError(t, err)
+		assert.Equal(t, packageName, p.Name)
+		assert.Equal(t, packageVersion, p.Version)
+		assert.Equal(t, "module gitea.com/go-gitea/gitea", p.GoMod)
+	})
+
+	t.Run("Valid", func(t *testing.T) {
+		data := createArchive(map[string][]byte{
+			packageName + "@" + packageVersion + "/subdir/go.mod": []byte("invalid"),
+			packageName + "@" + packageVersion + "/go.mod":        []byte("valid"),
+		})
+
+		p, err := ParsePackage(data, int64(data.Len()))
+		assert.NotNil(t, p)
+		assert.NoError(t, err)
+		assert.Equal(t, packageName, p.Name)
+		assert.Equal(t, packageVersion, p.Version)
+		assert.Equal(t, "valid", p.GoMod)
+	})
+}

modules/setting/packages.go

@@ -33,6 +33,7 @@ var (
 		LimitSizeContainer   int64
 		LimitSizeDebian      int64
 		LimitSizeGeneric     int64
+		LimitSizeGo          int64
 		LimitSizeHelm        int64
 		LimitSizeMaven       int64
 		LimitSizeNpm         int64
@@ -79,6 +80,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) {
 	Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
 	Packages.LimitSizeDebian = mustBytes(sec, "LIMIT_SIZE_DEBIAN")
 	Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
+	Packages.LimitSizeGo = mustBytes(sec, "LIMIT_SIZE_GO")
 	Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM")
 	Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN")
 	Packages.LimitSizeNpm = mustBytes(sec, "LIMIT_SIZE_NPM")