aboutsummaryrefslogtreecommitdiffstats
path: root/modules
diff options
context:
space:
mode:
Diffstat (limited to 'modules')
-rw-r--r--modules/packages/debian/metadata.go216
-rw-r--r--modules/packages/debian/metadata_test.go171
-rw-r--r--modules/packages/hashed_buffer.go22
-rw-r--r--modules/packages/hashed_buffer_test.go2
-rw-r--r--modules/packages/nuget/symbol_extractor.go2
-rw-r--r--modules/setting/packages.go2
-rw-r--r--modules/util/filebuffer/file_backed_buffer.go5
7 files changed, 410 insertions, 10 deletions
diff --git a/modules/packages/debian/metadata.go b/modules/packages/debian/metadata.go
new file mode 100644
index 0000000000..08daaf082e
--- /dev/null
+++ b/modules/packages/debian/metadata.go
@@ -0,0 +1,216 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package debian
+
+import (
+ "archive/tar"
+ "bufio"
+ "compress/gzip"
+ "io"
+ "net/mail"
+ "regexp"
+ "strings"
+
+ "code.gitea.io/gitea/modules/util"
+ "code.gitea.io/gitea/modules/validation"
+
+ "github.com/blakesmith/ar"
+ "github.com/klauspost/compress/zstd"
+ "github.com/ulikunitz/xz"
+)
+
+const (
+ PropertyDistribution = "debian.distribution"
+ PropertyComponent = "debian.component"
+ PropertyArchitecture = "debian.architecture"
+ PropertyControl = "debian.control"
+ PropertyRepositoryIncludeInRelease = "debian.repository.include_in_release"
+
+ SettingKeyPrivate = "debian.key.private"
+ SettingKeyPublic = "debian.key.public"
+
+ RepositoryPackage = "_debian"
+ RepositoryVersion = "_repository"
+)
+
+var (
+ ErrMissingControlFile = util.NewInvalidArgumentErrorf("control file is missing")
+ ErrUnsupportedCompression = util.NewInvalidArgumentErrorf("unsupported compression algorithmn")
+ ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid")
+ ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid")
+ ErrInvalidArchitecture = util.NewInvalidArgumentErrorf("package architecture is invalid")
+
+ // https://www.debian.org/doc/debian-policy/ch-controlfields.html#source
+ namePattern = regexp.MustCompile(`\A[a-z0-9][a-z0-9+-.]+\z`)
+ // https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
+ versionPattern = regexp.MustCompile(`\A(?:[0-9]:)?[a-zA-Z0-9.+~]+(?:-[a-zA-Z0-9.+-~]+)?\z`)
+)
+
+type Package struct {
+ Name string
+ Version string
+ Architecture string
+ Control string
+ Metadata *Metadata
+}
+
+type Metadata struct {
+ Maintainer string `json:"maintainer,omitempty"`
+ ProjectURL string `json:"project_url,omitempty"`
+ Description string `json:"description,omitempty"`
+ Dependencies []string `json:"dependencies,omitempty"`
+}
+
+// ParsePackage parses the Debian package file
+// https://manpages.debian.org/bullseye/dpkg-dev/deb.5.en.html
+func ParsePackage(r io.Reader) (*Package, error) {
+ arr := ar.NewReader(r)
+
+ for {
+ hd, err := arr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ if strings.HasPrefix(hd.Name, "control.tar") {
+ var inner io.Reader
+ switch hd.Name[11:] {
+ case "":
+ inner = arr
+ case ".gz":
+ gzr, err := gzip.NewReader(arr)
+ if err != nil {
+ return nil, err
+ }
+ defer gzr.Close()
+
+ inner = gzr
+ case ".xz":
+ xzr, err := xz.NewReader(arr)
+ if err != nil {
+ return nil, err
+ }
+
+ inner = xzr
+ case ".zst":
+ zr, err := zstd.NewReader(arr)
+ if err != nil {
+ return nil, err
+ }
+ defer zr.Close()
+
+ inner = zr
+ default:
+ return nil, ErrUnsupportedCompression
+ }
+
+ tr := tar.NewReader(inner)
+ for {
+ hd, err := tr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ if hd.Typeflag != tar.TypeReg {
+ continue
+ }
+
+ if hd.FileInfo().Name() == "control" {
+ return ParseControlFile(tr)
+ }
+ }
+ }
+ }
+
+ return nil, ErrMissingControlFile
+}
+
+// ParseControlFile parses a Debian control file to retrieve the metadata
+func ParseControlFile(r io.Reader) (*Package, error) {
+ p := &Package{
+ Metadata: &Metadata{},
+ }
+
+ key := ""
+ var depends strings.Builder
+ var control strings.Builder
+
+ s := bufio.NewScanner(io.TeeReader(r, &control))
+ for s.Scan() {
+ line := s.Text()
+
+ trimmed := strings.TrimSpace(line)
+ if trimmed == "" {
+ continue
+ }
+
+ if line[0] == ' ' || line[0] == '\t' {
+ switch key {
+ case "Description":
+ p.Metadata.Description += line
+ case "Depends":
+ depends.WriteString(trimmed)
+ }
+ } else {
+ parts := strings.SplitN(trimmed, ":", 2)
+ if len(parts) < 2 {
+ continue
+ }
+
+ key = parts[0]
+ value := strings.TrimSpace(parts[1])
+ switch key {
+ case "Package":
+ if !namePattern.MatchString(value) {
+ return nil, ErrInvalidName
+ }
+ p.Name = value
+ case "Version":
+ if !versionPattern.MatchString(value) {
+ return nil, ErrInvalidVersion
+ }
+ p.Version = value
+ case "Architecture":
+ if value == "" {
+ return nil, ErrInvalidArchitecture
+ }
+ p.Architecture = value
+ case "Maintainer":
+ a, err := mail.ParseAddress(value)
+ if err != nil || a.Name == "" {
+ p.Metadata.Maintainer = value
+ } else {
+ p.Metadata.Maintainer = a.Name
+ }
+ case "Description":
+ p.Metadata.Description = value
+ case "Depends":
+ depends.WriteString(value)
+ case "Homepage":
+ if validation.IsValidURL(value) {
+ p.Metadata.ProjectURL = value
+ }
+ }
+ }
+ }
+ if err := s.Err(); err != nil {
+ return nil, err
+ }
+
+ dependencies := strings.Split(depends.String(), ",")
+ for i := range dependencies {
+ dependencies[i] = strings.TrimSpace(dependencies[i])
+ }
+ p.Metadata.Dependencies = dependencies
+
+ p.Control = control.String()
+
+ return p, nil
+}
diff --git a/modules/packages/debian/metadata_test.go b/modules/packages/debian/metadata_test.go
new file mode 100644
index 0000000000..69fd51ea79
--- /dev/null
+++ b/modules/packages/debian/metadata_test.go
@@ -0,0 +1,171 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package debian
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "io"
+ "testing"
+
+ "github.com/blakesmith/ar"
+ "github.com/klauspost/compress/zstd"
+ "github.com/stretchr/testify/assert"
+ "github.com/ulikunitz/xz"
+)
+
+const (
+ packageName = "gitea"
+ packageVersion = "0:1.0.1-te~st"
+ packageArchitecture = "amd64"
+ packageAuthor = "KN4CK3R"
+ description = "Description with multiple lines."
+ projectURL = "https://gitea.io"
+)
+
+func TestParsePackage(t *testing.T) {
+ createArchive := func(files map[string][]byte) io.Reader {
+ var buf bytes.Buffer
+ aw := ar.NewWriter(&buf)
+ aw.WriteGlobalHeader()
+ for filename, content := range files {
+ hdr := &ar.Header{
+ Name: filename,
+ Mode: 0o600,
+ Size: int64(len(content)),
+ }
+ aw.WriteHeader(hdr)
+ aw.Write(content)
+ }
+ return &buf
+ }
+
+ t.Run("MissingControlFile", func(t *testing.T) {
+ data := createArchive(map[string][]byte{"dummy.txt": {}})
+
+ p, err := ParsePackage(data)
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrMissingControlFile)
+ })
+
+ t.Run("Compression", func(t *testing.T) {
+ t.Run("Unsupported", func(t *testing.T) {
+ data := createArchive(map[string][]byte{"control.tar.foo": {}})
+
+ p, err := ParsePackage(data)
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrUnsupportedCompression)
+ })
+
+ var buf bytes.Buffer
+ tw := tar.NewWriter(&buf)
+ tw.WriteHeader(&tar.Header{
+ Name: "control",
+ Mode: 0o600,
+ Size: 50,
+ })
+ tw.Write([]byte("Package: gitea\nVersion: 1.0.0\nArchitecture: amd64\n"))
+ tw.Close()
+
+ t.Run("None", func(t *testing.T) {
+ data := createArchive(map[string][]byte{"control.tar": buf.Bytes()})
+
+ p, err := ParsePackage(data)
+ assert.NotNil(t, p)
+ assert.NoError(t, err)
+ assert.Equal(t, "gitea", p.Name)
+ })
+
+ t.Run("gz", func(t *testing.T) {
+ var zbuf bytes.Buffer
+ zw := gzip.NewWriter(&zbuf)
+ zw.Write(buf.Bytes())
+ zw.Close()
+
+ data := createArchive(map[string][]byte{"control.tar.gz": zbuf.Bytes()})
+
+ p, err := ParsePackage(data)
+ assert.NotNil(t, p)
+ assert.NoError(t, err)
+ assert.Equal(t, "gitea", p.Name)
+ })
+
+ t.Run("xz", func(t *testing.T) {
+ var xbuf bytes.Buffer
+ xw, _ := xz.NewWriter(&xbuf)
+ xw.Write(buf.Bytes())
+ xw.Close()
+
+ data := createArchive(map[string][]byte{"control.tar.xz": xbuf.Bytes()})
+
+ p, err := ParsePackage(data)
+ assert.NotNil(t, p)
+ assert.NoError(t, err)
+ assert.Equal(t, "gitea", p.Name)
+ })
+
+ t.Run("zst", func(t *testing.T) {
+ var zbuf bytes.Buffer
+ zw, _ := zstd.NewWriter(&zbuf)
+ zw.Write(buf.Bytes())
+ zw.Close()
+
+ data := createArchive(map[string][]byte{"control.tar.zst": zbuf.Bytes()})
+
+ p, err := ParsePackage(data)
+ assert.NotNil(t, p)
+ assert.NoError(t, err)
+ assert.Equal(t, "gitea", p.Name)
+ })
+ })
+}
+
+func TestParseControlFile(t *testing.T) {
+ buildContent := func(name, version, architecture string) *bytes.Buffer {
+ var buf bytes.Buffer
+ buf.WriteString("Package: " + name + "\nVersion: " + version + "\nArchitecture: " + architecture + "\nMaintainer: " + packageAuthor + " <kn4ck3r@gitea.io>\nHomepage: " + projectURL + "\nDepends: a,\n b\nDescription: Description\n with multiple\n lines.")
+ return &buf
+ }
+
+ t.Run("InvalidName", func(t *testing.T) {
+ for _, name := range []string{"", "-cd"} {
+ p, err := ParseControlFile(buildContent(name, packageVersion, packageArchitecture))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidName)
+ }
+ })
+
+ t.Run("InvalidVersion", func(t *testing.T) {
+ for _, version := range []string{"", "1-", ":1.0", "1_0"} {
+ p, err := ParseControlFile(buildContent(packageName, version, packageArchitecture))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidVersion)
+ }
+ })
+
+ t.Run("InvalidArchitecture", func(t *testing.T) {
+ p, err := ParseControlFile(buildContent(packageName, packageVersion, ""))
+ assert.Nil(t, p)
+ assert.ErrorIs(t, err, ErrInvalidArchitecture)
+ })
+
+ t.Run("Valid", func(t *testing.T) {
+ content := buildContent(packageName, packageVersion, packageArchitecture)
+ full := content.String()
+
+ p, err := ParseControlFile(content)
+ assert.NoError(t, err)
+ assert.NotNil(t, p)
+
+ assert.Equal(t, packageName, p.Name)
+ assert.Equal(t, packageVersion, p.Version)
+ assert.Equal(t, packageArchitecture, p.Architecture)
+ assert.Equal(t, description, p.Metadata.Description)
+ assert.Equal(t, projectURL, p.Metadata.ProjectURL)
+ assert.Equal(t, packageAuthor, p.Metadata.Maintainer)
+ assert.Equal(t, []string{"a", "b"}, p.Metadata.Dependencies)
+ assert.Equal(t, full, p.Control)
+ })
+}
diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go
index ef00a45057..017ddf1c8f 100644
--- a/modules/packages/hashed_buffer.go
+++ b/modules/packages/hashed_buffer.go
@@ -25,8 +25,15 @@ type HashedBuffer struct {
combinedWriter io.Writer
}
-// NewHashedBuffer creates a hashed buffer with a specific maximum memory size
-func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) {
+const DefaultMemorySize = 32 * 1024 * 1024
+
+// NewHashedBuffer creates a hashed buffer with the default memory size
+func NewHashedBuffer() (*HashedBuffer, error) {
+ return NewHashedBufferWithSize(DefaultMemorySize)
+}
+
+// NewHashedBuffer creates a hashed buffer with a specific memory size
+func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) {
b, err := filebuffer.New(maxMemorySize)
if err != nil {
return nil, err
@@ -43,9 +50,14 @@ func NewHashedBuffer(maxMemorySize int) (*HashedBuffer, error) {
}, nil
}
-// CreateHashedBufferFromReader creates a hashed buffer and copies the provided reader data into it.
-func CreateHashedBufferFromReader(r io.Reader, maxMemorySize int) (*HashedBuffer, error) {
- b, err := NewHashedBuffer(maxMemorySize)
+// CreateHashedBufferFromReader creates a hashed buffer with the default memory size and copies the provided reader data into it.
+func CreateHashedBufferFromReader(r io.Reader) (*HashedBuffer, error) {
+ return CreateHashedBufferFromReaderWithSize(r, DefaultMemorySize)
+}
+
+// CreateHashedBufferFromReaderWithSize creates a hashed buffer and copies the provided reader data into it.
+func CreateHashedBufferFromReaderWithSize(r io.Reader, maxMemorySize int) (*HashedBuffer, error) {
+ b, err := NewHashedBufferWithSize(maxMemorySize)
if err != nil {
return nil, err
}
diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go
index e907aa0605..564e782f18 100644
--- a/modules/packages/hashed_buffer_test.go
+++ b/modules/packages/hashed_buffer_test.go
@@ -26,7 +26,7 @@ func TestHashedBuffer(t *testing.T) {
}
for _, c := range cases {
- buf, err := CreateHashedBufferFromReader(strings.NewReader(c.Data), c.MaxMemorySize)
+ buf, err := CreateHashedBufferFromReaderWithSize(strings.NewReader(c.Data), c.MaxMemorySize)
assert.NoError(t, err)
assert.EqualValues(t, len(c.Data), buf.Size())
diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go
index b709eac4c1..81bf0371a0 100644
--- a/modules/packages/nuget/symbol_extractor.go
+++ b/modules/packages/nuget/symbol_extractor.go
@@ -63,7 +63,7 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) {
return err
}
- buf, err := packages.CreateHashedBufferFromReader(f, 32*1024*1024)
+ buf, err := packages.CreateHashedBufferFromReader(f)
f.Close()
diff --git a/modules/setting/packages.go b/modules/setting/packages.go
index 89601c3b99..b52bbf40c7 100644
--- a/modules/setting/packages.go
+++ b/modules/setting/packages.go
@@ -30,6 +30,7 @@ var (
LimitSizeConan int64
LimitSizeConda int64
LimitSizeContainer int64
+ LimitSizeDebian int64
LimitSizeGeneric int64
LimitSizeHelm int64
LimitSizeMaven int64
@@ -73,6 +74,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) {
Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA")
Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
+ Packages.LimitSizeDebian = mustBytes(sec, "LIMIT_SIZE_DEBIAN")
Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM")
Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN")
diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go
index bfddf90e92..6b07bd0413 100644
--- a/modules/util/filebuffer/file_backed_buffer.go
+++ b/modules/util/filebuffer/file_backed_buffer.go
@@ -7,11 +7,10 @@ import (
"bytes"
"errors"
"io"
+ "math"
"os"
)
-const maxInt = int(^uint(0) >> 1) // taken from bytes.Buffer
-
var (
// ErrInvalidMemorySize occurs if the memory size is not in a valid range
ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32")
@@ -37,7 +36,7 @@ type FileBackedBuffer struct {
// New creates a file backed buffer with a specific maximum memory size
func New(maxMemorySize int) (*FileBackedBuffer, error) {
- if maxMemorySize < 0 || maxMemorySize > maxInt {
+ if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 {
return nil, ErrInvalidMemorySize
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-26 17:03:16 +0000