mirrors
/
gitea
mirror of https://github.com/go-gitea/gitea.git
Watch 1
Star 0
Fork 0
Code Issues 0 Releases 210 Wiki Activity
Browse Source

Fix all possible setting error related storages and added some tests (#23911) (#25244) 

Backport #23911 by @lunny

Follow up #22405

Fix #20703 

This PR rewrites storage configuration read sequences with some breaks
and tests. It becomes more strict than before and also fixed some
inherit problems.

- Move storage's MinioConfig struct into setting, so after the
configuration loading, the values will be stored into the struct but not
still on some section.
- All storages configurations should be stored on one section,
configuration items cannot be overrided by multiple sections. The
prioioty of configuration is `[attachment]` > `[storage.attachments]` |
`[storage.customized]` > `[storage]` > `default`
- For extra override configuration items, currently are `SERVE_DIRECT`,
`MINIO_BASE_PATH`, `MINIO_BUCKET`, which could be configured in another
section. The prioioty of the override configuration is `[attachment]` >
`[storage.attachments]` > `default`.
- Add more tests for storages configurations.
- Update the storage documentations.

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
tags/v1.20.0-rc1
Giteabot 11 months ago
parent
22948048b2
commit
21cd5c2f3d
No account linked to committer's email address
41 changed files with 1154 additions and 454 deletions
Split View Show Diff Stats
  1. 3
    3
      cmd/dump.go
  2. 15
    13
      cmd/migrate_storage.go
  3. 2
    1
      cmd/migrate_storage_test.go
  4. 21
    0
      custom/conf/app.example.ini
  5. 57
    4
      docs/content/doc/administration/config-cheat-sheet.en-us.md
  6. 53
    4
      docs/content/doc/administration/config-cheat-sheet.zh-cn.md
  7. 1
    1
      models/migrations/v1_10/v96.go
  8. 1
    1
      models/migrations/v1_11/v112.go
  9. 4
    4
      models/migrations/v1_11/v115.go
  10. 17
    10
      modules/setting/actions.go
  11. 97
    0
      modules/setting/actions_test.go
  12. 12
    10
      modules/setting/attachment.go
  13. 133
    0
      modules/setting/attachment_test.go
  14. 19
    0
      modules/setting/config_provider.go
  15. 30
    26
      modules/setting/lfs.go
  16. 77
    0
      modules/setting/lfs_test.go
  17. 17
    9
      modules/setting/packages.go
  18. 167
    0
      modules/setting/packages_test.go
  19. 19
    11
      modules/setting/picture.go
  20. 3
    5
      modules/setting/repository.go
  21. 25
    0
      modules/setting/repository_archive.go
  22. 111
    0
      modules/setting/repository_archive_test.go
  23. 25
    9
      modules/setting/setting.go
  24. 152
    57
      modules/setting/storage.go
  25. 42
    154
      modules/setting/storage_test.go
  26. 0
    56
      modules/storage/helper.go
  27. 3
    17
      modules/storage/local.go
  28. 3
    1
      modules/storage/local_test.go
  29. 5
    25
      modules/storage/minio.go
  30. 10
    6
      modules/storage/minio_test.go
  31. 14
    15
      modules/storage/storage.go
  32. 3
    1
      modules/storage/storage_test.go
  33. 2
    2
      routers/api/v1/repo/file.go
  34. 1
    1
      routers/install/install.go
  35. 2
    2
      routers/web/base.go
  36. 1
    1
      routers/web/repo/attachment.go
  37. 1
    1
      routers/web/repo/download.go
  38. 1
    1
      routers/web/repo/repo.go
  39. 1
    1
      services/lfs/server.go
  40. 1
    1
      templates/admin/config.tmpl
  41. 3
    1
      tests/test_utils.go

+ 3
- 3
cmd/dump.go View File

@@ -353,9 +353,9 @@ func runDump(ctx *cli.Context) error {
}

excludes = append(excludes, setting.RepoRootPath)
excludes = append(excludes, setting.LFS.Path)
excludes = append(excludes, setting.Attachment.Path)
excludes = append(excludes, setting.Packages.Path)
excludes = append(excludes, setting.LFS.Storage.Path)
excludes = append(excludes, setting.Attachment.Storage.Path)
excludes = append(excludes, setting.Packages.Storage.Path)
excludes = append(excludes, setting.Log.RootPath)
excludes = append(excludes, absFileName)
if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {

+ 15
- 13
cmd/migrate_storage.go View File

@@ -179,7 +179,7 @@ func runMigrateStorage(ctx *cli.Context) error {
switch strings.ToLower(ctx.String("storage")) {
case "":
fallthrough
case string(storage.LocalStorageType):
case string(setting.LocalStorageType):
p := ctx.String("path")
if p == "" {
log.Fatal("Path must be given when storage is loal")
@@ -187,22 +187,24 @@ func runMigrateStorage(ctx *cli.Context) error {
}
dstStorage, err = storage.NewLocalStorage(
stdCtx,
storage.LocalStorageConfig{
&setting.Storage{
Path: p,
})
case string(storage.MinioStorageType):
case string(setting.MinioStorageType):
dstStorage, err = storage.NewMinioStorage(
stdCtx,
storage.MinioStorageConfig{
Endpoint: ctx.String("minio-endpoint"),
AccessKeyID: ctx.String("minio-access-key-id"),
SecretAccessKey: ctx.String("minio-secret-access-key"),
Bucket: ctx.String("minio-bucket"),
Location: ctx.String("minio-location"),
BasePath: ctx.String("minio-base-path"),
UseSSL: ctx.Bool("minio-use-ssl"),
InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"),
ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"),
&setting.Storage{
MinioConfig: setting.MinioStorageConfig{
Endpoint: ctx.String("minio-endpoint"),
AccessKeyID: ctx.String("minio-access-key-id"),
SecretAccessKey: ctx.String("minio-secret-access-key"),
Bucket: ctx.String("minio-bucket"),
Location: ctx.String("minio-location"),
BasePath: ctx.String("minio-base-path"),
UseSSL: ctx.Bool("minio-use-ssl"),
InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"),
ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"),
},
})
default:
return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))

+ 2
- 1
cmd/migrate_storage_test.go View File

@@ -13,6 +13,7 @@ import (
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
packages_service "code.gitea.io/gitea/services/packages"

@@ -57,7 +58,7 @@ func TestMigratePackages(t *testing.T) {

dstStorage, err := storage.NewLocalStorage(
ctx,
storage.LocalStorageConfig{
&setting.Storage{
Path: p,
})
assert.NoError(t, err)

+ 21
- 0
custom/conf/app.example.ini View File

@@ -2392,6 +2392,10 @@ LEVEL = Info
;; Enable/Disable package registry capabilities
;ENABLED = true
;;
;STORAGE_TYPE = local
;; override the minio base path if storage type is minio
;MINIO_BASE_PATH = packages/
;;
;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload`
;CHUNKED_UPLOAD_PATH = tmp/package-upload
;;
@@ -2452,6 +2456,19 @@ LEVEL = Info
;; storage type
;STORAGE_TYPE = local

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; repo-archive storage will override storage
;;
;[repo-archive]
;STORAGE_TYPE = local
;;
;; Where your lfs files reside, default is data/lfs.
;PATH = data/repo-archive
;;
;; override the minio base path if storage type is minio
;MINIO_BASE_PATH = repo-archive/

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; settings for repository archives, will override storage setting
@@ -2471,6 +2488,9 @@ LEVEL = Info
;;
;; Where your lfs files reside, default is data/lfs.
;PATH = data/lfs
;;
;; override the minio base path if storage type is minio
;MINIO_BASE_PATH = lfs/

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2520,6 +2540,7 @@ LEVEL = Info
; [actions]
;; Enable/Disable actions capabilities
;ENABLED = false
;;
;; Default address to get action plugins, e.g. the default value means downloading from "https://gitea.com/actions/checkout" for "uses: actions/checkout@v3"
;DEFAULT_ACTIONS_URL = https://gitea.com


+ 57
- 4
docs/content/doc/administration/config-cheat-sheet.en-us.md View File

@@ -1254,8 +1254,9 @@ is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`.

## Storage (`storage`)

Default storage configuration for attachments, lfs, avatars and etc.
Default storage configuration for attachments, lfs, avatars, repo-avatars, repo-archive, packages, actions_log, actions_artifact.

- `STORAGE_TYPE`: **local**: Storage type, `local` for local disk or `minio` for s3 compatible object storage service.
- `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
@@ -1265,10 +1266,10 @@ Default storage configuration for attachments, lfs, avatars and etc.
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
- `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio`

And you can also define a customize storage like below:
The recommanded storage configuration for minio like below:

```ini
[storage.my_minio]
[storage]
STORAGE_TYPE = minio
; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
MINIO_ENDPOINT = localhost:9000
@@ -1284,9 +1285,54 @@ MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
; Minio skip SSL verification available when STORAGE_TYPE is `minio`
MINIO_INSECURE_SKIP_VERIFY = false
SERVE_DIRECT = true
```

Defaultly every storage has their default base path like below

| storage | default base path |
| ----------------- | ------------------ |
| attachments | attachments/ |
| lfs | lfs/ |
| avatars | avatars/ |
| repo-avatars | repo-avatars/ |
| repo-archive | repo-archive/ |
| packages | packages/ |
| actions_log | actions_log/ |
| actions_artifacts | actions_artifacts/ |

And bucket, basepath or `SERVE_DIRECT` could be special or overrided, if you want to use a different you can:

```ini
[storage.actions_log]
MINIO_BUCKET = gitea_actions_log
SERVE_DIRECT = true
MINIO_BASE_PATH = my_actions_log/ ; default is actions_log/ if blank
```

And used by `[attachment]`, `[lfs]` and etc. as `STORAGE_TYPE`.
If you want to customerize a different storage for `lfs` if above default storage defined

```ini
[lfs]
STORAGE_TYPE = my_minio

[storage.my_minio]
STORAGE_TYPE = minio
; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
MINIO_ENDPOINT = localhost:9000
; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
MINIO_ACCESS_KEY_ID =
; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
MINIO_SECRET_ACCESS_KEY =
; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
MINIO_BUCKET = gitea
; Minio location to create bucket only available when STORAGE_TYPE is `minio`
MINIO_LOCATION = us-east-1
; Minio enabled ssl only available when STORAGE_TYPE is `minio`
MINIO_USE_SSL = false
; Minio skip SSL verification available when STORAGE_TYPE is `minio`
MINIO_INSECURE_SKIP_VERIFY = false
```

## Repository Archive Storage (`storage.repo-archive`)

@@ -1306,6 +1352,11 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`.
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
- `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio`

## Repository Archives (`repo-archive`)

- `STORAGE_TYPE`: **local**: Storage type for actions logs, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]`
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`

## Proxy (`proxy`)

- `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy
@@ -1324,6 +1375,8 @@ PROXY_HOSTS = *.github.com

- `ENABLED`: **false**: Enable/Disable actions capabilities
- `DEFAULT_ACTIONS_URL`: **https://gitea.com**: Default address to get action plugins, e.g. the default value means downloading from "<https://gitea.com/actions/checkout>" for "uses: actions/checkout@v3"
- `STORAGE_TYPE`: **local**: Storage type for actions logs, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]`
- `MINIO_BASE_PATH`: **actions_log/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`

`DEFAULT_ACTIONS_URL` indicates where should we find the relative path action plugin. i.e. when use an action in a workflow file like


+ 53
- 4
docs/content/doc/administration/config-cheat-sheet.zh-cn.md View File

@@ -414,7 +414,7 @@ LFS 的存储配置。 如果 `STORAGE_TYPE` 为空，则此配置将从 `[stora

## Storage (`storage`)

Attachments, lfs, avatars and etc 的默认存储配置。
Attachments, lfs, avatars, repo-avatars, repo-archive, packages, actions_log, actions_artifact 的默认存储配置。

- `STORAGE_TYPE`: **local**: 附件存储类型，`local` 将存储到本地文件夹， `minio` 将存储到 s3 兼容的对象存储服务中。
- `SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前，仅 Minio/S3 是支持的。
@@ -425,11 +425,13 @@ Attachments, lfs, avatars and etc 的默认存储配置。
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket，仅当 `STORAGE_TYPE` 是 `minio` 时有效。
- `MINIO_USE_SSL`: **false**: Minio enabled ssl，仅当 `STORAGE_TYPE` 是 `minio` 时有效。

你也可以自定义一个存储的名字如下：
以下为推荐的 recommanded storage configuration for minio like below:

```ini
[storage.my_minio]
[storage]
STORAGE_TYPE = minio
; uncomment when STORAGE_TYPE = local
; PATH = storage root path
; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
MINIO_ENDPOINT = localhost:9000
; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
@@ -444,9 +446,56 @@ MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
; Minio skip SSL verification available when STORAGE_TYPE is `minio`
MINIO_INSECURE_SKIP_VERIFY = false
SERVE_DIRECT = true
```

默认的，每一个存储都会有各自默认的 BasePath 在同一个minio中，默认值如下：

| storage | default base path |
| ----------------- | ------------------ |
| attachments | attachments/ |
| lfs | lfs/ |
| avatars | avatars/ |
| repo-avatars | repo-avatars/ |
| repo-archive | repo-archive/ |
| packages | packages/ |
| actions_log | actions_log/ |
| actions_artifacts | actions_artifacts/ |

同时 bucket, basepath or `SERVE_DIRECT` 是可以被覆写的，像如下所示：

```ini
[storage.actions_log]
MINIO_BUCKET = gitea_actions_log
SERVE_DIRECT = true
MINIO_BASE_PATH = my_actions_log/ ; default is actions_log/ if blank
```

然后你在 `[attachment]`, `[lfs]` 等中可以把这个名字用作 `STORAGE_TYPE` 的值。
当然你也可以完全自定义，像如下

```ini
[lfs]
STORAGE_TYPE = my_minio
MINIO_BASE_PATH = my_lfs_basepath

[storage.my_minio]
STORAGE_TYPE = minio
; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
MINIO_ENDPOINT = localhost:9000
; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
MINIO_ACCESS_KEY_ID =
; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
MINIO_SECRET_ACCESS_KEY =
; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
MINIO_BUCKET = gitea
; Minio location to create bucket only available when STORAGE_TYPE is `minio`
MINIO_LOCATION = us-east-1
; Minio enabled ssl only available when STORAGE_TYPE is `minio`
MINIO_USE_SSL = false
; Minio skip SSL verification available when STORAGE_TYPE is `minio`
MINIO_INSECURE_SKIP_VERIFY = false
SERVE_DIRECT = true
```

## Repository Archive Storage (`storage.repo-archive`)


+ 1
- 1
models/migrations/v1_10/v96.go View File

@@ -53,7 +53,7 @@ func DeleteOrphanedAttachments(x *xorm.Engine) error {

for _, attachment := range attachments {
uuid := attachment.UUID
if err := util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
if err := util.RemoveAll(filepath.Join(setting.Attachment.Storage.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
return err
}
}

+ 1
- 1
models/migrations/v1_11/v112.go View File

@@ -30,7 +30,7 @@ func RemoveAttachmentMissedRepo(x *xorm.Engine) error {

for i := 0; i < len(attachments); i++ {
uuid := attachments[i].UUID
if err = util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
if err = util.RemoveAll(filepath.Join(setting.Attachment.Storage.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
fmt.Printf("Error: %v", err) //nolint:forbidigo
}
}

+ 4
- 4
models/migrations/v1_11/v115.go View File

@@ -61,7 +61,7 @@ func RenameExistingUserAvatarName(x *xorm.Engine) error {
for _, user := range users {
oldAvatar := user.Avatar

if stat, err := os.Stat(filepath.Join(setting.Avatar.Path, oldAvatar)); err != nil || !stat.Mode().IsRegular() {
if stat, err := os.Stat(filepath.Join(setting.Avatar.Storage.Path, oldAvatar)); err != nil || !stat.Mode().IsRegular() {
if err == nil {
err = fmt.Errorf("Error: \"%s\" is not a regular file", oldAvatar)
}
@@ -86,7 +86,7 @@ func RenameExistingUserAvatarName(x *xorm.Engine) error {
return fmt.Errorf("[user: %s] user table update: %w", user.LowerName, err)
}

deleteList.Add(filepath.Join(setting.Avatar.Path, oldAvatar))
deleteList.Add(filepath.Join(setting.Avatar.Storage.Path, oldAvatar))
migrated++
select {
case <-ticker.C:
@@ -135,7 +135,7 @@ func RenameExistingUserAvatarName(x *xorm.Engine) error {
// copyOldAvatarToNewLocation copies oldAvatar to newAvatarLocation
// and returns newAvatar location
func copyOldAvatarToNewLocation(userID int64, oldAvatar string) (string, error) {
fr, err := os.Open(filepath.Join(setting.Avatar.Path, oldAvatar))
fr, err := os.Open(filepath.Join(setting.Avatar.Storage.Path, oldAvatar))
if err != nil {
return "", fmt.Errorf("os.Open: %w", err)
}
@@ -151,7 +151,7 @@ func copyOldAvatarToNewLocation(userID int64, oldAvatar string) (string, error)
return newAvatar, nil
}

if err := os.WriteFile(filepath.Join(setting.Avatar.Path, newAvatar), data, 0o666); err != nil {
if err := os.WriteFile(filepath.Join(setting.Avatar.Storage.Path, newAvatar), data, 0o666); err != nil {
return "", fmt.Errorf("os.WriteFile: %w", err)
}


+ 17
- 10
modules/setting/actions.go View File

@@ -4,14 +4,14 @@
package setting

import (
"code.gitea.io/gitea/modules/log"
"fmt"
)

// Actions settings
var (
Actions = struct {
LogStorage Storage // how the created logs should be stored
ArtifactStorage Storage // how the created artifacts should be stored
LogStorage *Storage // how the created logs should be stored
ArtifactStorage *Storage // how the created artifacts should be stored
Enabled bool
DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"`
}{
@@ -20,15 +20,22 @@ var (
}
)

func loadActionsFrom(rootCfg ConfigProvider) {
func loadActionsFrom(rootCfg ConfigProvider) error {
sec := rootCfg.Section("actions")
if err := sec.MapTo(&Actions); err != nil {
log.Fatal("Failed to map Actions settings: %v", err)
err := sec.MapTo(&Actions)
if err != nil {
return fmt.Errorf("failed to map Actions settings: %v", err)
}

actionsSec := rootCfg.Section("actions.artifacts")
storageType := actionsSec.Key("STORAGE_TYPE").MustString("")
// don't support to read configuration from [actions]
Actions.LogStorage, err = getStorage(rootCfg, "actions_log", "", nil)
if err != nil {
return err
}

actionsSec, _ := rootCfg.GetSection("actions.artifacts")

Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec)

Actions.LogStorage = getStorage(rootCfg, "actions_log", "", nil)
Actions.ArtifactStorage = getStorage(rootCfg, "actions_artifacts", storageType, actionsSec)
return err
}

+ 97
- 0
modules/setting/actions_test.go View File

@@ -0,0 +1,97 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import (
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
)

func Test_getStorageInheritNameSectionTypeForActions(t *testing.T) {
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "minio", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath)
assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath)

iniStr = `
[storage.actions_log]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "minio", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath)
assert.EqualValues(t, "local", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path))

iniStr = `
[storage.actions_log]
STORAGE_TYPE = my_storage

[storage.my_storage]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "minio", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath)
assert.EqualValues(t, "local", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path))

iniStr = `
[storage.actions_artifacts]
STORAGE_TYPE = my_storage

[storage.my_storage]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "local", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path))
assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath)

iniStr = `
[storage.actions_artifacts]
STORAGE_TYPE = my_storage

[storage.my_storage]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "local", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path))
assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath)

iniStr = ``
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "local", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path))
assert.EqualValues(t, "local", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path))
}

+ 12
- 10
modules/setting/attachment.go View File

@@ -5,29 +5,31 @@ package setting

// Attachment settings
var Attachment = struct {
Storage
Storage *Storage
AllowedTypes string
MaxSize int64
MaxFiles int
Enabled bool
}{
Storage: Storage{
ServeDirect: false,
},
AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
Storage: &Storage{},
AllowedTypes: ".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip",
MaxSize: 4,
MaxFiles: 5,
Enabled: true,
}

func loadAttachmentFrom(rootCfg ConfigProvider) {
sec := rootCfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")

Attachment.Storage = getStorage(rootCfg, "attachments", storageType, sec)
func loadAttachmentFrom(rootCfg ConfigProvider) (err error) {
sec, _ := rootCfg.GetSection("attachment")
if sec == nil {
Attachment.Storage, err = getStorage(rootCfg, "attachments", "", nil)
return err
}

Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip")
Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4)
Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5)
Attachment.Enabled = sec.Key("ENABLED").MustBool(true)

Attachment.Storage, err = getStorage(rootCfg, "attachments", "", sec)
return err
}

+ 133
- 0
modules/setting/attachment_test.go View File

@@ -0,0 +1,133 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import (
"testing"

"github.com/stretchr/testify/assert"
)

func Test_getStorageCustomType(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = my_minio
MINIO_BUCKET = gitea-attachment

[storage.my_minio]
STORAGE_TYPE = minio
MINIO_ENDPOINT = my_minio:9000
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

assert.EqualValues(t, "minio", Attachment.Storage.Type)
assert.EqualValues(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint)
assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath)
}

func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio

[storage.minio]
MINIO_BUCKET = gitea-minio

[storage]
MINIO_BUCKET = gitea
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

assert.EqualValues(t, "minio", Attachment.Storage.Type)
assert.EqualValues(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath)
}

func Test_getStorageSpecificOverridesStorage(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio
MINIO_BUCKET = gitea-attachment

[storage.attachments]
MINIO_BUCKET = gitea

[storage]
STORAGE_TYPE = local
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

assert.EqualValues(t, "minio", Attachment.Storage.Type)
assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath)
}

func Test_getStorageGetDefaults(t *testing.T) {
cfg, err := NewConfigProviderFromData("")
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

// default storage is local, so bucket is empty
assert.EqualValues(t, "", Attachment.Storage.MinioConfig.Bucket)
}

func Test_getStorageInheritNameSectionType(t *testing.T) {
iniStr := `
[storage.attachments]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

assert.EqualValues(t, "minio", Attachment.Storage.Type)
}

func Test_AttachmentStorage(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))
storage := Attachment.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
}

func Test_AttachmentStorage1(t *testing.T) {
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))
assert.EqualValues(t, "minio", Attachment.Storage.Type)
assert.EqualValues(t, "gitea", Attachment.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath)
}

+ 19
- 0
modules/setting/config_provider.go View File

@@ -7,6 +7,7 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"

@@ -95,6 +96,18 @@ func ConfigSectionKeyString(sec ConfigSection, key string, def ...string) string
return ""
}

func ConfigSectionKeyBool(sec ConfigSection, key string, def ...bool) bool {
k := ConfigSectionKey(sec, key)
if k != nil && k.String() != "" {
b, _ := strconv.ParseBool(k.String())
return b
}
if len(def) > 0 {
return def[0]
}
return false
}

// ConfigInheritedKey works like ini.Section.Key(), but it always returns a new key instance, it is O(n) because NewKey is O(n)
// and the returned key is safe to be used with "MustXxx", it doesn't change the parent's values.
// Otherwise, ini.Section.Key().MustXxx would pollute the parent section's keys.
@@ -287,6 +300,12 @@ func deprecatedSetting(rootCfg ConfigProvider, oldSection, oldKey, newSection, n
}
}

func deprecatedSettingFatal(rootCfg ConfigProvider, oldSection, oldKey, newSection, newKey, version string) {
if rootCfg.Section(oldSection).HasKey(oldKey) {
log.Fatal("Deprecated fallback `[%s]` `%s` present. Use `[%s]` `%s` instead. This fallback will be/has been removed in %s", oldSection, oldKey, newSection, newKey, version)
}
}

// deprecatedSettingDB add a hint that the configuration has been moved to database but still kept in app.ini
func deprecatedSettingDB(rootCfg ConfigProvider, oldSection, oldKey string) {
if rootCfg.Section(oldSection).HasKey(oldKey) {

+ 30
- 26
modules/setting/lfs.go View File

@@ -5,10 +5,10 @@ package setting

import (
"encoding/base64"
"fmt"
"time"

"code.gitea.io/gitea/modules/generate"
"code.gitea.io/gitea/modules/log"
)

// LFS represents the configuration for Git LFS
@@ -20,25 +20,27 @@ var LFS = struct {
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`
LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"`

Storage
Storage *Storage
}{}

func loadLFSFrom(rootCfg ConfigProvider) {
func loadLFSFrom(rootCfg ConfigProvider) error {
sec := rootCfg.Section("server")
if err := sec.MapTo(&LFS); err != nil {
log.Fatal("Failed to map LFS settings: %v", err)
return fmt.Errorf("failed to map LFS settings: %v", err)
}

lfsSec := rootCfg.Section("lfs")
storageType := lfsSec.Key("STORAGE_TYPE").MustString("")
lfsSec, _ := rootCfg.GetSection("lfs")

// Specifically default PATH to LFS_CONTENT_PATH
// DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version
// if these are removed, the warning will not be shown
deprecatedSetting(rootCfg, "server", "LFS_CONTENT_PATH", "lfs", "PATH", "v1.19.0")
lfsSec.Key("PATH").MustString(sec.Key("LFS_CONTENT_PATH").String())
deprecatedSettingFatal(rootCfg, "server", "LFS_CONTENT_PATH", "lfs", "PATH", "v1.19.0")

LFS.Storage = getStorage(rootCfg, "lfs", storageType, lfsSec)
var err error
LFS.Storage, err = getStorage(rootCfg, "lfs", "", lfsSec)
if err != nil {
return err
}

// Rest of LFS service settings
if LFS.LocksPagingNum == 0 {
@@ -47,23 +49,25 @@ func loadLFSFrom(rootCfg ConfigProvider) {

LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(24 * time.Hour)

if LFS.StartServer {
LFS.JWTSecretBytes = make([]byte, 32)
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
if err != nil || n != 32 {
LFS.JWTSecretBase64, err = generate.NewJwtSecretBase64()
if err != nil {
log.Fatal("Error generating JWT Secret for custom config: %v", err)
return
}
// Save secret
sec.Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
if err := rootCfg.Save(); err != nil {
log.Fatal("Error saving JWT Secret for custom config: %v", err)
return
}
if !LFS.StartServer {
return nil
}
LFS.JWTSecretBytes = make([]byte, 32)
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
if err != nil || n != 32 {
LFS.JWTSecretBase64, err = generate.NewJwtSecretBase64()
if err != nil {
return fmt.Errorf("Error generating JWT Secret for custom config: %v", err)
}
// Save secret
sec.Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
if err := rootCfg.Save(); err != nil {
return fmt.Errorf("Error saving JWT Secret for custom config: %v", err)
}
}

return nil
}

+ 77
- 0
modules/setting/lfs_test.go View File

@@ -0,0 +1,77 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import (
"testing"

"github.com/stretchr/testify/assert"
)

func Test_getStorageInheritNameSectionTypeForLFS(t *testing.T) {
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))

assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath)

iniStr = `
[storage.lfs]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))

assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath)

iniStr = `
[lfs]
STORAGE_TYPE = my_minio

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))

assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath)

iniStr = `
[lfs]
STORAGE_TYPE = my_minio
MINIO_BASE_PATH = my_lfs/

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))

assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath)
}

func Test_LFSStorage1(t *testing.T) {
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadLFSFrom(cfg))
assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "gitea", LFS.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath)
}

+ 17
- 9
modules/setting/packages.go View File

@@ -4,20 +4,19 @@
package setting

import (
"fmt"
"math"
"net/url"
"os"
"path/filepath"

"code.gitea.io/gitea/modules/log"

"github.com/dustin/go-humanize"
)

// Package registry settings
var (
Packages = struct {
Storage
Storage *Storage
Enabled bool
ChunkedUploadPath string
RegistryHost string
@@ -51,13 +50,21 @@ var (
}
)

func loadPackagesFrom(rootCfg ConfigProvider) {
sec := rootCfg.Section("packages")
if err := sec.MapTo(&Packages); err != nil {
log.Fatal("Failed to map Packages settings: %v", err)
func loadPackagesFrom(rootCfg ConfigProvider) (err error) {
sec, _ := rootCfg.GetSection("packages")
if sec == nil {
Packages.Storage, err = getStorage(rootCfg, "packages", "", nil)
return err
}

if err = sec.MapTo(&Packages); err != nil {
return fmt.Errorf("failed to map Packages settings: %v", err)
}

Packages.Storage = getStorage(rootCfg, "packages", "", nil)
Packages.Storage, err = getStorage(rootCfg, "packages", "", sec)
if err != nil {
return err
}

appURL, _ := url.Parse(AppURL)
Packages.RegistryHost = appURL.Host
@@ -68,7 +75,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) {
}

if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil {
log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
return fmt.Errorf("unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
}

Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
@@ -93,6 +100,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) {
Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS")
Packages.LimitSizeSwift = mustBytes(sec, "LIMIT_SIZE_SWIFT")
Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT")
return nil
}

func mustBytes(section ConfigSection, key string) int64 {

+ 167
- 0
modules/setting/packages_test.go View File

@@ -29,3 +29,170 @@ func TestMustBytes(t *testing.T) {
assert.EqualValues(t, 1782579, test("1.7mib"))
assert.EqualValues(t, -1, test("1 yib")) // too large
}

func Test_getStorageInheritNameSectionTypeForPackages(t *testing.T) {
// packages storage inherits from storage if nothing configured
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadPackagesFrom(cfg))

assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath)

// we can also configure packages storage directly
iniStr = `
[storage.packages]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadPackagesFrom(cfg))

assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath)

// or we can indicate the storage type in the packages section
iniStr = `
[packages]
STORAGE_TYPE = my_minio

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadPackagesFrom(cfg))

assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath)

// or we can indicate the storage type and minio base path in the packages section
iniStr = `
[packages]
STORAGE_TYPE = my_minio
MINIO_BASE_PATH = my_packages/

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadPackagesFrom(cfg))

assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "my_packages/", Packages.Storage.MinioConfig.BasePath)
}

func Test_PackageStorage1(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[packages]
MINIO_BASE_PATH = packages/
SERVE_DIRECT = true
[storage]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadPackagesFrom(cfg))
storage := Packages.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath)
assert.True(t, storage.MinioConfig.ServeDirect)
}

func Test_PackageStorage2(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage.packages]
MINIO_BASE_PATH = packages/
SERVE_DIRECT = true
[storage]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadPackagesFrom(cfg))
storage := Packages.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath)
assert.True(t, storage.MinioConfig.ServeDirect)
}

func Test_PackageStorage3(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[packages]
STORAGE_TYPE = my_cfg
MINIO_BASE_PATH = my_packages/
SERVE_DIRECT = true
[storage.my_cfg]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadPackagesFrom(cfg))
storage := Packages.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath)
assert.True(t, storage.MinioConfig.ServeDirect)
}

func Test_PackageStorage4(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage.packages]
STORAGE_TYPE = my_cfg
MINIO_BASE_PATH = my_packages/
SERVE_DIRECT = true
[storage.my_cfg]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadPackagesFrom(cfg))
storage := Packages.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath)
assert.True(t, storage.MinioConfig.ServeDirect)
}

+ 19
- 11
modules/setting/picture.go View File

@@ -7,7 +7,7 @@ package setting

var (
Avatar = struct {
Storage
Storage *Storage

MaxWidth int
MaxHeight int
@@ -27,23 +27,26 @@ var (
EnableFederatedAvatar bool // Depreciated: migrated to database

RepoAvatar = struct {
Storage
Storage *Storage

Fallback string
FallbackImage string
}{}
)

func loadPictureFrom(rootCfg ConfigProvider) {
func loadAvatarsFrom(rootCfg ConfigProvider) error {
sec := rootCfg.Section("picture")

avatarSec := rootCfg.Section("avatar")
storageType := sec.Key("AVATAR_STORAGE_TYPE").MustString("")
// Specifically default PATH to AVATAR_UPLOAD_PATH
avatarSec.Key("PATH").MustString(
sec.Key("AVATAR_UPLOAD_PATH").String())
avatarSec.Key("PATH").MustString(sec.Key("AVATAR_UPLOAD_PATH").String())

Avatar.Storage = getStorage(rootCfg, "avatars", storageType, avatarSec)
var err error
Avatar.Storage, err = getStorage(rootCfg, "avatars", storageType, avatarSec)
if err != nil {
return err
}

Avatar.MaxWidth = sec.Key("AVATAR_MAX_WIDTH").MustInt(4096)
Avatar.MaxHeight = sec.Key("AVATAR_MAX_HEIGHT").MustInt(4096)
@@ -67,7 +70,7 @@ func loadPictureFrom(rootCfg ConfigProvider) {
EnableFederatedAvatar = sec.Key("ENABLE_FEDERATED_AVATAR").MustBool(GetDefaultEnableFederatedAvatar(DisableGravatar))
deprecatedSettingDB(rootCfg, "", "ENABLE_FEDERATED_AVATAR")

loadRepoAvatarFrom(rootCfg)
return nil
}

func GetDefaultDisableGravatar() bool {
@@ -85,17 +88,22 @@ func GetDefaultEnableFederatedAvatar(disableGravatar bool) bool {
return v
}

func loadRepoAvatarFrom(rootCfg ConfigProvider) {
func loadRepoAvatarFrom(rootCfg ConfigProvider) error {
sec := rootCfg.Section("picture")

repoAvatarSec := rootCfg.Section("repo-avatar")
storageType := sec.Key("REPOSITORY_AVATAR_STORAGE_TYPE").MustString("")
// Specifically default PATH to AVATAR_UPLOAD_PATH
repoAvatarSec.Key("PATH").MustString(
sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String())
repoAvatarSec.Key("PATH").MustString(sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String())

RepoAvatar.Storage = getStorage(rootCfg, "repo-avatars", storageType, repoAvatarSec)
var err error
RepoAvatar.Storage, err = getStorage(rootCfg, "repo-avatars", storageType, repoAvatarSec)
if err != nil {
return err
}

RepoAvatar.Fallback = sec.Key("REPOSITORY_AVATAR_FALLBACK").MustString("none")
RepoAvatar.FallbackImage = sec.Key("REPOSITORY_AVATAR_FALLBACK_IMAGE").MustString(AppSubURL + "/assets/img/repo_default.png")

return nil
}

+ 3
- 5
modules/setting/repository.go View File

@@ -265,10 +265,6 @@ var (
}
RepoRootPath string
ScriptType = "bash"

RepoArchive = struct {
Storage
}{}
)

func loadRepositoryFrom(rootCfg ConfigProvider) {
@@ -359,5 +355,7 @@ func loadRepositoryFrom(rootCfg ConfigProvider) {
Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath)
}

RepoArchive.Storage = getStorage(rootCfg, "repo-archive", "", nil)
if err := loadRepoArchiveFrom(rootCfg); err != nil {
log.Fatal("loadRepoArchiveFrom: %v", err)
}
}

+ 25
- 0
modules/setting/repository_archive.go View File

@@ -0,0 +1,25 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import "fmt"

var RepoArchive = struct {
Storage *Storage
}{}

func loadRepoArchiveFrom(rootCfg ConfigProvider) (err error) {
sec, _ := rootCfg.GetSection("repo-archive")
if sec == nil {
RepoArchive.Storage, err = getStorage(rootCfg, "repo-archive", "", nil)
return err
}

if err := sec.MapTo(&RepoArchive); err != nil {
return fmt.Errorf("mapto repoarchive failed: %v", err)
}

RepoArchive.Storage, err = getStorage(rootCfg, "repo-archive", "", sec)
return err
}

+ 111
- 0
modules/setting/repository_archive_test.go View File

@@ -0,0 +1,111 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import (
"testing"

"github.com/stretchr/testify/assert"
)

func Test_getStorageInheritNameSectionTypeForRepoArchive(t *testing.T) {
// packages storage inherits from storage if nothing configured
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))

assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath)

// we can also configure packages storage directly
iniStr = `
[storage.repo-archive]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))

assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath)

// or we can indicate the storage type in the packages section
iniStr = `
[repo-archive]
STORAGE_TYPE = my_minio

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))

assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath)

// or we can indicate the storage type and minio base path in the packages section
iniStr = `
[repo-archive]
STORAGE_TYPE = my_minio
MINIO_BASE_PATH = my_archive/

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))

assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath)
}

func Test_RepoArchiveStorage(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadRepoArchiveFrom(cfg))
storage := RepoArchive.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)

iniStr = `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage.repo-archive]
STORAGE_TYPE = s3
[storage.s3]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadRepoArchiveFrom(cfg))
storage = RepoArchive.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
}

+ 25
- 9
modules/setting/setting.go View File

@@ -203,16 +203,18 @@ func Init(opts *Options) {
var err error
CfgProvider, err = NewConfigProviderFromFile(opts)
if err != nil {
log.Fatal("Init[%v]: %v", opts, err)
log.Fatal("newConfigProviderFromFile[%v]: %v", opts, err)
}
if !opts.DisableLoadCommonSettings {
loadCommonSettingsFrom(CfgProvider)
if err := loadCommonSettingsFrom(CfgProvider); err != nil {
log.Fatal("loadCommonSettingsFrom[%v]: %v", opts, err)
}
}
}

// loadCommonSettingsFrom loads common configurations from a configuration provider.
func loadCommonSettingsFrom(cfg ConfigProvider) {
// WARNING: don't change the sequence except you know what you are doing.
func loadCommonSettingsFrom(cfg ConfigProvider) error {
// WARNNING: don't change the sequence except you know what you are doing.
loadRunModeFrom(cfg)
loadLogGlobalFrom(cfg)
loadServerFrom(cfg)
@@ -222,13 +224,26 @@ func loadCommonSettingsFrom(cfg ConfigProvider) {

loadOAuth2From(cfg)
loadSecurityFrom(cfg)
loadAttachmentFrom(cfg)
loadLFSFrom(cfg)
if err := loadAttachmentFrom(cfg); err != nil {
return err
}
if err := loadLFSFrom(cfg); err != nil {
return err
}
loadTimeFrom(cfg)
loadRepositoryFrom(cfg)
loadPictureFrom(cfg)
loadPackagesFrom(cfg)
loadActionsFrom(cfg)
if err := loadAvatarsFrom(cfg); err != nil {
return err
}
if err := loadRepoAvatarFrom(cfg); err != nil {
return err
}
if err := loadPackagesFrom(cfg); err != nil {
return err
}
if err := loadActionsFrom(cfg); err != nil {
return err
}
loadUIFrom(cfg)
loadAdminFrom(cfg)
loadAPIFrom(cfg)
@@ -239,6 +254,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) {
loadMirrorFrom(cfg)
loadMarkupFrom(cfg)
loadOtherFrom(cfg)
return nil
}

func loadRunModeFrom(rootCfg ConfigProvider) {

+ 152
- 57
modules/setting/storage.go View File

@@ -4,87 +4,182 @@
package setting

import (
"errors"
"fmt"
"path/filepath"
"reflect"
)

// StorageType is a type of Storage
type StorageType string

const (
// LocalStorageType is the type descriptor for local storage
LocalStorageType StorageType = "local"
// MinioStorageType is the type descriptor for minio storage
MinioStorageType StorageType = "minio"
)

var storageTypes = []StorageType{
LocalStorageType,
MinioStorageType,
}

// IsValidStorageType returns true if the given storage type is valid
func IsValidStorageType(storageType StorageType) bool {
for _, t := range storageTypes {
if t == storageType {
return true
}
}
return false
}

// MinioStorageConfig represents the configuration for a minio storage
type MinioStorageConfig struct {
Endpoint string `ini:"MINIO_ENDPOINT" json:",omitempty"`
AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID" json:",omitempty"`
SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY" json:",omitempty"`
Bucket string `ini:"MINIO_BUCKET" json:",omitempty"`
Location string `ini:"MINIO_LOCATION" json:",omitempty"`
BasePath string `ini:"MINIO_BASE_PATH" json:",omitempty"`
UseSSL bool `ini:"MINIO_USE_SSL"`
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM" json:",omitempty"`
ServeDirect bool `ini:"SERVE_DIRECT"`
}

// Storage represents configuration of storages
type Storage struct {
Type string
Path string
Section ConfigSection
ServeDirect bool
Type StorageType // local or minio
Path string `json:",omitempty"` // for local type
TemporaryPath string `json:",omitempty"`
MinioConfig MinioStorageConfig // for minio type
}

// MapTo implements the Mappable interface
func (s *Storage) MapTo(v interface{}) error {
pathValue := reflect.ValueOf(v).Elem().FieldByName("Path")
if pathValue.IsValid() && pathValue.Kind() == reflect.String {
pathValue.SetString(s.Path)
func (storage *Storage) ToShadowCopy() Storage {
shadowStorage := *storage
if shadowStorage.MinioConfig.AccessKeyID != "" {
shadowStorage.MinioConfig.AccessKeyID = "******"
}
if s.Section != nil {
return s.Section.MapTo(v)
if shadowStorage.MinioConfig.SecretAccessKey != "" {
shadowStorage.MinioConfig.SecretAccessKey = "******"
}
return nil
return shadowStorage
}

func getStorage(rootCfg ConfigProvider, name, typ string, targetSec ConfigSection) Storage {
const sectionName = "storage"
sec := rootCfg.Section(sectionName)
const storageSectionName = "storage"

func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection {
storageSec := rootCfg.Section(storageSectionName)
// Global Defaults
sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
sec.Key("MINIO_BUCKET").MustString("gitea")
sec.Key("MINIO_LOCATION").MustString("us-east-1")
sec.Key("MINIO_USE_SSL").MustBool(false)
sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
sec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default")
storageSec.Key("STORAGE_TYPE").MustString("local")
storageSec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
storageSec.Key("MINIO_ACCESS_KEY_ID").MustString("")
storageSec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
storageSec.Key("MINIO_BUCKET").MustString("gitea")
storageSec.Key("MINIO_LOCATION").MustString("us-east-1")
storageSec.Key("MINIO_USE_SSL").MustBool(false)
storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
storageSec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default")
return storageSec
}

if targetSec == nil {
targetSec, _ = rootCfg.NewSection(name)
func getStorage(rootCfg ConfigProvider, name, typ string, sec ConfigSection) (*Storage, error) {
if name == "" {
return nil, errors.New("no name for storage")
}

var storage Storage
storage.Section = targetSec
storage.Type = typ

overrides := make([]ConfigSection, 0, 3)
nameSec, err := rootCfg.GetSection(sectionName + "." + name)
if err == nil {
overrides = append(overrides, nameSec)
var targetSec ConfigSection
if typ != "" {
var err error
targetSec, err = rootCfg.GetSection(storageSectionName + "." + typ)
if err != nil {
if !IsValidStorageType(StorageType(typ)) {
return nil, fmt.Errorf("get section via storage type %q failed: %v", typ, err)
}
}
if targetSec != nil {
targetType := targetSec.Key("STORAGE_TYPE").String()
if targetType == "" {
if !IsValidStorageType(StorageType(typ)) {
return nil, fmt.Errorf("unknow storage type %q", typ)
}
targetSec.Key("STORAGE_TYPE").SetValue(typ)
} else if !IsValidStorageType(StorageType(targetType)) {
return nil, fmt.Errorf("unknow storage type %q for section storage.%v", targetType, typ)
}
}
}

typeSec, err := rootCfg.GetSection(sectionName + "." + typ)
if err == nil {
overrides = append(overrides, typeSec)
nextType := typeSec.Key("STORAGE_TYPE").String()
if len(nextType) > 0 {
storage.Type = nextType // Support custom STORAGE_TYPE
storageNameSec, _ := rootCfg.GetSection(storageSectionName + "." + name)

if targetSec == nil {
targetSec = sec
}
if targetSec == nil {
targetSec = storageNameSec
}
if targetSec == nil {
targetSec = getDefaultStorageSection(rootCfg)
} else {
targetType := targetSec.Key("STORAGE_TYPE").String()
switch {
case targetType == "":
if targetSec.Key("PATH").String() == "" {
targetSec = getDefaultStorageSection(rootCfg)
} else {
targetSec.Key("STORAGE_TYPE").SetValue("local")
}
default:
newTargetSec, _ := rootCfg.GetSection(storageSectionName + "." + targetType)
if newTargetSec == nil {
if !IsValidStorageType(StorageType(targetType)) {
return nil, fmt.Errorf("invalid storage section %s.%q", storageSectionName, targetType)
}
} else {
targetSec = newTargetSec
if IsValidStorageType(StorageType(targetType)) {
tp := targetSec.Key("STORAGE_TYPE").String()
if tp == "" {
targetSec.Key("STORAGE_TYPE").SetValue(targetType)
}
}
}
}
}
overrides = append(overrides, sec)

for _, override := range overrides {
for _, key := range override.Keys() {
if !targetSec.HasKey(key.Name()) {
_, _ = targetSec.NewKey(key.Name(), key.Value())
}
targetType := targetSec.Key("STORAGE_TYPE").String()
if !IsValidStorageType(StorageType(targetType)) {
return nil, fmt.Errorf("invalid storage type %q", targetType)
}

var storage Storage
storage.Type = StorageType(targetType)

switch targetType {
case string(LocalStorageType):
storage.Path = ConfigSectionKeyString(targetSec, "PATH", filepath.Join(AppDataPath, name))
if !filepath.IsAbs(storage.Path) {
storage.Path = filepath.Join(AppWorkPath, storage.Path)
}
if len(storage.Type) == 0 {
storage.Type = override.Key("STORAGE_TYPE").String()
case string(MinioStorageType):
storage.MinioConfig.BasePath = name + "/"

if err := targetSec.MapTo(&storage.MinioConfig); err != nil {
return nil, fmt.Errorf("map minio config failed: %v", err)
}
// extra config section will be read SERVE_DIRECT, PATH, MINIO_BASE_PATH to override the targetsec
extraConfigSec := sec
if extraConfigSec == nil {
extraConfigSec = storageNameSec
}
}
storage.ServeDirect = storage.Section.Key("SERVE_DIRECT").MustBool(false)

// Specific defaults
storage.Path = storage.Section.Key("PATH").MustString(filepath.Join(AppDataPath, name))
if !filepath.IsAbs(storage.Path) {
storage.Path = filepath.Join(AppWorkPath, storage.Path)
storage.Section.Key("PATH").SetValue(storage.Path)
if extraConfigSec != nil {
storage.MinioConfig.ServeDirect = ConfigSectionKeyBool(extraConfigSec, "SERVE_DIRECT", storage.MinioConfig.ServeDirect)
storage.MinioConfig.BasePath = ConfigSectionKeyString(extraConfigSec, "MINIO_BASE_PATH", storage.MinioConfig.BasePath)
storage.MinioConfig.Bucket = ConfigSectionKeyString(extraConfigSec, "MINIO_BUCKET", storage.MinioConfig.Bucket)
}
}
storage.Section.Key("MINIO_BASE_PATH").MustString(name + "/")

return storage
return &storage, nil
}

+ 42
- 154
modules/setting/storage_test.go View File

@@ -9,106 +9,6 @@ import (
"github.com/stretchr/testify/assert"
)

func Test_getStorageCustomType(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = my_minio
MINIO_BUCKET = gitea-attachment

[storage.my_minio]
STORAGE_TYPE = minio
MINIO_ENDPOINT = my_minio:9000
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "my_minio:9000", storage.Section.Key("MINIO_ENDPOINT").String())
assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageNameSectionOverridesTypeSection(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio

[storage.attachments]
MINIO_BUCKET = gitea-attachment

[storage.minio]
MINIO_BUCKET = gitea
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio

[storage.minio]
MINIO_BUCKET = gitea-minio

[storage]
MINIO_BUCKET = gitea
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea-minio", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageSpecificOverridesStorage(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio
MINIO_BUCKET = gitea-attachment

[storage.attachments]
MINIO_BUCKET = gitea

[storage]
STORAGE_TYPE = local
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageGetDefaults(t *testing.T) {
cfg, err := NewConfigProviderFromData("")
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "gitea", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageMultipleName(t *testing.T) {
iniStr := `
[lfs]
@@ -118,32 +18,20 @@ MINIO_BUCKET = gitea-lfs
MINIO_BUCKET = gitea-attachment

[storage]
STORAGE_TYPE = minio
MINIO_BUCKET = gitea-storage
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

{
sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String())
}
{
sec := cfg.Section("lfs")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "lfs", storageType, sec)

assert.EqualValues(t, "gitea-lfs", storage.Section.Key("MINIO_BUCKET").String())
}
{
sec := cfg.Section("avatar")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "avatars", storageType, sec)

assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String())
}
assert.NoError(t, loadAttachmentFrom(cfg))
assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket)

assert.NoError(t, loadLFSFrom(cfg))
assert.EqualValues(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket)

assert.NoError(t, loadAvatarsFrom(cfg))
assert.EqualValues(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket)
}

func Test_getStorageUseOtherNameAsType(t *testing.T) {
@@ -152,25 +40,17 @@ func Test_getStorageUseOtherNameAsType(t *testing.T) {
STORAGE_TYPE = lfs

[storage.lfs]
STORAGE_TYPE = minio
MINIO_BUCKET = gitea-storage
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

{
sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)
assert.NoError(t, loadAttachmentFrom(cfg))
assert.EqualValues(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket)

assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String())
}
{
sec := cfg.Section("lfs")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "lfs", storageType, sec)

assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String())
}
assert.NoError(t, loadLFSFrom(cfg))
assert.EqualValues(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket)
}

func Test_getStorageInheritStorageType(t *testing.T) {
@@ -181,24 +61,32 @@ STORAGE_TYPE = minio
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
}

func Test_getStorageInheritNameSectionType(t *testing.T) {
iniStr := `
[storage.attachments]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.NoError(t, loadPackagesFrom(cfg))
assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "gitea", Packages.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath)

assert.NoError(t, loadRepoArchiveFrom(cfg))
assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath)

assert.NoError(t, loadActionsFrom(cfg))
assert.EqualValues(t, "minio", Actions.LogStorage.Type)
assert.EqualValues(t, "gitea", Actions.LogStorage.MinioConfig.Bucket)
assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath)

assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket)
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath)

assert.NoError(t, loadAvatarsFrom(cfg))
assert.EqualValues(t, "minio", Avatar.Storage.Type)
assert.EqualValues(t, "gitea", Avatar.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath)

assert.NoError(t, loadRepoAvatarFrom(cfg))
assert.EqualValues(t, "minio", RepoAvatar.Storage.Type)
assert.EqualValues(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath)
}

+ 0
- 56
modules/storage/helper.go View File

@@ -8,64 +8,8 @@ import (
"io"
"net/url"
"os"
"reflect"

"code.gitea.io/gitea/modules/json"
)

// Mappable represents an interface that can MapTo another interface
type Mappable interface {
MapTo(v interface{}) error
}

// toConfig will attempt to convert a given configuration cfg into the provided exemplar type.
//
// It will tolerate the cfg being passed as a []byte or string of a json representation of the
// exemplar or the correct type of the exemplar itself
func toConfig(exemplar, cfg interface{}) (interface{}, error) {
// First of all check if we've got the same type as the exemplar - if so it's all fine.
if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) {
return cfg, nil
}

// Now if not - does it provide a MapTo function we can try?
if mappable, ok := cfg.(Mappable); ok {
newVal := reflect.New(reflect.TypeOf(exemplar))
if err := mappable.MapTo(newVal.Interface()); err == nil {
return newVal.Elem().Interface(), nil
}
// MapTo has failed us ... let's try the json route ...
}

// OK we've been passed a byte array right?
configBytes, ok := cfg.([]byte)
if !ok {
// oh ... it's a string then?
var configStr string

configStr, ok = cfg.(string)
configBytes = []byte(configStr)
}
if !ok {
// hmm ... can we marshal it to json?
var err error
configBytes, err = json.Marshal(cfg)
ok = err == nil
}
if !ok {
// no ... we've tried hard enough at this point - throw an error!
return nil, ErrInvalidConfiguration{cfg: cfg}
}

// OK unmarshal the byte array into a new copy of the exemplar
newVal := reflect.New(reflect.TypeOf(exemplar))
if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil {
// If we can't unmarshal it then return an error!
return nil, ErrInvalidConfiguration{cfg: cfg, err: err}
}
return newVal.Elem().Interface(), nil
}

var uninitializedStorage = discardStorage("uninitialized storage")

type discardStorage string

+ 3
- 17
modules/storage/local.go View File

@@ -12,20 +12,12 @@ import (
"path/filepath"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)

var _ ObjectStorage = &LocalStorage{}

// LocalStorageType is the type descriptor for local storage
const LocalStorageType Type = "local"

// LocalStorageConfig represents the configuration for a local storage
type LocalStorageConfig struct {
Path string `ini:"PATH"`
TemporaryPath string `ini:"TEMPORARY_PATH"`
}

// LocalStorage represents a local files storage
type LocalStorage struct {
ctx context.Context
@@ -34,13 +26,7 @@ type LocalStorage struct {
}

// NewLocalStorage returns a local files
func NewLocalStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) {
configInterface, err := toConfig(LocalStorageConfig{}, cfg)
if err != nil {
return nil, err
}
config := configInterface.(LocalStorageConfig)

func NewLocalStorage(ctx context.Context, config *setting.Storage) (ObjectStorage, error) {
if !filepath.IsAbs(config.Path) {
return nil, fmt.Errorf("LocalStorageConfig.Path should have been prepared by setting/storage.go and should be an absolute path, but not: %q", config.Path)
}
@@ -164,5 +150,5 @@ func (l *LocalStorage) IterateObjects(dirName string, fn func(path string, obj O
}

func init() {
RegisterStorageType(LocalStorageType, NewLocalStorage)
RegisterStorageType(setting.LocalStorageType, NewLocalStorage)
}

+ 3
- 1
modules/storage/local_test.go View File

@@ -8,6 +8,8 @@ import (
"path/filepath"
"testing"

"code.gitea.io/gitea/modules/setting"

"github.com/stretchr/testify/assert"
)

@@ -55,5 +57,5 @@ func TestBuildLocalPath(t *testing.T) {

func TestLocalStorageIterator(t *testing.T) {
dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir")
testStorageIterator(t, string(LocalStorageType), LocalStorageConfig{Path: dir})
testStorageIterator(t, setting.LocalStorageType, &setting.Storage{Path: dir})
}

+ 5
- 25
modules/storage/minio.go View File

@@ -16,6 +16,7 @@ import (
"time"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"

"github.com/minio/minio-go/v7"
@@ -41,25 +42,9 @@ func (m *minioObject) Stat() (os.FileInfo, error) {
return &minioFileInfo{oi}, nil
}

// MinioStorageType is the type descriptor for minio storage
const MinioStorageType Type = "minio"

// MinioStorageConfig represents the configuration for a minio storage
type MinioStorageConfig struct {
Endpoint string `ini:"MINIO_ENDPOINT"`
AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"`
SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"`
Bucket string `ini:"MINIO_BUCKET"`
Location string `ini:"MINIO_LOCATION"`
BasePath string `ini:"MINIO_BASE_PATH"`
UseSSL bool `ini:"MINIO_USE_SSL"`
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM"`
}

// MinioStorage returns a minio bucket storage
type MinioStorage struct {
cfg *MinioStorageConfig
cfg *setting.MinioStorageConfig
ctx context.Context
client *minio.Client
bucket string
@@ -87,13 +72,8 @@ func convertMinioErr(err error) error {
}

// NewMinioStorage returns a minio storage
func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) {
configInterface, err := toConfig(MinioStorageConfig{}, cfg)
if err != nil {
return nil, convertMinioErr(err)
}
config := configInterface.(MinioStorageConfig)

func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error) {
config := cfg.MinioConfig
if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
}
@@ -258,5 +238,5 @@ func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj O
}

func init() {
RegisterStorageType(MinioStorageType, NewMinioStorage)
RegisterStorageType(setting.MinioStorageType, NewMinioStorage)
}

+ 10
- 6
modules/storage/minio_test.go View File

@@ -6,6 +6,8 @@ package storage
import (
"os"
"testing"

"code.gitea.io/gitea/modules/setting"
)

func TestMinioStorageIterator(t *testing.T) {
@@ -13,11 +15,13 @@ func TestMinioStorageIterator(t *testing.T) {
t.Skip("minioStorage not present outside of CI")
return
}
testStorageIterator(t, string(MinioStorageType), MinioStorageConfig{
Endpoint: "127.0.0.1:9000",
AccessKeyID: "123456",
SecretAccessKey: "12345678",
Bucket: "gitea",
Location: "us-east-1",
testStorageIterator(t, setting.MinioStorageType, &setting.Storage{
MinioConfig: setting.MinioStorageConfig{
Endpoint: "127.0.0.1:9000",
AccessKeyID: "123456",
SecretAccessKey: "12345678",
Bucket: "gitea",
Location: "us-east-1",
},
})
}

+ 14
- 15
modules/storage/storage.go View File

@@ -37,16 +37,15 @@ func IsErrInvalidConfiguration(err error) bool {
return ok
}

// Type is a type of Storage
type Type string
type Type = setting.StorageType

// NewStorageFunc is a function that creates a storage
type NewStorageFunc func(ctx context.Context, cfg interface{}) (ObjectStorage, error)
type NewStorageFunc func(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error)

var storageMap = map[Type]NewStorageFunc{}

// RegisterStorageType registers a provided storage type with a function to create it
func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg interface{}) (ObjectStorage, error)) {
func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error)) {
storageMap[typ] = fn
}

@@ -151,11 +150,11 @@ func Init() error {
}

// NewStorage takes a storage type and some config and returns an ObjectStorage or an error
func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) {
func NewStorage(typStr Type, cfg *setting.Storage) (ObjectStorage, error) {
if len(typStr) == 0 {
typStr = string(LocalStorageType)
typStr = setting.LocalStorageType
}
fn, ok := storageMap[Type(typStr)]
fn, ok := storageMap[typStr]
if !ok {
return nil, fmt.Errorf("Unsupported storage type: %s", typStr)
}
@@ -165,7 +164,7 @@ func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) {

func initAvatars() (err error) {
log.Info("Initialising Avatar storage with type: %s", setting.Avatar.Storage.Type)
Avatars, err = NewStorage(setting.Avatar.Storage.Type, &setting.Avatar.Storage)
Avatars, err = NewStorage(setting.Avatar.Storage.Type, setting.Avatar.Storage)
return err
}

@@ -175,7 +174,7 @@ func initAttachments() (err error) {
return nil
}
log.Info("Initialising Attachment storage with type: %s", setting.Attachment.Storage.Type)
Attachments, err = NewStorage(setting.Attachment.Storage.Type, &setting.Attachment.Storage)
Attachments, err = NewStorage(setting.Attachment.Storage.Type, setting.Attachment.Storage)
return err
}

@@ -185,19 +184,19 @@ func initLFS() (err error) {
return nil
}
log.Info("Initialising LFS storage with type: %s", setting.LFS.Storage.Type)
LFS, err = NewStorage(setting.LFS.Storage.Type, &setting.LFS.Storage)
LFS, err = NewStorage(setting.LFS.Storage.Type, setting.LFS.Storage)
return err
}

func initRepoAvatars() (err error) {
log.Info("Initialising Repository Avatar storage with type: %s", setting.RepoAvatar.Storage.Type)
RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, &setting.RepoAvatar.Storage)
RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, setting.RepoAvatar.Storage)
return err
}

func initRepoArchives() (err error) {
log.Info("Initialising Repository Archive storage with type: %s", setting.RepoArchive.Storage.Type)
RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage)
RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, setting.RepoArchive.Storage)
return err
}

@@ -207,7 +206,7 @@ func initPackages() (err error) {
return nil
}
log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type)
Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage)
Packages, err = NewStorage(setting.Packages.Storage.Type, setting.Packages.Storage)
return err
}

@@ -218,10 +217,10 @@ func initActions() (err error) {
return nil
}
log.Info("Initialising Actions storage with type: %s", setting.Actions.LogStorage.Type)
if Actions, err = NewStorage(setting.Actions.LogStorage.Type, &setting.Actions.LogStorage); err != nil {
if Actions, err = NewStorage(setting.Actions.LogStorage.Type, setting.Actions.LogStorage); err != nil {
return err
}
log.Info("Initialising ActionsArtifacts storage with type: %s", setting.Actions.ArtifactStorage.Type)
ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, &setting.Actions.ArtifactStorage)
ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, setting.Actions.ArtifactStorage)
return err
}

+ 3
- 1
modules/storage/storage_test.go View File

@@ -7,10 +7,12 @@ import (
"bytes"
"testing"

"code.gitea.io/gitea/modules/setting"

"github.com/stretchr/testify/assert"
)

func testStorageIterator(t *testing.T, typStr string, cfg interface{}) {
func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) {
l, err := NewStorage(typStr, cfg)
assert.NoError(t, err)


+ 2
- 2
routers/api/v1/repo/file.go View File

@@ -200,7 +200,7 @@ func GetRawFileOrLFS(ctx *context.APIContext) {
return
}

if setting.LFS.ServeDirect {
if setting.LFS.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name())
if u != nil && err == nil {
@@ -320,7 +320,7 @@ func download(ctx *context.APIContext, archiveName string, archiver *repo_model.
downloadName := ctx.Repo.Repository.Name + "-" + archiveName

rPath := archiver.RelativePath()
if setting.RepoArchive.ServeDirect {
if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.RepoArchives.URL(rPath, downloadName)
if u != nil && err == nil {

+ 1
- 1
routers/install/install.go View File

@@ -116,7 +116,7 @@ func Install(ctx *context.Context) {
// Application general settings
form.AppName = setting.AppName
form.RepoRootPath = setting.RepoRootPath
form.LFSRootPath = setting.LFS.Path
form.LFSRootPath = setting.LFS.Storage.Path

// Note(unknown): it's hard for Windows users change a running user,
// so just use current one if config says default.

+ 2
- 2
routers/web/base.go View File

@@ -19,11 +19,11 @@ import (
"code.gitea.io/gitea/modules/web/routing"
)

func storageHandler(storageSetting setting.Storage, prefix string, objStore storage.ObjectStorage) func(next http.Handler) http.Handler {
func storageHandler(storageSetting *setting.Storage, prefix string, objStore storage.ObjectStorage) func(next http.Handler) http.Handler {
prefix = strings.Trim(prefix, "/")
funcInfo := routing.GetFuncInfo(storageHandler, prefix)
return func(next http.Handler) http.Handler {
if storageSetting.ServeDirect {
if storageSetting.MinioConfig.ServeDirect {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.Method != "GET" && req.Method != "HEAD" {
next.ServeHTTP(w, req)

+ 1
- 1
routers/web/repo/attachment.go View File

@@ -126,7 +126,7 @@ func ServeAttachment(ctx *context.Context, uuid string) {
return
}

if setting.Attachment.ServeDirect {
if setting.Attachment.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name)


+ 1
- 1
routers/web/repo/download.go View File

@@ -53,7 +53,7 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified time.Time
return nil
}

if setting.LFS.ServeDirect {
if setting.LFS.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name())
if u != nil && err == nil {

+ 1
- 1
routers/web/repo/repo.go View File

@@ -427,7 +427,7 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep
downloadName := ctx.Repo.Repository.Name + "-" + archiveName

rPath := archiver.RelativePath()
if setting.RepoArchive.ServeDirect {
if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.RepoArchives.URL(rPath, downloadName)
if u != nil && err == nil {

+ 1
- 1
services/lfs/server.go View File

@@ -452,7 +452,7 @@ func buildObjectResponse(rc *requestContext, pointer lfs_module.Pointer, downloa

if download {
var link *lfs_module.Link
if setting.LFS.ServeDirect {
if setting.LFS.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid)
if u != nil && err == nil {

+ 1
- 1
templates/admin/config.tmpl View File

@@ -102,7 +102,7 @@
<dd>{{if .LFS.StartServer}}{{svg "octicon-check"}}{{else}}{{svg "octicon-x"}}{{end}}</dd>
{{if .LFS.StartServer}}
<dt>{{.locale.Tr "admin.config.lfs_content_path"}}</dt>
<dd>{{.LFS.Path}}</dd>
<dd>{{JsonUtils.EncodeToString .LFS.Storage.ToShadowCopy}}</dd>
<dt>{{.locale.Tr "admin.config.lfs_http_auth_expiry"}}</dt>
<dd>{{.LFS.HTTPAuthExpiry}}</dd>
{{end}}

+ 3
- 1
tests/test_utils.go View File

@@ -214,7 +214,9 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() {

// load LFS object fixtures
// (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API)
lfsFixtures, err := storage.NewStorage("", storage.LocalStorageConfig{Path: path.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta")})
lfsFixtures, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{
Path: filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta"),
})
assert.NoError(t, err)
assert.NoError(t, storage.Clean(storage.LFS))
assert.NoError(t, lfsFixtures.IterateObjects("", func(path string, _ storage.Object) error {

Write Preview
Loading…
Cancel
Save