Browse Source

Fix all possible setting error related storages and added some tests (#23911) (#25244)

Backport #23911 by @lunny

Follow up #22405

Fix #20703 

This PR rewrites storage configuration read sequences with some breaks
and tests. It becomes more strict than before and also fixed some
inherit problems.

- Move storage's MinioConfig struct into setting, so after the
configuration loading, the values will be stored into the struct but not
still on some section.
- All storages configurations should be stored on one section,
configuration items cannot be overrided by multiple sections. The
prioioty of configuration is `[attachment]` > `[storage.attachments]` |
`[storage.customized]` > `[storage]` > `default`
- For extra override configuration items, currently are `SERVE_DIRECT`,
`MINIO_BASE_PATH`, `MINIO_BUCKET`, which could be configured in another
section. The prioioty of the override configuration is `[attachment]` >
`[storage.attachments]` > `default`.
- Add more tests for storages configurations.
- Update the storage documentations.

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
tags/v1.20.0-rc1
Giteabot 1 year ago
parent
commit
21cd5c2f3d
No account linked to committer's email address
41 changed files with 1154 additions and 454 deletions
  1. 3
    3
      cmd/dump.go
  2. 15
    13
      cmd/migrate_storage.go
  3. 2
    1
      cmd/migrate_storage_test.go
  4. 21
    0
      custom/conf/app.example.ini
  5. 57
    4
      docs/content/doc/administration/config-cheat-sheet.en-us.md
  6. 53
    4
      docs/content/doc/administration/config-cheat-sheet.zh-cn.md
  7. 1
    1
      models/migrations/v1_10/v96.go
  8. 1
    1
      models/migrations/v1_11/v112.go
  9. 4
    4
      models/migrations/v1_11/v115.go
  10. 17
    10
      modules/setting/actions.go
  11. 97
    0
      modules/setting/actions_test.go
  12. 12
    10
      modules/setting/attachment.go
  13. 133
    0
      modules/setting/attachment_test.go
  14. 19
    0
      modules/setting/config_provider.go
  15. 30
    26
      modules/setting/lfs.go
  16. 77
    0
      modules/setting/lfs_test.go
  17. 17
    9
      modules/setting/packages.go
  18. 167
    0
      modules/setting/packages_test.go
  19. 19
    11
      modules/setting/picture.go
  20. 3
    5
      modules/setting/repository.go
  21. 25
    0
      modules/setting/repository_archive.go
  22. 111
    0
      modules/setting/repository_archive_test.go
  23. 25
    9
      modules/setting/setting.go
  24. 152
    57
      modules/setting/storage.go
  25. 42
    154
      modules/setting/storage_test.go
  26. 0
    56
      modules/storage/helper.go
  27. 3
    17
      modules/storage/local.go
  28. 3
    1
      modules/storage/local_test.go
  29. 5
    25
      modules/storage/minio.go
  30. 10
    6
      modules/storage/minio_test.go
  31. 14
    15
      modules/storage/storage.go
  32. 3
    1
      modules/storage/storage_test.go
  33. 2
    2
      routers/api/v1/repo/file.go
  34. 1
    1
      routers/install/install.go
  35. 2
    2
      routers/web/base.go
  36. 1
    1
      routers/web/repo/attachment.go
  37. 1
    1
      routers/web/repo/download.go
  38. 1
    1
      routers/web/repo/repo.go
  39. 1
    1
      services/lfs/server.go
  40. 1
    1
      templates/admin/config.tmpl
  41. 3
    1
      tests/test_utils.go

+ 3
- 3
cmd/dump.go View File

} }


excludes = append(excludes, setting.RepoRootPath) excludes = append(excludes, setting.RepoRootPath)
excludes = append(excludes, setting.LFS.Path)
excludes = append(excludes, setting.Attachment.Path)
excludes = append(excludes, setting.Packages.Path)
excludes = append(excludes, setting.LFS.Storage.Path)
excludes = append(excludes, setting.Attachment.Storage.Path)
excludes = append(excludes, setting.Packages.Storage.Path)
excludes = append(excludes, setting.Log.RootPath) excludes = append(excludes, setting.Log.RootPath)
excludes = append(excludes, absFileName) excludes = append(excludes, absFileName)
if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil { if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {

+ 15
- 13
cmd/migrate_storage.go View File

switch strings.ToLower(ctx.String("storage")) { switch strings.ToLower(ctx.String("storage")) {
case "": case "":
fallthrough fallthrough
case string(storage.LocalStorageType):
case string(setting.LocalStorageType):
p := ctx.String("path") p := ctx.String("path")
if p == "" { if p == "" {
log.Fatal("Path must be given when storage is loal") log.Fatal("Path must be given when storage is loal")
} }
dstStorage, err = storage.NewLocalStorage( dstStorage, err = storage.NewLocalStorage(
stdCtx, stdCtx,
storage.LocalStorageConfig{
&setting.Storage{
Path: p, Path: p,
}) })
case string(storage.MinioStorageType):
case string(setting.MinioStorageType):
dstStorage, err = storage.NewMinioStorage( dstStorage, err = storage.NewMinioStorage(
stdCtx, stdCtx,
storage.MinioStorageConfig{
Endpoint: ctx.String("minio-endpoint"),
AccessKeyID: ctx.String("minio-access-key-id"),
SecretAccessKey: ctx.String("minio-secret-access-key"),
Bucket: ctx.String("minio-bucket"),
Location: ctx.String("minio-location"),
BasePath: ctx.String("minio-base-path"),
UseSSL: ctx.Bool("minio-use-ssl"),
InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"),
ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"),
&setting.Storage{
MinioConfig: setting.MinioStorageConfig{
Endpoint: ctx.String("minio-endpoint"),
AccessKeyID: ctx.String("minio-access-key-id"),
SecretAccessKey: ctx.String("minio-secret-access-key"),
Bucket: ctx.String("minio-bucket"),
Location: ctx.String("minio-location"),
BasePath: ctx.String("minio-base-path"),
UseSSL: ctx.Bool("minio-use-ssl"),
InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"),
ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"),
},
}) })
default: default:
return fmt.Errorf("unsupported storage type: %s", ctx.String("storage")) return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))

+ 2
- 1
cmd/migrate_storage_test.go View File

"code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
packages_module "code.gitea.io/gitea/modules/packages" packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/storage"
packages_service "code.gitea.io/gitea/services/packages" packages_service "code.gitea.io/gitea/services/packages"




dstStorage, err := storage.NewLocalStorage( dstStorage, err := storage.NewLocalStorage(
ctx, ctx,
storage.LocalStorageConfig{
&setting.Storage{
Path: p, Path: p,
}) })
assert.NoError(t, err) assert.NoError(t, err)

+ 21
- 0
custom/conf/app.example.ini View File

;; Enable/Disable package registry capabilities ;; Enable/Disable package registry capabilities
;ENABLED = true ;ENABLED = true
;; ;;
;STORAGE_TYPE = local
;; override the minio base path if storage type is minio
;MINIO_BASE_PATH = packages/
;;
;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload` ;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload`
;CHUNKED_UPLOAD_PATH = tmp/package-upload ;CHUNKED_UPLOAD_PATH = tmp/package-upload
;; ;;
;; storage type ;; storage type
;STORAGE_TYPE = local ;STORAGE_TYPE = local


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; repo-archive storage will override storage
;;
;[repo-archive]
;STORAGE_TYPE = local
;;
;; Where your lfs files reside, default is data/lfs.
;PATH = data/repo-archive
;;
;; override the minio base path if storage type is minio
;MINIO_BASE_PATH = repo-archive/

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; settings for repository archives, will override storage setting ;; settings for repository archives, will override storage setting
;; ;;
;; Where your lfs files reside, default is data/lfs. ;; Where your lfs files reside, default is data/lfs.
;PATH = data/lfs ;PATH = data/lfs
;;
;; override the minio base path if storage type is minio
;MINIO_BASE_PATH = lfs/


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; [actions] ; [actions]
;; Enable/Disable actions capabilities ;; Enable/Disable actions capabilities
;ENABLED = false ;ENABLED = false
;;
;; Default address to get action plugins, e.g. the default value means downloading from "https://gitea.com/actions/checkout" for "uses: actions/checkout@v3" ;; Default address to get action plugins, e.g. the default value means downloading from "https://gitea.com/actions/checkout" for "uses: actions/checkout@v3"
;DEFAULT_ACTIONS_URL = https://gitea.com ;DEFAULT_ACTIONS_URL = https://gitea.com



+ 57
- 4
docs/content/doc/administration/config-cheat-sheet.en-us.md View File



## Storage (`storage`) ## Storage (`storage`)


Default storage configuration for attachments, lfs, avatars and etc.
Default storage configuration for attachments, lfs, avatars, repo-avatars, repo-archive, packages, actions_log, actions_artifact.


- `STORAGE_TYPE`: **local**: Storage type, `local` for local disk or `minio` for s3 compatible object storage service.
- `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing. - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio` - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio` - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
- `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio`


And you can also define a customize storage like below:
The recommanded storage configuration for minio like below:


```ini ```ini
[storage.my_minio]
[storage]
STORAGE_TYPE = minio STORAGE_TYPE = minio
; Minio endpoint to connect only available when STORAGE_TYPE is `minio` ; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
MINIO_ENDPOINT = localhost:9000 MINIO_ENDPOINT = localhost:9000
MINIO_USE_SSL = false MINIO_USE_SSL = false
; Minio skip SSL verification available when STORAGE_TYPE is `minio` ; Minio skip SSL verification available when STORAGE_TYPE is `minio`
MINIO_INSECURE_SKIP_VERIFY = false MINIO_INSECURE_SKIP_VERIFY = false
SERVE_DIRECT = true
```

Defaultly every storage has their default base path like below

| storage | default base path |
| ----------------- | ------------------ |
| attachments | attachments/ |
| lfs | lfs/ |
| avatars | avatars/ |
| repo-avatars | repo-avatars/ |
| repo-archive | repo-archive/ |
| packages | packages/ |
| actions_log | actions_log/ |
| actions_artifacts | actions_artifacts/ |

And bucket, basepath or `SERVE_DIRECT` could be special or overrided, if you want to use a different you can:

```ini
[storage.actions_log]
MINIO_BUCKET = gitea_actions_log
SERVE_DIRECT = true
MINIO_BASE_PATH = my_actions_log/ ; default is actions_log/ if blank
``` ```


And used by `[attachment]`, `[lfs]` and etc. as `STORAGE_TYPE`.
If you want to customerize a different storage for `lfs` if above default storage defined

```ini
[lfs]
STORAGE_TYPE = my_minio

[storage.my_minio]
STORAGE_TYPE = minio
; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
MINIO_ENDPOINT = localhost:9000
; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
MINIO_ACCESS_KEY_ID =
; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
MINIO_SECRET_ACCESS_KEY =
; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
MINIO_BUCKET = gitea
; Minio location to create bucket only available when STORAGE_TYPE is `minio`
MINIO_LOCATION = us-east-1
; Minio enabled ssl only available when STORAGE_TYPE is `minio`
MINIO_USE_SSL = false
; Minio skip SSL verification available when STORAGE_TYPE is `minio`
MINIO_INSECURE_SKIP_VERIFY = false
```


## Repository Archive Storage (`storage.repo-archive`) ## Repository Archive Storage (`storage.repo-archive`)


- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
- `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio`


## Repository Archives (`repo-archive`)

- `STORAGE_TYPE`: **local**: Storage type for actions logs, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]`
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`

## Proxy (`proxy`) ## Proxy (`proxy`)


- `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy - `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy


- `ENABLED`: **false**: Enable/Disable actions capabilities - `ENABLED`: **false**: Enable/Disable actions capabilities
- `DEFAULT_ACTIONS_URL`: **https://gitea.com**: Default address to get action plugins, e.g. the default value means downloading from "<https://gitea.com/actions/checkout>" for "uses: actions/checkout@v3" - `DEFAULT_ACTIONS_URL`: **https://gitea.com**: Default address to get action plugins, e.g. the default value means downloading from "<https://gitea.com/actions/checkout>" for "uses: actions/checkout@v3"
- `STORAGE_TYPE`: **local**: Storage type for actions logs, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]`
- `MINIO_BASE_PATH`: **actions_log/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`


`DEFAULT_ACTIONS_URL` indicates where should we find the relative path action plugin. i.e. when use an action in a workflow file like `DEFAULT_ACTIONS_URL` indicates where should we find the relative path action plugin. i.e. when use an action in a workflow file like



+ 53
- 4
docs/content/doc/administration/config-cheat-sheet.zh-cn.md View File



## Storage (`storage`) ## Storage (`storage`)


Attachments, lfs, avatars and etc 的默认存储配置。
Attachments, lfs, avatars, repo-avatars, repo-archive, packages, actions_log, actions_artifact 的默认存储配置。


- `STORAGE_TYPE`: **local**: 附件存储类型,`local` 将存储到本地文件夹, `minio` 将存储到 s3 兼容的对象存储服务中。 - `STORAGE_TYPE`: **local**: 附件存储类型,`local` 将存储到本地文件夹, `minio` 将存储到 s3 兼容的对象存储服务中。
- `SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前,仅 Minio/S3 是支持的。 - `SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前,仅 Minio/S3 是支持的。
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket,仅当 `STORAGE_TYPE` 是 `minio` 时有效。 - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket,仅当 `STORAGE_TYPE` 是 `minio` 时有效。
- `MINIO_USE_SSL`: **false**: Minio enabled ssl,仅当 `STORAGE_TYPE` 是 `minio` 时有效。 - `MINIO_USE_SSL`: **false**: Minio enabled ssl,仅当 `STORAGE_TYPE` 是 `minio` 时有效。


你也可以自定义一个存储的名字如下:
以下为推荐的 recommanded storage configuration for minio like below:


```ini ```ini
[storage.my_minio]
[storage]
STORAGE_TYPE = minio STORAGE_TYPE = minio
; uncomment when STORAGE_TYPE = local
; PATH = storage root path
; Minio endpoint to connect only available when STORAGE_TYPE is `minio` ; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
MINIO_ENDPOINT = localhost:9000 MINIO_ENDPOINT = localhost:9000
; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio` ; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
MINIO_USE_SSL = false MINIO_USE_SSL = false
; Minio skip SSL verification available when STORAGE_TYPE is `minio` ; Minio skip SSL verification available when STORAGE_TYPE is `minio`
MINIO_INSECURE_SKIP_VERIFY = false MINIO_INSECURE_SKIP_VERIFY = false
SERVE_DIRECT = true
```

默认的,每一个存储都会有各自默认的 BasePath 在同一个minio中,默认值如下:

| storage | default base path |
| ----------------- | ------------------ |
| attachments | attachments/ |
| lfs | lfs/ |
| avatars | avatars/ |
| repo-avatars | repo-avatars/ |
| repo-archive | repo-archive/ |
| packages | packages/ |
| actions_log | actions_log/ |
| actions_artifacts | actions_artifacts/ |

同时 bucket, basepath or `SERVE_DIRECT` 是可以被覆写的,像如下所示:

```ini
[storage.actions_log]
MINIO_BUCKET = gitea_actions_log
SERVE_DIRECT = true
MINIO_BASE_PATH = my_actions_log/ ; default is actions_log/ if blank
``` ```


然后你在 `[attachment]`, `[lfs]` 等中可以把这个名字用作 `STORAGE_TYPE` 的值。
当然你也可以完全自定义,像如下

```ini
[lfs]
STORAGE_TYPE = my_minio
MINIO_BASE_PATH = my_lfs_basepath

[storage.my_minio]
STORAGE_TYPE = minio
; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
MINIO_ENDPOINT = localhost:9000
; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
MINIO_ACCESS_KEY_ID =
; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
MINIO_SECRET_ACCESS_KEY =
; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
MINIO_BUCKET = gitea
; Minio location to create bucket only available when STORAGE_TYPE is `minio`
MINIO_LOCATION = us-east-1
; Minio enabled ssl only available when STORAGE_TYPE is `minio`
MINIO_USE_SSL = false
; Minio skip SSL verification available when STORAGE_TYPE is `minio`
MINIO_INSECURE_SKIP_VERIFY = false
SERVE_DIRECT = true
```


## Repository Archive Storage (`storage.repo-archive`) ## Repository Archive Storage (`storage.repo-archive`)



+ 1
- 1
models/migrations/v1_10/v96.go View File



for _, attachment := range attachments { for _, attachment := range attachments {
uuid := attachment.UUID uuid := attachment.UUID
if err := util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
if err := util.RemoveAll(filepath.Join(setting.Attachment.Storage.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
return err return err
} }
} }

+ 1
- 1
models/migrations/v1_11/v112.go View File



for i := 0; i < len(attachments); i++ { for i := 0; i < len(attachments); i++ {
uuid := attachments[i].UUID uuid := attachments[i].UUID
if err = util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
if err = util.RemoveAll(filepath.Join(setting.Attachment.Storage.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
fmt.Printf("Error: %v", err) //nolint:forbidigo fmt.Printf("Error: %v", err) //nolint:forbidigo
} }
} }

+ 4
- 4
models/migrations/v1_11/v115.go View File

for _, user := range users { for _, user := range users {
oldAvatar := user.Avatar oldAvatar := user.Avatar


if stat, err := os.Stat(filepath.Join(setting.Avatar.Path, oldAvatar)); err != nil || !stat.Mode().IsRegular() {
if stat, err := os.Stat(filepath.Join(setting.Avatar.Storage.Path, oldAvatar)); err != nil || !stat.Mode().IsRegular() {
if err == nil { if err == nil {
err = fmt.Errorf("Error: \"%s\" is not a regular file", oldAvatar) err = fmt.Errorf("Error: \"%s\" is not a regular file", oldAvatar)
} }
return fmt.Errorf("[user: %s] user table update: %w", user.LowerName, err) return fmt.Errorf("[user: %s] user table update: %w", user.LowerName, err)
} }


deleteList.Add(filepath.Join(setting.Avatar.Path, oldAvatar))
deleteList.Add(filepath.Join(setting.Avatar.Storage.Path, oldAvatar))
migrated++ migrated++
select { select {
case <-ticker.C: case <-ticker.C:
// copyOldAvatarToNewLocation copies oldAvatar to newAvatarLocation // copyOldAvatarToNewLocation copies oldAvatar to newAvatarLocation
// and returns newAvatar location // and returns newAvatar location
func copyOldAvatarToNewLocation(userID int64, oldAvatar string) (string, error) { func copyOldAvatarToNewLocation(userID int64, oldAvatar string) (string, error) {
fr, err := os.Open(filepath.Join(setting.Avatar.Path, oldAvatar))
fr, err := os.Open(filepath.Join(setting.Avatar.Storage.Path, oldAvatar))
if err != nil { if err != nil {
return "", fmt.Errorf("os.Open: %w", err) return "", fmt.Errorf("os.Open: %w", err)
} }
return newAvatar, nil return newAvatar, nil
} }


if err := os.WriteFile(filepath.Join(setting.Avatar.Path, newAvatar), data, 0o666); err != nil {
if err := os.WriteFile(filepath.Join(setting.Avatar.Storage.Path, newAvatar), data, 0o666); err != nil {
return "", fmt.Errorf("os.WriteFile: %w", err) return "", fmt.Errorf("os.WriteFile: %w", err)
} }



+ 17
- 10
modules/setting/actions.go View File

package setting package setting


import ( import (
"code.gitea.io/gitea/modules/log"
"fmt"
) )


// Actions settings // Actions settings
var ( var (
Actions = struct { Actions = struct {
LogStorage Storage // how the created logs should be stored
ArtifactStorage Storage // how the created artifacts should be stored
LogStorage *Storage // how the created logs should be stored
ArtifactStorage *Storage // how the created artifacts should be stored
Enabled bool Enabled bool
DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"` DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"`
}{ }{
} }
) )


func loadActionsFrom(rootCfg ConfigProvider) {
func loadActionsFrom(rootCfg ConfigProvider) error {
sec := rootCfg.Section("actions") sec := rootCfg.Section("actions")
if err := sec.MapTo(&Actions); err != nil {
log.Fatal("Failed to map Actions settings: %v", err)
err := sec.MapTo(&Actions)
if err != nil {
return fmt.Errorf("failed to map Actions settings: %v", err)
} }


actionsSec := rootCfg.Section("actions.artifacts")
storageType := actionsSec.Key("STORAGE_TYPE").MustString("")
// don't support to read configuration from [actions]
Actions.LogStorage, err = getStorage(rootCfg, "actions_log", "", nil)
if err != nil {
return err
}

actionsSec, _ := rootCfg.GetSection("actions.artifacts")

Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec)


Actions.LogStorage = getStorage(rootCfg, "actions_log", "", nil)
Actions.ArtifactStorage = getStorage(rootCfg, "actions_artifacts", storageType, actionsSec)
return err
} }

+ 97
- 0
modules/setting/actions_test.go View File

// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import (
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
)

func Test_getStorageInheritNameSectionTypeForActions(t *testing.T) {
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "minio", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath)
assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath)

iniStr = `
[storage.actions_log]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "minio", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath)
assert.EqualValues(t, "local", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path))

iniStr = `
[storage.actions_log]
STORAGE_TYPE = my_storage

[storage.my_storage]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "minio", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath)
assert.EqualValues(t, "local", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path))

iniStr = `
[storage.actions_artifacts]
STORAGE_TYPE = my_storage

[storage.my_storage]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "local", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path))
assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath)

iniStr = `
[storage.actions_artifacts]
STORAGE_TYPE = my_storage

[storage.my_storage]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "local", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path))
assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath)

iniStr = ``
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadActionsFrom(cfg))

assert.EqualValues(t, "local", Actions.LogStorage.Type)
assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path))
assert.EqualValues(t, "local", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path))
}

+ 12
- 10
modules/setting/attachment.go View File



// Attachment settings // Attachment settings
var Attachment = struct { var Attachment = struct {
Storage
Storage *Storage
AllowedTypes string AllowedTypes string
MaxSize int64 MaxSize int64
MaxFiles int MaxFiles int
Enabled bool Enabled bool
}{ }{
Storage: Storage{
ServeDirect: false,
},
AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
Storage: &Storage{},
AllowedTypes: ".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip",
MaxSize: 4, MaxSize: 4,
MaxFiles: 5, MaxFiles: 5,
Enabled: true, Enabled: true,
} }


func loadAttachmentFrom(rootCfg ConfigProvider) {
sec := rootCfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")

Attachment.Storage = getStorage(rootCfg, "attachments", storageType, sec)
func loadAttachmentFrom(rootCfg ConfigProvider) (err error) {
sec, _ := rootCfg.GetSection("attachment")
if sec == nil {
Attachment.Storage, err = getStorage(rootCfg, "attachments", "", nil)
return err
}


Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip") Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip")
Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4) Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4)
Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5) Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5)
Attachment.Enabled = sec.Key("ENABLED").MustBool(true) Attachment.Enabled = sec.Key("ENABLED").MustBool(true)

Attachment.Storage, err = getStorage(rootCfg, "attachments", "", sec)
return err
} }

+ 133
- 0
modules/setting/attachment_test.go View File

// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import (
"testing"

"github.com/stretchr/testify/assert"
)

func Test_getStorageCustomType(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = my_minio
MINIO_BUCKET = gitea-attachment

[storage.my_minio]
STORAGE_TYPE = minio
MINIO_ENDPOINT = my_minio:9000
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

assert.EqualValues(t, "minio", Attachment.Storage.Type)
assert.EqualValues(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint)
assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath)
}

func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio

[storage.minio]
MINIO_BUCKET = gitea-minio

[storage]
MINIO_BUCKET = gitea
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

assert.EqualValues(t, "minio", Attachment.Storage.Type)
assert.EqualValues(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath)
}

func Test_getStorageSpecificOverridesStorage(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio
MINIO_BUCKET = gitea-attachment

[storage.attachments]
MINIO_BUCKET = gitea

[storage]
STORAGE_TYPE = local
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

assert.EqualValues(t, "minio", Attachment.Storage.Type)
assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath)
}

func Test_getStorageGetDefaults(t *testing.T) {
cfg, err := NewConfigProviderFromData("")
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

// default storage is local, so bucket is empty
assert.EqualValues(t, "", Attachment.Storage.MinioConfig.Bucket)
}

func Test_getStorageInheritNameSectionType(t *testing.T) {
iniStr := `
[storage.attachments]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))

assert.EqualValues(t, "minio", Attachment.Storage.Type)
}

func Test_AttachmentStorage(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))
storage := Attachment.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
}

func Test_AttachmentStorage1(t *testing.T) {
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadAttachmentFrom(cfg))
assert.EqualValues(t, "minio", Attachment.Storage.Type)
assert.EqualValues(t, "gitea", Attachment.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath)
}

+ 19
- 0
modules/setting/config_provider.go View File

"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"time" "time"


return "" return ""
} }


func ConfigSectionKeyBool(sec ConfigSection, key string, def ...bool) bool {
k := ConfigSectionKey(sec, key)
if k != nil && k.String() != "" {
b, _ := strconv.ParseBool(k.String())
return b
}
if len(def) > 0 {
return def[0]
}
return false
}

// ConfigInheritedKey works like ini.Section.Key(), but it always returns a new key instance, it is O(n) because NewKey is O(n) // ConfigInheritedKey works like ini.Section.Key(), but it always returns a new key instance, it is O(n) because NewKey is O(n)
// and the returned key is safe to be used with "MustXxx", it doesn't change the parent's values. // and the returned key is safe to be used with "MustXxx", it doesn't change the parent's values.
// Otherwise, ini.Section.Key().MustXxx would pollute the parent section's keys. // Otherwise, ini.Section.Key().MustXxx would pollute the parent section's keys.
} }
} }


func deprecatedSettingFatal(rootCfg ConfigProvider, oldSection, oldKey, newSection, newKey, version string) {
if rootCfg.Section(oldSection).HasKey(oldKey) {
log.Fatal("Deprecated fallback `[%s]` `%s` present. Use `[%s]` `%s` instead. This fallback will be/has been removed in %s", oldSection, oldKey, newSection, newKey, version)
}
}

// deprecatedSettingDB add a hint that the configuration has been moved to database but still kept in app.ini // deprecatedSettingDB add a hint that the configuration has been moved to database but still kept in app.ini
func deprecatedSettingDB(rootCfg ConfigProvider, oldSection, oldKey string) { func deprecatedSettingDB(rootCfg ConfigProvider, oldSection, oldKey string) {
if rootCfg.Section(oldSection).HasKey(oldKey) { if rootCfg.Section(oldSection).HasKey(oldKey) {

+ 30
- 26
modules/setting/lfs.go View File



import ( import (
"encoding/base64" "encoding/base64"
"fmt"
"time" "time"


"code.gitea.io/gitea/modules/generate" "code.gitea.io/gitea/modules/generate"
"code.gitea.io/gitea/modules/log"
) )


// LFS represents the configuration for Git LFS // LFS represents the configuration for Git LFS
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"` MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`
LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"` LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"`


Storage
Storage *Storage
}{} }{}


func loadLFSFrom(rootCfg ConfigProvider) {
func loadLFSFrom(rootCfg ConfigProvider) error {
sec := rootCfg.Section("server") sec := rootCfg.Section("server")
if err := sec.MapTo(&LFS); err != nil { if err := sec.MapTo(&LFS); err != nil {
log.Fatal("Failed to map LFS settings: %v", err)
return fmt.Errorf("failed to map LFS settings: %v", err)
} }


lfsSec := rootCfg.Section("lfs")
storageType := lfsSec.Key("STORAGE_TYPE").MustString("")
lfsSec, _ := rootCfg.GetSection("lfs")


// Specifically default PATH to LFS_CONTENT_PATH // Specifically default PATH to LFS_CONTENT_PATH
// DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version // DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version
// if these are removed, the warning will not be shown // if these are removed, the warning will not be shown
deprecatedSetting(rootCfg, "server", "LFS_CONTENT_PATH", "lfs", "PATH", "v1.19.0")
lfsSec.Key("PATH").MustString(sec.Key("LFS_CONTENT_PATH").String())
deprecatedSettingFatal(rootCfg, "server", "LFS_CONTENT_PATH", "lfs", "PATH", "v1.19.0")


LFS.Storage = getStorage(rootCfg, "lfs", storageType, lfsSec)
var err error
LFS.Storage, err = getStorage(rootCfg, "lfs", "", lfsSec)
if err != nil {
return err
}


// Rest of LFS service settings // Rest of LFS service settings
if LFS.LocksPagingNum == 0 { if LFS.LocksPagingNum == 0 {


LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(24 * time.Hour) LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(24 * time.Hour)


if LFS.StartServer {
LFS.JWTSecretBytes = make([]byte, 32)
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
if err != nil || n != 32 {
LFS.JWTSecretBase64, err = generate.NewJwtSecretBase64()
if err != nil {
log.Fatal("Error generating JWT Secret for custom config: %v", err)
return
}
// Save secret
sec.Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
if err := rootCfg.Save(); err != nil {
log.Fatal("Error saving JWT Secret for custom config: %v", err)
return
}
if !LFS.StartServer {
return nil
}
LFS.JWTSecretBytes = make([]byte, 32)
n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
if err != nil || n != 32 {
LFS.JWTSecretBase64, err = generate.NewJwtSecretBase64()
if err != nil {
return fmt.Errorf("Error generating JWT Secret for custom config: %v", err)
}
// Save secret
sec.Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
if err := rootCfg.Save(); err != nil {
return fmt.Errorf("Error saving JWT Secret for custom config: %v", err)
} }
} }

return nil
} }

+ 77
- 0
modules/setting/lfs_test.go View File

// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import (
"testing"

"github.com/stretchr/testify/assert"
)

func Test_getStorageInheritNameSectionTypeForLFS(t *testing.T) {
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))

assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath)

iniStr = `
[storage.lfs]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))

assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath)

iniStr = `
[lfs]
STORAGE_TYPE = my_minio

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))

assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath)

iniStr = `
[lfs]
STORAGE_TYPE = my_minio
MINIO_BASE_PATH = my_lfs/

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadLFSFrom(cfg))

assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath)
}

func Test_LFSStorage1(t *testing.T) {
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadLFSFrom(cfg))
assert.EqualValues(t, "minio", LFS.Storage.Type)
assert.EqualValues(t, "gitea", LFS.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath)
}

+ 17
- 9
modules/setting/packages.go View File

package setting package setting


import ( import (
"fmt"
"math" "math"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"


"code.gitea.io/gitea/modules/log"

"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
) )


// Package registry settings // Package registry settings
var ( var (
Packages = struct { Packages = struct {
Storage
Storage *Storage
Enabled bool Enabled bool
ChunkedUploadPath string ChunkedUploadPath string
RegistryHost string RegistryHost string
} }
) )


func loadPackagesFrom(rootCfg ConfigProvider) {
sec := rootCfg.Section("packages")
if err := sec.MapTo(&Packages); err != nil {
log.Fatal("Failed to map Packages settings: %v", err)
func loadPackagesFrom(rootCfg ConfigProvider) (err error) {
sec, _ := rootCfg.GetSection("packages")
if sec == nil {
Packages.Storage, err = getStorage(rootCfg, "packages", "", nil)
return err
}

if err = sec.MapTo(&Packages); err != nil {
return fmt.Errorf("failed to map Packages settings: %v", err)
} }


Packages.Storage = getStorage(rootCfg, "packages", "", nil)
Packages.Storage, err = getStorage(rootCfg, "packages", "", sec)
if err != nil {
return err
}


appURL, _ := url.Parse(AppURL) appURL, _ := url.Parse(AppURL)
Packages.RegistryHost = appURL.Host Packages.RegistryHost = appURL.Host
} }


if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil {
log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
return fmt.Errorf("unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
} }


Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS") Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS")
Packages.LimitSizeSwift = mustBytes(sec, "LIMIT_SIZE_SWIFT") Packages.LimitSizeSwift = mustBytes(sec, "LIMIT_SIZE_SWIFT")
Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT") Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT")
return nil
} }


func mustBytes(section ConfigSection, key string) int64 { func mustBytes(section ConfigSection, key string) int64 {

+ 167
- 0
modules/setting/packages_test.go View File

assert.EqualValues(t, 1782579, test("1.7mib")) assert.EqualValues(t, 1782579, test("1.7mib"))
assert.EqualValues(t, -1, test("1 yib")) // too large assert.EqualValues(t, -1, test("1 yib")) // too large
} }

func Test_getStorageInheritNameSectionTypeForPackages(t *testing.T) {
// packages storage inherits from storage if nothing configured
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadPackagesFrom(cfg))

assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath)

// we can also configure packages storage directly
iniStr = `
[storage.packages]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadPackagesFrom(cfg))

assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath)

// or we can indicate the storage type in the packages section
iniStr = `
[packages]
STORAGE_TYPE = my_minio

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadPackagesFrom(cfg))

assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath)

// or we can indicate the storage type and minio base path in the packages section
iniStr = `
[packages]
STORAGE_TYPE = my_minio
MINIO_BASE_PATH = my_packages/

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadPackagesFrom(cfg))

assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "my_packages/", Packages.Storage.MinioConfig.BasePath)
}

func Test_PackageStorage1(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[packages]
MINIO_BASE_PATH = packages/
SERVE_DIRECT = true
[storage]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadPackagesFrom(cfg))
storage := Packages.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath)
assert.True(t, storage.MinioConfig.ServeDirect)
}

func Test_PackageStorage2(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage.packages]
MINIO_BASE_PATH = packages/
SERVE_DIRECT = true
[storage]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadPackagesFrom(cfg))
storage := Packages.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath)
assert.True(t, storage.MinioConfig.ServeDirect)
}

func Test_PackageStorage3(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[packages]
STORAGE_TYPE = my_cfg
MINIO_BASE_PATH = my_packages/
SERVE_DIRECT = true
[storage.my_cfg]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadPackagesFrom(cfg))
storage := Packages.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath)
assert.True(t, storage.MinioConfig.ServeDirect)
}

func Test_PackageStorage4(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage.packages]
STORAGE_TYPE = my_cfg
MINIO_BASE_PATH = my_packages/
SERVE_DIRECT = true
[storage.my_cfg]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadPackagesFrom(cfg))
storage := Packages.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath)
assert.True(t, storage.MinioConfig.ServeDirect)
}

+ 19
- 11
modules/setting/picture.go View File



var ( var (
Avatar = struct { Avatar = struct {
Storage
Storage *Storage


MaxWidth int MaxWidth int
MaxHeight int MaxHeight int
EnableFederatedAvatar bool // Depreciated: migrated to database EnableFederatedAvatar bool // Depreciated: migrated to database


RepoAvatar = struct { RepoAvatar = struct {
Storage
Storage *Storage


Fallback string Fallback string
FallbackImage string FallbackImage string
}{} }{}
) )


func loadPictureFrom(rootCfg ConfigProvider) {
func loadAvatarsFrom(rootCfg ConfigProvider) error {
sec := rootCfg.Section("picture") sec := rootCfg.Section("picture")


avatarSec := rootCfg.Section("avatar") avatarSec := rootCfg.Section("avatar")
storageType := sec.Key("AVATAR_STORAGE_TYPE").MustString("") storageType := sec.Key("AVATAR_STORAGE_TYPE").MustString("")
// Specifically default PATH to AVATAR_UPLOAD_PATH // Specifically default PATH to AVATAR_UPLOAD_PATH
avatarSec.Key("PATH").MustString(
sec.Key("AVATAR_UPLOAD_PATH").String())
avatarSec.Key("PATH").MustString(sec.Key("AVATAR_UPLOAD_PATH").String())


Avatar.Storage = getStorage(rootCfg, "avatars", storageType, avatarSec)
var err error
Avatar.Storage, err = getStorage(rootCfg, "avatars", storageType, avatarSec)
if err != nil {
return err
}


Avatar.MaxWidth = sec.Key("AVATAR_MAX_WIDTH").MustInt(4096) Avatar.MaxWidth = sec.Key("AVATAR_MAX_WIDTH").MustInt(4096)
Avatar.MaxHeight = sec.Key("AVATAR_MAX_HEIGHT").MustInt(4096) Avatar.MaxHeight = sec.Key("AVATAR_MAX_HEIGHT").MustInt(4096)
EnableFederatedAvatar = sec.Key("ENABLE_FEDERATED_AVATAR").MustBool(GetDefaultEnableFederatedAvatar(DisableGravatar)) EnableFederatedAvatar = sec.Key("ENABLE_FEDERATED_AVATAR").MustBool(GetDefaultEnableFederatedAvatar(DisableGravatar))
deprecatedSettingDB(rootCfg, "", "ENABLE_FEDERATED_AVATAR") deprecatedSettingDB(rootCfg, "", "ENABLE_FEDERATED_AVATAR")


loadRepoAvatarFrom(rootCfg)
return nil
} }


func GetDefaultDisableGravatar() bool { func GetDefaultDisableGravatar() bool {
return v return v
} }


func loadRepoAvatarFrom(rootCfg ConfigProvider) {
func loadRepoAvatarFrom(rootCfg ConfigProvider) error {
sec := rootCfg.Section("picture") sec := rootCfg.Section("picture")


repoAvatarSec := rootCfg.Section("repo-avatar") repoAvatarSec := rootCfg.Section("repo-avatar")
storageType := sec.Key("REPOSITORY_AVATAR_STORAGE_TYPE").MustString("") storageType := sec.Key("REPOSITORY_AVATAR_STORAGE_TYPE").MustString("")
// Specifically default PATH to AVATAR_UPLOAD_PATH // Specifically default PATH to AVATAR_UPLOAD_PATH
repoAvatarSec.Key("PATH").MustString(
sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String())
repoAvatarSec.Key("PATH").MustString(sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String())


RepoAvatar.Storage = getStorage(rootCfg, "repo-avatars", storageType, repoAvatarSec)
var err error
RepoAvatar.Storage, err = getStorage(rootCfg, "repo-avatars", storageType, repoAvatarSec)
if err != nil {
return err
}


RepoAvatar.Fallback = sec.Key("REPOSITORY_AVATAR_FALLBACK").MustString("none") RepoAvatar.Fallback = sec.Key("REPOSITORY_AVATAR_FALLBACK").MustString("none")
RepoAvatar.FallbackImage = sec.Key("REPOSITORY_AVATAR_FALLBACK_IMAGE").MustString(AppSubURL + "/assets/img/repo_default.png") RepoAvatar.FallbackImage = sec.Key("REPOSITORY_AVATAR_FALLBACK_IMAGE").MustString(AppSubURL + "/assets/img/repo_default.png")

return nil
} }

+ 3
- 5
modules/setting/repository.go View File

} }
RepoRootPath string RepoRootPath string
ScriptType = "bash" ScriptType = "bash"

RepoArchive = struct {
Storage
}{}
) )


func loadRepositoryFrom(rootCfg ConfigProvider) { func loadRepositoryFrom(rootCfg ConfigProvider) {
Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath) Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath)
} }


RepoArchive.Storage = getStorage(rootCfg, "repo-archive", "", nil)
if err := loadRepoArchiveFrom(rootCfg); err != nil {
log.Fatal("loadRepoArchiveFrom: %v", err)
}
} }

+ 25
- 0
modules/setting/repository_archive.go View File

// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import "fmt"

var RepoArchive = struct {
Storage *Storage
}{}

func loadRepoArchiveFrom(rootCfg ConfigProvider) (err error) {
sec, _ := rootCfg.GetSection("repo-archive")
if sec == nil {
RepoArchive.Storage, err = getStorage(rootCfg, "repo-archive", "", nil)
return err
}

if err := sec.MapTo(&RepoArchive); err != nil {
return fmt.Errorf("mapto repoarchive failed: %v", err)
}

RepoArchive.Storage, err = getStorage(rootCfg, "repo-archive", "", sec)
return err
}

+ 111
- 0
modules/setting/repository_archive_test.go View File

// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import (
"testing"

"github.com/stretchr/testify/assert"
)

func Test_getStorageInheritNameSectionTypeForRepoArchive(t *testing.T) {
// packages storage inherits from storage if nothing configured
iniStr := `
[storage]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))

assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath)

// we can also configure packages storage directly
iniStr = `
[storage.repo-archive]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))

assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath)

// or we can indicate the storage type in the packages section
iniStr = `
[repo-archive]
STORAGE_TYPE = my_minio

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))

assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath)

// or we can indicate the storage type and minio base path in the packages section
iniStr = `
[repo-archive]
STORAGE_TYPE = my_minio
MINIO_BASE_PATH = my_archive/

[storage.my_minio]
STORAGE_TYPE = minio
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))

assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath)
}

func Test_RepoArchiveStorage(t *testing.T) {
iniStr := `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadRepoArchiveFrom(cfg))
storage := RepoArchive.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)

iniStr = `
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
[storage.repo-archive]
STORAGE_TYPE = s3
[storage.s3]
STORAGE_TYPE = minio
MINIO_ENDPOINT = s3.my-domain.net
MINIO_BUCKET = gitea
MINIO_LOCATION = homenet
MINIO_USE_SSL = true
MINIO_ACCESS_KEY_ID = correct_key
MINIO_SECRET_ACCESS_KEY = correct_key
`
cfg, err = NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

assert.NoError(t, loadRepoArchiveFrom(cfg))
storage = RepoArchive.Storage

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket)
}

+ 25
- 9
modules/setting/setting.go View File

var err error var err error
CfgProvider, err = NewConfigProviderFromFile(opts) CfgProvider, err = NewConfigProviderFromFile(opts)
if err != nil { if err != nil {
log.Fatal("Init[%v]: %v", opts, err)
log.Fatal("newConfigProviderFromFile[%v]: %v", opts, err)
} }
if !opts.DisableLoadCommonSettings { if !opts.DisableLoadCommonSettings {
loadCommonSettingsFrom(CfgProvider)
if err := loadCommonSettingsFrom(CfgProvider); err != nil {
log.Fatal("loadCommonSettingsFrom[%v]: %v", opts, err)
}
} }
} }


// loadCommonSettingsFrom loads common configurations from a configuration provider. // loadCommonSettingsFrom loads common configurations from a configuration provider.
func loadCommonSettingsFrom(cfg ConfigProvider) {
// WARNING: don't change the sequence except you know what you are doing.
func loadCommonSettingsFrom(cfg ConfigProvider) error {
// WARNNING: don't change the sequence except you know what you are doing.
loadRunModeFrom(cfg) loadRunModeFrom(cfg)
loadLogGlobalFrom(cfg) loadLogGlobalFrom(cfg)
loadServerFrom(cfg) loadServerFrom(cfg)


loadOAuth2From(cfg) loadOAuth2From(cfg)
loadSecurityFrom(cfg) loadSecurityFrom(cfg)
loadAttachmentFrom(cfg)
loadLFSFrom(cfg)
if err := loadAttachmentFrom(cfg); err != nil {
return err
}
if err := loadLFSFrom(cfg); err != nil {
return err
}
loadTimeFrom(cfg) loadTimeFrom(cfg)
loadRepositoryFrom(cfg) loadRepositoryFrom(cfg)
loadPictureFrom(cfg)
loadPackagesFrom(cfg)
loadActionsFrom(cfg)
if err := loadAvatarsFrom(cfg); err != nil {
return err
}
if err := loadRepoAvatarFrom(cfg); err != nil {
return err
}
if err := loadPackagesFrom(cfg); err != nil {
return err
}
if err := loadActionsFrom(cfg); err != nil {
return err
}
loadUIFrom(cfg) loadUIFrom(cfg)
loadAdminFrom(cfg) loadAdminFrom(cfg)
loadAPIFrom(cfg) loadAPIFrom(cfg)
loadMirrorFrom(cfg) loadMirrorFrom(cfg)
loadMarkupFrom(cfg) loadMarkupFrom(cfg)
loadOtherFrom(cfg) loadOtherFrom(cfg)
return nil
} }


func loadRunModeFrom(rootCfg ConfigProvider) { func loadRunModeFrom(rootCfg ConfigProvider) {

+ 152
- 57
modules/setting/storage.go View File

package setting package setting


import ( import (
"errors"
"fmt"
"path/filepath" "path/filepath"
"reflect"
) )


// StorageType is a type of Storage
type StorageType string

const (
// LocalStorageType is the type descriptor for local storage
LocalStorageType StorageType = "local"
// MinioStorageType is the type descriptor for minio storage
MinioStorageType StorageType = "minio"
)

var storageTypes = []StorageType{
LocalStorageType,
MinioStorageType,
}

// IsValidStorageType returns true if the given storage type is valid
func IsValidStorageType(storageType StorageType) bool {
for _, t := range storageTypes {
if t == storageType {
return true
}
}
return false
}

// MinioStorageConfig represents the configuration for a minio storage
type MinioStorageConfig struct {
Endpoint string `ini:"MINIO_ENDPOINT" json:",omitempty"`
AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID" json:",omitempty"`
SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY" json:",omitempty"`
Bucket string `ini:"MINIO_BUCKET" json:",omitempty"`
Location string `ini:"MINIO_LOCATION" json:",omitempty"`
BasePath string `ini:"MINIO_BASE_PATH" json:",omitempty"`
UseSSL bool `ini:"MINIO_USE_SSL"`
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM" json:",omitempty"`
ServeDirect bool `ini:"SERVE_DIRECT"`
}

// Storage represents configuration of storages // Storage represents configuration of storages
type Storage struct { type Storage struct {
Type string
Path string
Section ConfigSection
ServeDirect bool
Type StorageType // local or minio
Path string `json:",omitempty"` // for local type
TemporaryPath string `json:",omitempty"`
MinioConfig MinioStorageConfig // for minio type
} }


// MapTo implements the Mappable interface
func (s *Storage) MapTo(v interface{}) error {
pathValue := reflect.ValueOf(v).Elem().FieldByName("Path")
if pathValue.IsValid() && pathValue.Kind() == reflect.String {
pathValue.SetString(s.Path)
func (storage *Storage) ToShadowCopy() Storage {
shadowStorage := *storage
if shadowStorage.MinioConfig.AccessKeyID != "" {
shadowStorage.MinioConfig.AccessKeyID = "******"
} }
if s.Section != nil {
return s.Section.MapTo(v)
if shadowStorage.MinioConfig.SecretAccessKey != "" {
shadowStorage.MinioConfig.SecretAccessKey = "******"
} }
return nil
return shadowStorage
} }


func getStorage(rootCfg ConfigProvider, name, typ string, targetSec ConfigSection) Storage {
const sectionName = "storage"
sec := rootCfg.Section(sectionName)
const storageSectionName = "storage"


func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection {
storageSec := rootCfg.Section(storageSectionName)
// Global Defaults // Global Defaults
sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
sec.Key("MINIO_BUCKET").MustString("gitea")
sec.Key("MINIO_LOCATION").MustString("us-east-1")
sec.Key("MINIO_USE_SSL").MustBool(false)
sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
sec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default")
storageSec.Key("STORAGE_TYPE").MustString("local")
storageSec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
storageSec.Key("MINIO_ACCESS_KEY_ID").MustString("")
storageSec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
storageSec.Key("MINIO_BUCKET").MustString("gitea")
storageSec.Key("MINIO_LOCATION").MustString("us-east-1")
storageSec.Key("MINIO_USE_SSL").MustBool(false)
storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
storageSec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default")
return storageSec
}


if targetSec == nil {
targetSec, _ = rootCfg.NewSection(name)
func getStorage(rootCfg ConfigProvider, name, typ string, sec ConfigSection) (*Storage, error) {
if name == "" {
return nil, errors.New("no name for storage")
} }


var storage Storage
storage.Section = targetSec
storage.Type = typ

overrides := make([]ConfigSection, 0, 3)
nameSec, err := rootCfg.GetSection(sectionName + "." + name)
if err == nil {
overrides = append(overrides, nameSec)
var targetSec ConfigSection
if typ != "" {
var err error
targetSec, err = rootCfg.GetSection(storageSectionName + "." + typ)
if err != nil {
if !IsValidStorageType(StorageType(typ)) {
return nil, fmt.Errorf("get section via storage type %q failed: %v", typ, err)
}
}
if targetSec != nil {
targetType := targetSec.Key("STORAGE_TYPE").String()
if targetType == "" {
if !IsValidStorageType(StorageType(typ)) {
return nil, fmt.Errorf("unknow storage type %q", typ)
}
targetSec.Key("STORAGE_TYPE").SetValue(typ)
} else if !IsValidStorageType(StorageType(targetType)) {
return nil, fmt.Errorf("unknow storage type %q for section storage.%v", targetType, typ)
}
}
} }


typeSec, err := rootCfg.GetSection(sectionName + "." + typ)
if err == nil {
overrides = append(overrides, typeSec)
nextType := typeSec.Key("STORAGE_TYPE").String()
if len(nextType) > 0 {
storage.Type = nextType // Support custom STORAGE_TYPE
storageNameSec, _ := rootCfg.GetSection(storageSectionName + "." + name)

if targetSec == nil {
targetSec = sec
}
if targetSec == nil {
targetSec = storageNameSec
}
if targetSec == nil {
targetSec = getDefaultStorageSection(rootCfg)
} else {
targetType := targetSec.Key("STORAGE_TYPE").String()
switch {
case targetType == "":
if targetSec.Key("PATH").String() == "" {
targetSec = getDefaultStorageSection(rootCfg)
} else {
targetSec.Key("STORAGE_TYPE").SetValue("local")
}
default:
newTargetSec, _ := rootCfg.GetSection(storageSectionName + "." + targetType)
if newTargetSec == nil {
if !IsValidStorageType(StorageType(targetType)) {
return nil, fmt.Errorf("invalid storage section %s.%q", storageSectionName, targetType)
}
} else {
targetSec = newTargetSec
if IsValidStorageType(StorageType(targetType)) {
tp := targetSec.Key("STORAGE_TYPE").String()
if tp == "" {
targetSec.Key("STORAGE_TYPE").SetValue(targetType)
}
}
}
} }
} }
overrides = append(overrides, sec)


for _, override := range overrides {
for _, key := range override.Keys() {
if !targetSec.HasKey(key.Name()) {
_, _ = targetSec.NewKey(key.Name(), key.Value())
}
targetType := targetSec.Key("STORAGE_TYPE").String()
if !IsValidStorageType(StorageType(targetType)) {
return nil, fmt.Errorf("invalid storage type %q", targetType)
}

var storage Storage
storage.Type = StorageType(targetType)

switch targetType {
case string(LocalStorageType):
storage.Path = ConfigSectionKeyString(targetSec, "PATH", filepath.Join(AppDataPath, name))
if !filepath.IsAbs(storage.Path) {
storage.Path = filepath.Join(AppWorkPath, storage.Path)
} }
if len(storage.Type) == 0 {
storage.Type = override.Key("STORAGE_TYPE").String()
case string(MinioStorageType):
storage.MinioConfig.BasePath = name + "/"

if err := targetSec.MapTo(&storage.MinioConfig); err != nil {
return nil, fmt.Errorf("map minio config failed: %v", err)
}
// extra config section will be read SERVE_DIRECT, PATH, MINIO_BASE_PATH to override the targetsec
extraConfigSec := sec
if extraConfigSec == nil {
extraConfigSec = storageNameSec
} }
}
storage.ServeDirect = storage.Section.Key("SERVE_DIRECT").MustBool(false)


// Specific defaults
storage.Path = storage.Section.Key("PATH").MustString(filepath.Join(AppDataPath, name))
if !filepath.IsAbs(storage.Path) {
storage.Path = filepath.Join(AppWorkPath, storage.Path)
storage.Section.Key("PATH").SetValue(storage.Path)
if extraConfigSec != nil {
storage.MinioConfig.ServeDirect = ConfigSectionKeyBool(extraConfigSec, "SERVE_DIRECT", storage.MinioConfig.ServeDirect)
storage.MinioConfig.BasePath = ConfigSectionKeyString(extraConfigSec, "MINIO_BASE_PATH", storage.MinioConfig.BasePath)
storage.MinioConfig.Bucket = ConfigSectionKeyString(extraConfigSec, "MINIO_BUCKET", storage.MinioConfig.Bucket)
}
} }
storage.Section.Key("MINIO_BASE_PATH").MustString(name + "/")


return storage
return &storage, nil
} }

+ 42
- 154
modules/setting/storage_test.go View File

"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )


func Test_getStorageCustomType(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = my_minio
MINIO_BUCKET = gitea-attachment

[storage.my_minio]
STORAGE_TYPE = minio
MINIO_ENDPOINT = my_minio:9000
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "my_minio:9000", storage.Section.Key("MINIO_ENDPOINT").String())
assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageNameSectionOverridesTypeSection(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio

[storage.attachments]
MINIO_BUCKET = gitea-attachment

[storage.minio]
MINIO_BUCKET = gitea
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio

[storage.minio]
MINIO_BUCKET = gitea-minio

[storage]
MINIO_BUCKET = gitea
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea-minio", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageSpecificOverridesStorage(t *testing.T) {
iniStr := `
[attachment]
STORAGE_TYPE = minio
MINIO_BUCKET = gitea-attachment

[storage.attachments]
MINIO_BUCKET = gitea

[storage]
STORAGE_TYPE = local
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageGetDefaults(t *testing.T) {
cfg, err := NewConfigProviderFromData("")
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "gitea", storage.Section.Key("MINIO_BUCKET").String())
}

func Test_getStorageMultipleName(t *testing.T) { func Test_getStorageMultipleName(t *testing.T) {
iniStr := ` iniStr := `
[lfs] [lfs]
MINIO_BUCKET = gitea-attachment MINIO_BUCKET = gitea-attachment


[storage] [storage]
STORAGE_TYPE = minio
MINIO_BUCKET = gitea-storage MINIO_BUCKET = gitea-storage
` `
cfg, err := NewConfigProviderFromData(iniStr) cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err) assert.NoError(t, err)


{
sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String())
}
{
sec := cfg.Section("lfs")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "lfs", storageType, sec)

assert.EqualValues(t, "gitea-lfs", storage.Section.Key("MINIO_BUCKET").String())
}
{
sec := cfg.Section("avatar")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "avatars", storageType, sec)

assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String())
}
assert.NoError(t, loadAttachmentFrom(cfg))
assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket)

assert.NoError(t, loadLFSFrom(cfg))
assert.EqualValues(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket)

assert.NoError(t, loadAvatarsFrom(cfg))
assert.EqualValues(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket)
} }


func Test_getStorageUseOtherNameAsType(t *testing.T) { func Test_getStorageUseOtherNameAsType(t *testing.T) {
STORAGE_TYPE = lfs STORAGE_TYPE = lfs


[storage.lfs] [storage.lfs]
STORAGE_TYPE = minio
MINIO_BUCKET = gitea-storage MINIO_BUCKET = gitea-storage
` `
cfg, err := NewConfigProviderFromData(iniStr) cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err) assert.NoError(t, err)


{
sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)
assert.NoError(t, loadAttachmentFrom(cfg))
assert.EqualValues(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket)


assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String())
}
{
sec := cfg.Section("lfs")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "lfs", storageType, sec)

assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String())
}
assert.NoError(t, loadLFSFrom(cfg))
assert.EqualValues(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket)
} }


func Test_getStorageInheritStorageType(t *testing.T) { func Test_getStorageInheritStorageType(t *testing.T) {
cfg, err := NewConfigProviderFromData(iniStr) cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err) assert.NoError(t, err)


sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
}

func Test_getStorageInheritNameSectionType(t *testing.T) {
iniStr := `
[storage.attachments]
STORAGE_TYPE = minio
`
cfg, err := NewConfigProviderFromData(iniStr)
assert.NoError(t, err)

sec := cfg.Section("attachment")
storageType := sec.Key("STORAGE_TYPE").MustString("")
storage := getStorage(cfg, "attachments", storageType, sec)

assert.EqualValues(t, "minio", storage.Type)
assert.NoError(t, loadPackagesFrom(cfg))
assert.EqualValues(t, "minio", Packages.Storage.Type)
assert.EqualValues(t, "gitea", Packages.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath)

assert.NoError(t, loadRepoArchiveFrom(cfg))
assert.EqualValues(t, "minio", RepoArchive.Storage.Type)
assert.EqualValues(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath)

assert.NoError(t, loadActionsFrom(cfg))
assert.EqualValues(t, "minio", Actions.LogStorage.Type)
assert.EqualValues(t, "gitea", Actions.LogStorage.MinioConfig.Bucket)
assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath)

assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type)
assert.EqualValues(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket)
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath)

assert.NoError(t, loadAvatarsFrom(cfg))
assert.EqualValues(t, "minio", Avatar.Storage.Type)
assert.EqualValues(t, "gitea", Avatar.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath)

assert.NoError(t, loadRepoAvatarFrom(cfg))
assert.EqualValues(t, "minio", RepoAvatar.Storage.Type)
assert.EqualValues(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket)
assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath)
} }

+ 0
- 56
modules/storage/helper.go View File

"io" "io"
"net/url" "net/url"
"os" "os"
"reflect"

"code.gitea.io/gitea/modules/json"
) )


// Mappable represents an interface that can MapTo another interface
type Mappable interface {
MapTo(v interface{}) error
}

// toConfig will attempt to convert a given configuration cfg into the provided exemplar type.
//
// It will tolerate the cfg being passed as a []byte or string of a json representation of the
// exemplar or the correct type of the exemplar itself
func toConfig(exemplar, cfg interface{}) (interface{}, error) {
// First of all check if we've got the same type as the exemplar - if so it's all fine.
if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) {
return cfg, nil
}

// Now if not - does it provide a MapTo function we can try?
if mappable, ok := cfg.(Mappable); ok {
newVal := reflect.New(reflect.TypeOf(exemplar))
if err := mappable.MapTo(newVal.Interface()); err == nil {
return newVal.Elem().Interface(), nil
}
// MapTo has failed us ... let's try the json route ...
}

// OK we've been passed a byte array right?
configBytes, ok := cfg.([]byte)
if !ok {
// oh ... it's a string then?
var configStr string

configStr, ok = cfg.(string)
configBytes = []byte(configStr)
}
if !ok {
// hmm ... can we marshal it to json?
var err error
configBytes, err = json.Marshal(cfg)
ok = err == nil
}
if !ok {
// no ... we've tried hard enough at this point - throw an error!
return nil, ErrInvalidConfiguration{cfg: cfg}
}

// OK unmarshal the byte array into a new copy of the exemplar
newVal := reflect.New(reflect.TypeOf(exemplar))
if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil {
// If we can't unmarshal it then return an error!
return nil, ErrInvalidConfiguration{cfg: cfg, err: err}
}
return newVal.Elem().Interface(), nil
}

var uninitializedStorage = discardStorage("uninitialized storage") var uninitializedStorage = discardStorage("uninitialized storage")


type discardStorage string type discardStorage string

+ 3
- 17
modules/storage/local.go View File

"path/filepath" "path/filepath"


"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
) )


var _ ObjectStorage = &LocalStorage{} var _ ObjectStorage = &LocalStorage{}


// LocalStorageType is the type descriptor for local storage
const LocalStorageType Type = "local"

// LocalStorageConfig represents the configuration for a local storage
type LocalStorageConfig struct {
Path string `ini:"PATH"`
TemporaryPath string `ini:"TEMPORARY_PATH"`
}

// LocalStorage represents a local files storage // LocalStorage represents a local files storage
type LocalStorage struct { type LocalStorage struct {
ctx context.Context ctx context.Context
} }


// NewLocalStorage returns a local files // NewLocalStorage returns a local files
func NewLocalStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) {
configInterface, err := toConfig(LocalStorageConfig{}, cfg)
if err != nil {
return nil, err
}
config := configInterface.(LocalStorageConfig)

func NewLocalStorage(ctx context.Context, config *setting.Storage) (ObjectStorage, error) {
if !filepath.IsAbs(config.Path) { if !filepath.IsAbs(config.Path) {
return nil, fmt.Errorf("LocalStorageConfig.Path should have been prepared by setting/storage.go and should be an absolute path, but not: %q", config.Path) return nil, fmt.Errorf("LocalStorageConfig.Path should have been prepared by setting/storage.go and should be an absolute path, but not: %q", config.Path)
} }
} }


func init() { func init() {
RegisterStorageType(LocalStorageType, NewLocalStorage)
RegisterStorageType(setting.LocalStorageType, NewLocalStorage)
} }

+ 3
- 1
modules/storage/local_test.go View File

"path/filepath" "path/filepath"
"testing" "testing"


"code.gitea.io/gitea/modules/setting"

"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )




func TestLocalStorageIterator(t *testing.T) { func TestLocalStorageIterator(t *testing.T) {
dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir") dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir")
testStorageIterator(t, string(LocalStorageType), LocalStorageConfig{Path: dir})
testStorageIterator(t, setting.LocalStorageType, &setting.Storage{Path: dir})
} }

+ 5
- 25
modules/storage/minio.go View File

"time" "time"


"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"


"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
return &minioFileInfo{oi}, nil return &minioFileInfo{oi}, nil
} }


// MinioStorageType is the type descriptor for minio storage
const MinioStorageType Type = "minio"

// MinioStorageConfig represents the configuration for a minio storage
type MinioStorageConfig struct {
Endpoint string `ini:"MINIO_ENDPOINT"`
AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"`
SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"`
Bucket string `ini:"MINIO_BUCKET"`
Location string `ini:"MINIO_LOCATION"`
BasePath string `ini:"MINIO_BASE_PATH"`
UseSSL bool `ini:"MINIO_USE_SSL"`
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM"`
}

// MinioStorage returns a minio bucket storage // MinioStorage returns a minio bucket storage
type MinioStorage struct { type MinioStorage struct {
cfg *MinioStorageConfig
cfg *setting.MinioStorageConfig
ctx context.Context ctx context.Context
client *minio.Client client *minio.Client
bucket string bucket string
} }


// NewMinioStorage returns a minio storage // NewMinioStorage returns a minio storage
func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) {
configInterface, err := toConfig(MinioStorageConfig{}, cfg)
if err != nil {
return nil, convertMinioErr(err)
}
config := configInterface.(MinioStorageConfig)

func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error) {
config := cfg.MinioConfig
if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" { if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm) return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
} }
} }


func init() { func init() {
RegisterStorageType(MinioStorageType, NewMinioStorage)
RegisterStorageType(setting.MinioStorageType, NewMinioStorage)
} }

+ 10
- 6
modules/storage/minio_test.go View File

import ( import (
"os" "os"
"testing" "testing"

"code.gitea.io/gitea/modules/setting"
) )


func TestMinioStorageIterator(t *testing.T) { func TestMinioStorageIterator(t *testing.T) {
t.Skip("minioStorage not present outside of CI") t.Skip("minioStorage not present outside of CI")
return return
} }
testStorageIterator(t, string(MinioStorageType), MinioStorageConfig{
Endpoint: "127.0.0.1:9000",
AccessKeyID: "123456",
SecretAccessKey: "12345678",
Bucket: "gitea",
Location: "us-east-1",
testStorageIterator(t, setting.MinioStorageType, &setting.Storage{
MinioConfig: setting.MinioStorageConfig{
Endpoint: "127.0.0.1:9000",
AccessKeyID: "123456",
SecretAccessKey: "12345678",
Bucket: "gitea",
Location: "us-east-1",
},
}) })
} }

+ 14
- 15
modules/storage/storage.go View File

return ok return ok
} }


// Type is a type of Storage
type Type string
type Type = setting.StorageType


// NewStorageFunc is a function that creates a storage // NewStorageFunc is a function that creates a storage
type NewStorageFunc func(ctx context.Context, cfg interface{}) (ObjectStorage, error)
type NewStorageFunc func(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error)


var storageMap = map[Type]NewStorageFunc{} var storageMap = map[Type]NewStorageFunc{}


// RegisterStorageType registers a provided storage type with a function to create it // RegisterStorageType registers a provided storage type with a function to create it
func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg interface{}) (ObjectStorage, error)) {
func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error)) {
storageMap[typ] = fn storageMap[typ] = fn
} }


} }


// NewStorage takes a storage type and some config and returns an ObjectStorage or an error // NewStorage takes a storage type and some config and returns an ObjectStorage or an error
func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) {
func NewStorage(typStr Type, cfg *setting.Storage) (ObjectStorage, error) {
if len(typStr) == 0 { if len(typStr) == 0 {
typStr = string(LocalStorageType)
typStr = setting.LocalStorageType
} }
fn, ok := storageMap[Type(typStr)]
fn, ok := storageMap[typStr]
if !ok { if !ok {
return nil, fmt.Errorf("Unsupported storage type: %s", typStr) return nil, fmt.Errorf("Unsupported storage type: %s", typStr)
} }


func initAvatars() (err error) { func initAvatars() (err error) {
log.Info("Initialising Avatar storage with type: %s", setting.Avatar.Storage.Type) log.Info("Initialising Avatar storage with type: %s", setting.Avatar.Storage.Type)
Avatars, err = NewStorage(setting.Avatar.Storage.Type, &setting.Avatar.Storage)
Avatars, err = NewStorage(setting.Avatar.Storage.Type, setting.Avatar.Storage)
return err return err
} }


return nil return nil
} }
log.Info("Initialising Attachment storage with type: %s", setting.Attachment.Storage.Type) log.Info("Initialising Attachment storage with type: %s", setting.Attachment.Storage.Type)
Attachments, err = NewStorage(setting.Attachment.Storage.Type, &setting.Attachment.Storage)
Attachments, err = NewStorage(setting.Attachment.Storage.Type, setting.Attachment.Storage)
return err return err
} }


return nil return nil
} }
log.Info("Initialising LFS storage with type: %s", setting.LFS.Storage.Type) log.Info("Initialising LFS storage with type: %s", setting.LFS.Storage.Type)
LFS, err = NewStorage(setting.LFS.Storage.Type, &setting.LFS.Storage)
LFS, err = NewStorage(setting.LFS.Storage.Type, setting.LFS.Storage)
return err return err
} }


func initRepoAvatars() (err error) { func initRepoAvatars() (err error) {
log.Info("Initialising Repository Avatar storage with type: %s", setting.RepoAvatar.Storage.Type) log.Info("Initialising Repository Avatar storage with type: %s", setting.RepoAvatar.Storage.Type)
RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, &setting.RepoAvatar.Storage)
RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, setting.RepoAvatar.Storage)
return err return err
} }


func initRepoArchives() (err error) { func initRepoArchives() (err error) {
log.Info("Initialising Repository Archive storage with type: %s", setting.RepoArchive.Storage.Type) log.Info("Initialising Repository Archive storage with type: %s", setting.RepoArchive.Storage.Type)
RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage)
RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, setting.RepoArchive.Storage)
return err return err
} }


return nil return nil
} }
log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type) log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type)
Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage)
Packages, err = NewStorage(setting.Packages.Storage.Type, setting.Packages.Storage)
return err return err
} }


return nil return nil
} }
log.Info("Initialising Actions storage with type: %s", setting.Actions.LogStorage.Type) log.Info("Initialising Actions storage with type: %s", setting.Actions.LogStorage.Type)
if Actions, err = NewStorage(setting.Actions.LogStorage.Type, &setting.Actions.LogStorage); err != nil {
if Actions, err = NewStorage(setting.Actions.LogStorage.Type, setting.Actions.LogStorage); err != nil {
return err return err
} }
log.Info("Initialising ActionsArtifacts storage with type: %s", setting.Actions.ArtifactStorage.Type) log.Info("Initialising ActionsArtifacts storage with type: %s", setting.Actions.ArtifactStorage.Type)
ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, &setting.Actions.ArtifactStorage)
ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, setting.Actions.ArtifactStorage)
return err return err
} }

+ 3
- 1
modules/storage/storage_test.go View File

"bytes" "bytes"
"testing" "testing"


"code.gitea.io/gitea/modules/setting"

"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )


func testStorageIterator(t *testing.T, typStr string, cfg interface{}) {
func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) {
l, err := NewStorage(typStr, cfg) l, err := NewStorage(typStr, cfg)
assert.NoError(t, err) assert.NoError(t, err)



+ 2
- 2
routers/api/v1/repo/file.go View File

return return
} }


if setting.LFS.ServeDirect {
if setting.LFS.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly. // If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name()) u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name())
if u != nil && err == nil { if u != nil && err == nil {
downloadName := ctx.Repo.Repository.Name + "-" + archiveName downloadName := ctx.Repo.Repository.Name + "-" + archiveName


rPath := archiver.RelativePath() rPath := archiver.RelativePath()
if setting.RepoArchive.ServeDirect {
if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly. // If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.RepoArchives.URL(rPath, downloadName) u, err := storage.RepoArchives.URL(rPath, downloadName)
if u != nil && err == nil { if u != nil && err == nil {

+ 1
- 1
routers/install/install.go View File

// Application general settings // Application general settings
form.AppName = setting.AppName form.AppName = setting.AppName
form.RepoRootPath = setting.RepoRootPath form.RepoRootPath = setting.RepoRootPath
form.LFSRootPath = setting.LFS.Path
form.LFSRootPath = setting.LFS.Storage.Path


// Note(unknown): it's hard for Windows users change a running user, // Note(unknown): it's hard for Windows users change a running user,
// so just use current one if config says default. // so just use current one if config says default.

+ 2
- 2
routers/web/base.go View File

"code.gitea.io/gitea/modules/web/routing" "code.gitea.io/gitea/modules/web/routing"
) )


func storageHandler(storageSetting setting.Storage, prefix string, objStore storage.ObjectStorage) func(next http.Handler) http.Handler {
func storageHandler(storageSetting *setting.Storage, prefix string, objStore storage.ObjectStorage) func(next http.Handler) http.Handler {
prefix = strings.Trim(prefix, "/") prefix = strings.Trim(prefix, "/")
funcInfo := routing.GetFuncInfo(storageHandler, prefix) funcInfo := routing.GetFuncInfo(storageHandler, prefix)
return func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler {
if storageSetting.ServeDirect {
if storageSetting.MinioConfig.ServeDirect {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.Method != "GET" && req.Method != "HEAD" { if req.Method != "GET" && req.Method != "HEAD" {
next.ServeHTTP(w, req) next.ServeHTTP(w, req)

+ 1
- 1
routers/web/repo/attachment.go View File

return return
} }


if setting.Attachment.ServeDirect {
if setting.Attachment.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly. // If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name) u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name)



+ 1
- 1
routers/web/repo/download.go View File

return nil return nil
} }


if setting.LFS.ServeDirect {
if setting.LFS.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly. // If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name()) u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name())
if u != nil && err == nil { if u != nil && err == nil {

+ 1
- 1
routers/web/repo/repo.go View File

downloadName := ctx.Repo.Repository.Name + "-" + archiveName downloadName := ctx.Repo.Repository.Name + "-" + archiveName


rPath := archiver.RelativePath() rPath := archiver.RelativePath()
if setting.RepoArchive.ServeDirect {
if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly. // If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.RepoArchives.URL(rPath, downloadName) u, err := storage.RepoArchives.URL(rPath, downloadName)
if u != nil && err == nil { if u != nil && err == nil {

+ 1
- 1
services/lfs/server.go View File



if download { if download {
var link *lfs_module.Link var link *lfs_module.Link
if setting.LFS.ServeDirect {
if setting.LFS.Storage.MinioConfig.ServeDirect {
// If we have a signed url (S3, object storage), redirect to this directly. // If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid) u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid)
if u != nil && err == nil { if u != nil && err == nil {

+ 1
- 1
templates/admin/config.tmpl View File

<dd>{{if .LFS.StartServer}}{{svg "octicon-check"}}{{else}}{{svg "octicon-x"}}{{end}}</dd> <dd>{{if .LFS.StartServer}}{{svg "octicon-check"}}{{else}}{{svg "octicon-x"}}{{end}}</dd>
{{if .LFS.StartServer}} {{if .LFS.StartServer}}
<dt>{{.locale.Tr "admin.config.lfs_content_path"}}</dt> <dt>{{.locale.Tr "admin.config.lfs_content_path"}}</dt>
<dd>{{.LFS.Path}}</dd>
<dd>{{JsonUtils.EncodeToString .LFS.Storage.ToShadowCopy}}</dd>
<dt>{{.locale.Tr "admin.config.lfs_http_auth_expiry"}}</dt> <dt>{{.locale.Tr "admin.config.lfs_http_auth_expiry"}}</dt>
<dd>{{.LFS.HTTPAuthExpiry}}</dd> <dd>{{.LFS.HTTPAuthExpiry}}</dd>
{{end}} {{end}}

+ 3
- 1
tests/test_utils.go View File



// load LFS object fixtures // load LFS object fixtures
// (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API) // (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API)
lfsFixtures, err := storage.NewStorage("", storage.LocalStorageConfig{Path: path.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta")})
lfsFixtures, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{
Path: filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta"),
})
assert.NoError(t, err) assert.NoError(t, err)
assert.NoError(t, storage.Clean(storage.LFS)) assert.NoError(t, storage.Clean(storage.LFS))
assert.NoError(t, lfsFixtures.IterateObjects("", func(path string, _ storage.Object) error { assert.NoError(t, lfsFixtures.IterateObjects("", func(path string, _ storage.Object) error {

Loading…
Cancel
Save