diff options
author | Lunny Xiao <xiaolunwen@gmail.com> | 2023-06-14 11:42:38 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-06-14 11:42:38 +0800 |
commit | d6dd6d641b593c54fe1a1041c153111ce81dbc20 (patch) | |
tree | f9e7959356124bd830ce0b9e9e51c1373aa71932 /modules/setting | |
parent | dc0a7168f1450d45164fde63c56f04a1e5bbb949 (diff) | |
download | gitea-d6dd6d641b593c54fe1a1041c153111ce81dbc20.tar.gz gitea-d6dd6d641b593c54fe1a1041c153111ce81dbc20.zip |
Fix all possible setting error related storages and added some tests (#23911)
Follow up #22405
Fix #20703
This PR rewrites storage configuration read sequences with some breaks
and tests. It becomes more strict than before and also fixed some
inherit problems.
- Move storage's MinioConfig struct into setting, so after the
configuration loading, the values will be stored into the struct but not
still on some section.
- All storages configurations should be stored on one section,
configuration items cannot be overrided by multiple sections. The
prioioty of configuration is `[attachment]` > `[storage.attachments]` |
`[storage.customized]` > `[storage]` > `default`
- For extra override configuration items, currently are `SERVE_DIRECT`,
`MINIO_BASE_PATH`, `MINIO_BUCKET`, which could be configured in another
section. The prioioty of the override configuration is `[attachment]` >
`[storage.attachments]` > `default`.
- Add more tests for storages configurations.
- Update the storage documentations.
---------
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Diffstat (limited to 'modules/setting')
-rw-r--r-- | modules/setting/actions.go | 27 | ||||
-rw-r--r-- | modules/setting/actions_test.go | 97 | ||||
-rw-r--r-- | modules/setting/attachment.go | 22 | ||||
-rw-r--r-- | modules/setting/attachment_test.go | 133 | ||||
-rw-r--r-- | modules/setting/config_provider.go | 19 | ||||
-rw-r--r-- | modules/setting/lfs.go | 56 | ||||
-rw-r--r-- | modules/setting/lfs_test.go | 77 | ||||
-rw-r--r-- | modules/setting/packages.go | 26 | ||||
-rw-r--r-- | modules/setting/packages_test.go | 167 | ||||
-rw-r--r-- | modules/setting/picture.go | 30 | ||||
-rw-r--r-- | modules/setting/repository.go | 8 | ||||
-rw-r--r-- | modules/setting/repository_archive.go | 25 | ||||
-rw-r--r-- | modules/setting/repository_archive_test.go | 111 | ||||
-rw-r--r-- | modules/setting/setting.go | 34 | ||||
-rw-r--r-- | modules/setting/storage.go | 209 | ||||
-rw-r--r-- | modules/setting/storage_test.go | 196 |
16 files changed, 946 insertions, 291 deletions
diff --git a/modules/setting/actions.go b/modules/setting/actions.go index eb1b637a1c..1c8075cd6c 100644 --- a/modules/setting/actions.go +++ b/modules/setting/actions.go @@ -4,14 +4,14 @@ package setting import ( - "code.gitea.io/gitea/modules/log" + "fmt" ) // Actions settings var ( Actions = struct { - LogStorage Storage // how the created logs should be stored - ArtifactStorage Storage // how the created artifacts should be stored + LogStorage *Storage // how the created logs should be stored + ArtifactStorage *Storage // how the created artifacts should be stored Enabled bool DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"` }{ @@ -20,15 +20,22 @@ var ( } ) -func loadActionsFrom(rootCfg ConfigProvider) { +func loadActionsFrom(rootCfg ConfigProvider) error { sec := rootCfg.Section("actions") - if err := sec.MapTo(&Actions); err != nil { - log.Fatal("Failed to map Actions settings: %v", err) + err := sec.MapTo(&Actions) + if err != nil { + return fmt.Errorf("failed to map Actions settings: %v", err) } - actionsSec := rootCfg.Section("actions.artifacts") - storageType := actionsSec.Key("STORAGE_TYPE").MustString("") + // don't support to read configuration from [actions] + Actions.LogStorage, err = getStorage(rootCfg, "actions_log", "", nil) + if err != nil { + return err + } + + actionsSec, _ := rootCfg.GetSection("actions.artifacts") + + Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec) - Actions.LogStorage = getStorage(rootCfg, "actions_log", "", nil) - Actions.ArtifactStorage = getStorage(rootCfg, "actions_artifacts", storageType, actionsSec) + return err } diff --git a/modules/setting/actions_test.go b/modules/setting/actions_test.go new file mode 100644 index 0000000000..a1cc8fe333 --- /dev/null +++ b/modules/setting/actions_test.go @@ -0,0 +1,97 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getStorageInheritNameSectionTypeForActions(t *testing.T) { + iniStr := ` + [storage] + STORAGE_TYPE = minio + ` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "minio", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + + iniStr = ` +[storage.actions_log] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "minio", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + + iniStr = ` +[storage.actions_log] +STORAGE_TYPE = my_storage + +[storage.my_storage] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "minio", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + + iniStr = ` +[storage.actions_artifacts] +STORAGE_TYPE = my_storage + +[storage.my_storage] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "local", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + + iniStr = ` +[storage.actions_artifacts] +STORAGE_TYPE = my_storage + +[storage.my_storage] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "local", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + + iniStr = `` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadActionsFrom(cfg)) + + assert.EqualValues(t, "local", Actions.LogStorage.Type) + assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) +} diff --git a/modules/setting/attachment.go b/modules/setting/attachment.go index 4d4b8e3b02..491564c9dc 100644 --- a/modules/setting/attachment.go +++ b/modules/setting/attachment.go @@ -5,29 +5,31 @@ package setting // Attachment settings var Attachment = struct { - Storage + Storage *Storage AllowedTypes string MaxSize int64 MaxFiles int Enabled bool }{ - Storage: Storage{ - ServeDirect: false, - }, - AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip", + Storage: &Storage{}, + AllowedTypes: ".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip", MaxSize: 4, MaxFiles: 5, Enabled: true, } -func loadAttachmentFrom(rootCfg ConfigProvider) { - sec := rootCfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - - Attachment.Storage = getStorage(rootCfg, "attachments", storageType, sec) +func loadAttachmentFrom(rootCfg ConfigProvider) (err error) { + sec, _ := rootCfg.GetSection("attachment") + if sec == nil { + Attachment.Storage, err = getStorage(rootCfg, "attachments", "", nil) + return err + } Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip") Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4) Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5) Attachment.Enabled = sec.Key("ENABLED").MustBool(true) + + Attachment.Storage, err = getStorage(rootCfg, "attachments", "", sec) + return err } diff --git a/modules/setting/attachment_test.go b/modules/setting/attachment_test.go new file mode 100644 index 0000000000..3e8d2da4d9 --- /dev/null +++ b/modules/setting/attachment_test.go @@ -0,0 +1,133 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getStorageCustomType(t *testing.T) { + iniStr := ` +[attachment] +STORAGE_TYPE = my_minio +MINIO_BUCKET = gitea-attachment + +[storage.my_minio] +STORAGE_TYPE = minio +MINIO_ENDPOINT = my_minio:9000 +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + assert.EqualValues(t, "minio", Attachment.Storage.Type) + assert.EqualValues(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint) + assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) +} + +func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) { + iniStr := ` +[attachment] +STORAGE_TYPE = minio + +[storage.minio] +MINIO_BUCKET = gitea-minio + +[storage] +MINIO_BUCKET = gitea +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + assert.EqualValues(t, "minio", Attachment.Storage.Type) + assert.EqualValues(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) +} + +func Test_getStorageSpecificOverridesStorage(t *testing.T) { + iniStr := ` +[attachment] +STORAGE_TYPE = minio +MINIO_BUCKET = gitea-attachment + +[storage.attachments] +MINIO_BUCKET = gitea + +[storage] +STORAGE_TYPE = local +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + assert.EqualValues(t, "minio", Attachment.Storage.Type) + assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) +} + +func Test_getStorageGetDefaults(t *testing.T) { + cfg, err := NewConfigProviderFromData("") + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + // default storage is local, so bucket is empty + assert.EqualValues(t, "", Attachment.Storage.MinioConfig.Bucket) +} + +func Test_getStorageInheritNameSectionType(t *testing.T) { + iniStr := ` +[storage.attachments] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + + assert.EqualValues(t, "minio", Attachment.Storage.Type) +} + +func Test_AttachmentStorage(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + storage := Attachment.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) +} + +func Test_AttachmentStorage1(t *testing.T) { + iniStr := ` +[storage] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadAttachmentFrom(cfg)) + assert.EqualValues(t, "minio", Attachment.Storage.Type) + assert.EqualValues(t, "gitea", Attachment.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) +} diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index 8b317d94e3..526d69bbdc 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "time" @@ -95,6 +96,18 @@ func ConfigSectionKeyString(sec ConfigSection, key string, def ...string) string return "" } +func ConfigSectionKeyBool(sec ConfigSection, key string, def ...bool) bool { + k := ConfigSectionKey(sec, key) + if k != nil && k.String() != "" { + b, _ := strconv.ParseBool(k.String()) + return b + } + if len(def) > 0 { + return def[0] + } + return false +} + // ConfigInheritedKey works like ini.Section.Key(), but it always returns a new key instance, it is O(n) because NewKey is O(n) // and the returned key is safe to be used with "MustXxx", it doesn't change the parent's values. // Otherwise, ini.Section.Key().MustXxx would pollute the parent section's keys. @@ -287,6 +300,12 @@ func deprecatedSetting(rootCfg ConfigProvider, oldSection, oldKey, newSection, n } } +func deprecatedSettingFatal(rootCfg ConfigProvider, oldSection, oldKey, newSection, newKey, version string) { + if rootCfg.Section(oldSection).HasKey(oldKey) { + log.Fatal("Deprecated fallback `[%s]` `%s` present. Use `[%s]` `%s` instead. This fallback will be/has been removed in %s", oldSection, oldKey, newSection, newKey, version) + } +} + // deprecatedSettingDB add a hint that the configuration has been moved to database but still kept in app.ini func deprecatedSettingDB(rootCfg ConfigProvider, oldSection, oldKey string) { if rootCfg.Section(oldSection).HasKey(oldKey) { diff --git a/modules/setting/lfs.go b/modules/setting/lfs.go index a5c3631681..d68349be86 100644 --- a/modules/setting/lfs.go +++ b/modules/setting/lfs.go @@ -5,10 +5,10 @@ package setting import ( "encoding/base64" + "fmt" "time" "code.gitea.io/gitea/modules/generate" - "code.gitea.io/gitea/modules/log" ) // LFS represents the configuration for Git LFS @@ -20,25 +20,27 @@ var LFS = struct { MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"` LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"` - Storage + Storage *Storage }{} -func loadLFSFrom(rootCfg ConfigProvider) { +func loadLFSFrom(rootCfg ConfigProvider) error { sec := rootCfg.Section("server") if err := sec.MapTo(&LFS); err != nil { - log.Fatal("Failed to map LFS settings: %v", err) + return fmt.Errorf("failed to map LFS settings: %v", err) } - lfsSec := rootCfg.Section("lfs") - storageType := lfsSec.Key("STORAGE_TYPE").MustString("") + lfsSec, _ := rootCfg.GetSection("lfs") // Specifically default PATH to LFS_CONTENT_PATH // DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version // if these are removed, the warning will not be shown - deprecatedSetting(rootCfg, "server", "LFS_CONTENT_PATH", "lfs", "PATH", "v1.19.0") - lfsSec.Key("PATH").MustString(sec.Key("LFS_CONTENT_PATH").String()) + deprecatedSettingFatal(rootCfg, "server", "LFS_CONTENT_PATH", "lfs", "PATH", "v1.19.0") - LFS.Storage = getStorage(rootCfg, "lfs", storageType, lfsSec) + var err error + LFS.Storage, err = getStorage(rootCfg, "lfs", "", lfsSec) + if err != nil { + return err + } // Rest of LFS service settings if LFS.LocksPagingNum == 0 { @@ -47,23 +49,25 @@ func loadLFSFrom(rootCfg ConfigProvider) { LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(24 * time.Hour) - if LFS.StartServer { - LFS.JWTSecretBytes = make([]byte, 32) - n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) - - if err != nil || n != 32 { - LFS.JWTSecretBase64, err = generate.NewJwtSecretBase64() - if err != nil { - log.Fatal("Error generating JWT Secret for custom config: %v", err) - return - } - - // Save secret - sec.Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) - if err := rootCfg.Save(); err != nil { - log.Fatal("Error saving JWT Secret for custom config: %v", err) - return - } + if !LFS.StartServer { + return nil + } + + LFS.JWTSecretBytes = make([]byte, 32) + n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64)) + + if err != nil || n != 32 { + LFS.JWTSecretBase64, err = generate.NewJwtSecretBase64() + if err != nil { + return fmt.Errorf("Error generating JWT Secret for custom config: %v", err) + } + + // Save secret + sec.Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64) + if err := rootCfg.Save(); err != nil { + return fmt.Errorf("Error saving JWT Secret for custom config: %v", err) } } + + return nil } diff --git a/modules/setting/lfs_test.go b/modules/setting/lfs_test.go new file mode 100644 index 0000000000..3313cae0eb --- /dev/null +++ b/modules/setting/lfs_test.go @@ -0,0 +1,77 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getStorageInheritNameSectionTypeForLFS(t *testing.T) { + iniStr := ` + [storage] + STORAGE_TYPE = minio + ` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + + iniStr = ` +[storage.lfs] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + + iniStr = ` +[lfs] +STORAGE_TYPE = my_minio + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + + iniStr = ` +[lfs] +STORAGE_TYPE = my_minio +MINIO_BASE_PATH = my_lfs/ + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadLFSFrom(cfg)) + + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath) +} + +func Test_LFSStorage1(t *testing.T) { + iniStr := ` +[storage] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadLFSFrom(cfg)) + assert.EqualValues(t, "minio", LFS.Storage.Type) + assert.EqualValues(t, "gitea", LFS.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 5e64d7fe9f..dc8d98d29f 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -4,20 +4,19 @@ package setting import ( + "fmt" "math" "net/url" "os" "path/filepath" - "code.gitea.io/gitea/modules/log" - "github.com/dustin/go-humanize" ) // Package registry settings var ( Packages = struct { - Storage + Storage *Storage Enabled bool ChunkedUploadPath string RegistryHost string @@ -51,13 +50,21 @@ var ( } ) -func loadPackagesFrom(rootCfg ConfigProvider) { - sec := rootCfg.Section("packages") - if err := sec.MapTo(&Packages); err != nil { - log.Fatal("Failed to map Packages settings: %v", err) +func loadPackagesFrom(rootCfg ConfigProvider) (err error) { + sec, _ := rootCfg.GetSection("packages") + if sec == nil { + Packages.Storage, err = getStorage(rootCfg, "packages", "", nil) + return err + } + + if err = sec.MapTo(&Packages); err != nil { + return fmt.Errorf("failed to map Packages settings: %v", err) } - Packages.Storage = getStorage(rootCfg, "packages", "", nil) + Packages.Storage, err = getStorage(rootCfg, "packages", "", sec) + if err != nil { + return err + } appURL, _ := url.Parse(AppURL) Packages.RegistryHost = appURL.Host @@ -68,7 +75,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) { } if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { - log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) + return fmt.Errorf("unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) } Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") @@ -93,6 +100,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) { Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS") Packages.LimitSizeSwift = mustBytes(sec, "LIMIT_SIZE_SWIFT") Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT") + return nil } func mustBytes(section ConfigSection, key string) int64 { diff --git a/modules/setting/packages_test.go b/modules/setting/packages_test.go index d9f6e10528..87de276041 100644 --- a/modules/setting/packages_test.go +++ b/modules/setting/packages_test.go @@ -29,3 +29,170 @@ func TestMustBytes(t *testing.T) { assert.EqualValues(t, 1782579, test("1.7mib")) assert.EqualValues(t, -1, test("1 yib")) // too large } + +func Test_getStorageInheritNameSectionTypeForPackages(t *testing.T) { + // packages storage inherits from storage if nothing configured + iniStr := ` +[storage] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadPackagesFrom(cfg)) + + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + + // we can also configure packages storage directly + iniStr = ` +[storage.packages] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadPackagesFrom(cfg)) + + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + + // or we can indicate the storage type in the packages section + iniStr = ` +[packages] +STORAGE_TYPE = my_minio + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadPackagesFrom(cfg)) + + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + + // or we can indicate the storage type and minio base path in the packages section + iniStr = ` +[packages] +STORAGE_TYPE = my_minio +MINIO_BASE_PATH = my_packages/ + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadPackagesFrom(cfg)) + + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "my_packages/", Packages.Storage.MinioConfig.BasePath) +} + +func Test_PackageStorage1(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[packages] +MINIO_BASE_PATH = packages/ +SERVE_DIRECT = true +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadPackagesFrom(cfg)) + storage := Packages.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.True(t, storage.MinioConfig.ServeDirect) +} + +func Test_PackageStorage2(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage.packages] +MINIO_BASE_PATH = packages/ +SERVE_DIRECT = true +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadPackagesFrom(cfg)) + storage := Packages.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.True(t, storage.MinioConfig.ServeDirect) +} + +func Test_PackageStorage3(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[packages] +STORAGE_TYPE = my_cfg +MINIO_BASE_PATH = my_packages/ +SERVE_DIRECT = true +[storage.my_cfg] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadPackagesFrom(cfg)) + storage := Packages.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.True(t, storage.MinioConfig.ServeDirect) +} + +func Test_PackageStorage4(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage.packages] +STORAGE_TYPE = my_cfg +MINIO_BASE_PATH = my_packages/ +SERVE_DIRECT = true +[storage.my_cfg] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadPackagesFrom(cfg)) + storage := Packages.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.True(t, storage.MinioConfig.ServeDirect) +} diff --git a/modules/setting/picture.go b/modules/setting/picture.go index 64d9a608e6..fafae45bab 100644 --- a/modules/setting/picture.go +++ b/modules/setting/picture.go @@ -7,7 +7,7 @@ package setting var ( Avatar = struct { - Storage + Storage *Storage MaxWidth int MaxHeight int @@ -27,23 +27,26 @@ var ( EnableFederatedAvatar bool // Depreciated: migrated to database RepoAvatar = struct { - Storage + Storage *Storage Fallback string FallbackImage string }{} ) -func loadPictureFrom(rootCfg ConfigProvider) { +func loadAvatarsFrom(rootCfg ConfigProvider) error { sec := rootCfg.Section("picture") avatarSec := rootCfg.Section("avatar") storageType := sec.Key("AVATAR_STORAGE_TYPE").MustString("") // Specifically default PATH to AVATAR_UPLOAD_PATH - avatarSec.Key("PATH").MustString( - sec.Key("AVATAR_UPLOAD_PATH").String()) + avatarSec.Key("PATH").MustString(sec.Key("AVATAR_UPLOAD_PATH").String()) - Avatar.Storage = getStorage(rootCfg, "avatars", storageType, avatarSec) + var err error + Avatar.Storage, err = getStorage(rootCfg, "avatars", storageType, avatarSec) + if err != nil { + return err + } Avatar.MaxWidth = sec.Key("AVATAR_MAX_WIDTH").MustInt(4096) Avatar.MaxHeight = sec.Key("AVATAR_MAX_HEIGHT").MustInt(4096) @@ -67,7 +70,7 @@ func loadPictureFrom(rootCfg ConfigProvider) { EnableFederatedAvatar = sec.Key("ENABLE_FEDERATED_AVATAR").MustBool(GetDefaultEnableFederatedAvatar(DisableGravatar)) deprecatedSettingDB(rootCfg, "", "ENABLE_FEDERATED_AVATAR") - loadRepoAvatarFrom(rootCfg) + return nil } func GetDefaultDisableGravatar() bool { @@ -85,17 +88,22 @@ func GetDefaultEnableFederatedAvatar(disableGravatar bool) bool { return v } -func loadRepoAvatarFrom(rootCfg ConfigProvider) { +func loadRepoAvatarFrom(rootCfg ConfigProvider) error { sec := rootCfg.Section("picture") repoAvatarSec := rootCfg.Section("repo-avatar") storageType := sec.Key("REPOSITORY_AVATAR_STORAGE_TYPE").MustString("") // Specifically default PATH to AVATAR_UPLOAD_PATH - repoAvatarSec.Key("PATH").MustString( - sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String()) + repoAvatarSec.Key("PATH").MustString(sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").String()) - RepoAvatar.Storage = getStorage(rootCfg, "repo-avatars", storageType, repoAvatarSec) + var err error + RepoAvatar.Storage, err = getStorage(rootCfg, "repo-avatars", storageType, repoAvatarSec) + if err != nil { + return err + } RepoAvatar.Fallback = sec.Key("REPOSITORY_AVATAR_FALLBACK").MustString("none") RepoAvatar.FallbackImage = sec.Key("REPOSITORY_AVATAR_FALLBACK_IMAGE").MustString(AppSubURL + "/assets/img/repo_default.png") + + return nil } diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 406068b59d..42ffb99138 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -265,10 +265,6 @@ var ( } RepoRootPath string ScriptType = "bash" - - RepoArchive = struct { - Storage - }{} ) func loadRepositoryFrom(rootCfg ConfigProvider) { @@ -359,5 +355,7 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath) } - RepoArchive.Storage = getStorage(rootCfg, "repo-archive", "", nil) + if err := loadRepoArchiveFrom(rootCfg); err != nil { + log.Fatal("loadRepoArchiveFrom: %v", err) + } } diff --git a/modules/setting/repository_archive.go b/modules/setting/repository_archive.go new file mode 100644 index 0000000000..9d24afa9e7 --- /dev/null +++ b/modules/setting/repository_archive.go @@ -0,0 +1,25 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import "fmt" + +var RepoArchive = struct { + Storage *Storage +}{} + +func loadRepoArchiveFrom(rootCfg ConfigProvider) (err error) { + sec, _ := rootCfg.GetSection("repo-archive") + if sec == nil { + RepoArchive.Storage, err = getStorage(rootCfg, "repo-archive", "", nil) + return err + } + + if err := sec.MapTo(&RepoArchive); err != nil { + return fmt.Errorf("mapto repoarchive failed: %v", err) + } + + RepoArchive.Storage, err = getStorage(rootCfg, "repo-archive", "", sec) + return err +} diff --git a/modules/setting/repository_archive_test.go b/modules/setting/repository_archive_test.go new file mode 100644 index 0000000000..a0f91f0da1 --- /dev/null +++ b/modules/setting/repository_archive_test.go @@ -0,0 +1,111 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getStorageInheritNameSectionTypeForRepoArchive(t *testing.T) { + // packages storage inherits from storage if nothing configured + iniStr := ` +[storage] +STORAGE_TYPE = minio +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + + // we can also configure packages storage directly + iniStr = ` +[storage.repo-archive] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + + // or we can indicate the storage type in the packages section + iniStr = ` +[repo-archive] +STORAGE_TYPE = my_minio + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + + // or we can indicate the storage type and minio base path in the packages section + iniStr = ` +[repo-archive] +STORAGE_TYPE = my_minio +MINIO_BASE_PATH = my_archive/ + +[storage.my_minio] +STORAGE_TYPE = minio +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + assert.NoError(t, loadRepoArchiveFrom(cfg)) + + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath) +} + +func Test_RepoArchiveStorage(t *testing.T) { + iniStr := ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadRepoArchiveFrom(cfg)) + storage := RepoArchive.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + + iniStr = ` +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[storage.repo-archive] +STORAGE_TYPE = s3 +[storage.s3] +STORAGE_TYPE = minio +MINIO_ENDPOINT = s3.my-domain.net +MINIO_BUCKET = gitea +MINIO_LOCATION = homenet +MINIO_USE_SSL = true +MINIO_ACCESS_KEY_ID = correct_key +MINIO_SECRET_ACCESS_KEY = correct_key +` + cfg, err = NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadRepoArchiveFrom(cfg)) + storage = RepoArchive.Storage + + assert.EqualValues(t, "minio", storage.Type) + assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 1967d9e79a..293333a95b 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -203,16 +203,18 @@ func Init(opts *Options) { var err error CfgProvider, err = NewConfigProviderFromFile(opts) if err != nil { - log.Fatal("Init[%v]: %v", opts, err) + log.Fatal("newConfigProviderFromFile[%v]: %v", opts, err) } if !opts.DisableLoadCommonSettings { - loadCommonSettingsFrom(CfgProvider) + if err := loadCommonSettingsFrom(CfgProvider); err != nil { + log.Fatal("loadCommonSettingsFrom[%v]: %v", opts, err) + } } } // loadCommonSettingsFrom loads common configurations from a configuration provider. -func loadCommonSettingsFrom(cfg ConfigProvider) { - // WARNING: don't change the sequence except you know what you are doing. +func loadCommonSettingsFrom(cfg ConfigProvider) error { + // WARNNING: don't change the sequence except you know what you are doing. loadRunModeFrom(cfg) loadLogGlobalFrom(cfg) loadServerFrom(cfg) @@ -222,13 +224,26 @@ func loadCommonSettingsFrom(cfg ConfigProvider) { loadOAuth2From(cfg) loadSecurityFrom(cfg) - loadAttachmentFrom(cfg) - loadLFSFrom(cfg) + if err := loadAttachmentFrom(cfg); err != nil { + return err + } + if err := loadLFSFrom(cfg); err != nil { + return err + } loadTimeFrom(cfg) loadRepositoryFrom(cfg) - loadPictureFrom(cfg) - loadPackagesFrom(cfg) - loadActionsFrom(cfg) + if err := loadAvatarsFrom(cfg); err != nil { + return err + } + if err := loadRepoAvatarFrom(cfg); err != nil { + return err + } + if err := loadPackagesFrom(cfg); err != nil { + return err + } + if err := loadActionsFrom(cfg); err != nil { + return err + } loadUIFrom(cfg) loadAdminFrom(cfg) loadAPIFrom(cfg) @@ -239,6 +254,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) { loadMirrorFrom(cfg) loadMarkupFrom(cfg) loadOtherFrom(cfg) + return nil } func loadRunModeFrom(rootCfg ConfigProvider) { diff --git a/modules/setting/storage.go b/modules/setting/storage.go index 6da52807ec..ed804a910a 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -4,87 +4,182 @@ package setting import ( + "errors" + "fmt" "path/filepath" - "reflect" ) +// StorageType is a type of Storage +type StorageType string + +const ( + // LocalStorageType is the type descriptor for local storage + LocalStorageType StorageType = "local" + // MinioStorageType is the type descriptor for minio storage + MinioStorageType StorageType = "minio" +) + +var storageTypes = []StorageType{ + LocalStorageType, + MinioStorageType, +} + +// IsValidStorageType returns true if the given storage type is valid +func IsValidStorageType(storageType StorageType) bool { + for _, t := range storageTypes { + if t == storageType { + return true + } + } + return false +} + +// MinioStorageConfig represents the configuration for a minio storage +type MinioStorageConfig struct { + Endpoint string `ini:"MINIO_ENDPOINT" json:",omitempty"` + AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID" json:",omitempty"` + SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY" json:",omitempty"` + Bucket string `ini:"MINIO_BUCKET" json:",omitempty"` + Location string `ini:"MINIO_LOCATION" json:",omitempty"` + BasePath string `ini:"MINIO_BASE_PATH" json:",omitempty"` + UseSSL bool `ini:"MINIO_USE_SSL"` + InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"` + ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM" json:",omitempty"` + ServeDirect bool `ini:"SERVE_DIRECT"` +} + // Storage represents configuration of storages type Storage struct { - Type string - Path string - Section ConfigSection - ServeDirect bool + Type StorageType // local or minio + Path string `json:",omitempty"` // for local type + TemporaryPath string `json:",omitempty"` + MinioConfig MinioStorageConfig // for minio type } -// MapTo implements the Mappable interface -func (s *Storage) MapTo(v interface{}) error { - pathValue := reflect.ValueOf(v).Elem().FieldByName("Path") - if pathValue.IsValid() && pathValue.Kind() == reflect.String { - pathValue.SetString(s.Path) +func (storage *Storage) ToShadowCopy() Storage { + shadowStorage := *storage + if shadowStorage.MinioConfig.AccessKeyID != "" { + shadowStorage.MinioConfig.AccessKeyID = "******" } - if s.Section != nil { - return s.Section.MapTo(v) + if shadowStorage.MinioConfig.SecretAccessKey != "" { + shadowStorage.MinioConfig.SecretAccessKey = "******" } - return nil + return shadowStorage } -func getStorage(rootCfg ConfigProvider, name, typ string, targetSec ConfigSection) Storage { - const sectionName = "storage" - sec := rootCfg.Section(sectionName) +const storageSectionName = "storage" +func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection { + storageSec := rootCfg.Section(storageSectionName) // Global Defaults - sec.Key("MINIO_ENDPOINT").MustString("localhost:9000") - sec.Key("MINIO_ACCESS_KEY_ID").MustString("") - sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") - sec.Key("MINIO_BUCKET").MustString("gitea") - sec.Key("MINIO_LOCATION").MustString("us-east-1") - sec.Key("MINIO_USE_SSL").MustBool(false) - sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false) - sec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default") + storageSec.Key("STORAGE_TYPE").MustString("local") + storageSec.Key("MINIO_ENDPOINT").MustString("localhost:9000") + storageSec.Key("MINIO_ACCESS_KEY_ID").MustString("") + storageSec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") + storageSec.Key("MINIO_BUCKET").MustString("gitea") + storageSec.Key("MINIO_LOCATION").MustString("us-east-1") + storageSec.Key("MINIO_USE_SSL").MustBool(false) + storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false) + storageSec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default") + return storageSec +} - if targetSec == nil { - targetSec, _ = rootCfg.NewSection(name) +func getStorage(rootCfg ConfigProvider, name, typ string, sec ConfigSection) (*Storage, error) { + if name == "" { + return nil, errors.New("no name for storage") } - var storage Storage - storage.Section = targetSec - storage.Type = typ - - overrides := make([]ConfigSection, 0, 3) - nameSec, err := rootCfg.GetSection(sectionName + "." + name) - if err == nil { - overrides = append(overrides, nameSec) + var targetSec ConfigSection + if typ != "" { + var err error + targetSec, err = rootCfg.GetSection(storageSectionName + "." + typ) + if err != nil { + if !IsValidStorageType(StorageType(typ)) { + return nil, fmt.Errorf("get section via storage type %q failed: %v", typ, err) + } + } + if targetSec != nil { + targetType := targetSec.Key("STORAGE_TYPE").String() + if targetType == "" { + if !IsValidStorageType(StorageType(typ)) { + return nil, fmt.Errorf("unknow storage type %q", typ) + } + targetSec.Key("STORAGE_TYPE").SetValue(typ) + } else if !IsValidStorageType(StorageType(targetType)) { + return nil, fmt.Errorf("unknow storage type %q for section storage.%v", targetType, typ) + } + } } - typeSec, err := rootCfg.GetSection(sectionName + "." + typ) - if err == nil { - overrides = append(overrides, typeSec) - nextType := typeSec.Key("STORAGE_TYPE").String() - if len(nextType) > 0 { - storage.Type = nextType // Support custom STORAGE_TYPE + storageNameSec, _ := rootCfg.GetSection(storageSectionName + "." + name) + + if targetSec == nil { + targetSec = sec + } + if targetSec == nil { + targetSec = storageNameSec + } + if targetSec == nil { + targetSec = getDefaultStorageSection(rootCfg) + } else { + targetType := targetSec.Key("STORAGE_TYPE").String() + switch { + case targetType == "": + if targetSec.Key("PATH").String() == "" { + targetSec = getDefaultStorageSection(rootCfg) + } else { + targetSec.Key("STORAGE_TYPE").SetValue("local") + } + default: + newTargetSec, _ := rootCfg.GetSection(storageSectionName + "." + targetType) + if newTargetSec == nil { + if !IsValidStorageType(StorageType(targetType)) { + return nil, fmt.Errorf("invalid storage section %s.%q", storageSectionName, targetType) + } + } else { + targetSec = newTargetSec + if IsValidStorageType(StorageType(targetType)) { + tp := targetSec.Key("STORAGE_TYPE").String() + if tp == "" { + targetSec.Key("STORAGE_TYPE").SetValue(targetType) + } + } + } } } - overrides = append(overrides, sec) - for _, override := range overrides { - for _, key := range override.Keys() { - if !targetSec.HasKey(key.Name()) { - _, _ = targetSec.NewKey(key.Name(), key.Value()) - } + targetType := targetSec.Key("STORAGE_TYPE").String() + if !IsValidStorageType(StorageType(targetType)) { + return nil, fmt.Errorf("invalid storage type %q", targetType) + } + + var storage Storage + storage.Type = StorageType(targetType) + + switch targetType { + case string(LocalStorageType): + storage.Path = ConfigSectionKeyString(targetSec, "PATH", filepath.Join(AppDataPath, name)) + if !filepath.IsAbs(storage.Path) { + storage.Path = filepath.Join(AppWorkPath, storage.Path) } - if len(storage.Type) == 0 { - storage.Type = override.Key("STORAGE_TYPE").String() + case string(MinioStorageType): + storage.MinioConfig.BasePath = name + "/" + + if err := targetSec.MapTo(&storage.MinioConfig); err != nil { + return nil, fmt.Errorf("map minio config failed: %v", err) + } + // extra config section will be read SERVE_DIRECT, PATH, MINIO_BASE_PATH to override the targetsec + extraConfigSec := sec + if extraConfigSec == nil { + extraConfigSec = storageNameSec } - } - storage.ServeDirect = storage.Section.Key("SERVE_DIRECT").MustBool(false) - // Specific defaults - storage.Path = storage.Section.Key("PATH").MustString(filepath.Join(AppDataPath, name)) - if !filepath.IsAbs(storage.Path) { - storage.Path = filepath.Join(AppWorkPath, storage.Path) - storage.Section.Key("PATH").SetValue(storage.Path) + if extraConfigSec != nil { + storage.MinioConfig.ServeDirect = ConfigSectionKeyBool(extraConfigSec, "SERVE_DIRECT", storage.MinioConfig.ServeDirect) + storage.MinioConfig.BasePath = ConfigSectionKeyString(extraConfigSec, "MINIO_BASE_PATH", storage.MinioConfig.BasePath) + storage.MinioConfig.Bucket = ConfigSectionKeyString(extraConfigSec, "MINIO_BUCKET", storage.MinioConfig.Bucket) + } } - storage.Section.Key("MINIO_BASE_PATH").MustString(name + "/") - return storage + return &storage, nil } diff --git a/modules/setting/storage_test.go b/modules/setting/storage_test.go index 5e213606e3..4eda7646e8 100644 --- a/modules/setting/storage_test.go +++ b/modules/setting/storage_test.go @@ -9,106 +9,6 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_getStorageCustomType(t *testing.T) { - iniStr := ` -[attachment] -STORAGE_TYPE = my_minio -MINIO_BUCKET = gitea-attachment - -[storage.my_minio] -STORAGE_TYPE = minio -MINIO_ENDPOINT = my_minio:9000 -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "my_minio:9000", storage.Section.Key("MINIO_ENDPOINT").String()) - assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String()) -} - -func Test_getStorageNameSectionOverridesTypeSection(t *testing.T) { - iniStr := ` -[attachment] -STORAGE_TYPE = minio - -[storage.attachments] -MINIO_BUCKET = gitea-attachment - -[storage.minio] -MINIO_BUCKET = gitea -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String()) -} - -func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) { - iniStr := ` -[attachment] -STORAGE_TYPE = minio - -[storage.minio] -MINIO_BUCKET = gitea-minio - -[storage] -MINIO_BUCKET = gitea -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea-minio", storage.Section.Key("MINIO_BUCKET").String()) -} - -func Test_getStorageSpecificOverridesStorage(t *testing.T) { - iniStr := ` -[attachment] -STORAGE_TYPE = minio -MINIO_BUCKET = gitea-attachment - -[storage.attachments] -MINIO_BUCKET = gitea - -[storage] -STORAGE_TYPE = local -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String()) -} - -func Test_getStorageGetDefaults(t *testing.T) { - cfg, err := NewConfigProviderFromData("") - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "gitea", storage.Section.Key("MINIO_BUCKET").String()) -} - func Test_getStorageMultipleName(t *testing.T) { iniStr := ` [lfs] @@ -118,32 +18,20 @@ MINIO_BUCKET = gitea-lfs MINIO_BUCKET = gitea-attachment [storage] +STORAGE_TYPE = minio MINIO_BUCKET = gitea-storage ` cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) - { - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "gitea-attachment", storage.Section.Key("MINIO_BUCKET").String()) - } - { - sec := cfg.Section("lfs") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "lfs", storageType, sec) - - assert.EqualValues(t, "gitea-lfs", storage.Section.Key("MINIO_BUCKET").String()) - } - { - sec := cfg.Section("avatar") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "avatars", storageType, sec) - - assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String()) - } + assert.NoError(t, loadAttachmentFrom(cfg)) + assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + + assert.NoError(t, loadLFSFrom(cfg)) + assert.EqualValues(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket) + + assert.NoError(t, loadAvatarsFrom(cfg)) + assert.EqualValues(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket) } func Test_getStorageUseOtherNameAsType(t *testing.T) { @@ -152,25 +40,17 @@ func Test_getStorageUseOtherNameAsType(t *testing.T) { STORAGE_TYPE = lfs [storage.lfs] +STORAGE_TYPE = minio MINIO_BUCKET = gitea-storage ` cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) - { - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) + assert.NoError(t, loadAttachmentFrom(cfg)) + assert.EqualValues(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String()) - } - { - sec := cfg.Section("lfs") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "lfs", storageType, sec) - - assert.EqualValues(t, "gitea-storage", storage.Section.Key("MINIO_BUCKET").String()) - } + assert.NoError(t, loadLFSFrom(cfg)) + assert.EqualValues(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket) } func Test_getStorageInheritStorageType(t *testing.T) { @@ -181,24 +61,32 @@ STORAGE_TYPE = minio cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) -} - -func Test_getStorageInheritNameSectionType(t *testing.T) { - iniStr := ` -[storage.attachments] -STORAGE_TYPE = minio -` - cfg, err := NewConfigProviderFromData(iniStr) - assert.NoError(t, err) - - sec := cfg.Section("attachment") - storageType := sec.Key("STORAGE_TYPE").MustString("") - storage := getStorage(cfg, "attachments", storageType, sec) - - assert.EqualValues(t, "minio", storage.Type) + assert.NoError(t, loadPackagesFrom(cfg)) + assert.EqualValues(t, "minio", Packages.Storage.Type) + assert.EqualValues(t, "gitea", Packages.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + + assert.NoError(t, loadRepoArchiveFrom(cfg)) + assert.EqualValues(t, "minio", RepoArchive.Storage.Type) + assert.EqualValues(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + + assert.NoError(t, loadActionsFrom(cfg)) + assert.EqualValues(t, "minio", Actions.LogStorage.Type) + assert.EqualValues(t, "gitea", Actions.LogStorage.MinioConfig.Bucket) + assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + + assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) + assert.EqualValues(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket) + assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + + assert.NoError(t, loadAvatarsFrom(cfg)) + assert.EqualValues(t, "minio", Avatar.Storage.Type) + assert.EqualValues(t, "gitea", Avatar.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) + + assert.NoError(t, loadRepoAvatarFrom(cfg)) + assert.EqualValues(t, "minio", RepoAvatar.Storage.Type) + assert.EqualValues(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket) + assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath) } |