Add default storage configurations (#12813)

Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: zeripath <art27@cantab.net>

4 files changed, 141 insertions, 66 deletions

diff --git a/modules/setting/attachment.go b/modules/setting/attachment.go
index 4c7368eb68..56ccf5bc57 100644
--- a/modules/setting/attachment.go
+++ b/modules/setting/attachment.go
@@ -5,42 +5,25 @@
 package setting
 
 import (
-	"path"
 	"path/filepath"
 	"strings"
+
+	"code.gitea.io/gitea/modules/log"
 )
 
 var (
 	// Attachment settings
 	Attachment = struct {
-		StoreType   string
-		Path        string
-		ServeDirect bool
-		Minio       struct {
-			Endpoint        string
-			AccessKeyID     string
-			SecretAccessKey string
-			UseSSL          bool
-			Bucket          string
-			Location        string
-			BasePath        string
-		}
+		Storage
 		AllowedTypes string
 		MaxSize      int64
 		MaxFiles     int
 		Enabled      bool
 	}{
-		StoreType:   "local",
-		ServeDirect: false,
-		Minio: struct {
-			Endpoint        string
-			AccessKeyID     string
-			SecretAccessKey string
-			UseSSL          bool
-			Bucket          string
-			Location        string
-			BasePath        string
-		}{},
+		Storage: Storage{
+			Type:        LocalStorageType,
+			ServeDirect: false,
+		},
 		AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
 		MaxSize:      4,
 		MaxFiles:     5,
@@ -50,22 +33,36 @@ var (
 
 func newAttachmentService() {
 	sec := Cfg.Section("attachment")
-	Attachment.StoreType = sec.Key("STORE_TYPE").MustString("local")
-	Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
-	switch Attachment.StoreType {
-	case "local":
-		Attachment.Path = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
+	Attachment.Storage.Type = sec.Key("STORAGE_TYPE").MustString("")
+	if Attachment.Storage.Type == "" {
+		Attachment.Storage.Type = "default"
+	}
+
+	if Attachment.Storage.Type != LocalStorageType && Attachment.Storage.Type != MinioStorageType {
+		storage, ok := storages[Attachment.Storage.Type]
+		if !ok {
+			log.Fatal("Failed to get attachment storage type: %s", Attachment.Storage.Type)
+		}
+		Attachment.Storage = storage
+	}
+
+	// Override
+	Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(Attachment.ServeDirect)
+
+	switch Attachment.Storage.Type {
+	case LocalStorageType:
+		Attachment.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "attachments"))
 		if !filepath.IsAbs(Attachment.Path) {
-			Attachment.Path = path.Join(AppWorkPath, Attachment.Path)
+			Attachment.Path = filepath.Join(AppWorkPath, Attachment.Path)
 		}
-	case "minio":
-		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
-		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
-		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
-		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
-		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
+	case MinioStorageType:
+		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString(Attachment.Minio.Endpoint)
+		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString(Attachment.Minio.AccessKeyID)
+		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString(Attachment.Minio.SecretAccessKey)
+		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString(Attachment.Minio.Bucket)
+		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString(Attachment.Minio.Location)
+		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(Attachment.Minio.UseSSL)
 		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
-		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
 	}
 
 	Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
diff --git a/modules/setting/lfs.go b/modules/setting/lfs.go
index a740a6d629..da34d3a5ff 100644
--- a/modules/setting/lfs.go
+++ b/modules/setting/lfs.go
@@ -21,27 +21,14 @@ import (
 // LFS represents the configuration for Git LFS
 var LFS = struct {
 	StartServer     bool          `ini:"LFS_START_SERVER"`
-	ContentPath     string        `ini:"LFS_CONTENT_PATH"`
 	JWTSecretBase64 string        `ini:"LFS_JWT_SECRET"`
 	JWTSecretBytes  []byte        `ini:"-"`
 	HTTPAuthExpiry  time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
 	MaxFileSize     int64         `ini:"LFS_MAX_FILE_SIZE"`
 	LocksPagingNum  int           `ini:"LFS_LOCKS_PAGING_NUM"`
 
-	StoreType   string
-	ServeDirect bool
-	Minio       struct {
-		Endpoint        string
-		AccessKeyID     string
-		SecretAccessKey string
-		UseSSL          bool
-		Bucket          string
-		Location        string
-		BasePath        string
-	}
-}{
-	StoreType: "local",
-}
+	Storage
+}{}
 
 func newLFSService() {
 	sec := Cfg.Section("server")
@@ -49,10 +36,41 @@ func newLFSService() {
 		log.Fatal("Failed to map LFS settings: %v", err)
 	}
 
-	LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
-	if !filepath.IsAbs(LFS.ContentPath) {
-		LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath)
+	lfsSec := Cfg.Section("lfs")
+	LFS.Storage.Type = lfsSec.Key("STORAGE_TYPE").MustString("")
+	if LFS.Storage.Type == "" {
+		LFS.Storage.Type = "default"
+	}
+
+	if LFS.Storage.Type != LocalStorageType && LFS.Storage.Type != MinioStorageType {
+		storage, ok := storages[LFS.Storage.Type]
+		if !ok {
+			log.Fatal("Failed to get lfs storage type: %s", LFS.Storage.Type)
+		}
+		LFS.Storage = storage
 	}
+
+	// Override
+	LFS.ServeDirect = lfsSec.Key("SERVE_DIRECT").MustBool(LFS.ServeDirect)
+	switch LFS.Storage.Type {
+	case LocalStorageType:
+		// keep compatible
+		LFS.Path = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
+		LFS.Path = lfsSec.Key("PATH").MustString(LFS.Path)
+		if !filepath.IsAbs(LFS.Path) {
+			LFS.Path = filepath.Join(AppWorkPath, LFS.Path)
+		}
+
+	case MinioStorageType:
+		LFS.Minio.Endpoint = lfsSec.Key("MINIO_ENDPOINT").MustString(LFS.Minio.Endpoint)
+		LFS.Minio.AccessKeyID = lfsSec.Key("MINIO_ACCESS_KEY_ID").MustString(LFS.Minio.AccessKeyID)
+		LFS.Minio.SecretAccessKey = lfsSec.Key("MINIO_SECRET_ACCESS_KEY").MustString(LFS.Minio.SecretAccessKey)
+		LFS.Minio.Bucket = lfsSec.Key("MINIO_BUCKET").MustString(LFS.Minio.Bucket)
+		LFS.Minio.Location = lfsSec.Key("MINIO_LOCATION").MustString(LFS.Minio.Location)
+		LFS.Minio.UseSSL = lfsSec.Key("MINIO_USE_SSL").MustBool(LFS.Minio.UseSSL)
+		LFS.Minio.BasePath = lfsSec.Key("MINIO_BASE_PATH").MustString("lfs/")
+	}
+
 	if LFS.LocksPagingNum == 0 {
 		LFS.LocksPagingNum = 50
 	}
@@ -92,14 +110,6 @@ func newLFSService() {
 	}
 }
 
-func ensureLFSDirectory() {
-	if LFS.StartServer {
-		if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
-			log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err)
-		}
-	}
-}
-
 // CheckLFSVersion will check lfs version, if not satisfied, then disable it.
 func CheckLFSVersion() {
 	if LFS.StartServer {
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 7d7eacba6f..26f226fe53 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -691,8 +691,6 @@ func NewContext() {
 	SSH.CreateAuthorizedKeysFile = sec.Key("SSH_CREATE_AUTHORIZED_KEYS_FILE").MustBool(true)
 	SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false)
 
-	newLFSService()
-
 	if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil {
 		log.Fatal("Failed to OAuth2 settings: %v", err)
 		return
@@ -761,7 +759,9 @@ func NewContext() {
 		}
 	}
 
+	newStorageService()
 	newAttachmentService()
+	newLFSService()
 
 	timeFormatKey := Cfg.Section("time").Key("FORMAT").MustString("")
 	if timeFormatKey != "" {
@@ -1017,7 +1017,6 @@ func NewServices() {
 	InitDBConfig()
 	newService()
 	NewLogServices(false)
-	ensureLFSDirectory()
 	newCacheService()
 	newSessionService()
 	newCORSService()
diff --git a/modules/setting/storage.go b/modules/setting/storage.go
new file mode 100644
index 0000000000..c678a08f5b
--- /dev/null
+++ b/modules/setting/storage.go
@@ -0,0 +1,69 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	ini "gopkg.in/ini.v1"
+)
+
+// enumerate all storage types
+const (
+	LocalStorageType = "local"
+	MinioStorageType = "minio"
+)
+
+// Storage represents configuration of storages
+type Storage struct {
+	Type        string
+	Path        string
+	ServeDirect bool
+	Minio       struct {
+		Endpoint        string
+		AccessKeyID     string
+		SecretAccessKey string
+		UseSSL          bool
+		Bucket          string
+		Location        string
+		BasePath        string
+	}
+}
+
+var (
+	storages = make(map[string]Storage)
+)
+
+func getStorage(sec *ini.Section) Storage {
+	var storage Storage
+	storage.Type = sec.Key("STORAGE_TYPE").MustString(LocalStorageType)
+	storage.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
+	switch storage.Type {
+	case LocalStorageType:
+	case MinioStorageType:
+		storage.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
+		storage.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
+		storage.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
+		storage.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
+		storage.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
+		storage.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
+	}
+	return storage
+}
+
+func newStorageService() {
+	sec := Cfg.Section("storage")
+	storages["default"] = getStorage(sec)
+
+	for _, sec := range Cfg.Section("storage").ChildSections() {
+		name := strings.TrimPrefix(sec.Name(), "storage.")
+		if name == "default" || name == LocalStorageType || name == MinioStorageType {
+			log.Error("storage name %s is system reserved!", name)
+			continue
+		}
+		storages[name] = getStorage(sec)
+	}
+}