Add default storage configurations (#12813)

Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: zeripath <art27@cantab.net>

7 files changed, 241 insertions, 113 deletions

diff --git a/modules/setting/attachment.go b/modules/setting/attachment.go
index 4c7368eb68..56ccf5bc57 100644
--- a/modules/setting/attachment.go
+++ b/modules/setting/attachment.go
@@ -5,42 +5,25 @@
 package setting
 
 import (
-	"path"
 	"path/filepath"
 	"strings"
+
+	"code.gitea.io/gitea/modules/log"
 )
 
 var (
 	// Attachment settings
 	Attachment = struct {
-		StoreType   string
-		Path        string
-		ServeDirect bool
-		Minio       struct {
-			Endpoint        string
-			AccessKeyID     string
-			SecretAccessKey string
-			UseSSL          bool
-			Bucket          string
-			Location        string
-			BasePath        string
-		}
+		Storage
 		AllowedTypes string
 		MaxSize      int64
 		MaxFiles     int
 		Enabled      bool
 	}{
-		StoreType:   "local",
-		ServeDirect: false,
-		Minio: struct {
-			Endpoint        string
-			AccessKeyID     string
-			SecretAccessKey string
-			UseSSL          bool
-			Bucket          string
-			Location        string
-			BasePath        string
-		}{},
+		Storage: Storage{
+			Type:        LocalStorageType,
+			ServeDirect: false,
+		},
 		AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
 		MaxSize:      4,
 		MaxFiles:     5,
@@ -50,22 +33,36 @@ var (
 
 func newAttachmentService() {
 	sec := Cfg.Section("attachment")
-	Attachment.StoreType = sec.Key("STORE_TYPE").MustString("local")
-	Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
-	switch Attachment.StoreType {
-	case "local":
-		Attachment.Path = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
+	Attachment.Storage.Type = sec.Key("STORAGE_TYPE").MustString("")
+	if Attachment.Storage.Type == "" {
+		Attachment.Storage.Type = "default"
+	}
+
+	if Attachment.Storage.Type != LocalStorageType && Attachment.Storage.Type != MinioStorageType {
+		storage, ok := storages[Attachment.Storage.Type]
+		if !ok {
+			log.Fatal("Failed to get attachment storage type: %s", Attachment.Storage.Type)
+		}
+		Attachment.Storage = storage
+	}
+
+	// Override
+	Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(Attachment.ServeDirect)
+
+	switch Attachment.Storage.Type {
+	case LocalStorageType:
+		Attachment.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "attachments"))
 		if !filepath.IsAbs(Attachment.Path) {
-			Attachment.Path = path.Join(AppWorkPath, Attachment.Path)
+			Attachment.Path = filepath.Join(AppWorkPath, Attachment.Path)
 		}
-	case "minio":
-		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
-		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
-		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
-		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
-		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
+	case MinioStorageType:
+		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString(Attachment.Minio.Endpoint)
+		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString(Attachment.Minio.AccessKeyID)
+		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString(Attachment.Minio.SecretAccessKey)
+		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString(Attachment.Minio.Bucket)
+		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString(Attachment.Minio.Location)
+		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(Attachment.Minio.UseSSL)
 		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
-		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
 	}
 
 	Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
diff --git a/modules/setting/lfs.go b/modules/setting/lfs.go
index a740a6d629..da34d3a5ff 100644
--- a/modules/setting/lfs.go
+++ b/modules/setting/lfs.go
@@ -21,27 +21,14 @@ import (
 // LFS represents the configuration for Git LFS
 var LFS = struct {
 	StartServer     bool          `ini:"LFS_START_SERVER"`
-	ContentPath     string        `ini:"LFS_CONTENT_PATH"`
 	JWTSecretBase64 string        `ini:"LFS_JWT_SECRET"`
 	JWTSecretBytes  []byte        `ini:"-"`
 	HTTPAuthExpiry  time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
 	MaxFileSize     int64         `ini:"LFS_MAX_FILE_SIZE"`
 	LocksPagingNum  int           `ini:"LFS_LOCKS_PAGING_NUM"`
 
-	StoreType   string
-	ServeDirect bool
-	Minio       struct {
-		Endpoint        string
-		AccessKeyID     string
-		SecretAccessKey string
-		UseSSL          bool
-		Bucket          string
-		Location        string
-		BasePath        string
-	}
-}{
-	StoreType: "local",
-}
+	Storage
+}{}
 
 func newLFSService() {
 	sec := Cfg.Section("server")
@@ -49,10 +36,41 @@ func newLFSService() {
 		log.Fatal("Failed to map LFS settings: %v", err)
 	}
 
-	LFS.ContentPath = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
-	if !filepath.IsAbs(LFS.ContentPath) {
-		LFS.ContentPath = filepath.Join(AppWorkPath, LFS.ContentPath)
+	lfsSec := Cfg.Section("lfs")
+	LFS.Storage.Type = lfsSec.Key("STORAGE_TYPE").MustString("")
+	if LFS.Storage.Type == "" {
+		LFS.Storage.Type = "default"
+	}
+
+	if LFS.Storage.Type != LocalStorageType && LFS.Storage.Type != MinioStorageType {
+		storage, ok := storages[LFS.Storage.Type]
+		if !ok {
+			log.Fatal("Failed to get lfs storage type: %s", LFS.Storage.Type)
+		}
+		LFS.Storage = storage
 	}
+
+	// Override
+	LFS.ServeDirect = lfsSec.Key("SERVE_DIRECT").MustBool(LFS.ServeDirect)
+	switch LFS.Storage.Type {
+	case LocalStorageType:
+		// keep compatible
+		LFS.Path = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
+		LFS.Path = lfsSec.Key("PATH").MustString(LFS.Path)
+		if !filepath.IsAbs(LFS.Path) {
+			LFS.Path = filepath.Join(AppWorkPath, LFS.Path)
+		}
+
+	case MinioStorageType:
+		LFS.Minio.Endpoint = lfsSec.Key("MINIO_ENDPOINT").MustString(LFS.Minio.Endpoint)
+		LFS.Minio.AccessKeyID = lfsSec.Key("MINIO_ACCESS_KEY_ID").MustString(LFS.Minio.AccessKeyID)
+		LFS.Minio.SecretAccessKey = lfsSec.Key("MINIO_SECRET_ACCESS_KEY").MustString(LFS.Minio.SecretAccessKey)
+		LFS.Minio.Bucket = lfsSec.Key("MINIO_BUCKET").MustString(LFS.Minio.Bucket)
+		LFS.Minio.Location = lfsSec.Key("MINIO_LOCATION").MustString(LFS.Minio.Location)
+		LFS.Minio.UseSSL = lfsSec.Key("MINIO_USE_SSL").MustBool(LFS.Minio.UseSSL)
+		LFS.Minio.BasePath = lfsSec.Key("MINIO_BASE_PATH").MustString("lfs/")
+	}
+
 	if LFS.LocksPagingNum == 0 {
 		LFS.LocksPagingNum = 50
 	}
@@ -92,14 +110,6 @@ func newLFSService() {
 	}
 }
 
-func ensureLFSDirectory() {
-	if LFS.StartServer {
-		if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
-			log.Fatal("Failed to create '%s': %v", LFS.ContentPath, err)
-		}
-	}
-}
-
 // CheckLFSVersion will check lfs version, if not satisfied, then disable it.
 func CheckLFSVersion() {
 	if LFS.StartServer {
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 7d7eacba6f..26f226fe53 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -691,8 +691,6 @@ func NewContext() {
 	SSH.CreateAuthorizedKeysFile = sec.Key("SSH_CREATE_AUTHORIZED_KEYS_FILE").MustBool(true)
 	SSH.ExposeAnonymous = sec.Key("SSH_EXPOSE_ANONYMOUS").MustBool(false)
 
-	newLFSService()
-
 	if err = Cfg.Section("oauth2").MapTo(&OAuth2); err != nil {
 		log.Fatal("Failed to OAuth2 settings: %v", err)
 		return
@@ -761,7 +759,9 @@ func NewContext() {
 		}
 	}
 
+	newStorageService()
 	newAttachmentService()
+	newLFSService()
 
 	timeFormatKey := Cfg.Section("time").Key("FORMAT").MustString("")
 	if timeFormatKey != "" {
@@ -1017,7 +1017,6 @@ func NewServices() {
 	InitDBConfig()
 	newService()
 	NewLogServices(false)
-	ensureLFSDirectory()
 	newCacheService()
 	newSessionService()
 	newCORSService()
diff --git a/modules/setting/storage.go b/modules/setting/storage.go
new file mode 100644
index 0000000000..c678a08f5b
--- /dev/null
+++ b/modules/setting/storage.go
@@ -0,0 +1,69 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	ini "gopkg.in/ini.v1"
+)
+
+// enumerate all storage types
+const (
+	LocalStorageType = "local"
+	MinioStorageType = "minio"
+)
+
+// Storage represents configuration of storages
+type Storage struct {
+	Type        string
+	Path        string
+	ServeDirect bool
+	Minio       struct {
+		Endpoint        string
+		AccessKeyID     string
+		SecretAccessKey string
+		UseSSL          bool
+		Bucket          string
+		Location        string
+		BasePath        string
+	}
+}
+
+var (
+	storages = make(map[string]Storage)
+)
+
+func getStorage(sec *ini.Section) Storage {
+	var storage Storage
+	storage.Type = sec.Key("STORAGE_TYPE").MustString(LocalStorageType)
+	storage.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
+	switch storage.Type {
+	case LocalStorageType:
+	case MinioStorageType:
+		storage.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
+		storage.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
+		storage.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
+		storage.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
+		storage.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
+		storage.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
+	}
+	return storage
+}
+
+func newStorageService() {
+	sec := Cfg.Section("storage")
+	storages["default"] = getStorage(sec)
+
+	for _, sec := range Cfg.Section("storage").ChildSections() {
+		name := strings.TrimPrefix(sec.Name(), "storage.")
+		if name == "default" || name == LocalStorageType || name == MinioStorageType {
+			log.Error("storage name %s is system reserved!", name)
+			continue
+		}
+		storages[name] = getStorage(sec)
+	}
+}
diff --git a/modules/storage/local.go b/modules/storage/local.go
index a937a831f1..4185981664 100644
--- a/modules/storage/local.go
+++ b/modules/storage/local.go
@@ -59,7 +59,7 @@ func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) {
 }
 
 // Stat returns the info of the file
-func (l *LocalStorage) Stat(path string) (ObjectInfo, error) {
+func (l *LocalStorage) Stat(path string) (os.FileInfo, error) {
 	return os.Stat(filepath.Join(l.dir, path))
 }
 
@@ -73,3 +73,28 @@ func (l *LocalStorage) Delete(path string) error {
 func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
 	return nil, ErrURLNotSupported
 }
+
+// IterateObjects iterates across the objects in the local storage
+func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error {
+	return filepath.Walk(l.dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if path == l.dir {
+			return nil
+		}
+		if info.IsDir() {
+			return nil
+		}
+		relPath, err := filepath.Rel(l.dir, path)
+		if err != nil {
+			return err
+		}
+		obj, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer obj.Close()
+		return fn(relPath, obj)
+	})
+}
diff --git a/modules/storage/minio.go b/modules/storage/minio.go
index 30751755f3..d205eff7fd 100644
--- a/modules/storage/minio.go
+++ b/modules/storage/minio.go
@@ -22,6 +22,19 @@ var (
 	quoteEscaper               = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
 )
 
+type minioObject struct {
+	*minio.Object
+}
+
+func (m *minioObject) Stat() (os.FileInfo, error) {
+	oi, err := m.Object.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	return &minioFileInfo{oi}, nil
+}
+
 // MinioStorage returns a minio bucket storage
 type MinioStorage struct {
 	ctx      context.Context
@@ -69,7 +82,7 @@ func (m *MinioStorage) Open(path string) (Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	return object, nil
+	return &minioObject{object}, nil
 }
 
 // Save save a file to minio
@@ -104,8 +117,20 @@ func (m minioFileInfo) ModTime() time.Time {
 	return m.LastModified
 }
 
+func (m minioFileInfo) IsDir() bool {
+	return strings.HasSuffix(m.ObjectInfo.Key, "/")
+}
+
+func (m minioFileInfo) Mode() os.FileMode {
+	return os.ModePerm
+}
+
+func (m minioFileInfo) Sys() interface{} {
+	return nil
+}
+
 // Stat returns the stat information of the object
-func (m *MinioStorage) Stat(path string) (ObjectInfo, error) {
+func (m *MinioStorage) Stat(path string) (os.FileInfo, error) {
 	info, err := m.client.StatObject(
 		m.ctx,
 		m.bucket,
@@ -135,3 +160,26 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
 	reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
 	return m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
 }
+
+// IterateObjects iterates across the objects in the miniostorage
+func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) error {
+	var opts = minio.GetObjectOptions{}
+	lobjectCtx, cancel := context.WithCancel(m.ctx)
+	defer cancel()
+	for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{
+		Prefix:    m.basePath,
+		Recursive: true,
+	}) {
+		object, err := m.client.GetObject(lobjectCtx, m.bucket, mObjInfo.Key, opts)
+		if err != nil {
+			return err
+		}
+		if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
+			defer object.Close()
+			return fn(strings.TrimPrefix(m.basePath, mObjInfo.Key), &minioObject{object})
+		}(object, fn); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/modules/storage/storage.go b/modules/storage/storage.go
index e355d2459f..2cf7b17b49 100644
--- a/modules/storage/storage.go
+++ b/modules/storage/storage.go
@@ -10,7 +10,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
-	"time"
+	"os"
 
 	"code.gitea.io/gitea/modules/setting"
 )
@@ -18,28 +18,25 @@ import (
 var (
 	// ErrURLNotSupported represents url is not supported
 	ErrURLNotSupported = errors.New("url method not supported")
+	// ErrIterateObjectsNotSupported represents IterateObjects not supported
+	ErrIterateObjectsNotSupported = errors.New("iterateObjects method not supported")
 )
 
 // Object represents the object on the storage
 type Object interface {
 	io.ReadCloser
 	io.Seeker
-}
-
-// ObjectInfo represents the object info on the storage
-type ObjectInfo interface {
-	Name() string
-	Size() int64
-	ModTime() time.Time
+	Stat() (os.FileInfo, error)
 }
 
 // ObjectStorage represents an object storage to handle a bucket and files
 type ObjectStorage interface {
 	Open(path string) (Object, error)
 	Save(path string, r io.Reader) (int64, error)
-	Stat(path string) (ObjectInfo, error)
+	Stat(path string) (os.FileInfo, error)
 	Delete(path string) error
 	URL(path, name string) (*url.URL, error)
+	IterateObjects(func(path string, obj Object) error) error
 }
 
 // Copy copys a file from source ObjectStorage to dest ObjectStorage
@@ -70,14 +67,15 @@ func Init() error {
 	return initLFS()
 }
 
-func initAttachments() error {
+func initStorage(storageCfg setting.Storage) (ObjectStorage, error) {
 	var err error
-	switch setting.Attachment.StoreType {
-	case "local":
-		Attachments, err = NewLocalStorage(setting.Attachment.Path)
-	case "minio":
-		minio := setting.Attachment.Minio
-		Attachments, err = NewMinioStorage(
+	var s ObjectStorage
+	switch storageCfg.Type {
+	case setting.LocalStorageType:
+		s, err = NewLocalStorage(storageCfg.Path)
+	case setting.MinioStorageType:
+		minio := storageCfg.Minio
+		s, err = NewMinioStorage(
 			context.Background(),
 			minio.Endpoint,
 			minio.AccessKeyID,
@@ -88,40 +86,22 @@ func initAttachments() error {
 			minio.UseSSL,
 		)
 	default:
-		return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType)
+		return nil, fmt.Errorf("Unsupported attachment store type: %s", storageCfg.Type)
 	}
 
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	return nil
+	return s, nil
 }
 
-func initLFS() error {
-	var err error
-	switch setting.LFS.StoreType {
-	case "local":
-		LFS, err = NewLocalStorage(setting.LFS.ContentPath)
-	case "minio":
-		minio := setting.LFS.Minio
-		LFS, err = NewMinioStorage(
-			context.Background(),
-			minio.Endpoint,
-			minio.AccessKeyID,
-			minio.SecretAccessKey,
-			minio.Bucket,
-			minio.Location,
-			minio.BasePath,
-			minio.UseSSL,
-		)
-	default:
-		return fmt.Errorf("Unsupported LFS store type: %s", setting.LFS.StoreType)
-	}
-
-	if err != nil {
-		return err
-	}
+func initAttachments() (err error) {
+	Attachments, err = initStorage(setting.Attachment.Storage)
+	return
+}
 
-	return nil
+func initLFS() (err error) {
+	LFS, err = initStorage(setting.LFS.Storage)
+	return
 }