Add a storage layer for attachments (#11387)

* Add a storage layer for attachments

* Fix some bug

* fix test

* Fix copyright head and lint

* Fix bug

* Add setting for minio and flags for migrate-storage

* Add documents

* fix lint

* Add test for minio store type on attachments

* fix test

* fix test

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add warning when storage migrated successfully

* Fix drone

* fix test

* rebase

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* remove log on xorm

* Fi download bug

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* Add URL function to serve attachments directly from S3/Minio

* Add ability to enable/disable redirection in attachment configuration

* Fix typo

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* don't change unrelated files

* Fix lint

* Fix build

* update go.mod and go.sum

* Use github.com/minio/minio-go/v6

* Remove unused function

* Upgrade minio to v7 and some other improvements

* fix lint

* Fix go mod

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: Tyler <tystuyfzand@gmail.com>

2 files changed, 76 insertions, 16 deletions

diff --git a/modules/setting/attachment.go b/modules/setting/attachment.go
new file mode 100644
index 0000000000..4c7368eb68
--- /dev/null
+++ b/modules/setting/attachment.go
@@ -0,0 +1,75 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+var (
+	// Attachment settings
+	Attachment = struct {
+		StoreType   string
+		Path        string
+		ServeDirect bool
+		Minio       struct {
+			Endpoint        string
+			AccessKeyID     string
+			SecretAccessKey string
+			UseSSL          bool
+			Bucket          string
+			Location        string
+			BasePath        string
+		}
+		AllowedTypes string
+		MaxSize      int64
+		MaxFiles     int
+		Enabled      bool
+	}{
+		StoreType:   "local",
+		ServeDirect: false,
+		Minio: struct {
+			Endpoint        string
+			AccessKeyID     string
+			SecretAccessKey string
+			UseSSL          bool
+			Bucket          string
+			Location        string
+			BasePath        string
+		}{},
+		AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
+		MaxSize:      4,
+		MaxFiles:     5,
+		Enabled:      true,
+	}
+)
+
+func newAttachmentService() {
+	sec := Cfg.Section("attachment")
+	Attachment.StoreType = sec.Key("STORE_TYPE").MustString("local")
+	Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
+	switch Attachment.StoreType {
+	case "local":
+		Attachment.Path = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
+		if !filepath.IsAbs(Attachment.Path) {
+			Attachment.Path = path.Join(AppWorkPath, Attachment.Path)
+		}
+	case "minio":
+		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
+		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
+		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
+		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
+		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
+		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
+		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
+	}
+
+	Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
+	Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4)
+	Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5)
+	Attachment.Enabled = sec.Key("ENABLED").MustBool(true)
+}
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 33854a2300..c4cbe1dd98 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -299,13 +299,6 @@ var (
 	AccessLogTemplate  string
 	EnableXORMLog      bool
 
-	// Attachment settings
-	AttachmentPath         string
-	AttachmentAllowedTypes string
-	AttachmentMaxSize      int64
-	AttachmentMaxFiles     int
-	AttachmentEnabled      bool
-
 	// Time settings
 	TimeFormat string
 	// UILocation is the location on the UI, so that we can display the time on UI.
@@ -840,15 +833,7 @@ func NewContext() {
 		}
 	}
 
-	sec = Cfg.Section("attachment")
-	AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
-	if !filepath.IsAbs(AttachmentPath) {
-		AttachmentPath = path.Join(AppWorkPath, AttachmentPath)
-	}
-	AttachmentAllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
-	AttachmentMaxSize = sec.Key("MAX_SIZE").MustInt64(4)
-	AttachmentMaxFiles = sec.Key("MAX_FILES").MustInt(5)
-	AttachmentEnabled = sec.Key("ENABLED").MustBool(true)
+	newAttachmentService()
 
 	timeFormatKey := Cfg.Section("time").Key("FORMAT").MustString("")
 	if timeFormatKey != "" {