Add a storage layer for attachments (#11387)

* Add a storage layer for attachments

* Fix some bug

* fix test

* Fix copyright head and lint

* Fix bug

* Add setting for minio and flags for migrate-storage

* Add documents

* fix lint

* Add test for minio store type on attachments

* fix test

* fix test

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add warning when storage migrated successfully

* Fix drone

* fix test

* rebase

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* remove log on xorm

* Fi download bug

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* Add URL function to serve attachments directly from S3/Minio

* Add ability to enable/disable redirection in attachment configuration

* Fix typo

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* don't change unrelated files

* Fix lint

* Fix build

* update go.mod and go.sum

* Use github.com/minio/minio-go/v6

* Remove unused function

* Upgrade minio to v7 and some other improvements

* fix lint

* Fix go mod

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: Tyler <tystuyfzand@gmail.com>

7 files changed, 323 insertions, 30 deletions

diff --git a/modules/context/context.go b/modules/context/context.go
index cc68fb093b..1405860e07 100644
--- a/modules/context/context.go
+++ b/modules/context/context.go
@@ -319,7 +319,7 @@ func Contexter() macaron.Handler {
 
 		// If request sends files, parse them here otherwise the Query() can't be parsed and the CsrfToken will be invalid.
 		if ctx.Req.Method == "POST" && strings.Contains(ctx.Req.Header.Get("Content-Type"), "multipart/form-data") {
-			if err := ctx.Req.ParseMultipartForm(setting.AttachmentMaxSize << 20); err != nil && !strings.Contains(err.Error(), "EOF") { // 32MB max size
+			if err := ctx.Req.ParseMultipartForm(setting.Attachment.MaxSize << 20); err != nil && !strings.Contains(err.Error(), "EOF") { // 32MB max size
 				ctx.ServerError("ParseMultipartForm", err)
 				return
 			}
diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go
index 7d446fb408..8c097e143c 100644
--- a/modules/migrations/gitea.go
+++ b/modules/migrations/gitea.go
@@ -13,7 +13,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -26,6 +25,7 @@ import (
 	"code.gitea.io/gitea/modules/repository"
 	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 
@@ -275,18 +275,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 				}
 				defer resp.Body.Close()
 
-				localPath := attach.LocalPath()
-				if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
-					return fmt.Errorf("MkdirAll: %v", err)
-				}
-
-				fw, err := os.Create(localPath)
-				if err != nil {
-					return fmt.Errorf("Create: %v", err)
-				}
-				defer fw.Close()
-
-				_, err = io.Copy(fw, resp.Body)
+				_, err = storage.Attachments.Save(attach.RelativePath(), resp.Body)
 				return err
 			}()
 			if err != nil {
diff --git a/modules/setting/attachment.go b/modules/setting/attachment.go
new file mode 100644
index 0000000000..4c7368eb68
--- /dev/null
+++ b/modules/setting/attachment.go
@@ -0,0 +1,75 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+var (
+	// Attachment settings
+	Attachment = struct {
+		StoreType   string
+		Path        string
+		ServeDirect bool
+		Minio       struct {
+			Endpoint        string
+			AccessKeyID     string
+			SecretAccessKey string
+			UseSSL          bool
+			Bucket          string
+			Location        string
+			BasePath        string
+		}
+		AllowedTypes string
+		MaxSize      int64
+		MaxFiles     int
+		Enabled      bool
+	}{
+		StoreType:   "local",
+		ServeDirect: false,
+		Minio: struct {
+			Endpoint        string
+			AccessKeyID     string
+			SecretAccessKey string
+			UseSSL          bool
+			Bucket          string
+			Location        string
+			BasePath        string
+		}{},
+		AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
+		MaxSize:      4,
+		MaxFiles:     5,
+		Enabled:      true,
+	}
+)
+
+func newAttachmentService() {
+	sec := Cfg.Section("attachment")
+	Attachment.StoreType = sec.Key("STORE_TYPE").MustString("local")
+	Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
+	switch Attachment.StoreType {
+	case "local":
+		Attachment.Path = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
+		if !filepath.IsAbs(Attachment.Path) {
+			Attachment.Path = path.Join(AppWorkPath, Attachment.Path)
+		}
+	case "minio":
+		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
+		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
+		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
+		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
+		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
+		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
+		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
+	}
+
+	Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
+	Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4)
+	Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5)
+	Attachment.Enabled = sec.Key("ENABLED").MustBool(true)
+}
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 33854a2300..c4cbe1dd98 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -299,13 +299,6 @@ var (
 	AccessLogTemplate  string
 	EnableXORMLog      bool
 
-	// Attachment settings
-	AttachmentPath         string
-	AttachmentAllowedTypes string
-	AttachmentMaxSize      int64
-	AttachmentMaxFiles     int
-	AttachmentEnabled      bool
-
 	// Time settings
 	TimeFormat string
 	// UILocation is the location on the UI, so that we can display the time on UI.
@@ -840,15 +833,7 @@ func NewContext() {
 		}
 	}
 
-	sec = Cfg.Section("attachment")
-	AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
-	if !filepath.IsAbs(AttachmentPath) {
-		AttachmentPath = path.Join(AppWorkPath, AttachmentPath)
-	}
-	AttachmentAllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
-	AttachmentMaxSize = sec.Key("MAX_SIZE").MustInt64(4)
-	AttachmentMaxFiles = sec.Key("MAX_FILES").MustInt(5)
-	AttachmentEnabled = sec.Key("ENABLED").MustBool(true)
+	newAttachmentService()
 
 	timeFormatKey := Cfg.Section("time").Key("FORMAT").MustString("")
 	if timeFormatKey != "" {
diff --git a/modules/storage/local.go b/modules/storage/local.go
new file mode 100644
index 0000000000..4c830211d9
--- /dev/null
+++ b/modules/storage/local.go
@@ -0,0 +1,70 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package storage
+
+import (
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+var (
+	_ ObjectStorage = &LocalStorage{}
+)
+
+// LocalStorage represents a local files storage
+type LocalStorage struct {
+	dir string
+}
+
+// NewLocalStorage returns a local files
+func NewLocalStorage(bucket string) (*LocalStorage, error) {
+	if err := os.MkdirAll(bucket, os.ModePerm); err != nil {
+		return nil, err
+	}
+
+	return &LocalStorage{
+		dir: bucket,
+	}, nil
+}
+
+// Open a file
+func (l *LocalStorage) Open(path string) (io.ReadCloser, error) {
+	return os.Open(filepath.Join(l.dir, path))
+}
+
+// Save a file
+func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) {
+	p := filepath.Join(l.dir, path)
+	if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
+		return 0, err
+	}
+
+	// always override
+	if err := util.Remove(p); err != nil {
+		return 0, err
+	}
+
+	f, err := os.Create(p)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	return io.Copy(f, r)
+}
+
+// Delete delete a file
+func (l *LocalStorage) Delete(path string) error {
+	p := filepath.Join(l.dir, path)
+	return util.Remove(p)
+}
+
+// URL gets the redirect URL to a file
+func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
+	return nil, ErrURLNotSupported
+}
diff --git a/modules/storage/minio.go b/modules/storage/minio.go
new file mode 100644
index 0000000000..77d24e6b73
--- /dev/null
+++ b/modules/storage/minio.go
@@ -0,0 +1,101 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package storage
+
+import (
+	"context"
+	"io"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+var (
+	_            ObjectStorage = &MinioStorage{}
+	quoteEscaper               = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+)
+
+// MinioStorage returns a minio bucket storage
+type MinioStorage struct {
+	ctx      context.Context
+	client   *minio.Client
+	bucket   string
+	basePath string
+}
+
+// NewMinioStorage returns a minio storage
+func NewMinioStorage(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucket, location, basePath string, useSSL bool) (*MinioStorage, error) {
+	minioClient, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: useSSL,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if err := minioClient.MakeBucket(ctx, bucket, minio.MakeBucketOptions{
+		Region: location,
+	}); err != nil {
+		// Check to see if we already own this bucket (which happens if you run this twice)
+		exists, errBucketExists := minioClient.BucketExists(ctx, bucket)
+		if !exists || errBucketExists != nil {
+			return nil, err
+		}
+	}
+
+	return &MinioStorage{
+		ctx:      ctx,
+		client:   minioClient,
+		bucket:   bucket,
+		basePath: basePath,
+	}, nil
+}
+
+func (m *MinioStorage) buildMinioPath(p string) string {
+	return strings.TrimPrefix(path.Join(m.basePath, p), "/")
+}
+
+// Open open a file
+func (m *MinioStorage) Open(path string) (io.ReadCloser, error) {
+	var opts = minio.GetObjectOptions{}
+	object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
+	if err != nil {
+		return nil, err
+	}
+	return object, nil
+}
+
+// Save save a file to minio
+func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) {
+	uploadInfo, err := m.client.PutObject(
+		m.ctx,
+		m.bucket,
+		m.buildMinioPath(path),
+		r,
+		-1,
+		minio.PutObjectOptions{ContentType: "application/octet-stream"},
+	)
+	if err != nil {
+		return 0, err
+	}
+	return uploadInfo.Size, nil
+}
+
+// Delete delete a file
+func (m *MinioStorage) Delete(path string) error {
+	return m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{})
+}
+
+// URL gets the redirect URL to a file. The presigned link is valid for 5 minutes.
+func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
+	reqParams := make(url.Values)
+	// TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we?
+	reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
+	return m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
+}
diff --git a/modules/storage/storage.go b/modules/storage/storage.go
new file mode 100644
index 0000000000..8528ebc5cb
--- /dev/null
+++ b/modules/storage/storage.go
@@ -0,0 +1,73 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package storage
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var (
+	// ErrURLNotSupported represents url is not supported
+	ErrURLNotSupported = errors.New("url method not supported")
+)
+
+// ObjectStorage represents an object storage to handle a bucket and files
+type ObjectStorage interface {
+	Save(path string, r io.Reader) (int64, error)
+	Open(path string) (io.ReadCloser, error)
+	Delete(path string) error
+	URL(path, name string) (*url.URL, error)
+}
+
+// Copy copys a file from source ObjectStorage to dest ObjectStorage
+func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, srcPath string) (int64, error) {
+	f, err := srcStorage.Open(srcPath)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+
+	return dstStorage.Save(dstPath, f)
+}
+
+var (
+	// Attachments represents attachments storage
+	Attachments ObjectStorage
+)
+
+// Init init the stoarge
+func Init() error {
+	var err error
+	switch setting.Attachment.StoreType {
+	case "local":
+		Attachments, err = NewLocalStorage(setting.Attachment.Path)
+	case "minio":
+		minio := setting.Attachment.Minio
+		Attachments, err = NewMinioStorage(
+			context.Background(),
+			minio.Endpoint,
+			minio.AccessKeyID,
+			minio.SecretAccessKey,
+			minio.Bucket,
+			minio.Location,
+			minio.BasePath,
+			minio.UseSSL,
+		)
+	default:
+		return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}