Add a storage layer for attachments (#11387)

* Add a storage layer for attachments

* Fix some bug

* fix test

* Fix copyright head and lint

* Fix bug

* Add setting for minio and flags for migrate-storage

* Add documents

* fix lint

* Add test for minio store type on attachments

* fix test

* fix test

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add warning when storage migrated successfully

* Fix drone

* fix test

* rebase

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* remove log on xorm

* Fi download bug

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* Add URL function to serve attachments directly from S3/Minio

* Add ability to enable/disable redirection in attachment configuration

* Fix typo

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* don't change unrelated files

* Fix lint

* Fix build

* update go.mod and go.sum

* Use github.com/minio/minio-go/v6

* Remove unused function

* Upgrade minio to v7 and some other improvements

* fix lint

* Fix go mod

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: Tyler <tystuyfzand@gmail.com>

5 files changed, 34 insertions, 15 deletions

diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go
index 51725100b8..d3201faec0 100644
--- a/routers/repo/attachment.go
+++ b/routers/repo/attachment.go
@@ -7,26 +7,27 @@ package repo
 import (
 	"fmt"
 	"net/http"
-	"os"
 	"strings"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/upload"
 )
 
 func renderAttachmentSettings(ctx *context.Context) {
-	ctx.Data["IsAttachmentEnabled"] = setting.AttachmentEnabled
-	ctx.Data["AttachmentAllowedTypes"] = setting.AttachmentAllowedTypes
-	ctx.Data["AttachmentMaxSize"] = setting.AttachmentMaxSize
-	ctx.Data["AttachmentMaxFiles"] = setting.AttachmentMaxFiles
+	ctx.Data["IsAttachmentEnabled"] = setting.Attachment.Enabled
+	ctx.Data["AttachmentStoreType"] = setting.Attachment.StoreType
+	ctx.Data["AttachmentAllowedTypes"] = setting.Attachment.AllowedTypes
+	ctx.Data["AttachmentMaxSize"] = setting.Attachment.MaxSize
+	ctx.Data["AttachmentMaxFiles"] = setting.Attachment.MaxFiles
 }
 
 // UploadAttachment response for uploading issue's attachment
 func UploadAttachment(ctx *context.Context) {
-	if !setting.AttachmentEnabled {
+	if !setting.Attachment.Enabled {
 		ctx.Error(404, "attachment is not enabled")
 		return
 	}
@@ -44,7 +45,7 @@ func UploadAttachment(ctx *context.Context) {
 		buf = buf[:n]
 	}
 
-	err = upload.VerifyAllowedContentType(buf, strings.Split(setting.AttachmentAllowedTypes, ","))
+	err = upload.VerifyAllowedContentType(buf, strings.Split(setting.Attachment.AllowedTypes, ","))
 	if err != nil {
 		ctx.Error(400, err.Error())
 		return
@@ -122,8 +123,23 @@ func GetAttachment(ctx *context.Context) {
 		}
 	}
 
+	if setting.Attachment.ServeDirect {
+		//If we have a signed url (S3, object storage), redirect to this directly.
+		u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name)
+
+		if u != nil && err == nil {
+			if err := attach.IncreaseDownloadCount(); err != nil {
+				ctx.ServerError("Update", err)
+				return
+			}
+
+			ctx.Redirect(u.String())
+			return
+		}
+	}
+
 	//If we have matched and access to release or issue
-	fr, err := os.Open(attach.LocalPath())
+	fr, err := storage.Attachments.Open(attach.RelativePath())
 	if err != nil {
 		ctx.ServerError("Open", err)
 		return
diff --git a/routers/repo/download.go b/routers/repo/download.go
index 5ae9475ae7..326f097cbc 100644
--- a/routers/repo/download.go
+++ b/routers/repo/download.go
@@ -22,7 +22,10 @@ import (
 // ServeData download file from io.Reader
 func ServeData(ctx *context.Context, name string, reader io.Reader) error {
 	buf := make([]byte, 1024)
-	n, _ := reader.Read(buf)
+	n, err := reader.Read(buf)
+	if err != nil && err != io.EOF {
+		return err
+	}
 	if n >= 0 {
 		buf = buf[:n]
 	}
@@ -48,7 +51,7 @@ func ServeData(ctx *context.Context, name string, reader io.Reader) error {
 		ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
 	}
 
-	_, err := ctx.Resp.Write(buf)
+	_, err = ctx.Resp.Write(buf)
 	if err != nil {
 		return err
 	}
diff --git a/routers/repo/issue.go b/routers/repo/issue.go
index 2eabd6ab6c..d671f8de74 100644
--- a/routers/repo/issue.go
+++ b/routers/repo/issue.go
@@ -692,7 +692,7 @@ func NewIssuePost(ctx *context.Context, form auth.CreateIssueForm) {
 		return
 	}
 
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachments = form.Files
 	}
 
@@ -1633,7 +1633,7 @@ func NewComment(ctx *context.Context, form auth.CreateCommentForm) {
 	}
 
 	var attachments []string
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachments = form.Files
 	}
 
diff --git a/routers/repo/pull.go b/routers/repo/pull.go
index ed70ec13a8..a19dbb5cb3 100644
--- a/routers/repo/pull.go
+++ b/routers/repo/pull.go
@@ -911,7 +911,7 @@ func CompareAndPullRequestPost(ctx *context.Context, form auth.CreateIssueForm)
 		return
 	}
 
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachments = form.Files
 	}
 
diff --git a/routers/repo/release.go b/routers/repo/release.go
index 02fbcaccaf..c93c8f5d61 100644
--- a/routers/repo/release.go
+++ b/routers/repo/release.go
@@ -212,7 +212,7 @@ func NewReleasePost(ctx *context.Context, form auth.NewReleaseForm) {
 	}
 
 	var attachmentUUIDs []string
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachmentUUIDs = form.Files
 	}
 
@@ -333,7 +333,7 @@ func EditReleasePost(ctx *context.Context, form auth.EditReleaseForm) {
 	}
 
 	var attachmentUUIDs []string
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachmentUUIDs = form.Files
 	}