Add a storage layer for attachments (#11387)

* Add a storage layer for attachments

* Fix some bug

* fix test

* Fix copyright head and lint

* Fix bug

* Add setting for minio and flags for migrate-storage

* Add documents

* fix lint

* Add test for minio store type on attachments

* fix test

* fix test

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add warning when storage migrated successfully

* Fix drone

* fix test

* rebase

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* remove log on xorm

* Fi download bug

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* Add URL function to serve attachments directly from S3/Minio

* Add ability to enable/disable redirection in attachment configuration

* Fix typo

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* don't change unrelated files

* Fix lint

* Fix build

* update go.mod and go.sum

* Use github.com/minio/minio-go/v6

* Remove unused function

* Upgrade minio to v7 and some other improvements

* fix lint

* Fix go mod

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: Tyler <tystuyfzand@gmail.com>

1 files changed, 33 insertions, 30 deletions

diff --git a/models/attachment.go b/models/attachment.go
index 6215de95b5..26f466a400 100644
--- a/models/attachment.go
+++ b/models/attachment.go
@@ -5,15 +5,15 @@
 package models
 
 import (
+	"bytes"
 	"fmt"
 	"io"
-	"os"
 	"path"
 
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
-	"code.gitea.io/gitea/modules/util"
 
 	gouuid "github.com/google/uuid"
 	"xorm.io/xorm"
@@ -56,15 +56,14 @@ func (a *Attachment) APIFormat() *api.Attachment {
 	}
 }
 
-// AttachmentLocalPath returns where attachment is stored in local file
-// system based on given UUID.
-func AttachmentLocalPath(uuid string) string {
-	return path.Join(setting.AttachmentPath, uuid[0:1], uuid[1:2], uuid)
+// AttachmentRelativePath returns the relative path
+func AttachmentRelativePath(uuid string) string {
+	return path.Join(uuid[0:1], uuid[1:2], uuid)
 }
 
-// LocalPath returns where attachment is stored in local file system.
-func (a *Attachment) LocalPath() string {
-	return AttachmentLocalPath(a.UUID)
+// RelativePath returns the relative path of the attachment
+func (a *Attachment) RelativePath() string {
+	return AttachmentRelativePath(a.UUID)
 }
 
 // DownloadURL returns the download url of the attached file
@@ -100,29 +99,11 @@ func (a *Attachment) LinkedRepository() (*Repository, UnitType, error) {
 func NewAttachment(attach *Attachment, buf []byte, file io.Reader) (_ *Attachment, err error) {
 	attach.UUID = gouuid.New().String()
 
-	localPath := attach.LocalPath()
-	if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
-		return nil, fmt.Errorf("MkdirAll: %v", err)
-	}
-
-	fw, err := os.Create(localPath)
+	size, err := storage.Attachments.Save(attach.RelativePath(), io.MultiReader(bytes.NewReader(buf), file))
 	if err != nil {
 		return nil, fmt.Errorf("Create: %v", err)
 	}
-	defer fw.Close()
-
-	if _, err = fw.Write(buf); err != nil {
-		return nil, fmt.Errorf("Write: %v", err)
-	} else if _, err = io.Copy(fw, file); err != nil {
-		return nil, fmt.Errorf("Copy: %v", err)
-	}
-
-	// Update file size
-	var fi os.FileInfo
-	if fi, err = fw.Stat(); err != nil {
-		return nil, fmt.Errorf("file size: %v", err)
-	}
-	attach.Size = fi.Size()
+	attach.Size = size
 
 	if _, err := x.Insert(attach); err != nil {
 		return nil, err
@@ -238,7 +219,7 @@ func DeleteAttachments(attachments []*Attachment, remove bool) (int, error) {
 
 	if remove {
 		for i, a := range attachments {
-			if err := util.Remove(a.LocalPath()); err != nil {
+			if err := storage.Attachments.Delete(a.RelativePath()); err != nil {
 				return i, err
 			}
 		}
@@ -290,3 +271,25 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
 	_, err := x.Where("release_id = ?", releaseID).Delete(&Attachment{})
 	return err
 }
+
+// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users.
+func IterateAttachment(f func(attach *Attachment) error) error {
+	var start int
+	const batchSize = 100
+	for {
+		var attachments = make([]*Attachment, 0, batchSize)
+		if err := x.Limit(batchSize, start).Find(&attachments); err != nil {
+			return err
+		}
+		if len(attachments) == 0 {
+			return nil
+		}
+		start += len(attachments)
+
+		for _, attach := range attachments {
+			if err := f(attach); err != nil {
+				return err
+			}
+		}
+	}
+}