Add a storage layer for attachments (#11387)

* Add a storage layer for attachments

* Fix some bug

* fix test

* Fix copyright head and lint

* Fix bug

* Add setting for minio and flags for migrate-storage

* Add documents

* fix lint

* Add test for minio store type on attachments

* fix test

* fix test

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add warning when storage migrated successfully

* Fix drone

* fix test

* rebase

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* remove log on xorm

* Fi download bug

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* Add URL function to serve attachments directly from S3/Minio

* Add ability to enable/disable redirection in attachment configuration

* Fix typo

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* don't change unrelated files

* Fix lint

* Fix build

* update go.mod and go.sum

* Use github.com/minio/minio-go/v6

* Remove unused function

* Upgrade minio to v7 and some other improvements

* fix lint

* Fix go mod

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: Tyler <tystuyfzand@gmail.com>

7 files changed, 68 insertions, 47 deletions

diff --git a/models/admin.go b/models/admin.go
index ebd01419a8..9ed9c44ebb 100644
--- a/models/admin.go
+++ b/models/admin.go
@@ -1,4 +1,5 @@
 // Copyright 2014 The Gogs Authors. All rights reserved.
+// Copyright 2020 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
@@ -8,6 +9,7 @@ import (
 	"fmt"
 
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
 
@@ -65,6 +67,18 @@ func RemoveAllWithNotice(title, path string) {
 	removeAllWithNotice(x, title, path)
 }
 
+// RemoveStorageWithNotice removes a file from the storage and
+// creates a system notice when error occurs.
+func RemoveStorageWithNotice(bucket storage.ObjectStorage, title, path string) {
+	if err := bucket.Delete(path); err != nil {
+		desc := fmt.Sprintf("%s [%s]: %v", title, path, err)
+		log.Warn(title+" [%s]: %v", path, err)
+		if err = createNotice(x, NoticeRepository, desc); err != nil {
+			log.Error("CreateRepositoryNotice: %v", err)
+		}
+	}
+}
+
 func removeAllWithNotice(e Engine, title, path string) {
 	if err := util.RemoveAll(path); err != nil {
 		desc := fmt.Sprintf("%s [%s]: %v", title, path, err)
diff --git a/models/attachment.go b/models/attachment.go
index 6215de95b5..26f466a400 100644
--- a/models/attachment.go
+++ b/models/attachment.go
@@ -5,15 +5,15 @@
 package models
 
 import (
+	"bytes"
 	"fmt"
 	"io"
-	"os"
 	"path"
 
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
-	"code.gitea.io/gitea/modules/util"
 
 	gouuid "github.com/google/uuid"
 	"xorm.io/xorm"
@@ -56,15 +56,14 @@ func (a *Attachment) APIFormat() *api.Attachment {
 	}
 }
 
-// AttachmentLocalPath returns where attachment is stored in local file
-// system based on given UUID.
-func AttachmentLocalPath(uuid string) string {
-	return path.Join(setting.AttachmentPath, uuid[0:1], uuid[1:2], uuid)
+// AttachmentRelativePath returns the relative path
+func AttachmentRelativePath(uuid string) string {
+	return path.Join(uuid[0:1], uuid[1:2], uuid)
 }
 
-// LocalPath returns where attachment is stored in local file system.
-func (a *Attachment) LocalPath() string {
-	return AttachmentLocalPath(a.UUID)
+// RelativePath returns the relative path of the attachment
+func (a *Attachment) RelativePath() string {
+	return AttachmentRelativePath(a.UUID)
 }
 
 // DownloadURL returns the download url of the attached file
@@ -100,29 +99,11 @@ func (a *Attachment) LinkedRepository() (*Repository, UnitType, error) {
 func NewAttachment(attach *Attachment, buf []byte, file io.Reader) (_ *Attachment, err error) {
 	attach.UUID = gouuid.New().String()
 
-	localPath := attach.LocalPath()
-	if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
-		return nil, fmt.Errorf("MkdirAll: %v", err)
-	}
-
-	fw, err := os.Create(localPath)
+	size, err := storage.Attachments.Save(attach.RelativePath(), io.MultiReader(bytes.NewReader(buf), file))
 	if err != nil {
 		return nil, fmt.Errorf("Create: %v", err)
 	}
-	defer fw.Close()
-
-	if _, err = fw.Write(buf); err != nil {
-		return nil, fmt.Errorf("Write: %v", err)
-	} else if _, err = io.Copy(fw, file); err != nil {
-		return nil, fmt.Errorf("Copy: %v", err)
-	}
-
-	// Update file size
-	var fi os.FileInfo
-	if fi, err = fw.Stat(); err != nil {
-		return nil, fmt.Errorf("file size: %v", err)
-	}
-	attach.Size = fi.Size()
+	attach.Size = size
 
 	if _, err := x.Insert(attach); err != nil {
 		return nil, err
@@ -238,7 +219,7 @@ func DeleteAttachments(attachments []*Attachment, remove bool) (int, error) {
 
 	if remove {
 		for i, a := range attachments {
-			if err := util.Remove(a.LocalPath()); err != nil {
+			if err := storage.Attachments.Delete(a.RelativePath()); err != nil {
 				return i, err
 			}
 		}
@@ -290,3 +271,25 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
 	_, err := x.Where("release_id = ?", releaseID).Delete(&Attachment{})
 	return err
 }
+
+// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users.
+func IterateAttachment(f func(attach *Attachment) error) error {
+	var start int
+	const batchSize = 100
+	for {
+		var attachments = make([]*Attachment, 0, batchSize)
+		if err := x.Limit(batchSize, start).Find(&attachments); err != nil {
+			return err
+		}
+		if len(attachments) == 0 {
+			return nil
+		}
+		start += len(attachments)
+
+		for _, attach := range attachments {
+			if err := f(attach); err != nil {
+				return err
+			}
+		}
+	}
+}
diff --git a/models/issue.go b/models/issue.go
index 07d7fc9956..2912f7e8ef 100644
--- a/models/issue.go
+++ b/models/issue.go
@@ -1983,8 +1983,9 @@ func deleteIssuesByRepoID(sess Engine, repoID int64) (attachmentPaths []string,
 		Find(&attachments); err != nil {
 		return
 	}
+
 	for j := range attachments {
-		attachmentPaths = append(attachmentPaths, attachments[j].LocalPath())
+		attachmentPaths = append(attachmentPaths, attachments[j].RelativePath())
 	}
 
 	if _, err = sess.In("issue_id", deleteCond).
diff --git a/models/migrations/v112.go b/models/migrations/v112.go
index 7e80037700..9da7d8a781 100644
--- a/models/migrations/v112.go
+++ b/models/migrations/v112.go
@@ -6,7 +6,7 @@ package migrations
 
 import (
 	"fmt"
-	"path"
+	"path/filepath"
 
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
@@ -31,7 +31,7 @@ func removeAttachmentMissedRepo(x *xorm.Engine) error {
 
 		for i := 0; i < len(attachments); i++ {
 			uuid := attachments[i].UUID
-			if err = util.RemoveAll(path.Join(setting.AttachmentPath, uuid[0:1], uuid[1:2], uuid)); err != nil {
+			if err = util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
 				fmt.Printf("Error: %v", err)
 			}
 		}
diff --git a/models/migrations/v96.go b/models/migrations/v96.go
index 5decb832cd..f471ac384d 100644
--- a/models/migrations/v96.go
+++ b/models/migrations/v96.go
@@ -5,7 +5,7 @@
 package migrations
 
 import (
-	"path"
+	"path/filepath"
 
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
@@ -14,7 +14,6 @@ import (
 )
 
 func deleteOrphanedAttachments(x *xorm.Engine) error {
-
 	type Attachment struct {
 		ID        int64  `xorm:"pk autoincr"`
 		UUID      string `xorm:"uuid UNIQUE"`
@@ -23,12 +22,6 @@ func deleteOrphanedAttachments(x *xorm.Engine) error {
 		CommentID int64
 	}
 
-	// AttachmentLocalPath returns where attachment is stored in local file
-	// system based on given UUID.
-	AttachmentLocalPath := func(uuid string) string {
-		return path.Join(setting.AttachmentPath, uuid[0:1], uuid[1:2], uuid)
-	}
-
 	sess := x.NewSession()
 	defer sess.Close()
 
@@ -53,12 +46,15 @@ func deleteOrphanedAttachments(x *xorm.Engine) error {
 		for _, attachment := range attachements {
 			ids = append(ids, attachment.ID)
 		}
-		if _, err := sess.In("id", ids).Delete(new(Attachment)); err != nil {
-			return err
+		if len(ids) > 0 {
+			if _, err := sess.In("id", ids).Delete(new(Attachment)); err != nil {
+				return err
+			}
 		}
 
 		for _, attachment := range attachements {
-			if err := util.RemoveAll(AttachmentLocalPath(attachment.UUID)); err != nil {
+			uuid := attachment.UUID
+			if err := util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
 				return err
 			}
 		}
diff --git a/models/repo.go b/models/repo.go
index 146868d876..b9ebec8be9 100644
--- a/models/repo.go
+++ b/models/repo.go
@@ -32,6 +32,7 @@ import (
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
@@ -1595,7 +1596,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
 	}
 	releaseAttachments := make([]string, 0, len(attachments))
 	for i := 0; i < len(attachments); i++ {
-		releaseAttachments = append(releaseAttachments, attachments[i].LocalPath())
+		releaseAttachments = append(releaseAttachments, attachments[i].RelativePath())
 	}
 
 	if _, err = sess.Exec("UPDATE `user` SET num_stars=num_stars-1 WHERE id IN (SELECT `uid` FROM `star` WHERE repo_id = ?)", repo.ID); err != nil {
@@ -1720,12 +1721,12 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
 
 	// Remove issue attachment files.
 	for i := range attachmentPaths {
-		removeAllWithNotice(x, "Delete issue attachment", attachmentPaths[i])
+		RemoveStorageWithNotice(storage.Attachments, "Delete issue attachment", attachmentPaths[i])
 	}
 
 	// Remove release attachment files.
 	for i := range releaseAttachments {
-		removeAllWithNotice(x, "Delete release attachment", releaseAttachments[i])
+		RemoveStorageWithNotice(storage.Attachments, "Delete release attachment", releaseAttachments[i])
 	}
 
 	if len(repo.Avatar) > 0 {
diff --git a/models/unit_tests.go b/models/unit_tests.go
index dca363ffac..11b7708ffd 100644
--- a/models/unit_tests.go
+++ b/models/unit_tests.go
@@ -15,6 +15,7 @@ import (
 
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/util"
 
 	"github.com/stretchr/testify/assert"
@@ -67,6 +68,11 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
 		fatalTestError("url.Parse: %v\n", err)
 	}
 
+	setting.Attachment.Path = filepath.Join(setting.AppDataPath, "attachments")
+	if err = storage.Init(); err != nil {
+		fatalTestError("storage.Init: %v\n", err)
+	}
+
 	if err = util.RemoveAll(setting.RepoRootPath); err != nil {
 		fatalTestError("util.RemoveAll: %v\n", err)
 	}