LFS support to be stored on minio (#12518)

* LFS support to be stored on minio

* Fix test

* Fix lint

* Fix lint

* Fix check

* Fix test

* Update documents and add migration for LFS

* Fix some bugs

3 files changed, 62 insertions, 69 deletions

diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go
index b0fa77e255..cf0a05d644 100644
--- a/modules/lfs/content_store.go
+++ b/modules/lfs/content_store.go
@@ -10,11 +10,10 @@ import (
 	"errors"
 	"io"
 	"os"
-	"path/filepath"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/modules/storage"
 )
 
 var (
@@ -24,17 +23,15 @@ var (
 
 // ContentStore provides a simple file system based storage.
 type ContentStore struct {
-	BasePath string
+	storage.ObjectStorage
 }
 
 // Get takes a Meta object and retrieves the content from the store, returning
 // it as an io.Reader. If fromByte > 0, the reader starts from that byte
 func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-
-	f, err := os.Open(path)
+	f, err := s.Open(meta.RelativePath())
 	if err != nil {
-		log.Error("Whilst trying to read LFS OID[%s]: Unable to open %s Error: %v", meta.Oid, path, err)
+		log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
 		return nil, err
 	}
 	if fromByte > 0 {
@@ -48,82 +45,55 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
 
 // Put takes a Meta object and an io.Reader and writes the content to the store.
 func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-	tmpPath := path + ".tmp"
-
-	dir := filepath.Dir(path)
-	if err := os.MkdirAll(dir, 0750); err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Unable to create the LFS directory: %s Error: %v", meta.Oid, dir, err)
-		return err
-	}
-
-	file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
-	if err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Unable to open temporary file for writing: %s Error: %v", tmpPath, err)
-		return err
-	}
-	defer func() {
-		if err := util.Remove(tmpPath); err != nil {
-			log.Warn("Unable to remove temporary path: %s: Error: %v", tmpPath, err)
-		}
-	}()
-
 	hash := sha256.New()
-	hw := io.MultiWriter(hash, file)
-
-	written, err := io.Copy(hw, r)
+	rd := io.TeeReader(r, hash)
+	p := meta.RelativePath()
+	written, err := s.Save(p, rd)
 	if err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, tmpPath, err)
-		file.Close()
+		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
 		return err
 	}
-	file.Close()
 
 	if written != meta.Size {
+		if err := s.Delete(p); err != nil {
+			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+		}
 		return errSizeMismatch
 	}
 
 	shaStr := hex.EncodeToString(hash.Sum(nil))
 	if shaStr != meta.Oid {
+		if err := s.Delete(p); err != nil {
+			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+		}
 		return errHashMismatch
 	}
 
-	if err := os.Rename(tmpPath, path); err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Unable to move tmp file to final destination: %s Error: %v", meta.Oid, path, err)
-		return err
-	}
-
 	return nil
 }
 
 // Exists returns true if the object exists in the content store.
-func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return false
+func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
+	_, err := s.ObjectStorage.Stat(meta.RelativePath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
 	}
-	return true
+	return true, nil
 }
 
 // Verify returns true if the object exists in the content store and size is correct.
 func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-
-	fi, err := os.Stat(path)
-	if os.IsNotExist(err) || err == nil && fi.Size() != meta.Size {
+	p := meta.RelativePath()
+	fi, err := s.ObjectStorage.Stat(p)
+	if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
 		return false, nil
 	} else if err != nil {
-		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", path, meta.Oid, err)
+		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
 		return false, err
 	}
 
 	return true, nil
 }
-
-func transformKey(key string) string {
-	if len(key) < 5 {
-		return key
-	}
-
-	return filepath.Join(key[0:2], key[2:4], key[4:])
-}
diff --git a/modules/lfs/pointers.go b/modules/lfs/pointers.go
index bc27ee37a7..c6fbf090e5 100644
--- a/modules/lfs/pointers.go
+++ b/modules/lfs/pointers.go
@@ -12,6 +12,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 )
 
 // ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
@@ -53,9 +54,10 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
 		return nil
 	}
 
-	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	contentStore := &ContentStore{ObjectStorage: storage.LFS}
 	meta := &models.LFSMetaObject{Oid: oid, Size: size}
-	if !contentStore.Exists(meta) {
+	exist, err := contentStore.Exists(meta)
+	if err != nil || !exist {
 		return nil
 	}
 
@@ -64,6 +66,6 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
 
 // ReadMetaObject will read a models.LFSMetaObject and return a reader
 func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
-	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	contentStore := &ContentStore{ObjectStorage: storage.LFS}
 	return contentStore.Get(meta, 0)
 }
diff --git a/modules/lfs/server.go b/modules/lfs/server.go
index f227ebe2eb..2801f8410c 100644
--- a/modules/lfs/server.go
+++ b/modules/lfs/server.go
@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 
 	"gitea.com/macaron/macaron"
 	"github.com/dgrijalva/jwt-go"
@@ -187,7 +188,7 @@ func getContentHandler(ctx *context.Context) {
 		}
 	}
 
-	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	contentStore := &ContentStore{ObjectStorage: storage.LFS}
 	content, err := contentStore.Get(meta, fromByte)
 	if err != nil {
 		// Errors are logged in contentStore.Get
@@ -288,8 +289,14 @@ func PostHandler(ctx *context.Context) {
 	ctx.Resp.Header().Set("Content-Type", metaMediaType)
 
 	sentStatus := 202
-	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
-	if meta.Existing && contentStore.Exists(meta) {
+	contentStore := &ContentStore{ObjectStorage: storage.LFS}
+	exist, err := contentStore.Exists(meta)
+	if err != nil {
+		log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err)
+		writeStatus(ctx, 500)
+		return
+	}
+	if meta.Existing && exist {
 		sentStatus = 200
 	}
 	ctx.Resp.WriteHeader(sentStatus)
@@ -343,12 +350,20 @@ func BatchHandler(ctx *context.Context) {
 			return
 		}
 
-		contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+		contentStore := &ContentStore{ObjectStorage: storage.LFS}
 
 		meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
-		if err == nil && contentStore.Exists(meta) { // Object is found and exists
-			responseObjects = append(responseObjects, Represent(object, meta, true, false))
-			continue
+		if err == nil { // Object is found and exists
+			exist, err := contentStore.Exists(meta)
+			if err != nil {
+				log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
+				writeStatus(ctx, 500)
+				return
+			}
+			if exist {
+				responseObjects = append(responseObjects, Represent(object, meta, true, false))
+				continue
+			}
 		}
 
 		if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
@@ -360,7 +375,13 @@ func BatchHandler(ctx *context.Context) {
 		// Object is not found
 		meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
 		if err == nil {
-			responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !contentStore.Exists(meta)))
+			exist, err := contentStore.Exists(meta)
+			if err != nil {
+				log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
+				writeStatus(ctx, 500)
+				return
+			}
+			responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist))
 		} else {
 			log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err)
 		}
@@ -387,7 +408,7 @@ func PutHandler(ctx *context.Context) {
 		return
 	}
 
-	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	contentStore := &ContentStore{ObjectStorage: storage.LFS}
 	bodyReader := ctx.Req.Body().ReadCloser()
 	defer bodyReader.Close()
 	if err := contentStore.Put(meta, bodyReader); err != nil {
@@ -429,7 +450,7 @@ func VerifyHandler(ctx *context.Context) {
 		return
 	}
 
-	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	contentStore := &ContentStore{ObjectStorage: storage.LFS}
 	ok, err := contentStore.Verify(meta)
 	if err != nil {
 		// Error will be logged in Verify