LFS support to be stored on minio (#12518)

* LFS support to be stored on minio

* Fix test

* Fix lint

* Fix lint

* Fix check

* Fix test

* Update documents and add migration for LFS

* Fix some bugs

1 files changed, 26 insertions, 56 deletions

diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go
index b0fa77e255..cf0a05d644 100644
--- a/modules/lfs/content_store.go
+++ b/modules/lfs/content_store.go
@@ -10,11 +10,10 @@ import (
 	"errors"
 	"io"
 	"os"
-	"path/filepath"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/modules/storage"
 )
 
 var (
@@ -24,17 +23,15 @@ var (
 
 // ContentStore provides a simple file system based storage.
 type ContentStore struct {
-	BasePath string
+	storage.ObjectStorage
 }
 
 // Get takes a Meta object and retrieves the content from the store, returning
 // it as an io.Reader. If fromByte > 0, the reader starts from that byte
 func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-
-	f, err := os.Open(path)
+	f, err := s.Open(meta.RelativePath())
 	if err != nil {
-		log.Error("Whilst trying to read LFS OID[%s]: Unable to open %s Error: %v", meta.Oid, path, err)
+		log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
 		return nil, err
 	}
 	if fromByte > 0 {
@@ -48,82 +45,55 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
 
 // Put takes a Meta object and an io.Reader and writes the content to the store.
 func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-	tmpPath := path + ".tmp"
-
-	dir := filepath.Dir(path)
-	if err := os.MkdirAll(dir, 0750); err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Unable to create the LFS directory: %s Error: %v", meta.Oid, dir, err)
-		return err
-	}
-
-	file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
-	if err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Unable to open temporary file for writing: %s Error: %v", tmpPath, err)
-		return err
-	}
-	defer func() {
-		if err := util.Remove(tmpPath); err != nil {
-			log.Warn("Unable to remove temporary path: %s: Error: %v", tmpPath, err)
-		}
-	}()
-
 	hash := sha256.New()
-	hw := io.MultiWriter(hash, file)
-
-	written, err := io.Copy(hw, r)
+	rd := io.TeeReader(r, hash)
+	p := meta.RelativePath()
+	written, err := s.Save(p, rd)
 	if err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, tmpPath, err)
-		file.Close()
+		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
 		return err
 	}
-	file.Close()
 
 	if written != meta.Size {
+		if err := s.Delete(p); err != nil {
+			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+		}
 		return errSizeMismatch
 	}
 
 	shaStr := hex.EncodeToString(hash.Sum(nil))
 	if shaStr != meta.Oid {
+		if err := s.Delete(p); err != nil {
+			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+		}
 		return errHashMismatch
 	}
 
-	if err := os.Rename(tmpPath, path); err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Unable to move tmp file to final destination: %s Error: %v", meta.Oid, path, err)
-		return err
-	}
-
 	return nil
 }
 
 // Exists returns true if the object exists in the content store.
-func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return false
+func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
+	_, err := s.ObjectStorage.Stat(meta.RelativePath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
 	}
-	return true
+	return true, nil
 }
 
 // Verify returns true if the object exists in the content store and size is correct.
 func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-
-	fi, err := os.Stat(path)
-	if os.IsNotExist(err) || err == nil && fi.Size() != meta.Size {
+	p := meta.RelativePath()
+	fi, err := s.ObjectStorage.Stat(p)
+	if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
 		return false, nil
 	} else if err != nil {
-		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", path, meta.Oid, err)
+		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
 		return false, err
 	}
 
 	return true, nil
 }
-
-func transformKey(key string) string {
-	if len(key) < 5 {
-		return key
-	}
-
-	return filepath.Join(key[0:2], key[2:4], key[4:])
-}