Add LFS Migration and Mirror (#14726)

* Implemented LFS client.

* Implemented scanning for pointer files.

* Implemented downloading of lfs files.

* Moved model-dependent code into services.

* Removed models dependency. Added TryReadPointerFromBuffer.

* Migrated code from service to module.

* Centralised storage creation.

* Removed dependency from models.

* Moved ContentStore into modules.

* Share structs between server and client.

* Moved method to services.

* Implemented lfs download on clone.

* Implemented LFS sync on clone and mirror update.

* Added form fields.

* Updated templates.

* Fixed condition.

* Use alternate endpoint.

* Added missing methods.

* Fixed typo and make linter happy.

* Detached pointer parser from gogit dependency.

* Fixed TestGetLFSRange test.

* Added context to support cancellation.

* Use ReadFull to probably read more data.

* Removed duplicated code from models.

* Moved scan implementation into pointer_scanner_nogogit.

* Changed method name.

* Added comments.

* Added more/specific log/error messages.

* Embedded lfs.Pointer into models.LFSMetaObject.

* Moved code from models to module.

* Moved code from models to module.

* Moved code from models to module.

* Reduced pointer usage.

* Embedded type.

* Use promoted fields.

* Fixed unexpected eof.

* Added unit tests.

* Implemented migration of local file paths.

* Show an error on invalid LFS endpoints.

* Hide settings if not used.

* Added LFS info to mirror struct.

* Fixed comment.

* Check LFS endpoint.

* Manage LFS settings from mirror page.

* Fixed selector.

* Adjusted selector.

* Added more tests.

* Added local filesystem migration test.

* Fixed typo.

* Reset settings.

* Added special windows path handling.

* Added unit test for HTTPClient.

* Added unit test for BasicTransferAdapter.

* Moved into util package.

* Test if LFS endpoint is allowed.

* Added support for git://

* Just use a static placeholder as the displayed url may be invalid.

* Reverted to original code.

* Added "Advanced Settings".

* Updated wording.

* Added discovery info link.

* Implemented suggestion.

* Fixed missing format parameter.

* Added Pointer.IsValid().

* Always remove model on error.

* Added suggestions.

* Use channel instead of array.

* Update routers/repo/migrate.go

* fmt

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>

7 files changed, 1057 insertions, 17 deletions

diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go
index d9eb06d194..55d1f6e3bc 100644
--- a/services/forms/repo_form.go
+++ b/services/forms/repo_form.go
@@ -71,6 +71,8 @@ type MigrateRepoForm struct {
 	// required: true
 	RepoName       string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
 	Mirror         bool   `json:"mirror"`
+	LFS            bool   `json:"lfs"`
+	LFSEndpoint    string `json:"lfs_endpoint"`
 	Private        bool   `json:"private"`
 	Description    string `json:"description" binding:"MaxSize(255)"`
 	Wiki           bool   `json:"wiki"`
@@ -118,6 +120,8 @@ type RepoSettingForm struct {
 	MirrorAddress  string
 	MirrorUsername string
 	MirrorPassword string
+	LFS            bool   `form:"mirror_lfs"`
+	LFSEndpoint    string `form:"mirror_lfs_endpoint"`
 	Private        bool
 	Template       bool
 	EnablePrune    bool
diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go
index 18d56c174a..2ca6bd957e 100644
--- a/services/gitdiff/gitdiff.go
+++ b/services/gitdiff/gitdiff.go
@@ -25,6 +25,7 @@ import (
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/highlight"
+	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
@@ -1077,12 +1078,12 @@ func parseHunks(curFile *DiffFile, maxLines, maxLineCharacters int, input *bufio
 		curSection.Lines[len(curSection.Lines)-1].Content = line
 
 		// handle LFS
-		if line[1:] == models.LFSMetaFileIdentifier {
+		if line[1:] == lfs.MetaFileIdentifier {
 			curFileLFSPrefix = true
-		} else if curFileLFSPrefix && strings.HasPrefix(line[1:], models.LFSMetaFileOidPrefix) {
-			oid := strings.TrimPrefix(line[1:], models.LFSMetaFileOidPrefix)
+		} else if curFileLFSPrefix && strings.HasPrefix(line[1:], lfs.MetaFileOidPrefix) {
+			oid := strings.TrimPrefix(line[1:], lfs.MetaFileOidPrefix)
 			if len(oid) == 64 {
-				m := &models.LFSMetaObject{Oid: oid}
+				m := &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}
 				count, err := models.Count(m)
 
 				if err == nil && count > 0 {
diff --git a/services/lfs/locks.go b/services/lfs/locks.go
new file mode 100644
index 0000000000..6bbe43d36b
--- /dev/null
+++ b/services/lfs/locks.go
@@ -0,0 +1,349 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/convert"
+	lfs_module "code.gitea.io/gitea/modules/lfs"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	api "code.gitea.io/gitea/modules/structs"
+	jsoniter "github.com/json-iterator/go"
+)
+
+//checkIsValidRequest check if it a valid request in case of bad request it write the response to ctx.
+func checkIsValidRequest(ctx *context.Context) bool {
+	if !setting.LFS.StartServer {
+		log.Debug("Attempt to access LFS server but LFS server is disabled")
+		writeStatus(ctx, http.StatusNotFound)
+		return false
+	}
+	if !MetaMatcher(ctx.Req) {
+		log.Info("Attempt access LOCKs without accepting the correct media type: %s", lfs_module.MediaType)
+		writeStatus(ctx, http.StatusBadRequest)
+		return false
+	}
+	if !ctx.IsSigned {
+		user, _, _, err := parseToken(ctx.Req.Header.Get("Authorization"))
+		if err != nil {
+			ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+			writeStatus(ctx, http.StatusUnauthorized)
+			return false
+		}
+		ctx.User = user
+	}
+	return true
+}
+
+func handleLockListOut(ctx *context.Context, repo *models.Repository, lock *models.LFSLock, err error) {
+	if err != nil {
+		if models.IsErrLFSLockNotExist(err) {
+			ctx.JSON(http.StatusOK, api.LFSLockList{
+				Locks: []*api.LFSLock{},
+			})
+			return
+		}
+		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+			Message: "unable to list locks : Internal Server Error",
+		})
+		return
+	}
+	if repo.ID != lock.RepoID {
+		ctx.JSON(http.StatusOK, api.LFSLockList{
+			Locks: []*api.LFSLock{},
+		})
+		return
+	}
+	ctx.JSON(http.StatusOK, api.LFSLockList{
+		Locks: []*api.LFSLock{convert.ToLFSLock(lock)},
+	})
+}
+
+// GetListLockHandler list locks
+func GetListLockHandler(ctx *context.Context) {
+	if !checkIsValidRequest(ctx) {
+		// Status is written in checkIsValidRequest
+		return
+	}
+	ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+	rv, _ := unpack(ctx)
+
+	repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
+	if err != nil {
+		log.Debug("Could not find repository: %s/%s - %s", rv.User, rv.Repo, err)
+		writeStatus(ctx, 404)
+		return
+	}
+	repository.MustOwner()
+
+	authenticated := authenticate(ctx, repository, rv.Authorization, false)
+	if !authenticated {
+		ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+		ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+			Message: "You must have pull access to list locks",
+		})
+		return
+	}
+
+	cursor := ctx.QueryInt("cursor")
+	if cursor < 0 {
+		cursor = 0
+	}
+	limit := ctx.QueryInt("limit")
+	if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 {
+		limit = setting.LFS.LocksPagingNum
+	} else if limit < 0 {
+		limit = 0
+	}
+	id := ctx.Query("id")
+	if id != "" { //Case where we request a specific id
+		v, err := strconv.ParseInt(id, 10, 64)
+		if err != nil {
+			ctx.JSON(http.StatusBadRequest, api.LFSLockError{
+				Message: "bad request : " + err.Error(),
+			})
+			return
+		}
+		lock, err := models.GetLFSLockByID(v)
+		if err != nil && !models.IsErrLFSLockNotExist(err) {
+			log.Error("Unable to get lock with ID[%s]: Error: %v", v, err)
+		}
+		handleLockListOut(ctx, repository, lock, err)
+		return
+	}
+
+	path := ctx.Query("path")
+	if path != "" { //Case where we request a specific id
+		lock, err := models.GetLFSLock(repository, path)
+		if err != nil && !models.IsErrLFSLockNotExist(err) {
+			log.Error("Unable to get lock for repository %-v with path %s: Error: %v", repository, path, err)
+		}
+		handleLockListOut(ctx, repository, lock, err)
+		return
+	}
+
+	//If no query params path or id
+	lockList, err := models.GetLFSLockByRepoID(repository.ID, cursor, limit)
+	if err != nil {
+		log.Error("Unable to list locks for repository ID[%d]: Error: %v", repository.ID, err)
+		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+			Message: "unable to list locks : Internal Server Error",
+		})
+		return
+	}
+	lockListAPI := make([]*api.LFSLock, len(lockList))
+	next := ""
+	for i, l := range lockList {
+		lockListAPI[i] = convert.ToLFSLock(l)
+	}
+	if limit > 0 && len(lockList) == limit {
+		next = strconv.Itoa(cursor + 1)
+	}
+	ctx.JSON(http.StatusOK, api.LFSLockList{
+		Locks: lockListAPI,
+		Next:  next,
+	})
+}
+
+// PostLockHandler create lock
+func PostLockHandler(ctx *context.Context) {
+	if !checkIsValidRequest(ctx) {
+		// Status is written in checkIsValidRequest
+		return
+	}
+	ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+	userName := ctx.Params("username")
+	repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
+	authorization := ctx.Req.Header.Get("Authorization")
+
+	repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
+	if err != nil {
+		log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
+		writeStatus(ctx, 404)
+		return
+	}
+	repository.MustOwner()
+
+	authenticated := authenticate(ctx, repository, authorization, true)
+	if !authenticated {
+		ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+		ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+			Message: "You must have push access to create locks",
+		})
+		return
+	}
+
+	var req api.LFSLockRequest
+	bodyReader := ctx.Req.Body
+	defer bodyReader.Close()
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	dec := json.NewDecoder(bodyReader)
+	if err := dec.Decode(&req); err != nil {
+		log.Warn("Failed to decode lock request as json. Error: %v", err)
+		writeStatus(ctx, 400)
+		return
+	}
+
+	lock, err := models.CreateLFSLock(&models.LFSLock{
+		Repo:  repository,
+		Path:  req.Path,
+		Owner: ctx.User,
+	})
+	if err != nil {
+		if models.IsErrLFSLockAlreadyExist(err) {
+			ctx.JSON(http.StatusConflict, api.LFSLockError{
+				Lock:    convert.ToLFSLock(lock),
+				Message: "already created lock",
+			})
+			return
+		}
+		if models.IsErrLFSUnauthorizedAction(err) {
+			ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+			ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+				Message: "You must have push access to create locks : " + err.Error(),
+			})
+			return
+		}
+		log.Error("Unable to CreateLFSLock in repository %-v at %s for user %-v: Error: %v", repository, req.Path, ctx.User, err)
+		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+			Message: "internal server error : Internal Server Error",
+		})
+		return
+	}
+	ctx.JSON(http.StatusCreated, api.LFSLockResponse{Lock: convert.ToLFSLock(lock)})
+}
+
+// VerifyLockHandler list locks for verification
+func VerifyLockHandler(ctx *context.Context) {
+	if !checkIsValidRequest(ctx) {
+		// Status is written in checkIsValidRequest
+		return
+	}
+	ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+	userName := ctx.Params("username")
+	repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
+	authorization := ctx.Req.Header.Get("Authorization")
+
+	repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
+	if err != nil {
+		log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
+		writeStatus(ctx, 404)
+		return
+	}
+	repository.MustOwner()
+
+	authenticated := authenticate(ctx, repository, authorization, true)
+	if !authenticated {
+		ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+		ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+			Message: "You must have push access to verify locks",
+		})
+		return
+	}
+
+	cursor := ctx.QueryInt("cursor")
+	if cursor < 0 {
+		cursor = 0
+	}
+	limit := ctx.QueryInt("limit")
+	if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 {
+		limit = setting.LFS.LocksPagingNum
+	} else if limit < 0 {
+		limit = 0
+	}
+	lockList, err := models.GetLFSLockByRepoID(repository.ID, cursor, limit)
+	if err != nil {
+		log.Error("Unable to list locks for repository ID[%d]: Error: %v", repository.ID, err)
+		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+			Message: "unable to list locks : Internal Server Error",
+		})
+		return
+	}
+	next := ""
+	if limit > 0 && len(lockList) == limit {
+		next = strconv.Itoa(cursor + 1)
+	}
+	lockOursListAPI := make([]*api.LFSLock, 0, len(lockList))
+	lockTheirsListAPI := make([]*api.LFSLock, 0, len(lockList))
+	for _, l := range lockList {
+		if l.Owner.ID == ctx.User.ID {
+			lockOursListAPI = append(lockOursListAPI, convert.ToLFSLock(l))
+		} else {
+			lockTheirsListAPI = append(lockTheirsListAPI, convert.ToLFSLock(l))
+		}
+	}
+	ctx.JSON(http.StatusOK, api.LFSLockListVerify{
+		Ours:   lockOursListAPI,
+		Theirs: lockTheirsListAPI,
+		Next:   next,
+	})
+}
+
+// UnLockHandler delete locks
+func UnLockHandler(ctx *context.Context) {
+	if !checkIsValidRequest(ctx) {
+		// Status is written in checkIsValidRequest
+		return
+	}
+	ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+	userName := ctx.Params("username")
+	repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
+	authorization := ctx.Req.Header.Get("Authorization")
+
+	repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
+	if err != nil {
+		log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
+		writeStatus(ctx, 404)
+		return
+	}
+	repository.MustOwner()
+
+	authenticated := authenticate(ctx, repository, authorization, true)
+	if !authenticated {
+		ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+		ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+			Message: "You must have push access to delete locks",
+		})
+		return
+	}
+
+	var req api.LFSLockDeleteRequest
+	bodyReader := ctx.Req.Body
+	defer bodyReader.Close()
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	dec := json.NewDecoder(bodyReader)
+	if err := dec.Decode(&req); err != nil {
+		log.Warn("Failed to decode lock request as json. Error: %v", err)
+		writeStatus(ctx, 400)
+		return
+	}
+
+	lock, err := models.DeleteLFSLockByID(ctx.ParamsInt64("lid"), ctx.User, req.Force)
+	if err != nil {
+		if models.IsErrLFSUnauthorizedAction(err) {
+			ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+			ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+				Message: "You must have push access to delete locks : " + err.Error(),
+			})
+			return
+		}
+		log.Error("Unable to DeleteLFSLockByID[%d] by user %-v with force %t: Error: %v", ctx.ParamsInt64("lid"), ctx.User, req.Force, err)
+		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+			Message: "unable to delete lock : Internal Server Error",
+		})
+		return
+	}
+	ctx.JSON(http.StatusOK, api.LFSLockResponse{Lock: convert.ToLFSLock(lock)})
+}
diff --git a/services/lfs/server.go b/services/lfs/server.go
new file mode 100644
index 0000000000..cd9a3fd7a1
--- /dev/null
+++ b/services/lfs/server.go
@@ -0,0 +1,666 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/context"
+	lfs_module "code.gitea.io/gitea/modules/lfs"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/dgrijalva/jwt-go"
+	jsoniter "github.com/json-iterator/go"
+)
+
+// requestContext contain variables from the HTTP request.
+type requestContext struct {
+	User          string
+	Repo          string
+	Authorization string
+}
+
+// Claims is a JWT Token Claims
+type Claims struct {
+	RepoID int64
+	Op     string
+	UserID int64
+	jwt.StandardClaims
+}
+
+// ObjectLink builds a URL linking to the object.
+func (rc *requestContext) ObjectLink(oid string) string {
+	return setting.AppURL + path.Join(rc.User, rc.Repo+".git", "info/lfs/objects", oid)
+}
+
+// VerifyLink builds a URL for verifying the object.
+func (rc *requestContext) VerifyLink() string {
+	return setting.AppURL + path.Join(rc.User, rc.Repo+".git", "info/lfs/verify")
+}
+
+var oidRegExp = regexp.MustCompile(`^[A-Fa-f0-9]+$`)
+
+func isOidValid(oid string) bool {
+	return oidRegExp.MatchString(oid)
+}
+
+// ObjectOidHandler is the main request routing entry point into LFS server functions
+func ObjectOidHandler(ctx *context.Context) {
+	if !setting.LFS.StartServer {
+		log.Debug("Attempt to access LFS server but LFS server is disabled")
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" {
+		if MetaMatcher(ctx.Req) {
+			getMetaHandler(ctx)
+			return
+		}
+
+		getContentHandler(ctx)
+		return
+	} else if ctx.Req.Method == "PUT" {
+		PutHandler(ctx)
+		return
+	}
+
+	log.Warn("Unhandled LFS method: %s for %s/%s OID[%s]", ctx.Req.Method, ctx.Params("username"), ctx.Params("reponame"), ctx.Params("oid"))
+	writeStatus(ctx, 404)
+}
+
+func getAuthenticatedRepoAndMeta(ctx *context.Context, rc *requestContext, p lfs_module.Pointer, requireWrite bool) (*models.LFSMetaObject, *models.Repository) {
+	if !isOidValid(p.Oid) {
+		log.Info("Attempt to access invalid LFS OID[%s] in %s/%s", p.Oid, rc.User, rc.Repo)
+		writeStatus(ctx, 404)
+		return nil, nil
+	}
+
+	repository, err := models.GetRepositoryByOwnerAndName(rc.User, rc.Repo)
+	if err != nil {
+		log.Error("Unable to get repository: %s/%s Error: %v", rc.User, rc.Repo, err)
+		writeStatus(ctx, 404)
+		return nil, nil
+	}
+
+	if !authenticate(ctx, repository, rc.Authorization, requireWrite) {
+		requireAuth(ctx)
+		return nil, nil
+	}
+
+	meta, err := repository.GetLFSMetaObjectByOid(p.Oid)
+	if err != nil {
+		log.Error("Unable to get LFS OID[%s] Error: %v", p.Oid, err)
+		writeStatus(ctx, 404)
+		return nil, nil
+	}
+
+	return meta, repository
+}
+
+// getContentHandler gets the content from the content store
+func getContentHandler(ctx *context.Context) {
+	rc, p := unpack(ctx)
+
+	meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, false)
+	if meta == nil {
+		// Status already written in getAuthenticatedRepoAndMeta
+		return
+	}
+
+	// Support resume download using Range header
+	var fromByte, toByte int64
+	toByte = meta.Size - 1
+	statusCode := 200
+	if rangeHdr := ctx.Req.Header.Get("Range"); rangeHdr != "" {
+		regex := regexp.MustCompile(`bytes=(\d+)\-(\d*).*`)
+		match := regex.FindStringSubmatch(rangeHdr)
+		if len(match) > 1 {
+			statusCode = 206
+			fromByte, _ = strconv.ParseInt(match[1], 10, 32)
+
+			if fromByte >= meta.Size {
+				writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
+				return
+			}
+
+			if match[2] != "" {
+				_toByte, _ := strconv.ParseInt(match[2], 10, 32)
+				if _toByte >= fromByte && _toByte < toByte {
+					toByte = _toByte
+				}
+			}
+
+			ctx.Resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, toByte, meta.Size-fromByte))
+			ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Range")
+		}
+	}
+
+	contentStore := lfs_module.NewContentStore()
+	content, err := contentStore.Get(meta.Pointer)
+	if err != nil {
+		// Errors are logged in contentStore.Get
+		writeStatus(ctx, http.StatusNotFound)
+		return
+	}
+	defer content.Close()
+
+	if fromByte > 0 {
+		_, err = content.Seek(fromByte, io.SeekStart)
+		if err != nil {
+			log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
+
+			writeStatus(ctx, http.StatusInternalServerError)
+			return
+		}
+	}
+
+	contentLength := toByte + 1 - fromByte
+	ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10))
+	ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+
+	filename := ctx.Params("filename")
+	if len(filename) > 0 {
+		decodedFilename, err := base64.RawURLEncoding.DecodeString(filename)
+		if err == nil {
+			ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+string(decodedFilename)+"\"")
+			ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
+		}
+	}
+
+	ctx.Resp.WriteHeader(statusCode)
+	if written, err := io.CopyN(ctx.Resp, content, contentLength); err != nil {
+		log.Error("Error whilst copying LFS OID[%s] to the response after %d bytes. Error: %v", meta.Oid, written, err)
+	}
+	logRequest(ctx.Req, statusCode)
+}
+
+// getMetaHandler retrieves metadata about the object
+func getMetaHandler(ctx *context.Context) {
+	rc, p := unpack(ctx)
+
+	meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, false)
+	if meta == nil {
+		// Status already written in getAuthenticatedRepoAndMeta
+		return
+	}
+
+	ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+	if ctx.Req.Method == "GET" {
+		json := jsoniter.ConfigCompatibleWithStandardLibrary
+		enc := json.NewEncoder(ctx.Resp)
+		if err := enc.Encode(represent(rc, meta.Pointer, true, false)); err != nil {
+			log.Error("Failed to encode representation as json. Error: %v", err)
+		}
+	}
+
+	logRequest(ctx.Req, 200)
+}
+
+// PostHandler instructs the client how to upload data
+func PostHandler(ctx *context.Context) {
+	if !setting.LFS.StartServer {
+		log.Debug("Attempt to access LFS server but LFS server is disabled")
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !MetaMatcher(ctx.Req) {
+		log.Info("Attempt to POST without accepting the correct media type: %s", lfs_module.MediaType)
+		writeStatus(ctx, 400)
+		return
+	}
+
+	rc, p := unpack(ctx)
+
+	repository, err := models.GetRepositoryByOwnerAndName(rc.User, rc.Repo)
+	if err != nil {
+		log.Error("Unable to get repository: %s/%s Error: %v", rc.User, rc.Repo, err)
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !authenticate(ctx, repository, rc.Authorization, true) {
+		requireAuth(ctx)
+		return
+	}
+
+	if !isOidValid(p.Oid) {
+		log.Info("Invalid LFS OID[%s] attempt to POST in %s/%s", p.Oid, rc.User, rc.Repo)
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if setting.LFS.MaxFileSize > 0 && p.Size > setting.LFS.MaxFileSize {
+		log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", p.Oid, p.Size, rc.User, rc.Repo, setting.LFS.MaxFileSize)
+		writeStatus(ctx, 413)
+		return
+	}
+
+	meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: p, RepositoryID: repository.ID})
+	if err != nil {
+		log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", p.Oid, p.Size, rc.User, rc.Repo, err)
+		writeStatus(ctx, 404)
+		return
+	}
+
+	ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+	sentStatus := 202
+	contentStore := lfs_module.NewContentStore()
+	exist, err := contentStore.Exists(p)
+	if err != nil {
+		log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", p.Oid, rc.User, rc.Repo, err)
+		writeStatus(ctx, 500)
+		return
+	}
+	if meta.Existing && exist {
+		sentStatus = 200
+	}
+	ctx.Resp.WriteHeader(sentStatus)
+
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	enc := json.NewEncoder(ctx.Resp)
+	if err := enc.Encode(represent(rc, meta.Pointer, meta.Existing, true)); err != nil {
+		log.Error("Failed to encode representation as json. Error: %v", err)
+	}
+	logRequest(ctx.Req, sentStatus)
+}
+
+// BatchHandler provides the batch api
+func BatchHandler(ctx *context.Context) {
+	if !setting.LFS.StartServer {
+		log.Debug("Attempt to access LFS server but LFS server is disabled")
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !MetaMatcher(ctx.Req) {
+		log.Info("Attempt to BATCH without accepting the correct media type: %s", lfs_module.MediaType)
+		writeStatus(ctx, 400)
+		return
+	}
+
+	bv := unpackbatch(ctx)
+
+	reqCtx := &requestContext{
+		User:          ctx.Params("username"),
+		Repo:          strings.TrimSuffix(ctx.Params("reponame"), ".git"),
+		Authorization: ctx.Req.Header.Get("Authorization"),
+	}
+
+	var responseObjects []*lfs_module.ObjectResponse
+
+	// Create a response object
+	for _, object := range bv.Objects {
+		if !isOidValid(object.Oid) {
+			log.Info("Invalid LFS OID[%s] attempt to BATCH in %s/%s", object.Oid, reqCtx.User, reqCtx.Repo)
+			continue
+		}
+
+		repository, err := models.GetRepositoryByOwnerAndName(reqCtx.User, reqCtx.Repo)
+		if err != nil {
+			log.Error("Unable to get repository: %s/%s Error: %v", reqCtx.User, reqCtx.Repo, err)
+			writeStatus(ctx, 404)
+			return
+		}
+
+		requireWrite := false
+		if bv.Operation == "upload" {
+			requireWrite = true
+		}
+
+		if !authenticate(ctx, repository, reqCtx.Authorization, requireWrite) {
+			requireAuth(ctx)
+			return
+		}
+
+		contentStore := lfs_module.NewContentStore()
+
+		meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
+		if err == nil { // Object is found and exists
+			exist, err := contentStore.Exists(meta.Pointer)
+			if err != nil {
+				log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err)
+				writeStatus(ctx, 500)
+				return
+			}
+			if exist {
+				responseObjects = append(responseObjects, represent(reqCtx, meta.Pointer, true, false))
+				continue
+			}
+		}
+
+		if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
+			log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, reqCtx.User, reqCtx.Repo, setting.LFS.MaxFileSize)
+			writeStatus(ctx, 413)
+			return
+		}
+
+		// Object is not found
+		meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: object, RepositoryID: repository.ID})
+		if err == nil {
+			exist, err := contentStore.Exists(meta.Pointer)
+			if err != nil {
+				log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err)
+				writeStatus(ctx, 500)
+				return
+			}
+			responseObjects = append(responseObjects, represent(reqCtx, meta.Pointer, meta.Existing, !exist))
+		} else {
+			log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, reqCtx.User, reqCtx.Repo, err)
+		}
+	}
+
+	ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+	respobj := &lfs_module.BatchResponse{Objects: responseObjects}
+
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	enc := json.NewEncoder(ctx.Resp)
+	if err := enc.Encode(respobj); err != nil {
+		log.Error("Failed to encode representation as json. Error: %v", err)
+	}
+	logRequest(ctx.Req, 200)
+}
+
+// PutHandler receives data from the client and puts it into the content store
+func PutHandler(ctx *context.Context) {
+	rc, p := unpack(ctx)
+
+	meta, repository := getAuthenticatedRepoAndMeta(ctx, rc, p, true)
+	if meta == nil {
+		// Status already written in getAuthenticatedRepoAndMeta
+		return
+	}
+
+	contentStore := lfs_module.NewContentStore()
+	defer ctx.Req.Body.Close()
+	if err := contentStore.Put(meta.Pointer, ctx.Req.Body); err != nil {
+		// Put will log the error itself
+		ctx.Resp.WriteHeader(500)
+		if err == lfs_module.ErrSizeMismatch || err == lfs_module.ErrHashMismatch {
+			fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
+		} else {
+			fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`)
+		}
+		if _, err = repository.RemoveLFSMetaObjectByOid(p.Oid); err != nil {
+			log.Error("Whilst removing metaobject for LFS OID[%s] due to preceding error there was another Error: %v", p.Oid, err)
+		}
+		return
+	}
+
+	logRequest(ctx.Req, 200)
+}
+
+// VerifyHandler verify oid and its size from the content store
+func VerifyHandler(ctx *context.Context) {
+	if !setting.LFS.StartServer {
+		log.Debug("Attempt to access LFS server but LFS server is disabled")
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !MetaMatcher(ctx.Req) {
+		log.Info("Attempt to VERIFY without accepting the correct media type: %s", lfs_module.MediaType)
+		writeStatus(ctx, 400)
+		return
+	}
+
+	rc, p := unpack(ctx)
+
+	meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, true)
+	if meta == nil {
+		// Status already written in getAuthenticatedRepoAndMeta
+		return
+	}
+
+	contentStore := lfs_module.NewContentStore()
+	ok, err := contentStore.Verify(meta.Pointer)
+	if err != nil {
+		// Error will be logged in Verify
+		ctx.Resp.WriteHeader(500)
+		fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`)
+		return
+	}
+	if !ok {
+		writeStatus(ctx, 422)
+		return
+	}
+
+	logRequest(ctx.Req, 200)
+}
+
+// represent takes a requestContext and Meta and turns it into a ObjectResponse suitable
+// for json encoding
+func represent(rc *requestContext, pointer lfs_module.Pointer, download, upload bool) *lfs_module.ObjectResponse {
+	rep := &lfs_module.ObjectResponse{
+		Pointer: pointer,
+		Actions: make(map[string]*lfs_module.Link),
+	}
+
+	header := make(map[string]string)
+
+	if rc.Authorization == "" {
+		//https://github.com/github/git-lfs/issues/1088
+		header["Authorization"] = "Authorization: Basic dummy"
+	} else {
+		header["Authorization"] = rc.Authorization
+	}
+
+	if download {
+		rep.Actions["download"] = &lfs_module.Link{Href: rc.ObjectLink(pointer.Oid), Header: header}
+	}
+
+	if upload {
+		rep.Actions["upload"] = &lfs_module.Link{Href: rc.ObjectLink(pointer.Oid), Header: header}
+	}
+
+	if upload && !download {
+		// Force client side verify action while gitea lacks proper server side verification
+		verifyHeader := make(map[string]string)
+		for k, v := range header {
+			verifyHeader[k] = v
+		}
+
+		// This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662
+		verifyHeader["Accept"] = lfs_module.MediaType
+
+		rep.Actions["verify"] = &lfs_module.Link{Href: rc.VerifyLink(), Header: verifyHeader}
+	}
+
+	return rep
+}
+
+// MetaMatcher provides a mux.MatcherFunc that only allows requests that contain
+// an Accept header with the lfs_module.MediaType
+func MetaMatcher(r *http.Request) bool {
+	mediaParts := strings.Split(r.Header.Get("Accept"), ";")
+	mt := mediaParts[0]
+	return mt == lfs_module.MediaType
+}
+
+func unpack(ctx *context.Context) (*requestContext, lfs_module.Pointer) {
+	r := ctx.Req
+	rc := &requestContext{
+		User:          ctx.Params("username"),
+		Repo:          strings.TrimSuffix(ctx.Params("reponame"), ".git"),
+		Authorization: r.Header.Get("Authorization"),
+	}
+	p := lfs_module.Pointer{Oid: ctx.Params("oid")}
+
+	if r.Method == "POST" { // Maybe also check if +json
+		var p2 lfs_module.Pointer
+		bodyReader := r.Body
+		defer bodyReader.Close()
+		json := jsoniter.ConfigCompatibleWithStandardLibrary
+		dec := json.NewDecoder(bodyReader)
+		err := dec.Decode(&p2)
+		if err != nil {
+			// The error is logged as a WARN here because this may represent misbehaviour rather than a true error
+			log.Warn("Unable to decode POST request vars for LFS OID[%s] in %s/%s: Error: %v", p.Oid, rc.User, rc.Repo, err)
+			return rc, p
+		}
+
+		p.Oid = p2.Oid
+		p.Size = p2.Size
+	}
+
+	return rc, p
+}
+
+// TODO cheap hack, unify with unpack
+func unpackbatch(ctx *context.Context) *lfs_module.BatchRequest {
+
+	r := ctx.Req
+	var bv lfs_module.BatchRequest
+
+	bodyReader := r.Body
+	defer bodyReader.Close()
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	dec := json.NewDecoder(bodyReader)
+	err := dec.Decode(&bv)
+	if err != nil {
+		// The error is logged as a WARN here because this may represent misbehaviour rather than a true error
+		log.Warn("Unable to decode BATCH request vars in %s/%s: Error: %v", ctx.Params("username"), strings.TrimSuffix(ctx.Params("reponame"), ".git"), err)
+		return &bv
+	}
+
+	return &bv
+}
+
+func writeStatus(ctx *context.Context, status int) {
+	message := http.StatusText(status)
+
+	mediaParts := strings.Split(ctx.Req.Header.Get("Accept"), ";")
+	mt := mediaParts[0]
+	if strings.HasSuffix(mt, "+json") {
+		message = `{"message":"` + message + `"}`
+	}
+
+	ctx.Resp.WriteHeader(status)
+	fmt.Fprint(ctx.Resp, message)
+	logRequest(ctx.Req, status)
+}
+
+func logRequest(r *http.Request, status int) {
+	log.Debug("LFS request - Method: %s, URL: %s, Status %d", r.Method, r.URL, status)
+}
+
+// authenticate uses the authorization string to determine whether
+// or not to proceed. This server assumes an HTTP Basic auth format.
+func authenticate(ctx *context.Context, repository *models.Repository, authorization string, requireWrite bool) bool {
+	accessMode := models.AccessModeRead
+	if requireWrite {
+		accessMode = models.AccessModeWrite
+	}
+
+	// ctx.IsSigned is unnecessary here, this will be checked in perm.CanAccess
+	perm, err := models.GetUserRepoPermission(repository, ctx.User)
+	if err != nil {
+		log.Error("Unable to GetUserRepoPermission for user %-v in repo %-v Error: %v", ctx.User, repository)
+		return false
+	}
+
+	canRead := perm.CanAccess(accessMode, models.UnitTypeCode)
+	if canRead {
+		return true
+	}
+
+	user, repo, opStr, err := parseToken(authorization)
+	if err != nil {
+		// Most of these are Warn level - the true internal server errors are logged in parseToken already
+		log.Warn("Authentication failure for provided token with Error: %v", err)
+		return false
+	}
+	ctx.User = user
+	if opStr == "basic" {
+		perm, err = models.GetUserRepoPermission(repository, ctx.User)
+		if err != nil {
+			log.Error("Unable to GetUserRepoPermission for user %-v in repo %-v Error: %v", ctx.User, repository)
+			return false
+		}
+		return perm.CanAccess(accessMode, models.UnitTypeCode)
+	}
+	if repository.ID == repo.ID {
+		if requireWrite && opStr != "upload" {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+func parseToken(authorization string) (*models.User, *models.Repository, string, error) {
+	if authorization == "" {
+		return nil, nil, "unknown", fmt.Errorf("No token")
+	}
+	if strings.HasPrefix(authorization, "Bearer ") {
+		token, err := jwt.ParseWithClaims(authorization[7:], &Claims{}, func(t *jwt.Token) (interface{}, error) {
+			if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
+				return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
+			}
+			return setting.LFS.JWTSecretBytes, nil
+		})
+		if err != nil {
+			// The error here is WARN level because it is caused by bad authorization rather than an internal server error
+			return nil, nil, "unknown", err
+		}
+		claims, claimsOk := token.Claims.(*Claims)
+		if !token.Valid || !claimsOk {
+			return nil, nil, "unknown", fmt.Errorf("Token claim invalid")
+		}
+		r, err := models.GetRepositoryByID(claims.RepoID)
+		if err != nil {
+			log.Error("Unable to GetRepositoryById[%d]: Error: %v", claims.RepoID, err)
+			return nil, nil, claims.Op, err
+		}
+		u, err := models.GetUserByID(claims.UserID)
+		if err != nil {
+			log.Error("Unable to GetUserById[%d]: Error: %v", claims.UserID, err)
+			return nil, r, claims.Op, err
+		}
+		return u, r, claims.Op, nil
+	}
+
+	if strings.HasPrefix(authorization, "Basic ") {
+		c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic "))
+		if err != nil {
+			return nil, nil, "basic", err
+		}
+		cs := string(c)
+		i := strings.IndexByte(cs, ':')
+		if i < 0 {
+			return nil, nil, "basic", fmt.Errorf("Basic auth invalid")
+		}
+		user, password := cs[:i], cs[i+1:]
+		u, err := models.GetUserByName(user)
+		if err != nil {
+			log.Error("Unable to GetUserByName[%d]: Error: %v", user, err)
+			return nil, nil, "basic", err
+		}
+		if !u.IsPasswordSet() || !u.ValidatePassword(password) {
+			return nil, nil, "basic", fmt.Errorf("Basic auth failed")
+		}
+		return u, nil, "basic", nil
+	}
+
+	return nil, nil, "unknown", fmt.Errorf("Token not found")
+}
+
+func requireAuth(ctx *context.Context) {
+	ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+	writeStatus(ctx, 401)
+}
diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go
index e4981b8c00..9e2dde85fc 100644
--- a/services/mirror/mirror.go
+++ b/services/mirror/mirror.go
@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/notification"
 	repo_module "code.gitea.io/gitea/modules/repository"
@@ -206,7 +207,7 @@ func parseRemoteUpdateOutput(output string) []*mirrorSyncResult {
 }
 
 // runSync returns true if sync finished without error.
-func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) {
+func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) {
 	repoPath := m.Repo.RepoPath()
 	wikiPath := m.Repo.WikiPath()
 	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
@@ -253,13 +254,21 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) {
 		log.Error("OpenRepository: %v", err)
 		return nil, false
 	}
+	defer gitRepo.Close()
 
 	log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo)
 	if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
-		gitRepo.Close()
 		log.Error("Failed to synchronize tags to releases for repository: %v", err)
 	}
-	gitRepo.Close()
+
+	if m.LFS && setting.LFS.StartServer {
+		log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
+		readAddress(m)
+		ep := lfs.DetermineEndpoint(m.Address, m.LFSEndpoint)
+		if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil {
+			log.Error("Failed to synchronize LFS objects for repository: %v", err)
+		}
+	}
 
 	log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo)
 	if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil {
@@ -378,12 +387,12 @@ func SyncMirrors(ctx context.Context) {
 			mirrorQueue.Close()
 			return
 		case repoID := <-mirrorQueue.Queue():
-			syncMirror(repoID)
+			syncMirror(ctx, repoID)
 		}
 	}
 }
 
-func syncMirror(repoID string) {
+func syncMirror(ctx context.Context, repoID string) {
 	log.Trace("SyncMirrors [repo_id: %v]", repoID)
 	defer func() {
 		err := recover()
@@ -403,7 +412,7 @@ func syncMirror(repoID string) {
 	}
 
 	log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo)
-	results, ok := runSync(m)
+	results, ok := runSync(ctx, m)
 	if !ok {
 		return
 	}
diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go
index 57628aa68d..20492c784b 100644
--- a/services/mirror/mirror_test.go
+++ b/services/mirror/mirror_test.go
@@ -48,7 +48,9 @@ func TestRelease_MirrorDelete(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	mirror, err := repository.MigrateRepositoryGitData(context.Background(), user, mirrorRepo, opts)
+	ctx := context.Background()
+
+	mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts)
 	assert.NoError(t, err)
 
 	gitRepo, err := git.OpenRepository(repoPath)
@@ -74,7 +76,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
 	err = mirror.GetMirror()
 	assert.NoError(t, err)
 
-	_, ok := runSync(mirror.Mirror)
+	_, ok := runSync(ctx, mirror.Mirror)
 	assert.True(t, ok)
 
 	count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
@@ -85,7 +87,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true))
 
-	_, ok = runSync(mirror.Mirror)
+	_, ok = runSync(ctx, mirror.Mirror)
 	assert.True(t, ok)
 
 	count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions)
diff --git a/services/pull/lfs.go b/services/pull/lfs.go
index a1981b8253..b902c63619 100644
--- a/services/pull/lfs.go
+++ b/services/pull/lfs.go
@@ -70,6 +70,8 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg
 	defer wg.Done()
 	defer catFileBatchReader.Close()
 
+	contentStore := lfs.NewContentStore()
+
 	bufferedReader := bufio.NewReader(catFileBatchReader)
 	buf := make([]byte, 1025)
 	for {
@@ -101,10 +103,16 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg
 		}
 		pointerBuf = pointerBuf[:size]
 		// Now we need to check if the pointerBuf is an LFS pointer
-		pointer := lfs.IsPointerFile(&pointerBuf)
-		if pointer == nil {
+		pointer, _ := lfs.ReadPointerFromBuffer(pointerBuf)
+		if !pointer.IsValid() {
 			continue
 		}
+
+		exist, _ := contentStore.Exists(pointer)
+		if !exist {
+			continue
+		}
+
 		// Then we need to check that this pointer is in the db
 		if _, err := pr.HeadRepo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
 			if err == models.ErrLFSObjectNotExist {
@@ -117,8 +125,9 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg
 		// OK we have a pointer that is associated with the head repo
 		// and is actually a file in the LFS
 		// Therefore it should be associated with the base repo
-		pointer.RepositoryID = pr.BaseRepoID
-		if _, err := models.NewLFSMetaObject(pointer); err != nil {
+		meta := &models.LFSMetaObject{Pointer: pointer}
+		meta.RepositoryID = pr.BaseRepoID
+		if _, err := models.NewLFSMetaObject(meta); err != nil {
 			_ = catFileBatchReader.CloseWithError(err)
 			break
 		}