Add LFS Migration and Mirror (#14726)

* Implemented LFS client.

* Implemented scanning for pointer files.

* Implemented downloading of lfs files.

* Moved model-dependent code into services.

* Removed models dependency. Added TryReadPointerFromBuffer.

* Migrated code from service to module.

* Centralised storage creation.

* Removed dependency from models.

* Moved ContentStore into modules.

* Share structs between server and client.

* Moved method to services.

* Implemented lfs download on clone.

* Implemented LFS sync on clone and mirror update.

* Added form fields.

* Updated templates.

* Fixed condition.

* Use alternate endpoint.

* Added missing methods.

* Fixed typo and make linter happy.

* Detached pointer parser from gogit dependency.

* Fixed TestGetLFSRange test.

* Added context to support cancellation.

* Use ReadFull to probably read more data.

* Removed duplicated code from models.

* Moved scan implementation into pointer_scanner_nogogit.

* Changed method name.

* Added comments.

* Added more/specific log/error messages.

* Embedded lfs.Pointer into models.LFSMetaObject.

* Moved code from models to module.

* Moved code from models to module.

* Moved code from models to module.

* Reduced pointer usage.

* Embedded type.

* Use promoted fields.

* Fixed unexpected eof.

* Added unit tests.

* Implemented migration of local file paths.

* Show an error on invalid LFS endpoints.

* Hide settings if not used.

* Added LFS info to mirror struct.

* Fixed comment.

* Check LFS endpoint.

* Manage LFS settings from mirror page.

* Fixed selector.

* Adjusted selector.

* Added more tests.

* Added local filesystem migration test.

* Fixed typo.

* Reset settings.

* Added special windows path handling.

* Added unit test for HTTPClient.

* Added unit test for BasicTransferAdapter.

* Moved into util package.

* Test if LFS endpoint is allowed.

* Added support for git://

* Just use a static placeholder as the displayed url may be invalid.

* Reverted to original code.

* Added "Advanced Settings".

* Updated wording.

* Added discovery info link.

* Implemented suggestion.

* Fixed missing format parameter.

* Added Pointer.IsValid().

* Always remove model on error.

* Added suggestions.

* Use channel instead of array.

* Update routers/repo/migrate.go

* fmt

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>

18 files changed, 1195 insertions, 1157 deletions

diff --git a/modules/lfs/client.go b/modules/lfs/client.go
new file mode 100644
index 0000000000..ae35919d77
--- /dev/null
+++ b/modules/lfs/client.go
@@ -0,0 +1,24 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"context"
+	"io"
+	"net/url"
+)
+
+// Client is used to communicate with a LFS source
+type Client interface {
+	Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error)
+}
+
+// NewClient creates a LFS client
+func NewClient(endpoint *url.URL) Client {
+	if endpoint.Scheme == "file" {
+		return newFilesystemClient(endpoint)
+	}
+	return newHTTPClient(endpoint)
+}
diff --git a/modules/lfs/client_test.go b/modules/lfs/client_test.go
new file mode 100644
index 0000000000..d4eb005469
--- /dev/null
+++ b/modules/lfs/client_test.go
@@ -0,0 +1,23 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"net/url"
+
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewClient(t *testing.T) {
+	u, _ := url.Parse("file:///test")
+	c := NewClient(u)
+	assert.IsType(t, &FilesystemClient{}, c)
+
+	u, _ = url.Parse("https://test.com/lfs")
+	c = NewClient(u)
+	assert.IsType(t, &HTTPClient{}, c)
+}
diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go
index 520caa4c99..9fa2c7e3b2 100644
--- a/modules/lfs/content_store.go
+++ b/modules/lfs/content_store.go
@@ -13,14 +13,15 @@ import (
 	"io"
 	"os"
 
-	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
 )
 
 var (
-	errHashMismatch = errors.New("Content hash does not match OID")
-	errSizeMismatch = errors.New("Content size does not match")
+	// ErrHashMismatch occurs if the content has does not match OID
+	ErrHashMismatch = errors.New("Content hash does not match OID")
+	// ErrSizeMismatch occurs if the content size does not match
+	ErrSizeMismatch = errors.New("Content size does not match")
 )
 
 // ErrRangeNotSatisfiable represents an error which request range is not satisfiable.
@@ -28,61 +29,67 @@ type ErrRangeNotSatisfiable struct {
 	FromByte int64
 }
 
-func (err ErrRangeNotSatisfiable) Error() string {
-	return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
-}
-
 // IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable
 func IsErrRangeNotSatisfiable(err error) bool {
 	_, ok := err.(ErrRangeNotSatisfiable)
 	return ok
 }
 
+func (err ErrRangeNotSatisfiable) Error() string {
+	return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
+}
+
 // ContentStore provides a simple file system based storage.
 type ContentStore struct {
 	storage.ObjectStorage
 }
 
+// NewContentStore creates the default ContentStore
+func NewContentStore() *ContentStore {
+	contentStore := &ContentStore{ObjectStorage: storage.LFS}
+	return contentStore
+}
+
 // Get takes a Meta object and retrieves the content from the store, returning
 // it as an io.ReadSeekCloser.
-func (s *ContentStore) Get(meta *models.LFSMetaObject) (storage.Object, error) {
-	f, err := s.Open(meta.RelativePath())
+func (s *ContentStore) Get(pointer Pointer) (storage.Object, error) {
+	f, err := s.Open(pointer.RelativePath())
 	if err != nil {
-		log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
+		log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", pointer.Oid, err)
 		return nil, err
 	}
 	return f, err
 }
 
 // Put takes a Meta object and an io.Reader and writes the content to the store.
-func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
-	p := meta.RelativePath()
+func (s *ContentStore) Put(pointer Pointer, r io.Reader) error {
+	p := pointer.RelativePath()
 
 	// Wrap the provided reader with an inline hashing and size checker
-	wrappedRd := newHashingReader(meta.Size, meta.Oid, r)
+	wrappedRd := newHashingReader(pointer.Size, pointer.Oid, r)
 
 	// now pass the wrapped reader to Save - if there is a size mismatch or hash mismatch then
 	// the errors returned by the newHashingReader should percolate up to here
-	written, err := s.Save(p, wrappedRd, meta.Size)
+	written, err := s.Save(p, wrappedRd, pointer.Size)
 	if err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
+		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", pointer.Oid, p, err)
 		return err
 	}
 
 	// This shouldn't happen but it is sensible to test
-	if written != meta.Size {
+	if written != pointer.Size {
 		if err := s.Delete(p); err != nil {
-			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+			log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err)
 		}
-		return errSizeMismatch
+		return ErrSizeMismatch
 	}
 
 	return nil
 }
 
 // Exists returns true if the object exists in the content store.
-func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
-	_, err := s.ObjectStorage.Stat(meta.RelativePath())
+func (s *ContentStore) Exists(pointer Pointer) (bool, error) {
+	_, err := s.ObjectStorage.Stat(pointer.RelativePath())
 	if err != nil {
 		if os.IsNotExist(err) {
 			return false, nil
@@ -93,19 +100,25 @@ func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
 }
 
 // Verify returns true if the object exists in the content store and size is correct.
-func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
-	p := meta.RelativePath()
+func (s *ContentStore) Verify(pointer Pointer) (bool, error) {
+	p := pointer.RelativePath()
 	fi, err := s.ObjectStorage.Stat(p)
-	if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
+	if os.IsNotExist(err) || (err == nil && fi.Size() != pointer.Size) {
 		return false, nil
 	} else if err != nil {
-		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
+		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, pointer.Oid, err)
 		return false, err
 	}
 
 	return true, nil
 }
 
+// ReadMetaObject will read a models.LFSMetaObject and return a reader
+func ReadMetaObject(pointer Pointer) (io.ReadCloser, error) {
+	contentStore := NewContentStore()
+	return contentStore.Get(pointer)
+}
+
 type hashingReader struct {
 	internal     io.Reader
 	currentSize  int64
@@ -127,12 +140,12 @@ func (r *hashingReader) Read(b []byte) (int, error) {
 
 	if err != nil && err == io.EOF {
 		if r.currentSize != r.expectedSize {
-			return n, errSizeMismatch
+			return n, ErrSizeMismatch
 		}
 
 		shaStr := hex.EncodeToString(r.hash.Sum(nil))
 		if shaStr != r.expectedHash {
-			return n, errHashMismatch
+			return n, ErrHashMismatch
 		}
 	}
 
diff --git a/modules/lfs/endpoint.go b/modules/lfs/endpoint.go
new file mode 100644
index 0000000000..add16ce9f1
--- /dev/null
+++ b/modules/lfs/endpoint.go
@@ -0,0 +1,106 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+// DetermineEndpoint determines an endpoint from the clone url or uses the specified LFS url.
+func DetermineEndpoint(cloneurl, lfsurl string) *url.URL {
+	if len(lfsurl) > 0 {
+		return endpointFromURL(lfsurl)
+	}
+	return endpointFromCloneURL(cloneurl)
+}
+
+func endpointFromCloneURL(rawurl string) *url.URL {
+	ep := endpointFromURL(rawurl)
+	if ep == nil {
+		return ep
+	}
+
+	if strings.HasSuffix(ep.Path, "/") {
+		ep.Path = ep.Path[:len(ep.Path)-1]
+	}
+
+	if ep.Scheme == "file" {
+		return ep
+	}
+
+	if path.Ext(ep.Path) == ".git" {
+		ep.Path += "/info/lfs"
+	} else {
+		ep.Path += ".git/info/lfs"
+	}
+
+	return ep
+}
+
+func endpointFromURL(rawurl string) *url.URL {
+	if strings.HasPrefix(rawurl, "/") {
+		return endpointFromLocalPath(rawurl)
+	}
+
+	u, err := url.Parse(rawurl)
+	if err != nil {
+		log.Error("lfs.endpointFromUrl: %v", err)
+		return nil
+	}
+
+	switch u.Scheme {
+	case "http", "https":
+		return u
+	case "git":
+		u.Scheme = "https"
+		return u
+	case "file":
+		return u
+	default:
+		if _, err := os.Stat(rawurl); err == nil {
+			return endpointFromLocalPath(rawurl)
+		}
+
+		log.Error("lfs.endpointFromUrl: unknown url")
+		return nil
+	}
+}
+
+func endpointFromLocalPath(path string) *url.URL {
+	var slash string
+	if abs, err := filepath.Abs(path); err == nil {
+		if !strings.HasPrefix(abs, "/") {
+			slash = "/"
+		}
+		path = abs
+	}
+
+	var gitpath string
+	if filepath.Base(path) == ".git" {
+		gitpath = path
+		path = filepath.Dir(path)
+	} else {
+		gitpath = filepath.Join(path, ".git")
+	}
+
+	if _, err := os.Stat(gitpath); err == nil {
+		path = gitpath
+	} else if _, err := os.Stat(path); err != nil {
+		return nil
+	}
+
+	path = fmt.Sprintf("file://%s%s", slash, filepath.ToSlash(path))
+
+	u, _ := url.Parse(path)
+
+	return u
+}
diff --git a/modules/lfs/endpoint_test.go b/modules/lfs/endpoint_test.go
new file mode 100644
index 0000000000..a7e8b1bfb7
--- /dev/null
+++ b/modules/lfs/endpoint_test.go
@@ -0,0 +1,75 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func str2url(raw string) *url.URL {
+	u, _ := url.Parse(raw)
+	return u
+}
+
+func TestDetermineEndpoint(t *testing.T) {
+	// Test cases
+	var cases = []struct {
+		cloneurl string
+		lfsurl   string
+		expected *url.URL
+	}{
+		// case 0
+		{
+			cloneurl: "",
+			lfsurl:   "",
+			expected: nil,
+		},
+		// case 1
+		{
+			cloneurl: "https://git.com/repo",
+			lfsurl:   "",
+			expected: str2url("https://git.com/repo.git/info/lfs"),
+		},
+		// case 2
+		{
+			cloneurl: "https://git.com/repo.git",
+			lfsurl:   "",
+			expected: str2url("https://git.com/repo.git/info/lfs"),
+		},
+		// case 3
+		{
+			cloneurl: "",
+			lfsurl:   "https://gitlfs.com/repo",
+			expected: str2url("https://gitlfs.com/repo"),
+		},
+		// case 4
+		{
+			cloneurl: "https://git.com/repo.git",
+			lfsurl:   "https://gitlfs.com/repo",
+			expected: str2url("https://gitlfs.com/repo"),
+		},
+		// case 5
+		{
+			cloneurl: "git://git.com/repo.git",
+			lfsurl:   "",
+			expected: str2url("https://git.com/repo.git/info/lfs"),
+		},
+		// case 6
+		{
+			cloneurl: "",
+			lfsurl:   "git://gitlfs.com/repo",
+			expected: str2url("https://gitlfs.com/repo"),
+		},
+	}
+
+	for n, c := range cases {
+		ep := DetermineEndpoint(c.cloneurl, c.lfsurl)
+
+		assert.Equal(t, c.expected, ep, "case %d: error should match", n)
+	}
+}
diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go
new file mode 100644
index 0000000000..3a51564a82
--- /dev/null
+++ b/modules/lfs/filesystem_client.go
@@ -0,0 +1,50 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"context"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+// FilesystemClient is used to read LFS data from a filesystem path
+type FilesystemClient struct {
+	lfsdir string
+}
+
+func newFilesystemClient(endpoint *url.URL) *FilesystemClient {
+	path, _ := util.FileURLToPath(endpoint)
+
+	lfsdir := filepath.Join(path, "lfs", "objects")
+
+	client := &FilesystemClient{lfsdir}
+
+	return client
+}
+
+func (c *FilesystemClient) objectPath(oid string) string {
+	return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid)
+}
+
+// Download reads the specific LFS object from the target repository
+func (c *FilesystemClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) {
+	objectPath := c.objectPath(oid)
+
+	if _, err := os.Stat(objectPath); os.IsNotExist(err) {
+		return nil, err
+	}
+
+	file, err := os.Open(objectPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return file, nil
+}
diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go
new file mode 100644
index 0000000000..fb45defda1
--- /dev/null
+++ b/modules/lfs/http_client.go
@@ -0,0 +1,129 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+// HTTPClient is used to communicate with the LFS server
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
+type HTTPClient struct {
+	client    *http.Client
+	endpoint  string
+	transfers map[string]TransferAdapter
+}
+
+func newHTTPClient(endpoint *url.URL) *HTTPClient {
+	hc := &http.Client{}
+
+	client := &HTTPClient{
+		client:    hc,
+		endpoint:  strings.TrimSuffix(endpoint.String(), "/"),
+		transfers: make(map[string]TransferAdapter),
+	}
+
+	basic := &BasicTransferAdapter{hc}
+
+	client.transfers[basic.Name()] = basic
+
+	return client
+}
+
+func (c *HTTPClient) transferNames() []string {
+	keys := make([]string, len(c.transfers))
+
+	i := 0
+	for k := range c.transfers {
+		keys[i] = k
+		i++
+	}
+
+	return keys
+}
+
+func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) {
+	url := fmt.Sprintf("%s/objects/batch", c.endpoint)
+
+	request := &BatchRequest{operation, c.transferNames(), nil, objects}
+
+	payload := new(bytes.Buffer)
+	err := json.NewEncoder(payload).Encode(request)
+	if err != nil {
+		return nil, fmt.Errorf("lfs.HTTPClient.batch json.Encode: %w", err)
+	}
+
+	log.Trace("lfs.HTTPClient.batch NewRequestWithContext: %s", url)
+
+	req, err := http.NewRequestWithContext(ctx, "POST", url, payload)
+	if err != nil {
+		return nil, fmt.Errorf("lfs.HTTPClient.batch http.NewRequestWithContext: %w", err)
+	}
+	req.Header.Set("Content-type", MediaType)
+	req.Header.Set("Accept", MediaType)
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+		return nil, fmt.Errorf("lfs.HTTPClient.batch http.Do: %w", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("lfs.HTTPClient.batch: Unexpected servers response: %s", res.Status)
+	}
+
+	var response BatchResponse
+	err = json.NewDecoder(res.Body).Decode(&response)
+	if err != nil {
+		return nil, fmt.Errorf("lfs.HTTPClient.batch json.Decode: %w", err)
+	}
+
+	if len(response.Transfer) == 0 {
+		response.Transfer = "basic"
+	}
+
+	return &response, nil
+}
+
+// Download reads the specific LFS object from the LFS server
+func (c *HTTPClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) {
+	var objects []Pointer
+	objects = append(objects, Pointer{oid, size})
+
+	result, err := c.batch(ctx, "download", objects)
+	if err != nil {
+		return nil, err
+	}
+
+	transferAdapter, ok := c.transfers[result.Transfer]
+	if !ok {
+		return nil, fmt.Errorf("lfs.HTTPClient.Download Transferadapter not found: %s", result.Transfer)
+	}
+
+	if len(result.Objects) == 0 {
+		return nil, errors.New("lfs.HTTPClient.Download: No objects in result")
+	}
+
+	content, err := transferAdapter.Download(ctx, result.Objects[0])
+	if err != nil {
+		return nil, err
+	}
+	return content, nil
+}
diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go
new file mode 100644
index 0000000000..043aa0214e
--- /dev/null
+++ b/modules/lfs/http_client_test.go
@@ -0,0 +1,144 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type RoundTripFunc func(req *http.Request) *http.Response
+
+func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req), nil
+}
+
+type DummyTransferAdapter struct {
+}
+
+func (a *DummyTransferAdapter) Name() string {
+	return "dummy"
+}
+
+func (a *DummyTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) {
+	return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil
+}
+
+func TestHTTPClientDownload(t *testing.T) {
+	oid := "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041"
+	size := int64(6)
+
+	roundTripHandler := func(req *http.Request) *http.Response {
+		url := req.URL.String()
+		if strings.Contains(url, "status-not-ok") {
+			return &http.Response{StatusCode: http.StatusBadRequest}
+		}
+		if strings.Contains(url, "invalid-json-response") {
+			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))}
+		}
+		if strings.Contains(url, "valid-batch-request-download") {
+			assert.Equal(t, "POST", req.Method)
+			assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url)
+			assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url)
+
+			var batchRequest BatchRequest
+			err := json.NewDecoder(req.Body).Decode(&batchRequest)
+			assert.NoError(t, err)
+
+			assert.Equal(t, "download", batchRequest.Operation)
+			assert.Equal(t, 1, len(batchRequest.Objects))
+			assert.Equal(t, oid, batchRequest.Objects[0].Oid)
+			assert.Equal(t, size, batchRequest.Objects[0].Size)
+
+			batchResponse := &BatchResponse{
+				Transfer: "dummy",
+				Objects:  make([]*ObjectResponse, 1),
+			}
+
+			payload := new(bytes.Buffer)
+			json.NewEncoder(payload).Encode(batchResponse)
+
+			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
+		}
+		if strings.Contains(url, "invalid-response-no-objects") {
+			batchResponse := &BatchResponse{Transfer: "dummy"}
+
+			payload := new(bytes.Buffer)
+			json.NewEncoder(payload).Encode(batchResponse)
+
+			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
+		}
+		if strings.Contains(url, "unknown-transfer-adapter") {
+			batchResponse := &BatchResponse{Transfer: "unknown_adapter"}
+
+			payload := new(bytes.Buffer)
+			json.NewEncoder(payload).Encode(batchResponse)
+
+			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
+		}
+
+		t.Errorf("Unknown test case: %s", url)
+
+		return nil
+	}
+
+	hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)}
+	dummy := &DummyTransferAdapter{}
+
+	var cases = []struct {
+		endpoint      string
+		expectederror string
+	}{
+		// case 0
+		{
+			endpoint:      "https://status-not-ok.io",
+			expectederror: "Unexpected servers response: ",
+		},
+		// case 1
+		{
+			endpoint:      "https://invalid-json-response.io",
+			expectederror: "json.Decode: ",
+		},
+		// case 2
+		{
+			endpoint:      "https://valid-batch-request-download.io",
+			expectederror: "",
+		},
+		// case 3
+		{
+			endpoint:      "https://invalid-response-no-objects.io",
+			expectederror: "No objects in result",
+		},
+		// case 4
+		{
+			endpoint:      "https://unknown-transfer-adapter.io",
+			expectederror: "Transferadapter not found: ",
+		},
+	}
+
+	for n, c := range cases {
+		client := &HTTPClient{
+			client:    hc,
+			endpoint:  c.endpoint,
+			transfers: make(map[string]TransferAdapter),
+		}
+		client.transfers["dummy"] = dummy
+
+		_, err := client.Download(context.Background(), oid, size)
+		if len(c.expectederror) > 0 {
+			assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
+		} else {
+			assert.NoError(t, err, "case %d", n)
+		}
+	}
+}
diff --git a/modules/lfs/locks.go b/modules/lfs/locks.go
deleted file mode 100644
index eaa8305cb4..0000000000
--- a/modules/lfs/locks.go
+++ /dev/null
@@ -1,348 +0,0 @@
-// Copyright 2017 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package lfs
-
-import (
-	"net/http"
-	"strconv"
-	"strings"
-
-	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/context"
-	"code.gitea.io/gitea/modules/convert"
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
-	api "code.gitea.io/gitea/modules/structs"
-	jsoniter "github.com/json-iterator/go"
-)
-
-//checkIsValidRequest check if it a valid request in case of bad request it write the response to ctx.
-func checkIsValidRequest(ctx *context.Context) bool {
-	if !setting.LFS.StartServer {
-		log.Debug("Attempt to access LFS server but LFS server is disabled")
-		writeStatus(ctx, http.StatusNotFound)
-		return false
-	}
-	if !MetaMatcher(ctx.Req) {
-		log.Info("Attempt access LOCKs without accepting the correct media type: %s", metaMediaType)
-		writeStatus(ctx, http.StatusBadRequest)
-		return false
-	}
-	if !ctx.IsSigned {
-		user, _, _, err := parseToken(ctx.Req.Header.Get("Authorization"))
-		if err != nil {
-			ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
-			writeStatus(ctx, http.StatusUnauthorized)
-			return false
-		}
-		ctx.User = user
-	}
-	return true
-}
-
-func handleLockListOut(ctx *context.Context, repo *models.Repository, lock *models.LFSLock, err error) {
-	if err != nil {
-		if models.IsErrLFSLockNotExist(err) {
-			ctx.JSON(http.StatusOK, api.LFSLockList{
-				Locks: []*api.LFSLock{},
-			})
-			return
-		}
-		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
-			Message: "unable to list locks : Internal Server Error",
-		})
-		return
-	}
-	if repo.ID != lock.RepoID {
-		ctx.JSON(http.StatusOK, api.LFSLockList{
-			Locks: []*api.LFSLock{},
-		})
-		return
-	}
-	ctx.JSON(http.StatusOK, api.LFSLockList{
-		Locks: []*api.LFSLock{convert.ToLFSLock(lock)},
-	})
-}
-
-// GetListLockHandler list locks
-func GetListLockHandler(ctx *context.Context) {
-	if !checkIsValidRequest(ctx) {
-		// Status is written in checkIsValidRequest
-		return
-	}
-	ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
-	rv := unpack(ctx)
-
-	repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
-	if err != nil {
-		log.Debug("Could not find repository: %s/%s - %s", rv.User, rv.Repo, err)
-		writeStatus(ctx, 404)
-		return
-	}
-	repository.MustOwner()
-
-	authenticated := authenticate(ctx, repository, rv.Authorization, false)
-	if !authenticated {
-		ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
-		ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
-			Message: "You must have pull access to list locks",
-		})
-		return
-	}
-
-	cursor := ctx.QueryInt("cursor")
-	if cursor < 0 {
-		cursor = 0
-	}
-	limit := ctx.QueryInt("limit")
-	if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 {
-		limit = setting.LFS.LocksPagingNum
-	} else if limit < 0 {
-		limit = 0
-	}
-	id := ctx.Query("id")
-	if id != "" { //Case where we request a specific id
-		v, err := strconv.ParseInt(id, 10, 64)
-		if err != nil {
-			ctx.JSON(http.StatusBadRequest, api.LFSLockError{
-				Message: "bad request : " + err.Error(),
-			})
-			return
-		}
-		lock, err := models.GetLFSLockByID(v)
-		if err != nil && !models.IsErrLFSLockNotExist(err) {
-			log.Error("Unable to get lock with ID[%s]: Error: %v", v, err)
-		}
-		handleLockListOut(ctx, repository, lock, err)
-		return
-	}
-
-	path := ctx.Query("path")
-	if path != "" { //Case where we request a specific id
-		lock, err := models.GetLFSLock(repository, path)
-		if err != nil && !models.IsErrLFSLockNotExist(err) {
-			log.Error("Unable to get lock for repository %-v with path %s: Error: %v", repository, path, err)
-		}
-		handleLockListOut(ctx, repository, lock, err)
-		return
-	}
-
-	//If no query params path or id
-	lockList, err := models.GetLFSLockByRepoID(repository.ID, cursor, limit)
-	if err != nil {
-		log.Error("Unable to list locks for repository ID[%d]: Error: %v", repository.ID, err)
-		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
-			Message: "unable to list locks : Internal Server Error",
-		})
-		return
-	}
-	lockListAPI := make([]*api.LFSLock, len(lockList))
-	next := ""
-	for i, l := range lockList {
-		lockListAPI[i] = convert.ToLFSLock(l)
-	}
-	if limit > 0 && len(lockList) == limit {
-		next = strconv.Itoa(cursor + 1)
-	}
-	ctx.JSON(http.StatusOK, api.LFSLockList{
-		Locks: lockListAPI,
-		Next:  next,
-	})
-}
-
-// PostLockHandler create lock
-func PostLockHandler(ctx *context.Context) {
-	if !checkIsValidRequest(ctx) {
-		// Status is written in checkIsValidRequest
-		return
-	}
-	ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
-	userName := ctx.Params("username")
-	repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
-	authorization := ctx.Req.Header.Get("Authorization")
-
-	repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
-	if err != nil {
-		log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
-		writeStatus(ctx, 404)
-		return
-	}
-	repository.MustOwner()
-
-	authenticated := authenticate(ctx, repository, authorization, true)
-	if !authenticated {
-		ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
-		ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
-			Message: "You must have push access to create locks",
-		})
-		return
-	}
-
-	var req api.LFSLockRequest
-	bodyReader := ctx.Req.Body
-	defer bodyReader.Close()
-	json := jsoniter.ConfigCompatibleWithStandardLibrary
-	dec := json.NewDecoder(bodyReader)
-	if err := dec.Decode(&req); err != nil {
-		log.Warn("Failed to decode lock request as json. Error: %v", err)
-		writeStatus(ctx, 400)
-		return
-	}
-
-	lock, err := models.CreateLFSLock(&models.LFSLock{
-		Repo:  repository,
-		Path:  req.Path,
-		Owner: ctx.User,
-	})
-	if err != nil {
-		if models.IsErrLFSLockAlreadyExist(err) {
-			ctx.JSON(http.StatusConflict, api.LFSLockError{
-				Lock:    convert.ToLFSLock(lock),
-				Message: "already created lock",
-			})
-			return
-		}
-		if models.IsErrLFSUnauthorizedAction(err) {
-			ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
-			ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
-				Message: "You must have push access to create locks : " + err.Error(),
-			})
-			return
-		}
-		log.Error("Unable to CreateLFSLock in repository %-v at %s for user %-v: Error: %v", repository, req.Path, ctx.User, err)
-		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
-			Message: "internal server error : Internal Server Error",
-		})
-		return
-	}
-	ctx.JSON(http.StatusCreated, api.LFSLockResponse{Lock: convert.ToLFSLock(lock)})
-}
-
-// VerifyLockHandler list locks for verification
-func VerifyLockHandler(ctx *context.Context) {
-	if !checkIsValidRequest(ctx) {
-		// Status is written in checkIsValidRequest
-		return
-	}
-	ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
-	userName := ctx.Params("username")
-	repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
-	authorization := ctx.Req.Header.Get("Authorization")
-
-	repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
-	if err != nil {
-		log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
-		writeStatus(ctx, 404)
-		return
-	}
-	repository.MustOwner()
-
-	authenticated := authenticate(ctx, repository, authorization, true)
-	if !authenticated {
-		ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
-		ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
-			Message: "You must have push access to verify locks",
-		})
-		return
-	}
-
-	cursor := ctx.QueryInt("cursor")
-	if cursor < 0 {
-		cursor = 0
-	}
-	limit := ctx.QueryInt("limit")
-	if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 {
-		limit = setting.LFS.LocksPagingNum
-	} else if limit < 0 {
-		limit = 0
-	}
-	lockList, err := models.GetLFSLockByRepoID(repository.ID, cursor, limit)
-	if err != nil {
-		log.Error("Unable to list locks for repository ID[%d]: Error: %v", repository.ID, err)
-		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
-			Message: "unable to list locks : Internal Server Error",
-		})
-		return
-	}
-	next := ""
-	if limit > 0 && len(lockList) == limit {
-		next = strconv.Itoa(cursor + 1)
-	}
-	lockOursListAPI := make([]*api.LFSLock, 0, len(lockList))
-	lockTheirsListAPI := make([]*api.LFSLock, 0, len(lockList))
-	for _, l := range lockList {
-		if l.Owner.ID == ctx.User.ID {
-			lockOursListAPI = append(lockOursListAPI, convert.ToLFSLock(l))
-		} else {
-			lockTheirsListAPI = append(lockTheirsListAPI, convert.ToLFSLock(l))
-		}
-	}
-	ctx.JSON(http.StatusOK, api.LFSLockListVerify{
-		Ours:   lockOursListAPI,
-		Theirs: lockTheirsListAPI,
-		Next:   next,
-	})
-}
-
-// UnLockHandler delete locks
-func UnLockHandler(ctx *context.Context) {
-	if !checkIsValidRequest(ctx) {
-		// Status is written in checkIsValidRequest
-		return
-	}
-	ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
-	userName := ctx.Params("username")
-	repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
-	authorization := ctx.Req.Header.Get("Authorization")
-
-	repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
-	if err != nil {
-		log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
-		writeStatus(ctx, 404)
-		return
-	}
-	repository.MustOwner()
-
-	authenticated := authenticate(ctx, repository, authorization, true)
-	if !authenticated {
-		ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
-		ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
-			Message: "You must have push access to delete locks",
-		})
-		return
-	}
-
-	var req api.LFSLockDeleteRequest
-	bodyReader := ctx.Req.Body
-	defer bodyReader.Close()
-	json := jsoniter.ConfigCompatibleWithStandardLibrary
-	dec := json.NewDecoder(bodyReader)
-	if err := dec.Decode(&req); err != nil {
-		log.Warn("Failed to decode lock request as json. Error: %v", err)
-		writeStatus(ctx, 400)
-		return
-	}
-
-	lock, err := models.DeleteLFSLockByID(ctx.ParamsInt64("lid"), ctx.User, req.Force)
-	if err != nil {
-		if models.IsErrLFSUnauthorizedAction(err) {
-			ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
-			ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
-				Message: "You must have push access to delete locks : " + err.Error(),
-			})
-			return
-		}
-		log.Error("Unable to DeleteLFSLockByID[%d] by user %-v with force %t: Error: %v", ctx.ParamsInt64("lid"), ctx.User, req.Force, err)
-		ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
-			Message: "unable to delete lock : Internal Server Error",
-		})
-		return
-	}
-	ctx.JSON(http.StatusOK, api.LFSLockResponse{Lock: convert.ToLFSLock(lock)})
-}
diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go
new file mode 100644
index 0000000000..975b5e7dc6
--- /dev/null
+++ b/modules/lfs/pointer.go
@@ -0,0 +1,123 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+const (
+	blobSizeCutoff = 1024
+
+	// MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
+	// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
+	MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
+
+	// MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
+	MetaFileOidPrefix = "oid sha256:"
+)
+
+var (
+	// ErrMissingPrefix occurs if the content lacks the LFS prefix
+	ErrMissingPrefix = errors.New("Content lacks the LFS prefix")
+
+	// ErrInvalidStructure occurs if the content has an invalid structure
+	ErrInvalidStructure = errors.New("Content has an invalid structure")
+
+	// ErrInvalidOIDFormat occurs if the oid has an invalid format
+	ErrInvalidOIDFormat = errors.New("OID has an invalid format")
+)
+
+// ReadPointer tries to read LFS pointer data from the reader
+func ReadPointer(reader io.Reader) (Pointer, error) {
+	buf := make([]byte, blobSizeCutoff)
+	n, err := io.ReadFull(reader, buf)
+	if err != nil && err != io.ErrUnexpectedEOF {
+		return Pointer{}, err
+	}
+	buf = buf[:n]
+
+	return ReadPointerFromBuffer(buf)
+}
+
+var oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`)
+
+// ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise.
+func ReadPointerFromBuffer(buf []byte) (Pointer, error) {
+	var p Pointer
+
+	headString := string(buf)
+	if !strings.HasPrefix(headString, MetaFileIdentifier) {
+		return p, ErrMissingPrefix
+	}
+
+	splitLines := strings.Split(headString, "\n")
+	if len(splitLines) < 3 {
+		return p, ErrInvalidStructure
+	}
+
+	oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix)
+	if len(oid) != 64 || !oidPattern.MatchString(oid) {
+		return p, ErrInvalidOIDFormat
+	}
+	size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
+	if err != nil {
+		return p, err
+	}
+
+	p.Oid = oid
+	p.Size = size
+
+	return p, nil
+}
+
+// IsValid checks if the pointer has a valid structure.
+// It doesn't check if the pointed-to-content exists.
+func (p Pointer) IsValid() bool {
+	if len(p.Oid) != 64 {
+		return false
+	}
+	if !oidPattern.MatchString(p.Oid) {
+		return false
+	}
+	if p.Size < 0 {
+		return false
+	}
+	return true
+}
+
+// StringContent returns the string representation of the pointer
+// https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#the-pointer
+func (p Pointer) StringContent() string {
+	return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size)
+}
+
+// RelativePath returns the relative storage path of the pointer
+func (p Pointer) RelativePath() string {
+	if len(p.Oid) < 5 {
+		return p.Oid
+	}
+
+	return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:])
+}
+
+// GeneratePointer generates a pointer for arbitrary content
+func GeneratePointer(content io.Reader) (Pointer, error) {
+	h := sha256.New()
+	c, err := io.Copy(h, content)
+	if err != nil {
+		return Pointer{}, err
+	}
+	sum := h.Sum(nil)
+	return Pointer{Oid: hex.EncodeToString(sum), Size: c}, nil
+}
diff --git a/modules/lfs/pointer_scanner_gogit.go b/modules/lfs/pointer_scanner_gogit.go
new file mode 100644
index 0000000000..abd882990c
--- /dev/null
+++ b/modules/lfs/pointer_scanner_gogit.go
@@ -0,0 +1,64 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package lfs
+
+import (
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/git"
+
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// SearchPointerBlobs scans the whole repository for LFS pointer files
+func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) {
+	gitRepo := repo.GoGitRepo()
+
+	err := func() error {
+		blobs, err := gitRepo.BlobObjects()
+		if err != nil {
+			return fmt.Errorf("lfs.SearchPointerBlobs BlobObjects: %w", err)
+		}
+
+		return blobs.ForEach(func(blob *object.Blob) error {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			default:
+			}
+
+			if blob.Size > blobSizeCutoff {
+				return nil
+			}
+
+			reader, err := blob.Reader()
+			if err != nil {
+				return fmt.Errorf("lfs.SearchPointerBlobs blob.Reader: %w", err)
+			}
+			defer reader.Close()
+
+			pointer, _ := ReadPointer(reader)
+			if pointer.IsValid() {
+				pointerChan <- PointerBlob{Hash: blob.Hash.String(), Pointer: pointer}
+			}
+
+			return nil
+		})
+	}()
+
+	if err != nil {
+		select {
+		case <-ctx.Done():
+		default:
+			errChan <- err
+		}
+	}
+
+	close(pointerChan)
+	close(errChan)
+}
diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner_nogogit.go
new file mode 100644
index 0000000000..28d4afba61
--- /dev/null
+++ b/modules/lfs/pointer_scanner_nogogit.go
@@ -0,0 +1,110 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package lfs
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"strconv"
+	"sync"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/git/pipeline"
+)
+
+// SearchPointerBlobs scans the whole repository for LFS pointer files
+func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) {
+	basePath := repo.Path
+
+	catFileCheckReader, catFileCheckWriter := io.Pipe()
+	shasToBatchReader, shasToBatchWriter := io.Pipe()
+	catFileBatchReader, catFileBatchWriter := io.Pipe()
+
+	wg := sync.WaitGroup{}
+	wg.Add(4)
+
+	// Create the go-routines in reverse order.
+
+	// 4. Take the output of cat-file --batch and check if each file in turn
+	// to see if they're pointers to files in the LFS store
+	go createPointerResultsFromCatFileBatch(ctx, catFileBatchReader, &wg, pointerChan)
+
+	// 3. Take the shas of the blobs and batch read them
+	go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
+
+	// 2. From the provided objects restrict to blobs <=1k
+	go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
+
+	// 1. Run batch-check on all objects in the repository
+	if git.CheckGitVersionAtLeast("2.6.0") != nil {
+		revListReader, revListWriter := io.Pipe()
+		shasToCheckReader, shasToCheckWriter := io.Pipe()
+		wg.Add(2)
+		go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
+		go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
+		go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
+	} else {
+		go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
+	}
+	wg.Wait()
+
+	close(pointerChan)
+	close(errChan)
+}
+
+func createPointerResultsFromCatFileBatch(ctx context.Context, catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- PointerBlob) {
+	defer wg.Done()
+	defer catFileBatchReader.Close()
+
+	bufferedReader := bufio.NewReader(catFileBatchReader)
+	buf := make([]byte, 1025)
+
+loop:
+	for {
+		select {
+		case <-ctx.Done():
+			break loop
+		default:
+		}
+
+		// File descriptor line: sha
+		sha, err := bufferedReader.ReadString(' ')
+		if err != nil {
+			_ = catFileBatchReader.CloseWithError(err)
+			break
+		}
+		// Throw away the blob
+		if _, err := bufferedReader.ReadString(' '); err != nil {
+			_ = catFileBatchReader.CloseWithError(err)
+			break
+		}
+		sizeStr, err := bufferedReader.ReadString('\n')
+		if err != nil {
+			_ = catFileBatchReader.CloseWithError(err)
+			break
+		}
+		size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
+		if err != nil {
+			_ = catFileBatchReader.CloseWithError(err)
+			break
+		}
+		pointerBuf := buf[:size+1]
+		if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
+			_ = catFileBatchReader.CloseWithError(err)
+			break
+		}
+		pointerBuf = pointerBuf[:size]
+		// Now we need to check if the pointerBuf is an LFS pointer
+		pointer, _ := ReadPointerFromBuffer(pointerBuf)
+		if !pointer.IsValid() {
+			continue
+		}
+
+		pointerChan <- PointerBlob{Hash: sha, Pointer: pointer}
+	}
+}
diff --git a/modules/lfs/pointer_test.go b/modules/lfs/pointer_test.go
new file mode 100644
index 0000000000..0ed6df2c6d
--- /dev/null
+++ b/modules/lfs/pointer_test.go
@@ -0,0 +1,103 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStringContent(t *testing.T) {
+	p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393", Size: 1234}
+	expected := "version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n"
+	assert.Equal(t, p.StringContent(), expected)
+}
+
+func TestRelativePath(t *testing.T) {
+	p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393"}
+	expected := path.Join("4d", "7a", "214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393")
+	assert.Equal(t, p.RelativePath(), expected)
+
+	p2 := Pointer{Oid: "4d7a"}
+	assert.Equal(t, p2.RelativePath(), "4d7a")
+}
+
+func TestIsValid(t *testing.T) {
+	p := Pointer{}
+	assert.False(t, p.IsValid())
+
+	p = Pointer{Oid: "123"}
+	assert.False(t, p.IsValid())
+
+	p = Pointer{Oid: "z4cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc"}
+	assert.False(t, p.IsValid())
+
+	p = Pointer{Oid: "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc"}
+	assert.True(t, p.IsValid())
+
+	p = Pointer{Oid: "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc", Size: -1}
+	assert.False(t, p.IsValid())
+}
+
+func TestGeneratePointer(t *testing.T) {
+	p, err := GeneratePointer(strings.NewReader("Gitea"))
+	assert.NoError(t, err)
+	assert.True(t, p.IsValid())
+	assert.Equal(t, p.Oid, "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc")
+	assert.Equal(t, p.Size, int64(5))
+}
+
+func TestReadPointerFromBuffer(t *testing.T) {
+	p, err := ReadPointerFromBuffer([]byte{})
+	assert.ErrorIs(t, err, ErrMissingPrefix)
+	assert.False(t, p.IsValid())
+
+	p, err = ReadPointerFromBuffer([]byte("test"))
+	assert.ErrorIs(t, err, ErrMissingPrefix)
+	assert.False(t, p.IsValid())
+
+	p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\n"))
+	assert.ErrorIs(t, err, ErrInvalidStructure)
+	assert.False(t, p.IsValid())
+
+	p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a\nsize 1234\n"))
+	assert.ErrorIs(t, err, ErrInvalidOIDFormat)
+	assert.False(t, p.IsValid())
+
+	p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a2146z4ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n"))
+	assert.ErrorIs(t, err, ErrInvalidOIDFormat)
+	assert.False(t, p.IsValid())
+
+	p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\ntest 1234\n"))
+	assert.Error(t, err)
+	assert.False(t, p.IsValid())
+
+	p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize test\n"))
+	assert.Error(t, err)
+	assert.False(t, p.IsValid())
+
+	p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n"))
+	assert.NoError(t, err)
+	assert.True(t, p.IsValid())
+	assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393")
+	assert.Equal(t, p.Size, int64(1234))
+
+	p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\ntest"))
+	assert.NoError(t, err)
+	assert.True(t, p.IsValid())
+	assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393")
+	assert.Equal(t, p.Size, int64(1234))
+}
+
+func TestReadPointer(t *testing.T) {
+	p, err := ReadPointer(strings.NewReader("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n"))
+	assert.NoError(t, err)
+	assert.True(t, p.IsValid())
+	assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393")
+	assert.Equal(t, p.Size, int64(1234))
+}
diff --git a/modules/lfs/pointers.go b/modules/lfs/pointers.go
deleted file mode 100644
index 692c81f583..0000000000
--- a/modules/lfs/pointers.go
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package lfs
-
-import (
-	"io"
-	"strconv"
-	"strings"
-
-	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/base"
-	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/storage"
-)
-
-// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
-func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) {
-	if !setting.LFS.StartServer {
-		return nil, nil
-	}
-
-	buf := make([]byte, 1024)
-	n, _ := reader.Read(buf)
-	buf = buf[:n]
-
-	if isTextFile := base.IsTextFile(buf); !isTextFile {
-		return nil, nil
-	}
-
-	return IsPointerFile(&buf), &buf
-}
-
-// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file
-func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
-	if !setting.LFS.StartServer {
-		return nil
-	}
-
-	headString := string(*buf)
-	if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) {
-		return nil
-	}
-
-	splitLines := strings.Split(headString, "\n")
-	if len(splitLines) < 3 {
-		return nil
-	}
-
-	oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix)
-	size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
-	if len(oid) != 64 || err != nil {
-		return nil
-	}
-
-	contentStore := &ContentStore{ObjectStorage: storage.LFS}
-	meta := &models.LFSMetaObject{Oid: oid, Size: size}
-	exist, err := contentStore.Exists(meta)
-	if err != nil || !exist {
-		return nil
-	}
-
-	return meta
-}
-
-// ReadMetaObject will read a models.LFSMetaObject and return a reader
-func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
-	contentStore := &ContentStore{ObjectStorage: storage.LFS}
-	return contentStore.Get(meta)
-}
diff --git a/modules/lfs/server.go b/modules/lfs/server.go
deleted file mode 100644
index f45423b851..0000000000
--- a/modules/lfs/server.go
+++ /dev/null
@@ -1,712 +0,0 @@
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package lfs
-
-import (
-	"encoding/base64"
-	"fmt"
-	"io"
-	"net/http"
-	"path"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-
-	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/context"
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/storage"
-
-	"github.com/dgrijalva/jwt-go"
-	jsoniter "github.com/json-iterator/go"
-)
-
-const (
-	metaMediaType = "application/vnd.git-lfs+json"
-)
-
-// RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and
-// some headers are stored.
-type RequestVars struct {
-	Oid           string
-	Size          int64
-	User          string
-	Password      string
-	Repo          string
-	Authorization string
-}
-
-// BatchVars contains multiple RequestVars processed in one batch operation.
-// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
-type BatchVars struct {
-	Transfers []string       `json:"transfers,omitempty"`
-	Operation string         `json:"operation"`
-	Objects   []*RequestVars `json:"objects"`
-}
-
-// BatchResponse contains multiple object metadata Representation structures
-// for use with the batch API.
-type BatchResponse struct {
-	Transfer string            `json:"transfer,omitempty"`
-	Objects  []*Representation `json:"objects"`
-}
-
-// Representation is object metadata as seen by clients of the lfs server.
-type Representation struct {
-	Oid     string           `json:"oid"`
-	Size    int64            `json:"size"`
-	Actions map[string]*link `json:"actions"`
-	Error   *ObjectError     `json:"error,omitempty"`
-}
-
-// ObjectError defines the JSON structure returned to the client in case of an error
-type ObjectError struct {
-	Code    int    `json:"code"`
-	Message string `json:"message"`
-}
-
-// Claims is a JWT Token Claims
-type Claims struct {
-	RepoID int64
-	Op     string
-	UserID int64
-	jwt.StandardClaims
-}
-
-// ObjectLink builds a URL linking to the object.
-func (v *RequestVars) ObjectLink() string {
-	return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/objects", v.Oid)
-}
-
-// VerifyLink builds a URL for verifying the object.
-func (v *RequestVars) VerifyLink() string {
-	return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/verify")
-}
-
-// link provides a structure used to build a hypermedia representation of an HTTP link.
-type link struct {
-	Href      string            `json:"href"`
-	Header    map[string]string `json:"header,omitempty"`
-	ExpiresAt time.Time         `json:"expires_at,omitempty"`
-}
-
-var oidRegExp = regexp.MustCompile(`^[A-Fa-f0-9]+$`)
-
-func isOidValid(oid string) bool {
-	return oidRegExp.MatchString(oid)
-}
-
-// ObjectOidHandler is the main request routing entry point into LFS server functions
-func ObjectOidHandler(ctx *context.Context) {
-	if !setting.LFS.StartServer {
-		log.Debug("Attempt to access LFS server but LFS server is disabled")
-		writeStatus(ctx, 404)
-		return
-	}
-
-	if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" {
-		if MetaMatcher(ctx.Req) {
-			getMetaHandler(ctx)
-			return
-		}
-
-		getContentHandler(ctx)
-		return
-	} else if ctx.Req.Method == "PUT" {
-		PutHandler(ctx)
-		return
-	}
-
-	log.Warn("Unhandled LFS method: %s for %s/%s OID[%s]", ctx.Req.Method, ctx.Params("username"), ctx.Params("reponame"), ctx.Params("oid"))
-	writeStatus(ctx, 404)
-}
-
-func getAuthenticatedRepoAndMeta(ctx *context.Context, rv *RequestVars, requireWrite bool) (*models.LFSMetaObject, *models.Repository) {
-	if !isOidValid(rv.Oid) {
-		log.Info("Attempt to access invalid LFS OID[%s] in %s/%s", rv.Oid, rv.User, rv.Repo)
-		writeStatus(ctx, 404)
-		return nil, nil
-	}
-
-	repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
-	if err != nil {
-		log.Error("Unable to get repository: %s/%s Error: %v", rv.User, rv.Repo, err)
-		writeStatus(ctx, 404)
-		return nil, nil
-	}
-
-	if !authenticate(ctx, repository, rv.Authorization, requireWrite) {
-		requireAuth(ctx)
-		return nil, nil
-	}
-
-	meta, err := repository.GetLFSMetaObjectByOid(rv.Oid)
-	if err != nil {
-		log.Error("Unable to get LFS OID[%s] Error: %v", rv.Oid, err)
-		writeStatus(ctx, 404)
-		return nil, nil
-	}
-
-	return meta, repository
-}
-
-// getContentHandler gets the content from the content store
-func getContentHandler(ctx *context.Context) {
-	rv := unpack(ctx)
-
-	meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false)
-	if meta == nil {
-		// Status already written in getAuthenticatedRepoAndMeta
-		return
-	}
-
-	// Support resume download using Range header
-	var fromByte, toByte int64
-	toByte = meta.Size - 1
-	statusCode := 200
-	if rangeHdr := ctx.Req.Header.Get("Range"); rangeHdr != "" {
-		regex := regexp.MustCompile(`bytes=(\d+)\-(\d*).*`)
-		match := regex.FindStringSubmatch(rangeHdr)
-		if len(match) > 1 {
-			statusCode = 206
-			fromByte, _ = strconv.ParseInt(match[1], 10, 32)
-
-			if fromByte >= meta.Size {
-				writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
-				return
-			}
-
-			if match[2] != "" {
-				_toByte, _ := strconv.ParseInt(match[2], 10, 32)
-				if _toByte >= fromByte && _toByte < toByte {
-					toByte = _toByte
-				}
-			}
-
-			ctx.Resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, toByte, meta.Size-fromByte))
-			ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Range")
-		}
-	}
-
-	contentStore := &ContentStore{ObjectStorage: storage.LFS}
-	content, err := contentStore.Get(meta)
-	if err != nil {
-		// Errors are logged in contentStore.Get
-		writeStatus(ctx, http.StatusNotFound)
-		return
-	}
-	defer content.Close()
-
-	if fromByte > 0 {
-		_, err = content.Seek(fromByte, io.SeekStart)
-		if err != nil {
-			log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
-
-			writeStatus(ctx, http.StatusInternalServerError)
-			return
-		}
-	}
-
-	contentLength := toByte + 1 - fromByte
-	ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10))
-	ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
-
-	filename := ctx.Params("filename")
-	if len(filename) > 0 {
-		decodedFilename, err := base64.RawURLEncoding.DecodeString(filename)
-		if err == nil {
-			ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+string(decodedFilename)+"\"")
-			ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
-		}
-	}
-
-	ctx.Resp.WriteHeader(statusCode)
-	if written, err := io.CopyN(ctx.Resp, content, contentLength); err != nil {
-		log.Error("Error whilst copying LFS OID[%s] to the response after %d bytes. Error: %v", meta.Oid, written, err)
-	}
-	logRequest(ctx.Req, statusCode)
-}
-
-// getMetaHandler retrieves metadata about the object
-func getMetaHandler(ctx *context.Context) {
-	rv := unpack(ctx)
-
-	meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false)
-	if meta == nil {
-		// Status already written in getAuthenticatedRepoAndMeta
-		return
-	}
-
-	ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
-	if ctx.Req.Method == "GET" {
-		json := jsoniter.ConfigCompatibleWithStandardLibrary
-		enc := json.NewEncoder(ctx.Resp)
-		if err := enc.Encode(Represent(rv, meta, true, false)); err != nil {
-			log.Error("Failed to encode representation as json. Error: %v", err)
-		}
-	}
-
-	logRequest(ctx.Req, 200)
-}
-
-// PostHandler instructs the client how to upload data
-func PostHandler(ctx *context.Context) {
-	if !setting.LFS.StartServer {
-		log.Debug("Attempt to access LFS server but LFS server is disabled")
-		writeStatus(ctx, 404)
-		return
-	}
-
-	if !MetaMatcher(ctx.Req) {
-		log.Info("Attempt to POST without accepting the correct media type: %s", metaMediaType)
-		writeStatus(ctx, 400)
-		return
-	}
-
-	rv := unpack(ctx)
-
-	repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
-	if err != nil {
-		log.Error("Unable to get repository: %s/%s Error: %v", rv.User, rv.Repo, err)
-		writeStatus(ctx, 404)
-		return
-	}
-
-	if !authenticate(ctx, repository, rv.Authorization, true) {
-		requireAuth(ctx)
-		return
-	}
-
-	if !isOidValid(rv.Oid) {
-		log.Info("Invalid LFS OID[%s] attempt to POST in %s/%s", rv.Oid, rv.User, rv.Repo)
-		writeStatus(ctx, 404)
-		return
-	}
-
-	if setting.LFS.MaxFileSize > 0 && rv.Size > setting.LFS.MaxFileSize {
-		log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", rv.Oid, rv.Size, rv.User, rv.Repo, setting.LFS.MaxFileSize)
-		writeStatus(ctx, 413)
-		return
-	}
-
-	meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID})
-	if err != nil {
-		log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", rv.Oid, rv.Size, rv.User, rv.Repo, err)
-		writeStatus(ctx, 404)
-		return
-	}
-
-	ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
-	sentStatus := 202
-	contentStore := &ContentStore{ObjectStorage: storage.LFS}
-	exist, err := contentStore.Exists(meta)
-	if err != nil {
-		log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err)
-		writeStatus(ctx, 500)
-		return
-	}
-	if meta.Existing && exist {
-		sentStatus = 200
-	}
-	ctx.Resp.WriteHeader(sentStatus)
-
-	json := jsoniter.ConfigCompatibleWithStandardLibrary
-	enc := json.NewEncoder(ctx.Resp)
-	if err := enc.Encode(Represent(rv, meta, meta.Existing, true)); err != nil {
-		log.Error("Failed to encode representation as json. Error: %v", err)
-	}
-	logRequest(ctx.Req, sentStatus)
-}
-
-// BatchHandler provides the batch api
-func BatchHandler(ctx *context.Context) {
-	if !setting.LFS.StartServer {
-		log.Debug("Attempt to access LFS server but LFS server is disabled")
-		writeStatus(ctx, 404)
-		return
-	}
-
-	if !MetaMatcher(ctx.Req) {
-		log.Info("Attempt to BATCH without accepting the correct media type: %s", metaMediaType)
-		writeStatus(ctx, 400)
-		return
-	}
-
-	bv := unpackbatch(ctx)
-
-	var responseObjects []*Representation
-
-	// Create a response object
-	for _, object := range bv.Objects {
-		if !isOidValid(object.Oid) {
-			log.Info("Invalid LFS OID[%s] attempt to BATCH in %s/%s", object.Oid, object.User, object.Repo)
-			continue
-		}
-
-		repository, err := models.GetRepositoryByOwnerAndName(object.User, object.Repo)
-		if err != nil {
-			log.Error("Unable to get repository: %s/%s Error: %v", object.User, object.Repo, err)
-			writeStatus(ctx, 404)
-			return
-		}
-
-		requireWrite := false
-		if bv.Operation == "upload" {
-			requireWrite = true
-		}
-
-		if !authenticate(ctx, repository, object.Authorization, requireWrite) {
-			requireAuth(ctx)
-			return
-		}
-
-		contentStore := &ContentStore{ObjectStorage: storage.LFS}
-
-		meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
-		if err == nil { // Object is found and exists
-			exist, err := contentStore.Exists(meta)
-			if err != nil {
-				log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
-				writeStatus(ctx, 500)
-				return
-			}
-			if exist {
-				responseObjects = append(responseObjects, Represent(object, meta, true, false))
-				continue
-			}
-		}
-
-		if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
-			log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, object.User, object.Repo, setting.LFS.MaxFileSize)
-			writeStatus(ctx, 413)
-			return
-		}
-
-		// Object is not found
-		meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
-		if err == nil {
-			exist, err := contentStore.Exists(meta)
-			if err != nil {
-				log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
-				writeStatus(ctx, 500)
-				return
-			}
-			responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist))
-		} else {
-			log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err)
-		}
-	}
-
-	ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
-	respobj := &BatchResponse{Objects: responseObjects}
-
-	json := jsoniter.ConfigCompatibleWithStandardLibrary
-	enc := json.NewEncoder(ctx.Resp)
-	if err := enc.Encode(respobj); err != nil {
-		log.Error("Failed to encode representation as json. Error: %v", err)
-	}
-	logRequest(ctx.Req, 200)
-}
-
-// PutHandler receives data from the client and puts it into the content store
-func PutHandler(ctx *context.Context) {
-	rv := unpack(ctx)
-
-	meta, repository := getAuthenticatedRepoAndMeta(ctx, rv, true)
-	if meta == nil {
-		// Status already written in getAuthenticatedRepoAndMeta
-		return
-	}
-
-	contentStore := &ContentStore{ObjectStorage: storage.LFS}
-	defer ctx.Req.Body.Close()
-	if err := contentStore.Put(meta, ctx.Req.Body); err != nil {
-		// Put will log the error itself
-		ctx.Resp.WriteHeader(500)
-		if err == errSizeMismatch || err == errHashMismatch {
-			fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
-		} else {
-			fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`)
-		}
-		if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
-			log.Error("Whilst removing metaobject for LFS OID[%s] due to preceding error there was another Error: %v", rv.Oid, err)
-		}
-		return
-	}
-
-	logRequest(ctx.Req, 200)
-}
-
-// VerifyHandler verify oid and its size from the content store
-func VerifyHandler(ctx *context.Context) {
-	if !setting.LFS.StartServer {
-		log.Debug("Attempt to access LFS server but LFS server is disabled")
-		writeStatus(ctx, 404)
-		return
-	}
-
-	if !MetaMatcher(ctx.Req) {
-		log.Info("Attempt to VERIFY without accepting the correct media type: %s", metaMediaType)
-		writeStatus(ctx, 400)
-		return
-	}
-
-	rv := unpack(ctx)
-
-	meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, true)
-	if meta == nil {
-		// Status already written in getAuthenticatedRepoAndMeta
-		return
-	}
-
-	contentStore := &ContentStore{ObjectStorage: storage.LFS}
-	ok, err := contentStore.Verify(meta)
-	if err != nil {
-		// Error will be logged in Verify
-		ctx.Resp.WriteHeader(500)
-		fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`)
-		return
-	}
-	if !ok {
-		writeStatus(ctx, 422)
-		return
-	}
-
-	logRequest(ctx.Req, 200)
-}
-
-// Represent takes a RequestVars and Meta and turns it into a Representation suitable
-// for json encoding
-func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation {
-	rep := &Representation{
-		Oid:     meta.Oid,
-		Size:    meta.Size,
-		Actions: make(map[string]*link),
-	}
-
-	header := make(map[string]string)
-
-	if rv.Authorization == "" {
-		//https://github.com/github/git-lfs/issues/1088
-		header["Authorization"] = "Authorization: Basic dummy"
-	} else {
-		header["Authorization"] = rv.Authorization
-	}
-
-	if download {
-		rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header}
-	}
-
-	if upload {
-		rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header}
-	}
-
-	if upload && !download {
-		// Force client side verify action while gitea lacks proper server side verification
-		verifyHeader := make(map[string]string)
-		for k, v := range header {
-			verifyHeader[k] = v
-		}
-
-		// This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662
-		verifyHeader["Accept"] = metaMediaType
-
-		rep.Actions["verify"] = &link{Href: rv.VerifyLink(), Header: verifyHeader}
-	}
-
-	return rep
-}
-
-// MetaMatcher provides a mux.MatcherFunc that only allows requests that contain
-// an Accept header with the metaMediaType
-func MetaMatcher(r *http.Request) bool {
-	mediaParts := strings.Split(r.Header.Get("Accept"), ";")
-	mt := mediaParts[0]
-	return mt == metaMediaType
-}
-
-func unpack(ctx *context.Context) *RequestVars {
-	r := ctx.Req
-	rv := &RequestVars{
-		User:          ctx.Params("username"),
-		Repo:          strings.TrimSuffix(ctx.Params("reponame"), ".git"),
-		Oid:           ctx.Params("oid"),
-		Authorization: r.Header.Get("Authorization"),
-	}
-
-	if r.Method == "POST" { // Maybe also check if +json
-		var p RequestVars
-		bodyReader := r.Body
-		defer bodyReader.Close()
-		json := jsoniter.ConfigCompatibleWithStandardLibrary
-		dec := json.NewDecoder(bodyReader)
-		err := dec.Decode(&p)
-		if err != nil {
-			// The error is logged as a WARN here because this may represent misbehaviour rather than a true error
-			log.Warn("Unable to decode POST request vars for LFS OID[%s] in %s/%s: Error: %v", rv.Oid, rv.User, rv.Repo, err)
-			return rv
-		}
-
-		rv.Oid = p.Oid
-		rv.Size = p.Size
-	}
-
-	return rv
-}
-
-// TODO cheap hack, unify with unpack
-func unpackbatch(ctx *context.Context) *BatchVars {
-
-	r := ctx.Req
-	var bv BatchVars
-
-	bodyReader := r.Body
-	defer bodyReader.Close()
-	json := jsoniter.ConfigCompatibleWithStandardLibrary
-	dec := json.NewDecoder(bodyReader)
-	err := dec.Decode(&bv)
-	if err != nil {
-		// The error is logged as a WARN here because this may represent misbehaviour rather than a true error
-		log.Warn("Unable to decode BATCH request vars in %s/%s: Error: %v", ctx.Params("username"), strings.TrimSuffix(ctx.Params("reponame"), ".git"), err)
-		return &bv
-	}
-
-	for i := 0; i < len(bv.Objects); i++ {
-		bv.Objects[i].User = ctx.Params("username")
-		bv.Objects[i].Repo = strings.TrimSuffix(ctx.Params("reponame"), ".git")
-		bv.Objects[i].Authorization = r.Header.Get("Authorization")
-	}
-
-	return &bv
-}
-
-func writeStatus(ctx *context.Context, status int) {
-	message := http.StatusText(status)
-
-	mediaParts := strings.Split(ctx.Req.Header.Get("Accept"), ";")
-	mt := mediaParts[0]
-	if strings.HasSuffix(mt, "+json") {
-		message = `{"message":"` + message + `"}`
-	}
-
-	ctx.Resp.WriteHeader(status)
-	fmt.Fprint(ctx.Resp, message)
-	logRequest(ctx.Req, status)
-}
-
-func logRequest(r *http.Request, status int) {
-	log.Debug("LFS request - Method: %s, URL: %s, Status %d", r.Method, r.URL, status)
-}
-
-// authenticate uses the authorization string to determine whether
-// or not to proceed. This server assumes an HTTP Basic auth format.
-func authenticate(ctx *context.Context, repository *models.Repository, authorization string, requireWrite bool) bool {
-	accessMode := models.AccessModeRead
-	if requireWrite {
-		accessMode = models.AccessModeWrite
-	}
-
-	// ctx.IsSigned is unnecessary here, this will be checked in perm.CanAccess
-	perm, err := models.GetUserRepoPermission(repository, ctx.User)
-	if err != nil {
-		log.Error("Unable to GetUserRepoPermission for user %-v in repo %-v Error: %v", ctx.User, repository)
-		return false
-	}
-
-	canRead := perm.CanAccess(accessMode, models.UnitTypeCode)
-	if canRead {
-		return true
-	}
-
-	user, repo, opStr, err := parseToken(authorization)
-	if err != nil {
-		// Most of these are Warn level - the true internal server errors are logged in parseToken already
-		log.Warn("Authentication failure for provided token with Error: %v", err)
-		return false
-	}
-	ctx.User = user
-	if opStr == "basic" {
-		perm, err = models.GetUserRepoPermission(repository, ctx.User)
-		if err != nil {
-			log.Error("Unable to GetUserRepoPermission for user %-v in repo %-v Error: %v", ctx.User, repository)
-			return false
-		}
-		return perm.CanAccess(accessMode, models.UnitTypeCode)
-	}
-	if repository.ID == repo.ID {
-		if requireWrite && opStr != "upload" {
-			return false
-		}
-		return true
-	}
-	return false
-}
-
-func parseToken(authorization string) (*models.User, *models.Repository, string, error) {
-	if authorization == "" {
-		return nil, nil, "unknown", fmt.Errorf("No token")
-	}
-	if strings.HasPrefix(authorization, "Bearer ") {
-		token, err := jwt.ParseWithClaims(authorization[7:], &Claims{}, func(t *jwt.Token) (interface{}, error) {
-			if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
-				return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
-			}
-			return setting.LFS.JWTSecretBytes, nil
-		})
-		if err != nil {
-			// The error here is WARN level because it is caused by bad authorization rather than an internal server error
-			return nil, nil, "unknown", err
-		}
-		claims, claimsOk := token.Claims.(*Claims)
-		if !token.Valid || !claimsOk {
-			return nil, nil, "unknown", fmt.Errorf("Token claim invalid")
-		}
-		r, err := models.GetRepositoryByID(claims.RepoID)
-		if err != nil {
-			log.Error("Unable to GetRepositoryById[%d]: Error: %v", claims.RepoID, err)
-			return nil, nil, claims.Op, err
-		}
-		u, err := models.GetUserByID(claims.UserID)
-		if err != nil {
-			log.Error("Unable to GetUserById[%d]: Error: %v", claims.UserID, err)
-			return nil, r, claims.Op, err
-		}
-		return u, r, claims.Op, nil
-	}
-
-	if strings.HasPrefix(authorization, "Basic ") {
-		c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic "))
-		if err != nil {
-			return nil, nil, "basic", err
-		}
-		cs := string(c)
-		i := strings.IndexByte(cs, ':')
-		if i < 0 {
-			return nil, nil, "basic", fmt.Errorf("Basic auth invalid")
-		}
-		user, password := cs[:i], cs[i+1:]
-		u, err := models.GetUserByName(user)
-		if err != nil {
-			log.Error("Unable to GetUserByName[%d]: Error: %v", user, err)
-			return nil, nil, "basic", err
-		}
-		if !u.IsPasswordSet() || !u.ValidatePassword(password) {
-			return nil, nil, "basic", fmt.Errorf("Basic auth failed")
-		}
-		return u, nil, "basic", nil
-	}
-
-	return nil, nil, "unknown", fmt.Errorf("Token not found")
-}
-
-func requireAuth(ctx *context.Context) {
-	ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
-	writeStatus(ctx, 401)
-}
diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go
new file mode 100644
index 0000000000..70b76d7512
--- /dev/null
+++ b/modules/lfs/shared.go
@@ -0,0 +1,69 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"time"
+)
+
+const (
+	// MediaType contains the media type for LFS server requests
+	MediaType = "application/vnd.git-lfs+json"
+)
+
+// BatchRequest contains multiple requests processed in one batch operation.
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#requests
+type BatchRequest struct {
+	Operation string     `json:"operation"`
+	Transfers []string   `json:"transfers,omitempty"`
+	Ref       *Reference `json:"ref,omitempty"`
+	Objects   []Pointer  `json:"objects"`
+}
+
+// Reference contains a git reference.
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#ref-property
+type Reference struct {
+	Name string `json:"name"`
+}
+
+// Pointer contains LFS pointer data
+type Pointer struct {
+	Oid  string `json:"oid" xorm:"UNIQUE(s) INDEX NOT NULL"`
+	Size int64  `json:"size" xorm:"NOT NULL"`
+}
+
+// BatchResponse contains multiple object metadata Representation structures
+// for use with the batch API.
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses
+type BatchResponse struct {
+	Transfer string            `json:"transfer,omitempty"`
+	Objects  []*ObjectResponse `json:"objects"`
+}
+
+// ObjectResponse is object metadata as seen by clients of the LFS server.
+type ObjectResponse struct {
+	Pointer
+	Actions map[string]*Link `json:"actions"`
+	Error   *ObjectError     `json:"error,omitempty"`
+}
+
+// Link provides a structure used to build a hypermedia representation of an HTTP link.
+type Link struct {
+	Href      string            `json:"href"`
+	Header    map[string]string `json:"header,omitempty"`
+	ExpiresAt time.Time         `json:"expires_at,omitempty"`
+}
+
+// ObjectError defines the JSON structure returned to the client in case of an error
+type ObjectError struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+}
+
+// PointerBlob associates a Git blob with a Pointer.
+type PointerBlob struct {
+	Hash string
+	Pointer
+}
diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go
new file mode 100644
index 0000000000..ea3aff0000
--- /dev/null
+++ b/modules/lfs/transferadapter.go
@@ -0,0 +1,58 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// TransferAdapter represents an adapter for downloading/uploading LFS objects
+type TransferAdapter interface {
+	Name() string
+	Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error)
+	//Upload(ctx context.Context, reader io.Reader) error
+}
+
+// BasicTransferAdapter implements the "basic" adapter
+type BasicTransferAdapter struct {
+	client *http.Client
+}
+
+// Name returns the name of the adapter
+func (a *BasicTransferAdapter) Name() string {
+	return "basic"
+}
+
+// Download reads the download location and downloads the data
+func (a *BasicTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) {
+	download, ok := r.Actions["download"]
+	if !ok {
+		return nil, errors.New("lfs.BasicTransferAdapter.Download: Action 'download' not found")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", download.Href, nil)
+	if err != nil {
+		return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.NewRequestWithContext: %w", err)
+	}
+	for key, value := range download.Header {
+		req.Header.Set(key, value)
+	}
+
+	res, err := a.client.Do(req)
+	if err != nil {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+		return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.Do: %w", err)
+	}
+
+	return res.Body, nil
+}
diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go
new file mode 100644
index 0000000000..0eabd3faee
--- /dev/null
+++ b/modules/lfs/transferadapter_test.go
@@ -0,0 +1,78 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBasicTransferAdapterName(t *testing.T) {
+	a := &BasicTransferAdapter{}
+
+	assert.Equal(t, "basic", a.Name())
+}
+
+func TestBasicTransferAdapterDownload(t *testing.T) {
+	roundTripHandler := func(req *http.Request) *http.Response {
+		url := req.URL.String()
+		if strings.Contains(url, "valid-download-request") {
+			assert.Equal(t, "GET", req.Method)
+			assert.Equal(t, "test-value", req.Header.Get("test-header"))
+
+			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("dummy"))}
+		}
+
+		t.Errorf("Unknown test case: %s", url)
+
+		return nil
+	}
+
+	hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)}
+	a := &BasicTransferAdapter{hc}
+
+	var cases = []struct {
+		response      *ObjectResponse
+		expectederror string
+	}{
+		// case 0
+		{
+			response:      &ObjectResponse{},
+			expectederror: "Action 'download' not found",
+		},
+		// case 1
+		{
+			response: &ObjectResponse{
+				Actions: map[string]*Link{"upload": nil},
+			},
+			expectederror: "Action 'download' not found",
+		},
+		// case 2
+		{
+			response: &ObjectResponse{
+				Actions: map[string]*Link{"download": {
+					Href:   "https://valid-download-request.io",
+					Header: map[string]string{"test-header": "test-value"},
+				}},
+			},
+			expectederror: "",
+		},
+	}
+
+	for n, c := range cases {
+		_, err := a.Download(context.Background(), c.response)
+		if len(c.expectederror) > 0 {
+			assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
+		} else {
+			assert.NoError(t, err, "case %d", n)
+		}
+	}
+}