Add basic repository lfs management (#7199)

This PR adds basic repository LFS management UI including the ability to find all possible pointers within the repository. Locks are not managed at present but would be addable through some simple additions.

* Add basic repository lfs management
* add auto-associate function
* Add functionality to find commits with this lfs file
* Add link to find commits on the lfs file view
* Adjust commit view to state the likely branch causing the commit
* Only read Oid from database

7 files changed, 205 insertions, 3 deletions

diff --git a/modules/git/pipeline/catfile.go b/modules/git/pipeline/catfile.go
new file mode 100644
index 0000000000..7293cf9d7f
--- /dev/null
+++ b/modules/git/pipeline/catfile.go
@@ -0,0 +1,94 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package pipeline
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+)
+
+// CatFileBatchCheck runs cat-file with --batch-check
+func CatFileBatchCheck(shasToCheckReader *io.PipeReader, catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
+	defer wg.Done()
+	defer shasToCheckReader.Close()
+	defer catFileCheckWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	cmd := git.NewCommand("cat-file", "--batch-check")
+	if err := cmd.RunInDirFullPipeline(tmpBasePath, catFileCheckWriter, stderr, shasToCheckReader); err != nil {
+		_ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
+	}
+}
+
+// CatFileBatchCheckAllObjects runs cat-file with --batch-check --batch-all
+func CatFileBatchCheckAllObjects(catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string, errChan chan<- error) {
+	defer wg.Done()
+	defer catFileCheckWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects")
+	if err := cmd.RunInDirPipeline(tmpBasePath, catFileCheckWriter, stderr); err != nil {
+		log.Error("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String())
+		err = fmt.Errorf("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String())
+		_ = catFileCheckWriter.CloseWithError(err)
+		errChan <- err
+	}
+}
+
+// CatFileBatch runs cat-file --batch
+func CatFileBatch(shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
+	defer wg.Done()
+	defer shasToBatchReader.Close()
+	defer catFileBatchWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil {
+		_ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
+	}
+}
+
+// BlobsLessThan1024FromCatFileBatchCheck reads a pipeline from cat-file --batch-check and returns the blobs <1024 in size
+func BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) {
+	defer wg.Done()
+	defer catFileCheckReader.Close()
+	scanner := bufio.NewScanner(catFileCheckReader)
+	defer func() {
+		_ = shasToBatchWriter.CloseWithError(scanner.Err())
+	}()
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
+		}
+		fields := strings.Split(line, " ")
+		if len(fields) < 3 || fields[1] != "blob" {
+			continue
+		}
+		size, _ := strconv.Atoi(fields[2])
+		if size > 1024 {
+			continue
+		}
+		toWrite := []byte(fields[0] + "\n")
+		for len(toWrite) > 0 {
+			n, err := shasToBatchWriter.Write(toWrite)
+			if err != nil {
+				_ = catFileCheckReader.CloseWithError(err)
+				break
+			}
+			toWrite = toWrite[n:]
+		}
+	}
+}
diff --git a/modules/git/pipeline/namerev.go b/modules/git/pipeline/namerev.go
new file mode 100644
index 0000000000..eebb53b0ca
--- /dev/null
+++ b/modules/git/pipeline/namerev.go
@@ -0,0 +1,28 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package pipeline
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/git"
+)
+
+// NameRevStdin runs name-rev --stdin
+func NameRevStdin(shasToNameReader *io.PipeReader, nameRevStdinWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
+	defer wg.Done()
+	defer shasToNameReader.Close()
+	defer nameRevStdinWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	if err := git.NewCommand("name-rev", "--stdin", "--name-only", "--always").RunInDirFullPipeline(tmpBasePath, nameRevStdinWriter, stderr, shasToNameReader); err != nil {
+		_ = shasToNameReader.CloseWithError(fmt.Errorf("git name-rev [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
+	}
+}
diff --git a/modules/git/pipeline/revlist.go b/modules/git/pipeline/revlist.go
new file mode 100644
index 0000000000..4e13e19444
--- /dev/null
+++ b/modules/git/pipeline/revlist.go
@@ -0,0 +1,75 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package pipeline
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+)
+
+// RevListAllObjects runs rev-list --objects --all and writes to a pipewriter
+func RevListAllObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, basePath string, errChan chan<- error) {
+	defer wg.Done()
+	defer revListWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	cmd := git.NewCommand("rev-list", "--objects", "--all")
+	if err := cmd.RunInDirPipeline(basePath, revListWriter, stderr); err != nil {
+		log.Error("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String())
+		err = fmt.Errorf("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String())
+		_ = revListWriter.CloseWithError(err)
+		errChan <- err
+	}
+}
+
+// RevListObjects run rev-list --objects from headSHA to baseSHA
+func RevListObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath, headSHA, baseSHA string, errChan chan<- error) {
+	defer wg.Done()
+	defer revListWriter.Close()
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	cmd := git.NewCommand("rev-list", "--objects", headSHA, "--not", baseSHA)
+	if err := cmd.RunInDirPipeline(tmpBasePath, revListWriter, stderr); err != nil {
+		log.Error("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())
+		errChan <- fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())
+	}
+}
+
+// BlobsFromRevListObjects reads a RevListAllObjects and only selects blobs
+func BlobsFromRevListObjects(revListReader *io.PipeReader, shasToCheckWriter *io.PipeWriter, wg *sync.WaitGroup) {
+	defer wg.Done()
+	defer revListReader.Close()
+	scanner := bufio.NewScanner(revListReader)
+	defer func() {
+		_ = shasToCheckWriter.CloseWithError(scanner.Err())
+	}()
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
+		}
+		fields := strings.Split(line, " ")
+		if len(fields) < 2 || len(fields[1]) == 0 {
+			continue
+		}
+		toWrite := []byte(fields[0] + "\n")
+		for len(toWrite) > 0 {
+			n, err := shasToCheckWriter.Write(toWrite)
+			if err != nil {
+				_ = revListReader.CloseWithError(err)
+				break
+			}
+			toWrite = toWrite[n:]
+		}
+	}
+}
diff --git a/modules/git/repo.go b/modules/git/repo.go
index dd886f3a2e..e1d75ca4aa 100644
--- a/modules/git/repo.go
+++ b/modules/git/repo.go
@@ -117,6 +117,11 @@ func OpenRepository(repoPath string) (*Repository, error) {
 	}, nil
 }
 
+// GoGitRepo gets the go-git repo representation
+func (repo *Repository) GoGitRepo() *gogit.Repository {
+	return repo.gogitRepo
+}
+
 // IsEmpty Check if repository is empty.
 func (repo *Repository) IsEmpty() (bool, error) {
 	var errbuf strings.Builder
diff --git a/modules/lfs/server.go b/modules/lfs/server.go
index 6fa97a2894..dc498a86c8 100644
--- a/modules/lfs/server.go
+++ b/modules/lfs/server.go
@@ -332,7 +332,7 @@ func PutHandler(ctx *context.Context) {
 	if err := contentStore.Put(meta, bodyReader); err != nil {
 		ctx.Resp.WriteHeader(500)
 		fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
-		if err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
+		if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
 			log.Error("RemoveLFSMetaObjectByOid: %v", err)
 		}
 		return
diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go
index 8a1e51730b..8e057700ab 100644
--- a/modules/repofiles/update.go
+++ b/modules/repofiles/update.go
@@ -385,7 +385,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
 		contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
 		if !contentStore.Exists(lfsMetaObject) {
 			if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
-				if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
+				if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
 					return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
 				}
 				return nil, err
diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go
index 202e66b89a..a2e7cc927c 100644
--- a/modules/repofiles/upload.go
+++ b/modules/repofiles/upload.go
@@ -36,7 +36,7 @@ func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, orig
 			continue
 		}
 		if !info.lfsMetaObject.Existing {
-			if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil {
+			if _, err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil {
 				original = fmt.Errorf("%v, %v", original, err)
 			}
 		}