summaryrefslogtreecommitdiffstats
path: root/routers/repo/lfs.go
diff options
context:
space:
mode:
Diffstat (limited to 'routers/repo/lfs.go')
-rw-r--r--routers/repo/lfs.go551
1 files changed, 551 insertions, 0 deletions
diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go
new file mode 100644
index 0000000000..de5020c944
--- /dev/null
+++ b/routers/repo/lfs.go
@@ -0,0 +1,551 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package repo
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ gotemplate "html/template"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/charset"
+ "code.gitea.io/gitea/modules/context"
+ "code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/git/pipeline"
+ "code.gitea.io/gitea/modules/lfs"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/setting"
+
+ "github.com/mcuadros/go-version"
+ "github.com/unknwon/com"
+ gogit "gopkg.in/src-d/go-git.v4"
+ "gopkg.in/src-d/go-git.v4/plumbing"
+ "gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+const (
+ tplSettingsLFS base.TplName = "repo/settings/lfs"
+ tplSettingsLFSFile base.TplName = "repo/settings/lfs_file"
+ tplSettingsLFSFileFind base.TplName = "repo/settings/lfs_file_find"
+ tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers"
+)
+
+// LFSFiles shows a repository's LFS files
+func LFSFiles(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ ctx.NotFound("LFSFiles", nil)
+ return
+ }
+ page := ctx.QueryInt("page")
+ if page <= 1 {
+ page = 1
+ }
+ total, err := ctx.Repo.Repository.CountLFSMetaObjects()
+ if err != nil {
+ ctx.ServerError("LFSFiles", err)
+ return
+ }
+
+ pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5)
+ ctx.Data["Title"] = ctx.Tr("repo.settings.lfs")
+ ctx.Data["PageIsSettingsLFS"] = true
+ lfsMetaObjects, err := ctx.Repo.Repository.GetLFSMetaObjects(pager.Paginater.Current(), setting.UI.ExplorePagingNum)
+ if err != nil {
+ ctx.ServerError("LFSFiles", err)
+ return
+ }
+ ctx.Data["LFSFiles"] = lfsMetaObjects
+ ctx.Data["Page"] = pager
+ ctx.HTML(200, tplSettingsLFS)
+}
+
+// LFSFileGet serves a single LFS file
+func LFSFileGet(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ ctx.NotFound("LFSFileGet", nil)
+ return
+ }
+ ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
+ oid := ctx.Params("oid")
+ ctx.Data["Title"] = oid
+ ctx.Data["PageIsSettingsLFS"] = true
+ meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid)
+ if err != nil {
+ if err == models.ErrLFSObjectNotExist {
+ ctx.NotFound("LFSFileGet", nil)
+ return
+ }
+ ctx.ServerError("LFSFileGet", err)
+ return
+ }
+ ctx.Data["LFSFile"] = meta
+ dataRc, err := lfs.ReadMetaObject(meta)
+ if err != nil {
+ ctx.ServerError("LFSFileGet", err)
+ return
+ }
+ defer dataRc.Close()
+ buf := make([]byte, 1024)
+ n, err := dataRc.Read(buf)
+ if err != nil {
+ ctx.ServerError("Data", err)
+ return
+ }
+ buf = buf[:n]
+
+ isTextFile := base.IsTextFile(buf)
+ ctx.Data["IsTextFile"] = isTextFile
+
+ fileSize := meta.Size
+ ctx.Data["FileSize"] = meta.Size
+ ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, "direct")
+ switch {
+ case isTextFile:
+ if fileSize >= setting.UI.MaxDisplayFileSize {
+ ctx.Data["IsFileTooLarge"] = true
+ break
+ }
+
+ d, _ := ioutil.ReadAll(dataRc)
+ buf = charset.ToUTF8WithFallback(append(buf, d...))
+
+ // Building code view blocks with line number on server side.
+ var fileContent string
+ if content, err := charset.ToUTF8WithErr(buf); err != nil {
+ log.Error("ToUTF8WithErr: %v", err)
+ fileContent = string(buf)
+ } else {
+ fileContent = content
+ }
+
+ var output bytes.Buffer
+ lines := strings.Split(fileContent, "\n")
+ //Remove blank line at the end of file
+ if len(lines) > 0 && lines[len(lines)-1] == "" {
+ lines = lines[:len(lines)-1]
+ }
+ for index, line := range lines {
+ line = gotemplate.HTMLEscapeString(line)
+ if index != len(lines)-1 {
+ line += "\n"
+ }
+ output.WriteString(fmt.Sprintf(`<li class="L%d" rel="L%d">%s</li>`, index+1, index+1, line))
+ }
+ ctx.Data["FileContent"] = gotemplate.HTML(output.String())
+
+ output.Reset()
+ for i := 0; i < len(lines); i++ {
+ output.WriteString(fmt.Sprintf(`<span id="L%d">%d</span>`, i+1, i+1))
+ }
+ ctx.Data["LineNums"] = gotemplate.HTML(output.String())
+
+ case base.IsPDFFile(buf):
+ ctx.Data["IsPDFFile"] = true
+ case base.IsVideoFile(buf):
+ ctx.Data["IsVideoFile"] = true
+ case base.IsAudioFile(buf):
+ ctx.Data["IsAudioFile"] = true
+ case base.IsImageFile(buf):
+ ctx.Data["IsImageFile"] = true
+ }
+ ctx.HTML(200, tplSettingsLFSFile)
+}
+
+// LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it
+func LFSDelete(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ ctx.NotFound("LFSDelete", nil)
+ return
+ }
+ oid := ctx.Params("oid")
+ count, err := ctx.Repo.Repository.RemoveLFSMetaObjectByOid(oid)
+ if err != nil {
+ ctx.ServerError("LFSDelete", err)
+ return
+ }
+ // FIXME: Warning: the LFS store is not locked - and can't be locked - there could be a race condition here
+ // Please note a similar condition happens in models/repo.go DeleteRepository
+ if count == 0 {
+ oidPath := filepath.Join(oid[0:2], oid[2:4], oid[4:])
+ err = os.Remove(filepath.Join(setting.LFS.ContentPath, oidPath))
+ if err != nil {
+ ctx.ServerError("LFSDelete", err)
+ return
+ }
+ }
+ ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
+}
+
+type lfsResult struct {
+ Name string
+ SHA string
+ Summary string
+ When time.Time
+ ParentHashes []plumbing.Hash
+ BranchName string
+ FullCommitName string
+}
+
+type lfsResultSlice []*lfsResult
+
+func (a lfsResultSlice) Len() int { return len(a) }
+func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
+
+// LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
+func LFSFileFind(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ ctx.NotFound("LFSFind", nil)
+ return
+ }
+ oid := ctx.Query("oid")
+ size := ctx.QueryInt64("size")
+ if len(oid) == 0 || size == 0 {
+ ctx.NotFound("LFSFind", nil)
+ return
+ }
+ sha := ctx.Query("sha")
+ ctx.Data["Title"] = oid
+ ctx.Data["PageIsSettingsLFS"] = true
+ var hash plumbing.Hash
+ if len(sha) == 0 {
+ meta := models.LFSMetaObject{Oid: oid, Size: size}
+ pointer := meta.Pointer()
+ hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer))
+ sha = hash.String()
+ } else {
+ hash = plumbing.NewHash(sha)
+ }
+ ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
+ ctx.Data["Oid"] = oid
+ ctx.Data["Size"] = size
+ ctx.Data["SHA"] = sha
+
+ resultsMap := map[string]*lfsResult{}
+ results := make([]*lfsResult, 0)
+
+ basePath := ctx.Repo.Repository.RepoPath()
+ gogitRepo := ctx.Repo.GitRepo.GoGitRepo()
+
+ commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
+ Order: gogit.LogOrderCommitterTime,
+ All: true,
+ })
+ if err != nil {
+ log.Error("Failed to get GoGit CommitsIter: %v", err)
+ ctx.ServerError("LFSFind: Iterate Commits", err)
+ return
+ }
+
+ err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
+ tree, err := gitCommit.Tree()
+ if err != nil {
+ return err
+ }
+ treeWalker := object.NewTreeWalker(tree, true, nil)
+ defer treeWalker.Close()
+ for {
+ name, entry, err := treeWalker.Next()
+ if err == io.EOF {
+ break
+ }
+ if entry.Hash == hash {
+ result := lfsResult{
+ Name: name,
+ SHA: gitCommit.Hash.String(),
+ Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
+ When: gitCommit.Author.When,
+ ParentHashes: gitCommit.ParentHashes,
+ }
+ resultsMap[gitCommit.Hash.String()+":"+name] = &result
+ }
+ }
+ return nil
+ })
+ if err != nil && err != io.EOF {
+ log.Error("Failure in CommitIter.ForEach: %v", err)
+ ctx.ServerError("LFSFind: IterateCommits ForEach", err)
+ return
+ }
+
+ for _, result := range resultsMap {
+ hasParent := false
+ for _, parentHash := range result.ParentHashes {
+ if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
+ break
+ }
+ }
+ if !hasParent {
+ results = append(results, result)
+ }
+ }
+
+ sort.Sort(lfsResultSlice(results))
+
+ // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
+ shasToNameReader, shasToNameWriter := io.Pipe()
+ nameRevStdinReader, nameRevStdinWriter := io.Pipe()
+ errChan := make(chan error, 1)
+ wg := sync.WaitGroup{}
+ wg.Add(3)
+
+ go func() {
+ defer wg.Done()
+ scanner := bufio.NewScanner(nameRevStdinReader)
+ i := 0
+ for scanner.Scan() {
+ line := scanner.Text()
+ if len(line) == 0 {
+ continue
+ }
+ result := results[i]
+ result.FullCommitName = line
+ result.BranchName = strings.Split(line, "~")[0]
+ i++
+ }
+ }()
+ go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
+ go func() {
+ defer wg.Done()
+ defer shasToNameWriter.Close()
+ for _, result := range results {
+ i := 0
+ if i < len(result.SHA) {
+ n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
+ if err != nil {
+ errChan <- err
+ break
+ }
+ i += n
+ }
+ n := 0
+ for n < 1 {
+ n, err = shasToNameWriter.Write([]byte{'\n'})
+ if err != nil {
+ errChan <- err
+ break
+ }
+
+ }
+
+ }
+ }()
+
+ wg.Wait()
+
+ select {
+ case err, has := <-errChan:
+ if has {
+ ctx.ServerError("LFSPointerFiles", err)
+ }
+ default:
+ }
+
+ ctx.Data["Results"] = results
+ ctx.HTML(200, tplSettingsLFSFileFind)
+}
+
+// LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store
+func LFSPointerFiles(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ ctx.NotFound("LFSFileGet", nil)
+ return
+ }
+ ctx.Data["PageIsSettingsLFS"] = true
+ binVersion, err := git.BinVersion()
+ if err != nil {
+ log.Fatal("Error retrieving git version: %v", err)
+ }
+ ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
+
+ basePath := ctx.Repo.Repository.RepoPath()
+
+ pointerChan := make(chan pointerResult)
+
+ catFileCheckReader, catFileCheckWriter := io.Pipe()
+ shasToBatchReader, shasToBatchWriter := io.Pipe()
+ catFileBatchReader, catFileBatchWriter := io.Pipe()
+ errChan := make(chan error, 1)
+ wg := sync.WaitGroup{}
+ wg.Add(5)
+
+ var numPointers, numAssociated, numNoExist, numAssociatable int
+
+ go func() {
+ defer wg.Done()
+ pointers := make([]pointerResult, 0, 50)
+ for pointer := range pointerChan {
+ pointers = append(pointers, pointer)
+ if pointer.InRepo {
+ numAssociated++
+ }
+ if !pointer.Exists {
+ numNoExist++
+ }
+ if !pointer.InRepo && pointer.Accessible {
+ numAssociatable++
+ }
+ }
+ numPointers = len(pointers)
+ ctx.Data["Pointers"] = pointers
+ ctx.Data["NumPointers"] = numPointers
+ ctx.Data["NumAssociated"] = numAssociated
+ ctx.Data["NumAssociatable"] = numAssociatable
+ ctx.Data["NumNoExist"] = numNoExist
+ ctx.Data["NumNotAssociated"] = numPointers - numAssociated
+ }()
+ go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User)
+ go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
+ go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
+ if !version.Compare(binVersion, "2.6.0", ">=") {
+ revListReader, revListWriter := io.Pipe()
+ shasToCheckReader, shasToCheckWriter := io.Pipe()
+ wg.Add(2)
+ go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
+ go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
+ go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
+ } else {
+ go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
+ }
+ wg.Wait()
+
+ select {
+ case err, has := <-errChan:
+ if has {
+ ctx.ServerError("LFSPointerFiles", err)
+ }
+ default:
+ }
+ ctx.HTML(200, tplSettingsLFSPointers)
+}
+
+type pointerResult struct {
+ SHA string
+ Oid string
+ Size int64
+ InRepo bool
+ Exists bool
+ Accessible bool
+}
+
+func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
+ defer wg.Done()
+ defer catFileBatchReader.Close()
+ contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath}
+
+ bufferedReader := bufio.NewReader(catFileBatchReader)
+ buf := make([]byte, 1025)
+ for {
+ // File descriptor line: sha
+ sha, err := bufferedReader.ReadString(' ')
+ if err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ // Throw away the blob
+ if _, err := bufferedReader.ReadString(' '); err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ sizeStr, err := bufferedReader.ReadString('\n')
+ if err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
+ if err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ pointerBuf := buf[:size+1]
+ if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ pointerBuf = pointerBuf[:size]
+ // Now we need to check if the pointerBuf is an LFS pointer
+ pointer := lfs.IsPointerFile(&pointerBuf)
+ if pointer == nil {
+ continue
+ }
+
+ result := pointerResult{
+ SHA: strings.TrimSpace(sha),
+ Oid: pointer.Oid,
+ Size: pointer.Size,
+ }
+
+ // Then we need to check that this pointer is in the db
+ if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
+ if err != models.ErrLFSObjectNotExist {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ } else {
+ result.InRepo = true
+ }
+
+ result.Exists = contentStore.Exists(pointer)
+
+ if result.Exists {
+ if !result.InRepo {
+ // Can we fix?
+ // OK well that's "simple"
+ // - we need to check whether current user has access to a repo that has access to the file
+ result.Accessible, err = models.LFSObjectAccessible(user, result.Oid)
+ if err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ } else {
+ result.Accessible = true
+ }
+ }
+ pointerChan <- result
+ }
+ close(pointerChan)
+}
+
+// LFSAutoAssociate auto associates accessible lfs files
+func LFSAutoAssociate(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ ctx.NotFound("LFSAutoAssociate", nil)
+ return
+ }
+ oids := ctx.QueryStrings("oid")
+ metas := make([]*models.LFSMetaObject, len(oids))
+ for i, oid := range oids {
+ idx := strings.IndexRune(oid, ' ')
+ if idx < 0 || idx+1 > len(oid) {
+ ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s", oid))
+ return
+ }
+ var err error
+ metas[i] = &models.LFSMetaObject{}
+ metas[i].Size, err = com.StrTo(oid[idx+1:]).Int64()
+ if err != nil {
+ ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s %v", oid, err))
+ return
+ }
+ metas[i].Oid = oid[:idx]
+ //metas[i].RepositoryID = ctx.Repo.Repository.ID
+ }
+ if err := models.LFSAutoAssociate(metas, ctx.User, ctx.Repo.Repository.ID); err != nil {
+ ctx.ServerError("LFSAutoAssociate", err)
+ return
+ }
+ ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
+}
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-21 00:36:57 +0000