You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

lfs.go 20KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727
  1. // Copyright 2019 The Gitea Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package repo
  5. import (
  6. "bufio"
  7. "bytes"
  8. "fmt"
  9. gotemplate "html/template"
  10. "io"
  11. "io/ioutil"
  12. "os"
  13. "path"
  14. "path/filepath"
  15. "sort"
  16. "strconv"
  17. "strings"
  18. "sync"
  19. "time"
  20. "code.gitea.io/gitea/models"
  21. "code.gitea.io/gitea/modules/base"
  22. "code.gitea.io/gitea/modules/charset"
  23. "code.gitea.io/gitea/modules/context"
  24. "code.gitea.io/gitea/modules/git"
  25. "code.gitea.io/gitea/modules/git/pipeline"
  26. "code.gitea.io/gitea/modules/lfs"
  27. "code.gitea.io/gitea/modules/log"
  28. "code.gitea.io/gitea/modules/setting"
  29. gogit "github.com/go-git/go-git/v5"
  30. "github.com/go-git/go-git/v5/plumbing"
  31. "github.com/go-git/go-git/v5/plumbing/object"
  32. "github.com/mcuadros/go-version"
  33. "github.com/unknwon/com"
  34. )
  35. const (
  36. tplSettingsLFS base.TplName = "repo/settings/lfs"
  37. tplSettingsLFSLocks base.TplName = "repo/settings/lfs_locks"
  38. tplSettingsLFSFile base.TplName = "repo/settings/lfs_file"
  39. tplSettingsLFSFileFind base.TplName = "repo/settings/lfs_file_find"
  40. tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers"
  41. )
  42. // LFSFiles shows a repository's LFS files
  43. func LFSFiles(ctx *context.Context) {
  44. if !setting.LFS.StartServer {
  45. ctx.NotFound("LFSFiles", nil)
  46. return
  47. }
  48. page := ctx.QueryInt("page")
  49. if page <= 1 {
  50. page = 1
  51. }
  52. total, err := ctx.Repo.Repository.CountLFSMetaObjects()
  53. if err != nil {
  54. ctx.ServerError("LFSFiles", err)
  55. return
  56. }
  57. ctx.Data["Total"] = total
  58. pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5)
  59. ctx.Data["Title"] = ctx.Tr("repo.settings.lfs")
  60. ctx.Data["PageIsSettingsLFS"] = true
  61. lfsMetaObjects, err := ctx.Repo.Repository.GetLFSMetaObjects(pager.Paginater.Current(), setting.UI.ExplorePagingNum)
  62. if err != nil {
  63. ctx.ServerError("LFSFiles", err)
  64. return
  65. }
  66. ctx.Data["LFSFiles"] = lfsMetaObjects
  67. ctx.Data["Page"] = pager
  68. ctx.HTML(200, tplSettingsLFS)
  69. }
  70. // LFSLocks shows a repository's LFS locks
  71. func LFSLocks(ctx *context.Context) {
  72. if !setting.LFS.StartServer {
  73. ctx.NotFound("LFSLocks", nil)
  74. return
  75. }
  76. ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
  77. page := ctx.QueryInt("page")
  78. if page <= 1 {
  79. page = 1
  80. }
  81. total, err := models.CountLFSLockByRepoID(ctx.Repo.Repository.ID)
  82. if err != nil {
  83. ctx.ServerError("LFSLocks", err)
  84. return
  85. }
  86. ctx.Data["Total"] = total
  87. pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5)
  88. ctx.Data["Title"] = ctx.Tr("repo.settings.lfs_locks")
  89. ctx.Data["PageIsSettingsLFS"] = true
  90. lfsLocks, err := models.GetLFSLockByRepoID(ctx.Repo.Repository.ID, pager.Paginater.Current(), setting.UI.ExplorePagingNum)
  91. if err != nil {
  92. ctx.ServerError("LFSLocks", err)
  93. return
  94. }
  95. ctx.Data["LFSLocks"] = lfsLocks
  96. if len(lfsLocks) == 0 {
  97. ctx.Data["Page"] = pager
  98. ctx.HTML(200, tplSettingsLFSLocks)
  99. return
  100. }
  101. // Clone base repo.
  102. tmpBasePath, err := models.CreateTemporaryPath("locks")
  103. if err != nil {
  104. log.Error("Failed to create temporary path: %v", err)
  105. ctx.ServerError("LFSLocks", err)
  106. return
  107. }
  108. defer func() {
  109. if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
  110. log.Error("LFSLocks: RemoveTemporaryPath: %v", err)
  111. }
  112. }()
  113. if err := git.Clone(ctx.Repo.Repository.RepoPath(), tmpBasePath, git.CloneRepoOptions{
  114. Bare: true,
  115. Shared: true,
  116. }); err != nil {
  117. log.Error("Failed to clone repository: %s (%v)", ctx.Repo.Repository.FullName(), err)
  118. ctx.ServerError("LFSLocks", fmt.Errorf("Failed to clone repository: %s (%v)", ctx.Repo.Repository.FullName(), err))
  119. }
  120. gitRepo, err := git.OpenRepository(tmpBasePath)
  121. if err != nil {
  122. log.Error("Unable to open temporary repository: %s (%v)", tmpBasePath, err)
  123. ctx.ServerError("LFSLocks", fmt.Errorf("Failed to open new temporary repository in: %s %v", tmpBasePath, err))
  124. }
  125. filenames := make([]string, len(lfsLocks))
  126. for i, lock := range lfsLocks {
  127. filenames[i] = lock.Path
  128. }
  129. if err := gitRepo.ReadTreeToIndex(ctx.Repo.Repository.DefaultBranch); err != nil {
  130. log.Error("Unable to read the default branch to the index: %s (%v)", ctx.Repo.Repository.DefaultBranch, err)
  131. ctx.ServerError("LFSLocks", fmt.Errorf("Unable to read the default branch to the index: %s (%v)", ctx.Repo.Repository.DefaultBranch, err))
  132. }
  133. name2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{
  134. Attributes: []string{"lockable"},
  135. Filenames: filenames,
  136. CachedOnly: true,
  137. })
  138. if err != nil {
  139. log.Error("Unable to check attributes in %s (%v)", tmpBasePath, err)
  140. ctx.ServerError("LFSLocks", err)
  141. }
  142. lockables := make([]bool, len(lfsLocks))
  143. for i, lock := range lfsLocks {
  144. attribute2info, has := name2attribute2info[lock.Path]
  145. if !has {
  146. continue
  147. }
  148. if attribute2info["lockable"] != "set" {
  149. continue
  150. }
  151. lockables[i] = true
  152. }
  153. ctx.Data["Lockables"] = lockables
  154. filelist, err := gitRepo.LsFiles(filenames...)
  155. if err != nil {
  156. log.Error("Unable to lsfiles in %s (%v)", tmpBasePath, err)
  157. ctx.ServerError("LFSLocks", err)
  158. }
  159. filemap := make(map[string]bool, len(filelist))
  160. for _, name := range filelist {
  161. filemap[name] = true
  162. }
  163. linkable := make([]bool, len(lfsLocks))
  164. for i, lock := range lfsLocks {
  165. linkable[i] = filemap[lock.Path]
  166. }
  167. ctx.Data["Linkable"] = linkable
  168. ctx.Data["Page"] = pager
  169. ctx.HTML(200, tplSettingsLFSLocks)
  170. }
  171. // LFSLockFile locks a file
  172. func LFSLockFile(ctx *context.Context) {
  173. if !setting.LFS.StartServer {
  174. ctx.NotFound("LFSLocks", nil)
  175. return
  176. }
  177. originalPath := ctx.Query("path")
  178. lockPath := originalPath
  179. if len(lockPath) == 0 {
  180. ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_locking_path", originalPath))
  181. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  182. return
  183. }
  184. if lockPath[len(lockPath)-1] == '/' {
  185. ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_lock_directory", originalPath))
  186. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  187. return
  188. }
  189. lockPath = path.Clean("/" + lockPath)[1:]
  190. if len(lockPath) == 0 {
  191. ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_locking_path", originalPath))
  192. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  193. return
  194. }
  195. _, err := models.CreateLFSLock(&models.LFSLock{
  196. Repo: ctx.Repo.Repository,
  197. Path: lockPath,
  198. Owner: ctx.User,
  199. })
  200. if err != nil {
  201. if models.IsErrLFSLockAlreadyExist(err) {
  202. ctx.Flash.Error(ctx.Tr("repo.settings.lfs_lock_already_exists", originalPath))
  203. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  204. return
  205. }
  206. ctx.ServerError("LFSLockFile", err)
  207. return
  208. }
  209. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  210. }
  211. // LFSUnlock forcibly unlocks an LFS lock
  212. func LFSUnlock(ctx *context.Context) {
  213. if !setting.LFS.StartServer {
  214. ctx.NotFound("LFSUnlock", nil)
  215. return
  216. }
  217. _, err := models.DeleteLFSLockByID(ctx.ParamsInt64("lid"), ctx.User, true)
  218. if err != nil {
  219. ctx.ServerError("LFSUnlock", err)
  220. return
  221. }
  222. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  223. }
  224. // LFSFileGet serves a single LFS file
  225. func LFSFileGet(ctx *context.Context) {
  226. if !setting.LFS.StartServer {
  227. ctx.NotFound("LFSFileGet", nil)
  228. return
  229. }
  230. ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
  231. oid := ctx.Params("oid")
  232. ctx.Data["Title"] = oid
  233. ctx.Data["PageIsSettingsLFS"] = true
  234. meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid)
  235. if err != nil {
  236. if err == models.ErrLFSObjectNotExist {
  237. ctx.NotFound("LFSFileGet", nil)
  238. return
  239. }
  240. ctx.ServerError("LFSFileGet", err)
  241. return
  242. }
  243. ctx.Data["LFSFile"] = meta
  244. dataRc, err := lfs.ReadMetaObject(meta)
  245. if err != nil {
  246. ctx.ServerError("LFSFileGet", err)
  247. return
  248. }
  249. defer dataRc.Close()
  250. buf := make([]byte, 1024)
  251. n, err := dataRc.Read(buf)
  252. if err != nil {
  253. ctx.ServerError("Data", err)
  254. return
  255. }
  256. buf = buf[:n]
  257. isTextFile := base.IsTextFile(buf)
  258. ctx.Data["IsTextFile"] = isTextFile
  259. fileSize := meta.Size
  260. ctx.Data["FileSize"] = meta.Size
  261. ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, "direct")
  262. switch {
  263. case isTextFile:
  264. if fileSize >= setting.UI.MaxDisplayFileSize {
  265. ctx.Data["IsFileTooLarge"] = true
  266. break
  267. }
  268. d, _ := ioutil.ReadAll(dataRc)
  269. buf = charset.ToUTF8WithFallback(append(buf, d...))
  270. // Building code view blocks with line number on server side.
  271. var fileContent string
  272. if content, err := charset.ToUTF8WithErr(buf); err != nil {
  273. log.Error("ToUTF8WithErr: %v", err)
  274. fileContent = string(buf)
  275. } else {
  276. fileContent = content
  277. }
  278. var output bytes.Buffer
  279. lines := strings.Split(fileContent, "\n")
  280. //Remove blank line at the end of file
  281. if len(lines) > 0 && lines[len(lines)-1] == "" {
  282. lines = lines[:len(lines)-1]
  283. }
  284. for index, line := range lines {
  285. line = gotemplate.HTMLEscapeString(line)
  286. if index != len(lines)-1 {
  287. line += "\n"
  288. }
  289. output.WriteString(fmt.Sprintf(`<li class="L%d" rel="L%d">%s</li>`, index+1, index+1, line))
  290. }
  291. ctx.Data["FileContent"] = gotemplate.HTML(output.String())
  292. output.Reset()
  293. for i := 0; i < len(lines); i++ {
  294. output.WriteString(fmt.Sprintf(`<span id="L%d">%d</span>`, i+1, i+1))
  295. }
  296. ctx.Data["LineNums"] = gotemplate.HTML(output.String())
  297. case base.IsPDFFile(buf):
  298. ctx.Data["IsPDFFile"] = true
  299. case base.IsVideoFile(buf):
  300. ctx.Data["IsVideoFile"] = true
  301. case base.IsAudioFile(buf):
  302. ctx.Data["IsAudioFile"] = true
  303. case base.IsImageFile(buf):
  304. ctx.Data["IsImageFile"] = true
  305. }
  306. ctx.HTML(200, tplSettingsLFSFile)
  307. }
  308. // LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it
  309. func LFSDelete(ctx *context.Context) {
  310. if !setting.LFS.StartServer {
  311. ctx.NotFound("LFSDelete", nil)
  312. return
  313. }
  314. oid := ctx.Params("oid")
  315. count, err := ctx.Repo.Repository.RemoveLFSMetaObjectByOid(oid)
  316. if err != nil {
  317. ctx.ServerError("LFSDelete", err)
  318. return
  319. }
  320. // FIXME: Warning: the LFS store is not locked - and can't be locked - there could be a race condition here
  321. // Please note a similar condition happens in models/repo.go DeleteRepository
  322. if count == 0 {
  323. oidPath := filepath.Join(oid[0:2], oid[2:4], oid[4:])
  324. err = os.Remove(filepath.Join(setting.LFS.ContentPath, oidPath))
  325. if err != nil {
  326. ctx.ServerError("LFSDelete", err)
  327. return
  328. }
  329. }
  330. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
  331. }
  332. type lfsResult struct {
  333. Name string
  334. SHA string
  335. Summary string
  336. When time.Time
  337. ParentHashes []plumbing.Hash
  338. BranchName string
  339. FullCommitName string
  340. }
  341. type lfsResultSlice []*lfsResult
  342. func (a lfsResultSlice) Len() int { return len(a) }
  343. func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
  344. func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
  345. // LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
  346. func LFSFileFind(ctx *context.Context) {
  347. if !setting.LFS.StartServer {
  348. ctx.NotFound("LFSFind", nil)
  349. return
  350. }
  351. oid := ctx.Query("oid")
  352. size := ctx.QueryInt64("size")
  353. if len(oid) == 0 || size == 0 {
  354. ctx.NotFound("LFSFind", nil)
  355. return
  356. }
  357. sha := ctx.Query("sha")
  358. ctx.Data["Title"] = oid
  359. ctx.Data["PageIsSettingsLFS"] = true
  360. var hash plumbing.Hash
  361. if len(sha) == 0 {
  362. meta := models.LFSMetaObject{Oid: oid, Size: size}
  363. pointer := meta.Pointer()
  364. hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer))
  365. sha = hash.String()
  366. } else {
  367. hash = plumbing.NewHash(sha)
  368. }
  369. ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
  370. ctx.Data["Oid"] = oid
  371. ctx.Data["Size"] = size
  372. ctx.Data["SHA"] = sha
  373. resultsMap := map[string]*lfsResult{}
  374. results := make([]*lfsResult, 0)
  375. basePath := ctx.Repo.Repository.RepoPath()
  376. gogitRepo := ctx.Repo.GitRepo.GoGitRepo()
  377. commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
  378. Order: gogit.LogOrderCommitterTime,
  379. All: true,
  380. })
  381. if err != nil {
  382. log.Error("Failed to get GoGit CommitsIter: %v", err)
  383. ctx.ServerError("LFSFind: Iterate Commits", err)
  384. return
  385. }
  386. err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
  387. tree, err := gitCommit.Tree()
  388. if err != nil {
  389. return err
  390. }
  391. treeWalker := object.NewTreeWalker(tree, true, nil)
  392. defer treeWalker.Close()
  393. for {
  394. name, entry, err := treeWalker.Next()
  395. if err == io.EOF {
  396. break
  397. }
  398. if entry.Hash == hash {
  399. result := lfsResult{
  400. Name: name,
  401. SHA: gitCommit.Hash.String(),
  402. Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
  403. When: gitCommit.Author.When,
  404. ParentHashes: gitCommit.ParentHashes,
  405. }
  406. resultsMap[gitCommit.Hash.String()+":"+name] = &result
  407. }
  408. }
  409. return nil
  410. })
  411. if err != nil && err != io.EOF {
  412. log.Error("Failure in CommitIter.ForEach: %v", err)
  413. ctx.ServerError("LFSFind: IterateCommits ForEach", err)
  414. return
  415. }
  416. for _, result := range resultsMap {
  417. hasParent := false
  418. for _, parentHash := range result.ParentHashes {
  419. if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
  420. break
  421. }
  422. }
  423. if !hasParent {
  424. results = append(results, result)
  425. }
  426. }
  427. sort.Sort(lfsResultSlice(results))
  428. // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
  429. shasToNameReader, shasToNameWriter := io.Pipe()
  430. nameRevStdinReader, nameRevStdinWriter := io.Pipe()
  431. errChan := make(chan error, 1)
  432. wg := sync.WaitGroup{}
  433. wg.Add(3)
  434. go func() {
  435. defer wg.Done()
  436. scanner := bufio.NewScanner(nameRevStdinReader)
  437. i := 0
  438. for scanner.Scan() {
  439. line := scanner.Text()
  440. if len(line) == 0 {
  441. continue
  442. }
  443. result := results[i]
  444. result.FullCommitName = line
  445. result.BranchName = strings.Split(line, "~")[0]
  446. i++
  447. }
  448. }()
  449. go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
  450. go func() {
  451. defer wg.Done()
  452. defer shasToNameWriter.Close()
  453. for _, result := range results {
  454. i := 0
  455. if i < len(result.SHA) {
  456. n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
  457. if err != nil {
  458. errChan <- err
  459. break
  460. }
  461. i += n
  462. }
  463. n := 0
  464. for n < 1 {
  465. n, err = shasToNameWriter.Write([]byte{'\n'})
  466. if err != nil {
  467. errChan <- err
  468. break
  469. }
  470. }
  471. }
  472. }()
  473. wg.Wait()
  474. select {
  475. case err, has := <-errChan:
  476. if has {
  477. ctx.ServerError("LFSPointerFiles", err)
  478. }
  479. default:
  480. }
  481. ctx.Data["Results"] = results
  482. ctx.HTML(200, tplSettingsLFSFileFind)
  483. }
  484. // LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store
  485. func LFSPointerFiles(ctx *context.Context) {
  486. if !setting.LFS.StartServer {
  487. ctx.NotFound("LFSFileGet", nil)
  488. return
  489. }
  490. ctx.Data["PageIsSettingsLFS"] = true
  491. binVersion, err := git.BinVersion()
  492. if err != nil {
  493. log.Fatal("Error retrieving git version: %v", err)
  494. }
  495. ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
  496. basePath := ctx.Repo.Repository.RepoPath()
  497. pointerChan := make(chan pointerResult)
  498. catFileCheckReader, catFileCheckWriter := io.Pipe()
  499. shasToBatchReader, shasToBatchWriter := io.Pipe()
  500. catFileBatchReader, catFileBatchWriter := io.Pipe()
  501. errChan := make(chan error, 1)
  502. wg := sync.WaitGroup{}
  503. wg.Add(5)
  504. var numPointers, numAssociated, numNoExist, numAssociatable int
  505. go func() {
  506. defer wg.Done()
  507. pointers := make([]pointerResult, 0, 50)
  508. for pointer := range pointerChan {
  509. pointers = append(pointers, pointer)
  510. if pointer.InRepo {
  511. numAssociated++
  512. }
  513. if !pointer.Exists {
  514. numNoExist++
  515. }
  516. if !pointer.InRepo && pointer.Accessible {
  517. numAssociatable++
  518. }
  519. }
  520. numPointers = len(pointers)
  521. ctx.Data["Pointers"] = pointers
  522. ctx.Data["NumPointers"] = numPointers
  523. ctx.Data["NumAssociated"] = numAssociated
  524. ctx.Data["NumAssociatable"] = numAssociatable
  525. ctx.Data["NumNoExist"] = numNoExist
  526. ctx.Data["NumNotAssociated"] = numPointers - numAssociated
  527. }()
  528. go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User)
  529. go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
  530. go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
  531. if !version.Compare(binVersion, "2.6.0", ">=") {
  532. revListReader, revListWriter := io.Pipe()
  533. shasToCheckReader, shasToCheckWriter := io.Pipe()
  534. wg.Add(2)
  535. go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
  536. go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
  537. go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
  538. } else {
  539. go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
  540. }
  541. wg.Wait()
  542. select {
  543. case err, has := <-errChan:
  544. if has {
  545. ctx.ServerError("LFSPointerFiles", err)
  546. }
  547. default:
  548. }
  549. ctx.HTML(200, tplSettingsLFSPointers)
  550. }
  551. type pointerResult struct {
  552. SHA string
  553. Oid string
  554. Size int64
  555. InRepo bool
  556. Exists bool
  557. Accessible bool
  558. }
  559. func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
  560. defer wg.Done()
  561. defer catFileBatchReader.Close()
  562. contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath}
  563. bufferedReader := bufio.NewReader(catFileBatchReader)
  564. buf := make([]byte, 1025)
  565. for {
  566. // File descriptor line: sha
  567. sha, err := bufferedReader.ReadString(' ')
  568. if err != nil {
  569. _ = catFileBatchReader.CloseWithError(err)
  570. break
  571. }
  572. // Throw away the blob
  573. if _, err := bufferedReader.ReadString(' '); err != nil {
  574. _ = catFileBatchReader.CloseWithError(err)
  575. break
  576. }
  577. sizeStr, err := bufferedReader.ReadString('\n')
  578. if err != nil {
  579. _ = catFileBatchReader.CloseWithError(err)
  580. break
  581. }
  582. size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
  583. if err != nil {
  584. _ = catFileBatchReader.CloseWithError(err)
  585. break
  586. }
  587. pointerBuf := buf[:size+1]
  588. if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
  589. _ = catFileBatchReader.CloseWithError(err)
  590. break
  591. }
  592. pointerBuf = pointerBuf[:size]
  593. // Now we need to check if the pointerBuf is an LFS pointer
  594. pointer := lfs.IsPointerFile(&pointerBuf)
  595. if pointer == nil {
  596. continue
  597. }
  598. result := pointerResult{
  599. SHA: strings.TrimSpace(sha),
  600. Oid: pointer.Oid,
  601. Size: pointer.Size,
  602. }
  603. // Then we need to check that this pointer is in the db
  604. if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
  605. if err != models.ErrLFSObjectNotExist {
  606. _ = catFileBatchReader.CloseWithError(err)
  607. break
  608. }
  609. } else {
  610. result.InRepo = true
  611. }
  612. result.Exists = contentStore.Exists(pointer)
  613. if result.Exists {
  614. if !result.InRepo {
  615. // Can we fix?
  616. // OK well that's "simple"
  617. // - we need to check whether current user has access to a repo that has access to the file
  618. result.Accessible, err = models.LFSObjectAccessible(user, result.Oid)
  619. if err != nil {
  620. _ = catFileBatchReader.CloseWithError(err)
  621. break
  622. }
  623. } else {
  624. result.Accessible = true
  625. }
  626. }
  627. pointerChan <- result
  628. }
  629. close(pointerChan)
  630. }
  631. // LFSAutoAssociate auto associates accessible lfs files
  632. func LFSAutoAssociate(ctx *context.Context) {
  633. if !setting.LFS.StartServer {
  634. ctx.NotFound("LFSAutoAssociate", nil)
  635. return
  636. }
  637. oids := ctx.QueryStrings("oid")
  638. metas := make([]*models.LFSMetaObject, len(oids))
  639. for i, oid := range oids {
  640. idx := strings.IndexRune(oid, ' ')
  641. if idx < 0 || idx+1 > len(oid) {
  642. ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s", oid))
  643. return
  644. }
  645. var err error
  646. metas[i] = &models.LFSMetaObject{}
  647. metas[i].Size, err = com.StrTo(oid[idx+1:]).Int64()
  648. if err != nil {
  649. ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s %v", oid, err))
  650. return
  651. }
  652. metas[i].Oid = oid[:idx]
  653. //metas[i].RepositoryID = ctx.Repo.Repository.ID
  654. }
  655. if err := models.LFSAutoAssociate(metas, ctx.User, ctx.Repo.Repository.ID); err != nil {
  656. ctx.ServerError("LFSAutoAssociate", err)
  657. return
  658. }
  659. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
  660. }