You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729
  1. // Copyright 2019 The Gitea Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package repo
  5. import (
  6. "bufio"
  7. "bytes"
  8. "fmt"
  9. gotemplate "html/template"
  10. "io"
  11. "io/ioutil"
  12. "path"
  13. "sort"
  14. "strconv"
  15. "strings"
  16. "sync"
  17. "time"
  18. "code.gitea.io/gitea/models"
  19. "code.gitea.io/gitea/modules/base"
  20. "code.gitea.io/gitea/modules/charset"
  21. "code.gitea.io/gitea/modules/context"
  22. "code.gitea.io/gitea/modules/git"
  23. "code.gitea.io/gitea/modules/git/pipeline"
  24. "code.gitea.io/gitea/modules/lfs"
  25. "code.gitea.io/gitea/modules/log"
  26. "code.gitea.io/gitea/modules/setting"
  27. "code.gitea.io/gitea/modules/storage"
  28. gogit "github.com/go-git/go-git/v5"
  29. "github.com/go-git/go-git/v5/plumbing"
  30. "github.com/go-git/go-git/v5/plumbing/object"
  31. "github.com/unknwon/com"
  32. )
  33. const (
  34. tplSettingsLFS base.TplName = "repo/settings/lfs"
  35. tplSettingsLFSLocks base.TplName = "repo/settings/lfs_locks"
  36. tplSettingsLFSFile base.TplName = "repo/settings/lfs_file"
  37. tplSettingsLFSFileFind base.TplName = "repo/settings/lfs_file_find"
  38. tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers"
  39. )
  40. // LFSFiles shows a repository's LFS files
  41. func LFSFiles(ctx *context.Context) {
  42. if !setting.LFS.StartServer {
  43. ctx.NotFound("LFSFiles", nil)
  44. return
  45. }
  46. page := ctx.QueryInt("page")
  47. if page <= 1 {
  48. page = 1
  49. }
  50. total, err := ctx.Repo.Repository.CountLFSMetaObjects()
  51. if err != nil {
  52. ctx.ServerError("LFSFiles", err)
  53. return
  54. }
  55. ctx.Data["Total"] = total
  56. pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5)
  57. ctx.Data["Title"] = ctx.Tr("repo.settings.lfs")
  58. ctx.Data["PageIsSettingsLFS"] = true
  59. lfsMetaObjects, err := ctx.Repo.Repository.GetLFSMetaObjects(pager.Paginater.Current(), setting.UI.ExplorePagingNum)
  60. if err != nil {
  61. ctx.ServerError("LFSFiles", err)
  62. return
  63. }
  64. ctx.Data["LFSFiles"] = lfsMetaObjects
  65. ctx.Data["Page"] = pager
  66. ctx.HTML(200, tplSettingsLFS)
  67. }
  68. // LFSLocks shows a repository's LFS locks
  69. func LFSLocks(ctx *context.Context) {
  70. if !setting.LFS.StartServer {
  71. ctx.NotFound("LFSLocks", nil)
  72. return
  73. }
  74. ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
  75. page := ctx.QueryInt("page")
  76. if page <= 1 {
  77. page = 1
  78. }
  79. total, err := models.CountLFSLockByRepoID(ctx.Repo.Repository.ID)
  80. if err != nil {
  81. ctx.ServerError("LFSLocks", err)
  82. return
  83. }
  84. ctx.Data["Total"] = total
  85. pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5)
  86. ctx.Data["Title"] = ctx.Tr("repo.settings.lfs_locks")
  87. ctx.Data["PageIsSettingsLFS"] = true
  88. lfsLocks, err := models.GetLFSLockByRepoID(ctx.Repo.Repository.ID, pager.Paginater.Current(), setting.UI.ExplorePagingNum)
  89. if err != nil {
  90. ctx.ServerError("LFSLocks", err)
  91. return
  92. }
  93. ctx.Data["LFSLocks"] = lfsLocks
  94. if len(lfsLocks) == 0 {
  95. ctx.Data["Page"] = pager
  96. ctx.HTML(200, tplSettingsLFSLocks)
  97. return
  98. }
  99. // Clone base repo.
  100. tmpBasePath, err := models.CreateTemporaryPath("locks")
  101. if err != nil {
  102. log.Error("Failed to create temporary path: %v", err)
  103. ctx.ServerError("LFSLocks", err)
  104. return
  105. }
  106. defer func() {
  107. if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
  108. log.Error("LFSLocks: RemoveTemporaryPath: %v", err)
  109. }
  110. }()
  111. if err := git.Clone(ctx.Repo.Repository.RepoPath(), tmpBasePath, git.CloneRepoOptions{
  112. Bare: true,
  113. Shared: true,
  114. }); err != nil {
  115. log.Error("Failed to clone repository: %s (%v)", ctx.Repo.Repository.FullName(), err)
  116. ctx.ServerError("LFSLocks", fmt.Errorf("Failed to clone repository: %s (%v)", ctx.Repo.Repository.FullName(), err))
  117. }
  118. gitRepo, err := git.OpenRepository(tmpBasePath)
  119. if err != nil {
  120. log.Error("Unable to open temporary repository: %s (%v)", tmpBasePath, err)
  121. ctx.ServerError("LFSLocks", fmt.Errorf("Failed to open new temporary repository in: %s %v", tmpBasePath, err))
  122. }
  123. filenames := make([]string, len(lfsLocks))
  124. for i, lock := range lfsLocks {
  125. filenames[i] = lock.Path
  126. }
  127. if err := gitRepo.ReadTreeToIndex(ctx.Repo.Repository.DefaultBranch); err != nil {
  128. log.Error("Unable to read the default branch to the index: %s (%v)", ctx.Repo.Repository.DefaultBranch, err)
  129. ctx.ServerError("LFSLocks", fmt.Errorf("Unable to read the default branch to the index: %s (%v)", ctx.Repo.Repository.DefaultBranch, err))
  130. }
  131. name2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{
  132. Attributes: []string{"lockable"},
  133. Filenames: filenames,
  134. CachedOnly: true,
  135. })
  136. if err != nil {
  137. log.Error("Unable to check attributes in %s (%v)", tmpBasePath, err)
  138. ctx.ServerError("LFSLocks", err)
  139. }
  140. lockables := make([]bool, len(lfsLocks))
  141. for i, lock := range lfsLocks {
  142. attribute2info, has := name2attribute2info[lock.Path]
  143. if !has {
  144. continue
  145. }
  146. if attribute2info["lockable"] != "set" {
  147. continue
  148. }
  149. lockables[i] = true
  150. }
  151. ctx.Data["Lockables"] = lockables
  152. filelist, err := gitRepo.LsFiles(filenames...)
  153. if err != nil {
  154. log.Error("Unable to lsfiles in %s (%v)", tmpBasePath, err)
  155. ctx.ServerError("LFSLocks", err)
  156. }
  157. filemap := make(map[string]bool, len(filelist))
  158. for _, name := range filelist {
  159. filemap[name] = true
  160. }
  161. linkable := make([]bool, len(lfsLocks))
  162. for i, lock := range lfsLocks {
  163. linkable[i] = filemap[lock.Path]
  164. }
  165. ctx.Data["Linkable"] = linkable
  166. ctx.Data["Page"] = pager
  167. ctx.HTML(200, tplSettingsLFSLocks)
  168. }
  169. // LFSLockFile locks a file
  170. func LFSLockFile(ctx *context.Context) {
  171. if !setting.LFS.StartServer {
  172. ctx.NotFound("LFSLocks", nil)
  173. return
  174. }
  175. originalPath := ctx.Query("path")
  176. lockPath := originalPath
  177. if len(lockPath) == 0 {
  178. ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_locking_path", originalPath))
  179. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  180. return
  181. }
  182. if lockPath[len(lockPath)-1] == '/' {
  183. ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_lock_directory", originalPath))
  184. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  185. return
  186. }
  187. lockPath = path.Clean("/" + lockPath)[1:]
  188. if len(lockPath) == 0 {
  189. ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_locking_path", originalPath))
  190. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  191. return
  192. }
  193. _, err := models.CreateLFSLock(&models.LFSLock{
  194. Repo: ctx.Repo.Repository,
  195. Path: lockPath,
  196. Owner: ctx.User,
  197. })
  198. if err != nil {
  199. if models.IsErrLFSLockAlreadyExist(err) {
  200. ctx.Flash.Error(ctx.Tr("repo.settings.lfs_lock_already_exists", originalPath))
  201. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  202. return
  203. }
  204. ctx.ServerError("LFSLockFile", err)
  205. return
  206. }
  207. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  208. }
  209. // LFSUnlock forcibly unlocks an LFS lock
  210. func LFSUnlock(ctx *context.Context) {
  211. if !setting.LFS.StartServer {
  212. ctx.NotFound("LFSUnlock", nil)
  213. return
  214. }
  215. _, err := models.DeleteLFSLockByID(ctx.ParamsInt64("lid"), ctx.User, true)
  216. if err != nil {
  217. ctx.ServerError("LFSUnlock", err)
  218. return
  219. }
  220. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks")
  221. }
  222. // LFSFileGet serves a single LFS file
  223. func LFSFileGet(ctx *context.Context) {
  224. if !setting.LFS.StartServer {
  225. ctx.NotFound("LFSFileGet", nil)
  226. return
  227. }
  228. ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
  229. oid := ctx.Params("oid")
  230. ctx.Data["Title"] = oid
  231. ctx.Data["PageIsSettingsLFS"] = true
  232. meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid)
  233. if err != nil {
  234. if err == models.ErrLFSObjectNotExist {
  235. ctx.NotFound("LFSFileGet", nil)
  236. return
  237. }
  238. ctx.ServerError("LFSFileGet", err)
  239. return
  240. }
  241. ctx.Data["LFSFile"] = meta
  242. dataRc, err := lfs.ReadMetaObject(meta)
  243. if err != nil {
  244. ctx.ServerError("LFSFileGet", err)
  245. return
  246. }
  247. defer dataRc.Close()
  248. buf := make([]byte, 1024)
  249. n, err := dataRc.Read(buf)
  250. if err != nil {
  251. ctx.ServerError("Data", err)
  252. return
  253. }
  254. buf = buf[:n]
  255. isTextFile := base.IsTextFile(buf)
  256. ctx.Data["IsTextFile"] = isTextFile
  257. fileSize := meta.Size
  258. ctx.Data["FileSize"] = meta.Size
  259. ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, "direct")
  260. switch {
  261. case isTextFile:
  262. if fileSize >= setting.UI.MaxDisplayFileSize {
  263. ctx.Data["IsFileTooLarge"] = true
  264. break
  265. }
  266. d, _ := ioutil.ReadAll(dataRc)
  267. buf = charset.ToUTF8WithFallback(append(buf, d...))
  268. // Building code view blocks with line number on server side.
  269. var fileContent string
  270. if content, err := charset.ToUTF8WithErr(buf); err != nil {
  271. log.Error("ToUTF8WithErr: %v", err)
  272. fileContent = string(buf)
  273. } else {
  274. fileContent = content
  275. }
  276. var output bytes.Buffer
  277. lines := strings.Split(fileContent, "\n")
  278. //Remove blank line at the end of file
  279. if len(lines) > 0 && lines[len(lines)-1] == "" {
  280. lines = lines[:len(lines)-1]
  281. }
  282. for index, line := range lines {
  283. line = gotemplate.HTMLEscapeString(line)
  284. if index != len(lines)-1 {
  285. line += "\n"
  286. }
  287. output.WriteString(fmt.Sprintf(`<li class="L%d" rel="L%d">%s</li>`, index+1, index+1, line))
  288. }
  289. ctx.Data["FileContent"] = gotemplate.HTML(output.String())
  290. output.Reset()
  291. for i := 0; i < len(lines); i++ {
  292. output.WriteString(fmt.Sprintf(`<span id="L%d">%d</span>`, i+1, i+1))
  293. }
  294. ctx.Data["LineNums"] = gotemplate.HTML(output.String())
  295. case base.IsPDFFile(buf):
  296. ctx.Data["IsPDFFile"] = true
  297. case base.IsVideoFile(buf):
  298. ctx.Data["IsVideoFile"] = true
  299. case base.IsAudioFile(buf):
  300. ctx.Data["IsAudioFile"] = true
  301. case base.IsImageFile(buf):
  302. ctx.Data["IsImageFile"] = true
  303. }
  304. ctx.HTML(200, tplSettingsLFSFile)
  305. }
  306. // LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it
  307. func LFSDelete(ctx *context.Context) {
  308. if !setting.LFS.StartServer {
  309. ctx.NotFound("LFSDelete", nil)
  310. return
  311. }
  312. oid := ctx.Params("oid")
  313. count, err := ctx.Repo.Repository.RemoveLFSMetaObjectByOid(oid)
  314. if err != nil {
  315. ctx.ServerError("LFSDelete", err)
  316. return
  317. }
  318. // FIXME: Warning: the LFS store is not locked - and can't be locked - there could be a race condition here
  319. // Please note a similar condition happens in models/repo.go DeleteRepository
  320. if count == 0 {
  321. oidPath := path.Join(oid[0:2], oid[2:4], oid[4:])
  322. err = storage.LFS.Delete(oidPath)
  323. if err != nil {
  324. ctx.ServerError("LFSDelete", err)
  325. return
  326. }
  327. }
  328. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
  329. }
  330. type lfsResult struct {
  331. Name string
  332. SHA string
  333. Summary string
  334. When time.Time
  335. ParentHashes []plumbing.Hash
  336. BranchName string
  337. FullCommitName string
  338. }
  339. type lfsResultSlice []*lfsResult
  340. func (a lfsResultSlice) Len() int { return len(a) }
  341. func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
  342. func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
  343. // LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
  344. func LFSFileFind(ctx *context.Context) {
  345. if !setting.LFS.StartServer {
  346. ctx.NotFound("LFSFind", nil)
  347. return
  348. }
  349. oid := ctx.Query("oid")
  350. size := ctx.QueryInt64("size")
  351. if len(oid) == 0 || size == 0 {
  352. ctx.NotFound("LFSFind", nil)
  353. return
  354. }
  355. sha := ctx.Query("sha")
  356. ctx.Data["Title"] = oid
  357. ctx.Data["PageIsSettingsLFS"] = true
  358. var hash plumbing.Hash
  359. if len(sha) == 0 {
  360. meta := models.LFSMetaObject{Oid: oid, Size: size}
  361. pointer := meta.Pointer()
  362. hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer))
  363. sha = hash.String()
  364. } else {
  365. hash = plumbing.NewHash(sha)
  366. }
  367. ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
  368. ctx.Data["Oid"] = oid
  369. ctx.Data["Size"] = size
  370. ctx.Data["SHA"] = sha
  371. resultsMap := map[string]*lfsResult{}
  372. results := make([]*lfsResult, 0)
  373. basePath := ctx.Repo.Repository.RepoPath()
  374. gogitRepo := ctx.Repo.GitRepo.GoGitRepo()
  375. commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
  376. Order: gogit.LogOrderCommitterTime,
  377. All: true,
  378. })
  379. if err != nil {
  380. log.Error("Failed to get GoGit CommitsIter: %v", err)
  381. ctx.ServerError("LFSFind: Iterate Commits", err)
  382. return
  383. }
  384. err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
  385. tree, err := gitCommit.Tree()
  386. if err != nil {
  387. return err
  388. }
  389. treeWalker := object.NewTreeWalker(tree, true, nil)
  390. defer treeWalker.Close()
  391. for {
  392. name, entry, err := treeWalker.Next()
  393. if err == io.EOF {
  394. break
  395. }
  396. if entry.Hash == hash {
  397. result := lfsResult{
  398. Name: name,
  399. SHA: gitCommit.Hash.String(),
  400. Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
  401. When: gitCommit.Author.When,
  402. ParentHashes: gitCommit.ParentHashes,
  403. }
  404. resultsMap[gitCommit.Hash.String()+":"+name] = &result
  405. }
  406. }
  407. return nil
  408. })
  409. if err != nil && err != io.EOF {
  410. log.Error("Failure in CommitIter.ForEach: %v", err)
  411. ctx.ServerError("LFSFind: IterateCommits ForEach", err)
  412. return
  413. }
  414. for _, result := range resultsMap {
  415. hasParent := false
  416. for _, parentHash := range result.ParentHashes {
  417. if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
  418. break
  419. }
  420. }
  421. if !hasParent {
  422. results = append(results, result)
  423. }
  424. }
  425. sort.Sort(lfsResultSlice(results))
  426. // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
  427. shasToNameReader, shasToNameWriter := io.Pipe()
  428. nameRevStdinReader, nameRevStdinWriter := io.Pipe()
  429. errChan := make(chan error, 1)
  430. wg := sync.WaitGroup{}
  431. wg.Add(3)
  432. go func() {
  433. defer wg.Done()
  434. scanner := bufio.NewScanner(nameRevStdinReader)
  435. i := 0
  436. for scanner.Scan() {
  437. line := scanner.Text()
  438. if len(line) == 0 {
  439. continue
  440. }
  441. result := results[i]
  442. result.FullCommitName = line
  443. result.BranchName = strings.Split(line, "~")[0]
  444. i++
  445. }
  446. }()
  447. go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
  448. go func() {
  449. defer wg.Done()
  450. defer shasToNameWriter.Close()
  451. for _, result := range results {
  452. i := 0
  453. if i < len(result.SHA) {
  454. n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
  455. if err != nil {
  456. errChan <- err
  457. break
  458. }
  459. i += n
  460. }
  461. n := 0
  462. for n < 1 {
  463. n, err = shasToNameWriter.Write([]byte{'\n'})
  464. if err != nil {
  465. errChan <- err
  466. break
  467. }
  468. }
  469. }
  470. }()
  471. wg.Wait()
  472. select {
  473. case err, has := <-errChan:
  474. if has {
  475. ctx.ServerError("LFSPointerFiles", err)
  476. }
  477. default:
  478. }
  479. ctx.Data["Results"] = results
  480. ctx.HTML(200, tplSettingsLFSFileFind)
  481. }
  482. // LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store
  483. func LFSPointerFiles(ctx *context.Context) {
  484. if !setting.LFS.StartServer {
  485. ctx.NotFound("LFSFileGet", nil)
  486. return
  487. }
  488. ctx.Data["PageIsSettingsLFS"] = true
  489. err := git.LoadGitVersion()
  490. if err != nil {
  491. log.Fatal("Error retrieving git version: %v", err)
  492. }
  493. ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
  494. basePath := ctx.Repo.Repository.RepoPath()
  495. pointerChan := make(chan pointerResult)
  496. catFileCheckReader, catFileCheckWriter := io.Pipe()
  497. shasToBatchReader, shasToBatchWriter := io.Pipe()
  498. catFileBatchReader, catFileBatchWriter := io.Pipe()
  499. errChan := make(chan error, 1)
  500. wg := sync.WaitGroup{}
  501. wg.Add(5)
  502. var numPointers, numAssociated, numNoExist, numAssociatable int
  503. go func() {
  504. defer wg.Done()
  505. pointers := make([]pointerResult, 0, 50)
  506. for pointer := range pointerChan {
  507. pointers = append(pointers, pointer)
  508. if pointer.InRepo {
  509. numAssociated++
  510. }
  511. if !pointer.Exists {
  512. numNoExist++
  513. }
  514. if !pointer.InRepo && pointer.Accessible {
  515. numAssociatable++
  516. }
  517. }
  518. numPointers = len(pointers)
  519. ctx.Data["Pointers"] = pointers
  520. ctx.Data["NumPointers"] = numPointers
  521. ctx.Data["NumAssociated"] = numAssociated
  522. ctx.Data["NumAssociatable"] = numAssociatable
  523. ctx.Data["NumNoExist"] = numNoExist
  524. ctx.Data["NumNotAssociated"] = numPointers - numAssociated
  525. }()
  526. go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User)
  527. go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
  528. go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
  529. if git.CheckGitVersionAtLeast("2.6.0") != nil {
  530. revListReader, revListWriter := io.Pipe()
  531. shasToCheckReader, shasToCheckWriter := io.Pipe()
  532. wg.Add(2)
  533. go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
  534. go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
  535. go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
  536. } else {
  537. go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
  538. }
  539. wg.Wait()
  540. select {
  541. case err, has := <-errChan:
  542. if has {
  543. ctx.ServerError("LFSPointerFiles", err)
  544. }
  545. default:
  546. }
  547. ctx.HTML(200, tplSettingsLFSPointers)
  548. }
  549. type pointerResult struct {
  550. SHA string
  551. Oid string
  552. Size int64
  553. InRepo bool
  554. Exists bool
  555. Accessible bool
  556. }
  557. func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
  558. defer wg.Done()
  559. defer catFileBatchReader.Close()
  560. contentStore := lfs.ContentStore{ObjectStorage: storage.LFS}
  561. bufferedReader := bufio.NewReader(catFileBatchReader)
  562. buf := make([]byte, 1025)
  563. for {
  564. // File descriptor line: sha
  565. sha, err := bufferedReader.ReadString(' ')
  566. if err != nil {
  567. _ = catFileBatchReader.CloseWithError(err)
  568. break
  569. }
  570. // Throw away the blob
  571. if _, err := bufferedReader.ReadString(' '); err != nil {
  572. _ = catFileBatchReader.CloseWithError(err)
  573. break
  574. }
  575. sizeStr, err := bufferedReader.ReadString('\n')
  576. if err != nil {
  577. _ = catFileBatchReader.CloseWithError(err)
  578. break
  579. }
  580. size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
  581. if err != nil {
  582. _ = catFileBatchReader.CloseWithError(err)
  583. break
  584. }
  585. pointerBuf := buf[:size+1]
  586. if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
  587. _ = catFileBatchReader.CloseWithError(err)
  588. break
  589. }
  590. pointerBuf = pointerBuf[:size]
  591. // Now we need to check if the pointerBuf is an LFS pointer
  592. pointer := lfs.IsPointerFile(&pointerBuf)
  593. if pointer == nil {
  594. continue
  595. }
  596. result := pointerResult{
  597. SHA: strings.TrimSpace(sha),
  598. Oid: pointer.Oid,
  599. Size: pointer.Size,
  600. }
  601. // Then we need to check that this pointer is in the db
  602. if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
  603. if err != models.ErrLFSObjectNotExist {
  604. _ = catFileBatchReader.CloseWithError(err)
  605. break
  606. }
  607. } else {
  608. result.InRepo = true
  609. }
  610. result.Exists, err = contentStore.Exists(pointer)
  611. if err != nil {
  612. _ = catFileBatchReader.CloseWithError(err)
  613. break
  614. }
  615. if result.Exists {
  616. if !result.InRepo {
  617. // Can we fix?
  618. // OK well that's "simple"
  619. // - we need to check whether current user has access to a repo that has access to the file
  620. result.Accessible, err = models.LFSObjectAccessible(user, result.Oid)
  621. if err != nil {
  622. _ = catFileBatchReader.CloseWithError(err)
  623. break
  624. }
  625. } else {
  626. result.Accessible = true
  627. }
  628. }
  629. pointerChan <- result
  630. }
  631. close(pointerChan)
  632. }
  633. // LFSAutoAssociate auto associates accessible lfs files
  634. func LFSAutoAssociate(ctx *context.Context) {
  635. if !setting.LFS.StartServer {
  636. ctx.NotFound("LFSAutoAssociate", nil)
  637. return
  638. }
  639. oids := ctx.QueryStrings("oid")
  640. metas := make([]*models.LFSMetaObject, len(oids))
  641. for i, oid := range oids {
  642. idx := strings.IndexRune(oid, ' ')
  643. if idx < 0 || idx+1 > len(oid) {
  644. ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s", oid))
  645. return
  646. }
  647. var err error
  648. metas[i] = &models.LFSMetaObject{}
  649. metas[i].Size, err = com.StrTo(oid[idx+1:]).Int64()
  650. if err != nil {
  651. ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s %v", oid, err))
  652. return
  653. }
  654. metas[i].Oid = oid[:idx]
  655. //metas[i].RepositoryID = ctx.Repo.Repository.ID
  656. }
  657. if err := models.LFSAutoAssociate(metas, ctx.User, ctx.Repo.Repository.ID); err != nil {
  658. ctx.ServerError("LFSAutoAssociate", err)
  659. return
  660. }
  661. ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
  662. }