You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

mirror_pull.go 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615
  1. // Copyright 2021 The Gitea Authors. All rights reserved.
  2. // SPDX-License-Identifier: MIT
  3. package mirror
  4. import (
  5. "context"
  6. "fmt"
  7. "strings"
  8. "time"
  9. repo_model "code.gitea.io/gitea/models/repo"
  10. system_model "code.gitea.io/gitea/models/system"
  11. "code.gitea.io/gitea/modules/cache"
  12. "code.gitea.io/gitea/modules/git"
  13. "code.gitea.io/gitea/modules/gitrepo"
  14. "code.gitea.io/gitea/modules/lfs"
  15. "code.gitea.io/gitea/modules/log"
  16. "code.gitea.io/gitea/modules/process"
  17. "code.gitea.io/gitea/modules/proxy"
  18. repo_module "code.gitea.io/gitea/modules/repository"
  19. "code.gitea.io/gitea/modules/setting"
  20. "code.gitea.io/gitea/modules/timeutil"
  21. "code.gitea.io/gitea/modules/util"
  22. notify_service "code.gitea.io/gitea/services/notify"
  23. )
  24. // gitShortEmptySha Git short empty SHA
  25. const gitShortEmptySha = "0000000"
  26. // UpdateAddress writes new address to Git repository and database
  27. func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error {
  28. remoteName := m.GetRemoteName()
  29. repoPath := m.GetRepository(ctx).RepoPath()
  30. // Remove old remote
  31. _, _, err := git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(remoteName).RunStdString(&git.RunOpts{Dir: repoPath})
  32. if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
  33. return err
  34. }
  35. cmd := git.NewCommand(ctx, "remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(addr)
  36. if strings.Contains(addr, "://") && strings.Contains(addr, "@") {
  37. cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=fetch %s [repo_path: %s]", remoteName, util.SanitizeCredentialURLs(addr), repoPath))
  38. } else {
  39. cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=fetch %s [repo_path: %s]", remoteName, addr, repoPath))
  40. }
  41. _, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath})
  42. if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
  43. return err
  44. }
  45. if m.Repo.HasWiki() {
  46. wikiPath := m.Repo.WikiPath()
  47. wikiRemotePath := repo_module.WikiRemoteURL(ctx, addr)
  48. // Remove old remote of wiki
  49. _, _, err = git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(remoteName).RunStdString(&git.RunOpts{Dir: wikiPath})
  50. if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
  51. return err
  52. }
  53. cmd = git.NewCommand(ctx, "remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(wikiRemotePath)
  54. if strings.Contains(wikiRemotePath, "://") && strings.Contains(wikiRemotePath, "@") {
  55. cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=fetch %s [repo_path: %s]", remoteName, util.SanitizeCredentialURLs(wikiRemotePath), wikiPath))
  56. } else {
  57. cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=fetch %s [repo_path: %s]", remoteName, wikiRemotePath, wikiPath))
  58. }
  59. _, _, err = cmd.RunStdString(&git.RunOpts{Dir: wikiPath})
  60. if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
  61. return err
  62. }
  63. }
  64. m.Repo.OriginalURL = addr
  65. return repo_model.UpdateRepositoryCols(ctx, m.Repo, "original_url")
  66. }
  67. // mirrorSyncResult contains information of a updated reference.
  68. // If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
  69. // If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
  70. type mirrorSyncResult struct {
  71. refName git.RefName
  72. oldCommitID string
  73. newCommitID string
  74. }
  75. // parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
  76. // possible output example:
  77. /*
  78. // * [new tag] v0.1.8 -> v0.1.8
  79. // * [new branch] master -> origin/master
  80. // - [deleted] (none) -> origin/test // delete a branch
  81. // - [deleted] (none) -> 1 // delete a tag
  82. // 957a993..a87ba5f test -> origin/test
  83. // + f895a1e...957a993 test -> origin/test (forced update)
  84. */
  85. // TODO: return whether it's a force update
  86. func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult {
  87. results := make([]*mirrorSyncResult, 0, 3)
  88. lines := strings.Split(output, "\n")
  89. for i := range lines {
  90. // Make sure reference name is presented before continue
  91. idx := strings.Index(lines[i], "-> ")
  92. if idx == -1 {
  93. continue
  94. }
  95. refName := strings.TrimSpace(lines[i][idx+3:])
  96. switch {
  97. case strings.HasPrefix(lines[i], " * [new tag]"): // new tag
  98. results = append(results, &mirrorSyncResult{
  99. refName: git.RefNameFromTag(refName),
  100. oldCommitID: gitShortEmptySha,
  101. })
  102. case strings.HasPrefix(lines[i], " * [new branch]"): // new branch
  103. refName = strings.TrimPrefix(refName, remoteName+"/")
  104. results = append(results, &mirrorSyncResult{
  105. refName: git.RefNameFromBranch(refName),
  106. oldCommitID: gitShortEmptySha,
  107. })
  108. case strings.HasPrefix(lines[i], " - "): // Delete reference
  109. isTag := !strings.HasPrefix(refName, remoteName+"/")
  110. var refFullName git.RefName
  111. if isTag {
  112. refFullName = git.RefNameFromTag(refName)
  113. } else {
  114. refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
  115. }
  116. results = append(results, &mirrorSyncResult{
  117. refName: refFullName,
  118. newCommitID: gitShortEmptySha,
  119. })
  120. case strings.HasPrefix(lines[i], " + "): // Force update
  121. if idx := strings.Index(refName, " "); idx > -1 {
  122. refName = refName[:idx]
  123. }
  124. delimIdx := strings.Index(lines[i][3:], " ")
  125. if delimIdx == -1 {
  126. log.Error("SHA delimiter not found: %q", lines[i])
  127. continue
  128. }
  129. shas := strings.Split(lines[i][3:delimIdx+3], "...")
  130. if len(shas) != 2 {
  131. log.Error("Expect two SHAs but not what found: %q", lines[i])
  132. continue
  133. }
  134. results = append(results, &mirrorSyncResult{
  135. refName: git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/")),
  136. oldCommitID: shas[0],
  137. newCommitID: shas[1],
  138. })
  139. case strings.HasPrefix(lines[i], " "): // New commits of a reference
  140. delimIdx := strings.Index(lines[i][3:], " ")
  141. if delimIdx == -1 {
  142. log.Error("SHA delimiter not found: %q", lines[i])
  143. continue
  144. }
  145. shas := strings.Split(lines[i][3:delimIdx+3], "..")
  146. if len(shas) != 2 {
  147. log.Error("Expect two SHAs but not what found: %q", lines[i])
  148. continue
  149. }
  150. results = append(results, &mirrorSyncResult{
  151. refName: git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/")),
  152. oldCommitID: shas[0],
  153. newCommitID: shas[1],
  154. })
  155. default:
  156. log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
  157. }
  158. }
  159. return results
  160. }
  161. func pruneBrokenReferences(ctx context.Context,
  162. m *repo_model.Mirror,
  163. repoPath string,
  164. timeout time.Duration,
  165. stdoutBuilder, stderrBuilder *strings.Builder,
  166. isWiki bool,
  167. ) error {
  168. wiki := ""
  169. if isWiki {
  170. wiki = "Wiki "
  171. }
  172. stderrBuilder.Reset()
  173. stdoutBuilder.Reset()
  174. pruneErr := git.NewCommand(ctx, "remote", "prune").AddDynamicArguments(m.GetRemoteName()).
  175. SetDescription(fmt.Sprintf("Mirror.runSync %ssPrune references: %s ", wiki, m.Repo.FullName())).
  176. Run(&git.RunOpts{
  177. Timeout: timeout,
  178. Dir: repoPath,
  179. Stdout: stdoutBuilder,
  180. Stderr: stderrBuilder,
  181. })
  182. if pruneErr != nil {
  183. stdout := stdoutBuilder.String()
  184. stderr := stderrBuilder.String()
  185. // sanitize the output, since it may contain the remote address, which may
  186. // contain a password
  187. stderrMessage := util.SanitizeCredentialURLs(stderr)
  188. stdoutMessage := util.SanitizeCredentialURLs(stdout)
  189. log.Error("Failed to prune mirror repository %s%-v references:\nStdout: %s\nStderr: %s\nErr: %v", wiki, m.Repo, stdoutMessage, stderrMessage, pruneErr)
  190. desc := fmt.Sprintf("Failed to prune mirror repository %s'%s' references: %s", wiki, repoPath, stderrMessage)
  191. if err := system_model.CreateRepositoryNotice(desc); err != nil {
  192. log.Error("CreateRepositoryNotice: %v", err)
  193. }
  194. // this if will only be reached on a successful prune so try to get the mirror again
  195. }
  196. return pruneErr
  197. }
  198. // runSync returns true if sync finished without error.
  199. func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) {
  200. repoPath := m.Repo.RepoPath()
  201. wikiPath := m.Repo.WikiPath()
  202. timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
  203. log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo)
  204. // use fetch but not remote update because git fetch support --tags but remote update doesn't
  205. cmd := git.NewCommand(ctx, "fetch")
  206. if m.EnablePrune {
  207. cmd.AddArguments("--prune")
  208. }
  209. cmd.AddArguments("--tags").AddDynamicArguments(m.GetRemoteName())
  210. remoteURL, remoteErr := git.GetRemoteURL(ctx, repoPath, m.GetRemoteName())
  211. if remoteErr != nil {
  212. log.Error("SyncMirrors [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr)
  213. return nil, false
  214. }
  215. envs := proxy.EnvWithProxy(remoteURL.URL)
  216. stdoutBuilder := strings.Builder{}
  217. stderrBuilder := strings.Builder{}
  218. if err := cmd.
  219. SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())).
  220. Run(&git.RunOpts{
  221. Timeout: timeout,
  222. Dir: repoPath,
  223. Env: envs,
  224. Stdout: &stdoutBuilder,
  225. Stderr: &stderrBuilder,
  226. }); err != nil {
  227. stdout := stdoutBuilder.String()
  228. stderr := stderrBuilder.String()
  229. // sanitize the output, since it may contain the remote address, which may contain a password
  230. stderrMessage := util.SanitizeCredentialURLs(stderr)
  231. stdoutMessage := util.SanitizeCredentialURLs(stdout)
  232. // Now check if the error is a resolve reference due to broken reference
  233. if strings.Contains(stderr, "unable to resolve reference") && strings.Contains(stderr, "reference broken") {
  234. log.Warn("SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err)
  235. err = nil
  236. // Attempt prune
  237. pruneErr := pruneBrokenReferences(ctx, m, repoPath, timeout, &stdoutBuilder, &stderrBuilder, false)
  238. if pruneErr == nil {
  239. // Successful prune - reattempt mirror
  240. stderrBuilder.Reset()
  241. stdoutBuilder.Reset()
  242. if err = cmd.
  243. SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())).
  244. Run(&git.RunOpts{
  245. Timeout: timeout,
  246. Dir: repoPath,
  247. Stdout: &stdoutBuilder,
  248. Stderr: &stderrBuilder,
  249. }); err != nil {
  250. stdout := stdoutBuilder.String()
  251. stderr := stderrBuilder.String()
  252. // sanitize the output, since it may contain the remote address, which may
  253. // contain a password
  254. stderrMessage = util.SanitizeCredentialURLs(stderr)
  255. stdoutMessage = util.SanitizeCredentialURLs(stdout)
  256. }
  257. }
  258. }
  259. // If there is still an error (or there always was an error)
  260. if err != nil {
  261. log.Error("SyncMirrors [repo: %-v]: failed to update mirror repository:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err)
  262. desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, stderrMessage)
  263. if err = system_model.CreateRepositoryNotice(desc); err != nil {
  264. log.Error("CreateRepositoryNotice: %v", err)
  265. }
  266. return nil, false
  267. }
  268. }
  269. output := stderrBuilder.String()
  270. if err := git.WriteCommitGraph(ctx, repoPath); err != nil {
  271. log.Error("SyncMirrors [repo: %-v]: %v", m.Repo, err)
  272. }
  273. gitRepo, err := gitrepo.OpenRepository(ctx, m.Repo)
  274. if err != nil {
  275. log.Error("SyncMirrors [repo: %-v]: failed to OpenRepository: %v", m.Repo, err)
  276. return nil, false
  277. }
  278. log.Trace("SyncMirrors [repo: %-v]: syncing branches...", m.Repo)
  279. if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, m.Repo, gitRepo, 0); err != nil {
  280. log.Error("SyncMirrors [repo: %-v]: failed to synchronize branches: %v", m.Repo, err)
  281. }
  282. log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo)
  283. if err = repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo); err != nil {
  284. log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err)
  285. }
  286. if m.LFS && setting.LFS.StartServer {
  287. log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
  288. endpoint := lfs.DetermineEndpoint(remoteURL.String(), m.LFSEndpoint)
  289. lfsClient := lfs.NewClient(endpoint, nil)
  290. if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil {
  291. log.Error("SyncMirrors [repo: %-v]: failed to synchronize LFS objects for repository: %v", m.Repo, err)
  292. }
  293. }
  294. gitRepo.Close()
  295. log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo)
  296. if err := repo_module.UpdateRepoSize(ctx, m.Repo); err != nil {
  297. log.Error("SyncMirrors [repo: %-v]: failed to update size for mirror repository: %v", m.Repo, err)
  298. }
  299. if m.Repo.HasWiki() {
  300. log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo)
  301. stderrBuilder.Reset()
  302. stdoutBuilder.Reset()
  303. if err := git.NewCommand(ctx, "remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).
  304. SetDescription(fmt.Sprintf("Mirror.runSync Wiki: %s ", m.Repo.FullName())).
  305. Run(&git.RunOpts{
  306. Timeout: timeout,
  307. Dir: wikiPath,
  308. Stdout: &stdoutBuilder,
  309. Stderr: &stderrBuilder,
  310. }); err != nil {
  311. stdout := stdoutBuilder.String()
  312. stderr := stderrBuilder.String()
  313. // sanitize the output, since it may contain the remote address, which may contain a password
  314. stderrMessage := util.SanitizeCredentialURLs(stderr)
  315. stdoutMessage := util.SanitizeCredentialURLs(stdout)
  316. // Now check if the error is a resolve reference due to broken reference
  317. if strings.Contains(stderrMessage, "unable to resolve reference") && strings.Contains(stderrMessage, "reference broken") {
  318. log.Warn("SyncMirrors [repo: %-v Wiki]: failed to update mirror wiki repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err)
  319. err = nil
  320. // Attempt prune
  321. pruneErr := pruneBrokenReferences(ctx, m, repoPath, timeout, &stdoutBuilder, &stderrBuilder, true)
  322. if pruneErr == nil {
  323. // Successful prune - reattempt mirror
  324. stderrBuilder.Reset()
  325. stdoutBuilder.Reset()
  326. if err = git.NewCommand(ctx, "remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).
  327. SetDescription(fmt.Sprintf("Mirror.runSync Wiki: %s ", m.Repo.FullName())).
  328. Run(&git.RunOpts{
  329. Timeout: timeout,
  330. Dir: wikiPath,
  331. Stdout: &stdoutBuilder,
  332. Stderr: &stderrBuilder,
  333. }); err != nil {
  334. stdout := stdoutBuilder.String()
  335. stderr := stderrBuilder.String()
  336. stderrMessage = util.SanitizeCredentialURLs(stderr)
  337. stdoutMessage = util.SanitizeCredentialURLs(stdout)
  338. }
  339. }
  340. }
  341. // If there is still an error (or there always was an error)
  342. if err != nil {
  343. log.Error("SyncMirrors [repo: %-v Wiki]: failed to update mirror repository wiki:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err)
  344. desc := fmt.Sprintf("Failed to update mirror repository wiki '%s': %s", wikiPath, stderrMessage)
  345. if err = system_model.CreateRepositoryNotice(desc); err != nil {
  346. log.Error("CreateRepositoryNotice: %v", err)
  347. }
  348. return nil, false
  349. }
  350. if err := git.WriteCommitGraph(ctx, wikiPath); err != nil {
  351. log.Error("SyncMirrors [repo: %-v]: %v", m.Repo, err)
  352. }
  353. }
  354. log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo)
  355. }
  356. log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo)
  357. branches, _, err := gitrepo.GetBranchesByPath(ctx, m.Repo, 0, 0)
  358. if err != nil {
  359. log.Error("SyncMirrors [repo: %-v]: failed to GetBranches: %v", m.Repo, err)
  360. return nil, false
  361. }
  362. for _, branch := range branches {
  363. cache.Remove(m.Repo.GetCommitsCountCacheKey(branch.Name, true))
  364. }
  365. m.UpdatedUnix = timeutil.TimeStampNow()
  366. return parseRemoteUpdateOutput(output, m.GetRemoteName()), true
  367. }
  368. // SyncPullMirror starts the sync of the pull mirror and schedules the next run.
  369. func SyncPullMirror(ctx context.Context, repoID int64) bool {
  370. log.Trace("SyncMirrors [repo_id: %v]", repoID)
  371. defer func() {
  372. err := recover()
  373. if err == nil {
  374. return
  375. }
  376. // There was a panic whilst syncMirrors...
  377. log.Error("PANIC whilst SyncMirrors[repo_id: %d] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2))
  378. }()
  379. m, err := repo_model.GetMirrorByRepoID(ctx, repoID)
  380. if err != nil {
  381. log.Error("SyncMirrors [repo_id: %v]: unable to GetMirrorByRepoID: %v", repoID, err)
  382. return false
  383. }
  384. _ = m.GetRepository(ctx) // force load repository of mirror
  385. ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing Mirror %s/%s", m.Repo.OwnerName, m.Repo.Name))
  386. defer finished()
  387. log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo)
  388. results, ok := runSync(ctx, m)
  389. if !ok {
  390. if err = repo_model.TouchMirror(ctx, m); err != nil {
  391. log.Error("SyncMirrors [repo: %-v]: failed to TouchMirror: %v", m.Repo, err)
  392. }
  393. return false
  394. }
  395. log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo)
  396. m.ScheduleNextUpdate()
  397. if err = repo_model.UpdateMirror(ctx, m); err != nil {
  398. log.Error("SyncMirrors [repo: %-v]: failed to UpdateMirror with next update date: %v", m.Repo, err)
  399. return false
  400. }
  401. var gitRepo *git.Repository
  402. if len(results) == 0 {
  403. log.Trace("SyncMirrors [repo: %-v]: no branches updated", m.Repo)
  404. } else {
  405. log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
  406. gitRepo, err = gitrepo.OpenRepository(ctx, m.Repo)
  407. if err != nil {
  408. log.Error("SyncMirrors [repo: %-v]: unable to OpenRepository: %v", m.Repo, err)
  409. return false
  410. }
  411. defer gitRepo.Close()
  412. if ok := checkAndUpdateEmptyRepository(ctx, m, gitRepo, results); !ok {
  413. return false
  414. }
  415. }
  416. for _, result := range results {
  417. // Discard GitHub pull requests, i.e. refs/pull/*
  418. if result.refName.IsPull() {
  419. continue
  420. }
  421. // Create reference
  422. if result.oldCommitID == gitShortEmptySha {
  423. commitID, err := gitRepo.GetRefCommitID(result.refName.String())
  424. if err != nil {
  425. log.Error("SyncMirrors [repo: %-v]: unable to GetRefCommitID [ref_name: %s]: %v", m.Repo, result.refName, err)
  426. continue
  427. }
  428. objectFormat := git.ObjectFormatFromName(m.Repo.ObjectFormatName)
  429. notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{
  430. RefFullName: result.refName,
  431. OldCommitID: objectFormat.EmptyObjectID().String(),
  432. NewCommitID: commitID,
  433. }, repo_module.NewPushCommits())
  434. notify_service.SyncCreateRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.refName, commitID)
  435. continue
  436. }
  437. // Delete reference
  438. if result.newCommitID == gitShortEmptySha {
  439. notify_service.SyncDeleteRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.refName)
  440. continue
  441. }
  442. // Push commits
  443. oldCommitID, err := git.GetFullCommitID(gitRepo.Ctx, gitRepo.Path, result.oldCommitID)
  444. if err != nil {
  445. log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID[%s]: %v", m.Repo, result.oldCommitID, err)
  446. continue
  447. }
  448. newCommitID, err := git.GetFullCommitID(gitRepo.Ctx, gitRepo.Path, result.newCommitID)
  449. if err != nil {
  450. log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID [%s]: %v", m.Repo, result.newCommitID, err)
  451. continue
  452. }
  453. commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
  454. if err != nil {
  455. log.Error("SyncMirrors [repo: %-v]: unable to get CommitsBetweenIDs [new_commit_id: %s, old_commit_id: %s]: %v", m.Repo, newCommitID, oldCommitID, err)
  456. continue
  457. }
  458. theCommits := repo_module.GitToPushCommits(commits)
  459. if len(theCommits.Commits) > setting.UI.FeedMaxCommitNum {
  460. theCommits.Commits = theCommits.Commits[:setting.UI.FeedMaxCommitNum]
  461. }
  462. if newCommit, err := gitRepo.GetCommit(newCommitID); err != nil {
  463. log.Error("SyncMirrors [repo: %-v]: unable to get commit %s: %v", m.Repo, newCommitID, err)
  464. continue
  465. } else {
  466. theCommits.HeadCommit = repo_module.CommitToPushCommit(newCommit)
  467. }
  468. theCommits.CompareURL = m.Repo.ComposeCompareURL(oldCommitID, newCommitID)
  469. notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{
  470. RefFullName: result.refName,
  471. OldCommitID: oldCommitID,
  472. NewCommitID: newCommitID,
  473. }, theCommits)
  474. }
  475. log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
  476. // Get latest commit date and update to current repository updated time
  477. commitDate, err := git.GetLatestCommitTime(ctx, m.Repo.RepoPath())
  478. if err != nil {
  479. log.Error("SyncMirrors [repo: %-v]: unable to GetLatestCommitDate: %v", m.Repo, err)
  480. return false
  481. }
  482. if err = repo_model.UpdateRepositoryUpdatedTime(ctx, m.RepoID, commitDate); err != nil {
  483. log.Error("SyncMirrors [repo: %-v]: unable to update repository 'updated_unix': %v", m.Repo, err)
  484. return false
  485. }
  486. log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo)
  487. return true
  488. }
  489. func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, gitRepo *git.Repository, results []*mirrorSyncResult) bool {
  490. if !m.Repo.IsEmpty {
  491. return true
  492. }
  493. hasDefault := false
  494. hasMaster := false
  495. hasMain := false
  496. defaultBranchName := m.Repo.DefaultBranch
  497. if len(defaultBranchName) == 0 {
  498. defaultBranchName = setting.Repository.DefaultBranch
  499. }
  500. firstName := ""
  501. for _, result := range results {
  502. if !result.refName.IsBranch() {
  503. continue
  504. }
  505. name := result.refName.BranchName()
  506. if len(firstName) == 0 {
  507. firstName = name
  508. }
  509. hasDefault = hasDefault || name == defaultBranchName
  510. hasMaster = hasMaster || name == "master"
  511. hasMain = hasMain || name == "main"
  512. }
  513. if len(firstName) > 0 {
  514. if hasDefault {
  515. m.Repo.DefaultBranch = defaultBranchName
  516. } else if hasMaster {
  517. m.Repo.DefaultBranch = "master"
  518. } else if hasMain {
  519. m.Repo.DefaultBranch = "main"
  520. } else {
  521. m.Repo.DefaultBranch = firstName
  522. }
  523. // Update the git repository default branch
  524. if err := gitrepo.SetDefaultBranch(ctx, m.Repo, m.Repo.DefaultBranch); err != nil {
  525. if !git.IsErrUnsupportedVersion(err) {
  526. log.Error("Failed to update default branch of underlying git repository %-v. Error: %v", m.Repo, err)
  527. desc := fmt.Sprintf("Failed to update default branch of underlying git repository '%s': %v", m.Repo.RepoPath(), err)
  528. if err = system_model.CreateRepositoryNotice(desc); err != nil {
  529. log.Error("CreateRepositoryNotice: %v", err)
  530. }
  531. return false
  532. }
  533. }
  534. m.Repo.IsEmpty = false
  535. // Update the is empty and default_branch columns
  536. if err := repo_model.UpdateRepositoryCols(ctx, m.Repo, "default_branch", "is_empty"); err != nil {
  537. log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err)
  538. desc := fmt.Sprintf("Failed to update default branch of repository '%s': %v", m.Repo.RepoPath(), err)
  539. if err = system_model.CreateRepositoryNotice(desc); err != nil {
  540. log.Error("CreateRepositoryNotice: %v", err)
  541. }
  542. return false
  543. }
  544. }
  545. return true
  546. }