From fa4663e61e39f0cef225ea92235a16e7da977b08 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Fri, 7 Sep 2018 05:06:09 +0300 Subject: Add push webhook support for mirrored repositories (#4127) --- Gopkg.lock | 4 +- models/action.go | 68 +++++++++++++++ models/repo_mirror.go | 147 +++++++++++++++++++++++++++++--- modules/templates/helper.go | 2 + options/locale/locale_en-US.ini | 3 + templates/user/dashboard/feeds.tmpl | 11 ++- vendor/code.gitea.io/git/commit.go | 24 +++++- vendor/code.gitea.io/git/repo_commit.go | 17 +++- 8 files changed, 257 insertions(+), 19 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 04c7d18334..7126f73642 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,11 +3,11 @@ [[projects]] branch = "master" - digest = "1:42f77a668e3bd06812ef254f334d0d0a62346969fbcd3fa3a613e75067343751" + digest = "1:835585f8450b4ec12252d032b0f13e6571ecf846e49076f69067f2503a7c1e07" name = "code.gitea.io/git" packages = ["."] pruneopts = "NUT" - revision = "31f4b8e8c805438ac6d8914b38accb1d8aaf695e" + revision = "6ef79e80b3b06ca13a1f3a7b940903ebc73b44cb" [[projects]] branch = "master" diff --git a/models/action.go b/models/action.go index adf30bb88b..f3f89d143f 100644 --- a/models/action.go +++ b/models/action.go @@ -47,6 +47,9 @@ const ( ActionReopenPullRequest // 15 ActionDeleteTag // 16 ActionDeleteBranch // 17 + ActionMirrorSyncPush // 18 + ActionMirrorSyncCreate // 19 + ActionMirrorSyncDelete // 20 ) var ( @@ -736,6 +739,71 @@ func MergePullRequestAction(actUser *User, repo *Repository, pull *Issue) error return mergePullRequestAction(x, actUser, repo, pull) } +func mirrorSyncAction(e Engine, opType ActionType, repo *Repository, refName string, data []byte) error { + if err := notifyWatchers(e, &Action{ + ActUserID: repo.OwnerID, + ActUser: repo.MustOwner(), + OpType: opType, + RepoID: repo.ID, + Repo: repo, + IsPrivate: repo.IsPrivate, + RefName: refName, + Content: string(data), + }); err != nil { + return fmt.Errorf("notifyWatchers: %v", err) + } + return nil +} + +// MirrorSyncPushActionOptions mirror synchronization action options. +type MirrorSyncPushActionOptions struct { + RefName string + OldCommitID string + NewCommitID string + Commits *PushCommits +} + +// MirrorSyncPushAction adds new action for mirror synchronization of pushed commits. +func MirrorSyncPushAction(repo *Repository, opts MirrorSyncPushActionOptions) error { + if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { + opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] + } + + apiCommits := opts.Commits.ToAPIPayloadCommits(repo.HTMLURL()) + + opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) + apiPusher := repo.MustOwner().APIFormat() + if err := PrepareWebhooks(repo, HookEventPush, &api.PushPayload{ + Ref: opts.RefName, + Before: opts.OldCommitID, + After: opts.NewCommitID, + CompareURL: setting.AppURL + opts.Commits.CompareURL, + Commits: apiCommits, + Repo: repo.APIFormat(AccessModeOwner), + Pusher: apiPusher, + Sender: apiPusher, + }); err != nil { + return fmt.Errorf("PrepareWebhooks: %v", err) + } + + data, err := json.Marshal(opts.Commits) + if err != nil { + return err + } + + return mirrorSyncAction(x, ActionMirrorSyncPush, repo, opts.RefName, data) +} + +// MirrorSyncCreateAction adds new action for mirror synchronization of new reference. +func MirrorSyncCreateAction(repo *Repository, refName string) error { + return mirrorSyncAction(x, ActionMirrorSyncCreate, repo, refName, nil) +} + +// MirrorSyncDeleteAction adds new action for mirror synchronization of delete reference. +func MirrorSyncDeleteAction(repo *Repository, refName string) error { + return mirrorSyncAction(x, ActionMirrorSyncDelete, repo, refName, nil) +} + // GetFeedsOptions options for retrieving feeds type GetFeedsOptions struct { RequestedUser *User diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 1431fcf916..447d055307 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -1,4 +1,5 @@ // Copyright 2016 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -6,6 +7,7 @@ package models import ( "fmt" + "strings" "time" "code.gitea.io/git" @@ -119,8 +121,68 @@ func (m *Mirror) SaveAddress(addr string) error { return cfg.SaveToIndent(configPath, "\t") } +// gitShortEmptySha Git short empty SHA +const gitShortEmptySha = "0000000" + +// mirrorSyncResult contains information of a updated reference. +// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. +// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. +type mirrorSyncResult struct { + refName string + oldCommitID string + newCommitID string +} + +// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream. +func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { + results := make([]*mirrorSyncResult, 0, 3) + lines := strings.Split(output, "\n") + for i := range lines { + // Make sure reference name is presented before continue + idx := strings.Index(lines[i], "-> ") + if idx == -1 { + continue + } + + refName := lines[i][idx+3:] + + switch { + case strings.HasPrefix(lines[i], " * "): // New reference + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " - "): // Delete reference + results = append(results, &mirrorSyncResult{ + refName: refName, + newCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " "): // New commits of a reference + delimIdx := strings.Index(lines[i][3:], " ") + if delimIdx == -1 { + log.Error(2, "SHA delimiter not found: %q", lines[i]) + continue + } + shas := strings.Split(lines[i][3:delimIdx+3], "..") + if len(shas) != 2 { + log.Error(2, "Expect two SHAs but not what found: %q", lines[i]) + continue + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: shas[0], + newCommitID: shas[1], + }) + + default: + log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i]) + } + } + return results +} + // runSync returns true if sync finished without error. -func (m *Mirror) runSync() bool { +func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() wikiPath := m.Repo.WikiPath() timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second @@ -130,28 +192,30 @@ func (m *Mirror) runSync() bool { gitArgs = append(gitArgs, "--prune") } - if _, stderr, err := process.GetManager().ExecDir( + _, stderr, err := process.GetManager().ExecDir( timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath), - "git", gitArgs...); err != nil { + "git", gitArgs...) + if err != nil { // sanitize the output, since it may contain the remote address, which may // contain a password message, err := sanitizeOutput(stderr, repoPath) if err != nil { log.Error(4, "sanitizeOutput: %v", err) - return false + return nil, false } desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message) log.Error(4, desc) if err = CreateRepositoryNotice(desc); err != nil { log.Error(4, "CreateRepositoryNotice: %v", err) } - return false + return nil, false } + output := stderr gitRepo, err := git.OpenRepository(repoPath) if err != nil { log.Error(4, "OpenRepository: %v", err) - return false + return nil, false } if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil { log.Error(4, "Failed to synchronize tags to releases for repository: %v", err) @@ -170,21 +234,21 @@ func (m *Mirror) runSync() bool { message, err := sanitizeOutput(stderr, wikiPath) if err != nil { log.Error(4, "sanitizeOutput: %v", err) - return false + return nil, false } desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message) log.Error(4, desc) if err = CreateRepositoryNotice(desc); err != nil { log.Error(4, "CreateRepositoryNotice: %v", err) } - return false + return nil, false } } branches, err := m.Repo.GetBranches() if err != nil { log.Error(4, "GetBranches: %v", err) - return false + return nil, false } for i := range branches { @@ -192,7 +256,7 @@ func (m *Mirror) runSync() bool { } m.UpdatedUnix = util.TimeStampNow() - return true + return parseRemoteUpdateOutput(output), true } func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) { @@ -268,7 +332,8 @@ func SyncMirrors() { continue } - if !m.runSync() { + results, ok := m.runSync() + if !ok { continue } @@ -278,6 +343,66 @@ func SyncMirrors() { continue } + var gitRepo *git.Repository + if len(results) == 0 { + log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID) + } else { + gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) + if err != nil { + log.Error(2, "OpenRepository [%d]: %v", m.RepoID, err) + continue + } + } + + for _, result := range results { + // Discard GitHub pull requests, i.e. refs/pull/* + if strings.HasPrefix(result.refName, "refs/pull/") { + continue + } + + // Create reference + if result.oldCommitID == gitShortEmptySha { + if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil { + log.Error(2, "MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err) + } + continue + } + + // Delete reference + if result.newCommitID == gitShortEmptySha { + if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil { + log.Error(2, "MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err) + } + continue + } + + // Push commits + oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID) + if err != nil { + log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID) + if err != nil { + log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) + if err != nil { + log.Error(2, "CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err) + continue + } + if err = MirrorSyncPushAction(m.Repo, MirrorSyncPushActionOptions{ + RefName: result.refName, + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + Commits: ListToPushCommits(commits), + }); err != nil { + log.Error(2, "MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err) + continue + } + } + // Get latest commit date and update to current repository updated time commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) if err != nil { diff --git a/modules/templates/helper.go b/modules/templates/helper.go index c46a293be8..d55c122df0 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -391,6 +391,8 @@ func ActionIcon(opType models.ActionType) string { return "issue-closed" case models.ActionReopenIssue, models.ActionReopenPullRequest: return "issue-reopened" + case models.ActionMirrorSyncPush, models.ActionMirrorSyncCreate, models.ActionMirrorSyncDelete: + return "repo-clone" default: return "invalid type" } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index cd9895be19..9bd6f2e597 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1663,6 +1663,9 @@ push_tag = pushed tag %[2]s to %[3]s delete_tag = deleted tag %[2]s from %[3]s delete_branch = deleted branch %[2]s from %[3]s compare_commits = Compare %d commits +mirror_sync_push = synced commits to %[3]s at %[4]s from mirror +mirror_sync_create = synced new reference %[2]s to %[3]s from mirror +mirror_sync_delete = synced and deleted reference %[2]s at %[3]s from mirror [tool] ago = %s ago diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index d9082a0f12..78950183b9 100644 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -5,7 +5,7 @@
-
+

{{.ShortActUserName}} {{if eq .GetOpType 1}} @@ -49,9 +49,16 @@ {{else if eq .GetOpType 17}} {{ $index := index .GetIssueInfos 0}} {{$.i18n.Tr "action.delete_branch" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}} + {{else if eq .GetOpType 18}} + {{ $branchLink := .GetBranch | EscapePound}} + {{$.i18n.Tr "action.mirror_sync_push" .GetRepoLink $branchLink .GetBranch .ShortRepoPath | Str2html}} + {{else if eq .GetOpType 19}} + {{$.i18n.Tr "action.mirror_sync_create" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}} + {{else if eq .GetOpType 20}} + {{$.i18n.Tr "action.mirror_sync_delete" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}} {{end}}

- {{if eq .GetOpType 5}} + {{if or (eq .GetOpType 5) (eq .GetOpType 18)}}
    {{ $push := ActionContent2Commits .}} diff --git a/vendor/code.gitea.io/git/commit.go b/vendor/code.gitea.io/git/commit.go index 299a2381b6..5e8c91d303 100644 --- a/vendor/code.gitea.io/git/commit.go +++ b/vendor/code.gitea.io/git/commit.go @@ -34,14 +34,18 @@ type CommitGPGSignature struct { } // similar to https://github.com/git/git/blob/3bc53220cb2dcf709f7a027a3f526befd021d858/commit.c#L1128 -func newGPGSignatureFromCommitline(data []byte, signatureStart int) (*CommitGPGSignature, error) { +func newGPGSignatureFromCommitline(data []byte, signatureStart int, tag bool) (*CommitGPGSignature, error) { sig := new(CommitGPGSignature) signatureEnd := bytes.LastIndex(data, []byte("-----END PGP SIGNATURE-----")) if signatureEnd == -1 { return nil, fmt.Errorf("end of commit signature not found") } sig.Signature = strings.Replace(string(data[signatureStart:signatureEnd+27]), "\n ", "\n", -1) - sig.Payload = string(data[:signatureStart-8]) + string(data[signatureEnd+27:]) + if tag { + sig.Payload = string(data[:signatureStart-1]) + } else { + sig.Payload = string(data[:signatureStart-8]) + string(data[signatureEnd+27:]) + } return sig, nil } @@ -274,3 +278,19 @@ func (c *Commit) GetSubModule(entryname string) (*SubModule, error) { } return nil, nil } + +// GetFullCommitID returns full length (40) of commit ID by given short SHA in a repository. +func GetFullCommitID(repoPath, shortID string) (string, error) { + if len(shortID) >= 40 { + return shortID, nil + } + + commitID, err := NewCommand("rev-parse", shortID).RunInDir(repoPath) + if err != nil { + if strings.Contains(err.Error(), "exit status 128") { + return "", ErrNotExist{shortID, ""} + } + return "", err + } + return strings.TrimSpace(commitID), nil +} diff --git a/vendor/code.gitea.io/git/repo_commit.go b/vendor/code.gitea.io/git/repo_commit.go index 1acdfffb34..d5cab8f873 100644 --- a/vendor/code.gitea.io/git/repo_commit.go +++ b/vendor/code.gitea.io/git/repo_commit.go @@ -78,7 +78,7 @@ l: } commit.Committer = sig case "gpgsig": - sig, err := newGPGSignatureFromCommitline(data, nextline+spacepos+1) + sig, err := newGPGSignatureFromCommitline(data, nextline+spacepos+1, false) if err != nil { return nil, err } @@ -86,7 +86,20 @@ l: } nextline += eol + 1 case eol == 0: - commit.CommitMessage = string(data[nextline+1:]) + cm := string(data[nextline+1:]) + + // Tag GPG signatures are stored below the commit message + sigindex := strings.Index(cm, "-----BEGIN PGP SIGNATURE-----") + if sigindex != -1 { + sig, err := newGPGSignatureFromCommitline(data, (nextline+1)+sigindex, true) + if err == nil && sig != nil { + // remove signature from commit message + cm = cm[:sigindex-1] + commit.Signature = sig + } + } + + commit.CommitMessage = cm break l default: break l -- cgit v1.2.3