diff options
author | Lauris BH <lauris@nix.lv> | 2017-09-20 08:26:49 +0300 |
---|---|---|
committer | Lunny Xiao <xiaolunwen@gmail.com> | 2017-09-20 13:26:49 +0800 |
commit | 7a0297819d3aececb8b633de5a2995c40b7bfcae (patch) | |
tree | 7aed967ad21f95df2b16359ec1551662d653a1fd /models | |
parent | 8b6236d67b21f2a3291207ef77d103a3ce7476ba (diff) | |
download | gitea-7a0297819d3aececb8b633de5a2995c40b7bfcae.tar.gz gitea-7a0297819d3aececb8b633de5a2995c40b7bfcae.zip |
Sync releases table with tags on push and for mirrors (#2459)
* Sync releases table with tags on push and for mirrors
* Code style fixes
* Fix api to return only releases
* Optimize release creation and update
Minimize posibility of race conditions
* Fix release lower tag name updating
* handle tag reference update by addionally comparing commit id
Diffstat (limited to 'models')
-rw-r--r-- | models/migrations/migrations.go | 2 | ||||
-rw-r--r-- | models/migrations/v42.go | 57 | ||||
-rw-r--r-- | models/release.go | 89 | ||||
-rw-r--r-- | models/repo.go | 4 | ||||
-rw-r--r-- | models/repo_mirror.go | 10 | ||||
-rw-r--r-- | models/update.go | 105 |
6 files changed, 248 insertions, 19 deletions
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index c36f6fabb7..d302b2c075 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -132,6 +132,8 @@ var migrations = []Migration{ NewMigration("migrate protected branch struct", migrateProtectedBranchStruct), // v41 -> v42 NewMigration("add default value to user prohibit_login", addDefaultValueToUserProhibitLogin), + // v42 -> v43 + NewMigration("add tags to releases and sync existing repositories", releaseAddColumnIsTagAndSyncTags), } // Migrate database to current version diff --git a/models/migrations/v42.go b/models/migrations/v42.go new file mode 100644 index 0000000000..3547ef1f9e --- /dev/null +++ b/models/migrations/v42.go @@ -0,0 +1,57 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + + "code.gitea.io/git" + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + + "github.com/go-xorm/xorm" +) + +// ReleaseV39 describes the added field for Release +type ReleaseV39 struct { + IsTag bool `xorm:"NOT NULL DEFAULT false"` +} + +// TableName will be invoked by XORM to customrize the table name +func (*ReleaseV39) TableName() string { + return "release" +} + +func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error { + if err := x.Sync2(new(ReleaseV39)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + + // For the sake of SQLite3, we can't use x.Iterate here. + offset := 0 + pageSize := 20 + for { + repos := make([]*models.Repository, 0, pageSize) + if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil { + return fmt.Errorf("select repos [offset: %d]: %v", offset, err) + } + for _, repo := range repos { + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { + log.Warn("OpenRepository: %v", err) + continue + } + + if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil { + log.Warn("SyncReleasesWithTags: %v", err) + } + } + if len(repos) < pageSize { + break + } + offset += pageSize + } + return nil +} diff --git a/models/release.go b/models/release.go index f12fc06840..42054131a3 100644 --- a/models/release.go +++ b/models/release.go @@ -34,7 +34,8 @@ type Release struct { NumCommitsBehind int64 `xorm:"-"` Note string `xorm:"TEXT"` IsDraft bool `xorm:"NOT NULL DEFAULT false"` - IsPrerelease bool + IsPrerelease bool `xorm:"NOT NULL DEFAULT false"` + IsTag bool `xorm:"NOT NULL DEFAULT false"` Attachments []*Attachment `xorm:"-"` @@ -139,17 +140,18 @@ func createTag(gitRepo *git.Repository, rel *Release) error { } return err } - } else { - commit, err := gitRepo.GetTagCommit(rel.TagName) - if err != nil { - return fmt.Errorf("GetTagCommit: %v", err) - } + rel.LowerTagName = strings.ToLower(rel.TagName) + } + commit, err := gitRepo.GetTagCommit(rel.TagName) + if err != nil { + return fmt.Errorf("GetTagCommit: %v", err) + } - rel.Sha1 = commit.ID.String() - rel.NumCommits, err = commit.CommitsCount() - if err != nil { - return fmt.Errorf("CommitsCount: %v", err) - } + rel.Sha1 = commit.ID.String() + rel.CreatedUnix = commit.Author.When.Unix() + rel.NumCommits, err = commit.CommitsCount() + if err != nil { + return fmt.Errorf("CommitsCount: %v", err) } } return nil @@ -236,6 +238,7 @@ func GetReleaseByID(id int64) (*Release, error) { // FindReleasesOptions describes the conditions to Find releases type FindReleasesOptions struct { IncludeDrafts bool + IncludeTags bool TagNames []string } @@ -246,6 +249,9 @@ func (opts *FindReleasesOptions) toConds(repoID int64) builder.Cond { if !opts.IncludeDrafts { cond = cond.And(builder.Eq{"is_draft": false}) } + if !opts.IncludeTags { + cond = cond.And(builder.Eq{"is_tag": false}) + } if len(opts.TagNames) > 0 { cond = cond.And(builder.In("tag_name", opts.TagNames)) } @@ -361,6 +367,8 @@ func UpdateRelease(gitRepo *git.Repository, rel *Release, attachmentUUIDs []stri if err = createTag(gitRepo, rel); err != nil { return err } + rel.LowerTagName = strings.ToLower(rel.TagName) + _, err = x.Id(rel.ID).AllCols().Update(rel) if err != nil { return err @@ -397,11 +405,64 @@ func DeleteReleaseByID(id int64, u *User, delTag bool) error { if err != nil && !strings.Contains(stderr, "not found") { return fmt.Errorf("git tag -d: %v - %s", err, stderr) } - } - if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil { - return fmt.Errorf("Delete: %v", err) + if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil { + return fmt.Errorf("Delete: %v", err) + } + } else { + rel.IsTag = true + rel.IsDraft = false + rel.IsPrerelease = false + rel.Title = "" + rel.Note = "" + + if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil { + return fmt.Errorf("Update: %v", err) + } } return nil } + +// SyncReleasesWithTags synchronizes release table with repository tags +func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error { + existingRelTags := make(map[string]struct{}) + opts := FindReleasesOptions{IncludeDrafts: true, IncludeTags: true} + for page := 1; ; page++ { + rels, err := GetReleasesByRepoID(repo.ID, opts, page, 100) + if err != nil { + return fmt.Errorf("GetReleasesByRepoID: %v", err) + } + if len(rels) == 0 { + break + } + for _, rel := range rels { + if rel.IsDraft { + continue + } + commitID, err := gitRepo.GetTagCommitID(rel.TagName) + if err != nil { + return fmt.Errorf("GetTagCommitID: %v", err) + } + if !gitRepo.IsTagExist(rel.TagName) || commitID != rel.Sha1 { + if err := pushUpdateDeleteTag(repo, gitRepo, rel.TagName); err != nil { + return fmt.Errorf("pushUpdateDeleteTag: %v", err) + } + } else { + existingRelTags[strings.ToLower(rel.TagName)] = struct{}{} + } + } + } + tags, err := gitRepo.GetTags() + if err != nil { + return fmt.Errorf("GetTags: %v", err) + } + for _, tagName := range tags { + if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok { + if err := pushUpdateAddTag(repo, gitRepo, tagName); err != nil { + return fmt.Errorf("pushUpdateAddTag: %v", err) + } + } + } + return nil +} diff --git a/models/repo.go b/models/repo.go index 4c844d9b56..cdaf943944 100644 --- a/models/repo.go +++ b/models/repo.go @@ -940,6 +940,10 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err if headBranch != nil { repo.DefaultBranch = headBranch.Name } + + if err = SyncReleasesWithTags(repo, gitRepo); err != nil { + log.Error(4, "Failed to synchronize tags to releases for repository: %v", err) + } } if err = repo.UpdateSize(); err != nil { diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 526b7cd02e..60d4f9f05a 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -13,6 +13,7 @@ import ( "github.com/go-xorm/xorm" "gopkg.in/ini.v1" + "code.gitea.io/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" @@ -156,6 +157,15 @@ func (m *Mirror) runSync() bool { return false } + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + log.Error(4, "OpenRepository: %v", err) + return false + } + if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil { + log.Error(4, "Failed to synchronize tags to releases for repository: %v", err) + } + if err := m.Repo.UpdateSize(); err != nil { log.Error(4, "Failed to update size for mirror repository: %v", err) } diff --git a/models/update.go b/models/update.go index e6cbba64a5..56a227b24a 100644 --- a/models/update.go +++ b/models/update.go @@ -81,6 +81,93 @@ func PushUpdate(branch string, opt PushUpdateOptions) error { return nil } +func pushUpdateDeleteTag(repo *Repository, gitRepo *git.Repository, tagName string) error { + rel, err := GetRelease(repo.ID, tagName) + if err != nil { + if IsErrReleaseNotExist(err) { + return nil + } + return fmt.Errorf("GetRelease: %v", err) + } + if rel.IsTag { + if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil { + return fmt.Errorf("Delete: %v", err) + } + } else { + rel.IsDraft = true + rel.NumCommits = 0 + rel.Sha1 = "" + if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil { + return fmt.Errorf("Update: %v", err) + } + } + + return nil +} + +func pushUpdateAddTag(repo *Repository, gitRepo *git.Repository, tagName string) error { + rel, err := GetRelease(repo.ID, tagName) + if err != nil && !IsErrReleaseNotExist(err) { + return fmt.Errorf("GetRelease: %v", err) + } + + tag, err := gitRepo.GetTag(tagName) + if err != nil { + return fmt.Errorf("GetTag: %v", err) + } + commit, err := tag.Commit() + if err != nil { + return fmt.Errorf("Commit: %v", err) + } + tagCreatedUnix := commit.Author.When.Unix() + + author, err := GetUserByEmail(commit.Author.Email) + if err != nil && !IsErrUserNotExist(err) { + return fmt.Errorf("GetUserByEmail: %v", err) + } + + commitsCount, err := commit.CommitsCount() + if err != nil { + return fmt.Errorf("CommitsCount: %v", err) + } + + if rel == nil { + rel = &Release{ + RepoID: repo.ID, + Title: "", + TagName: tagName, + LowerTagName: strings.ToLower(tagName), + Target: "", + Sha1: commit.ID.String(), + NumCommits: commitsCount, + Note: "", + IsDraft: false, + IsPrerelease: false, + IsTag: true, + CreatedUnix: tagCreatedUnix, + } + if author != nil { + rel.PublisherID = author.ID + } + + if _, err = x.InsertOne(rel); err != nil { + return fmt.Errorf("InsertOne: %v", err) + } + } else { + rel.Sha1 = commit.ID.String() + rel.CreatedUnix = tagCreatedUnix + rel.NumCommits = commitsCount + rel.IsDraft = false + if rel.IsTag && author != nil { + rel.PublisherID = author.ID + } + if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil { + return fmt.Errorf("Update: %v", err) + } + } + return nil +} + func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) { isNewRef := opts.OldCommitID == git.EmptySHA isDelRef := opts.NewCommitID == git.EmptySHA @@ -106,23 +193,31 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) { return nil, fmt.Errorf("GetRepositoryByName: %v", err) } + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + return nil, fmt.Errorf("OpenRepository: %v", err) + } + if isDelRef { + // Tag has been deleted + if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { + err = pushUpdateDeleteTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):]) + if err != nil { + return nil, fmt.Errorf("pushUpdateDeleteTag: %v", err) + } + } log.GitLogger.Info("Reference '%s' has been deleted from '%s/%s' by %s", opts.RefFullName, opts.RepoUserName, opts.RepoName, opts.PusherName) return repo, nil } - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - return nil, fmt.Errorf("OpenRepository: %v", err) - } - if err = repo.UpdateSize(); err != nil { log.Error(4, "Failed to update size for repository: %v", err) } // Push tags. if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { + pushUpdateAddTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):]) if err := CommitRepoAction(CommitRepoActionOptions{ PusherName: opts.PusherName, RepoOwnerID: owner.ID, |