* Add logging to long migrations Also fix v136 to not use models Signed-off-by: Andrew Thornton <art27@cantab.net>tags/v1.13.0-rc1
"crypto/md5" | "crypto/md5" | ||||
"fmt" | "fmt" | ||||
"io/ioutil" | "io/ioutil" | ||||
"math" | |||||
"os" | "os" | ||||
"path/filepath" | "path/filepath" | ||||
"time" | |||||
"code.gitea.io/gitea/modules/log" | "code.gitea.io/gitea/modules/log" | ||||
"code.gitea.io/gitea/modules/setting" | "code.gitea.io/gitea/modules/setting" | ||||
LowerName string `xorm:"UNIQUE NOT NULL"` | LowerName string `xorm:"UNIQUE NOT NULL"` | ||||
Avatar string | Avatar string | ||||
} | } | ||||
ticker := time.NewTicker(5 * time.Second) | |||||
defer ticker.Stop() | |||||
count, err := x.Count(new(User)) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
log.Info("%d User Avatar(s) to migrate ...", count) | |||||
deleteList := make(map[string]struct{}) | deleteList := make(map[string]struct{}) | ||||
start := 0 | start := 0 | ||||
migrated := 0 | |||||
for { | for { | ||||
if err := sess.Begin(); err != nil { | if err := sess.Begin(); err != nil { | ||||
return fmt.Errorf("session.Begin: %v", err) | return fmt.Errorf("session.Begin: %v", err) | ||||
} | } | ||||
deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{} | deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{} | ||||
migrated++ | |||||
select { | |||||
case <-ticker.C: | |||||
log.Info( | |||||
"%d/%d (%2.0f%%) User Avatar(s) migrated (%d old avatars to be deleted) in %d batches. %d Remaining ...", | |||||
migrated, | |||||
count, | |||||
float64(migrated)/float64(count)*100, | |||||
len(deleteList), | |||||
int(math.Ceil(float64(migrated)/float64(50))), | |||||
count-int64(migrated)) | |||||
default: | |||||
} | |||||
} | } | ||||
if err := sess.Commit(); err != nil { | if err := sess.Commit(); err != nil { | ||||
_ = sess.Rollback() | _ = sess.Rollback() | ||||
} | } | ||||
} | } | ||||
deleteCount := len(deleteList) | |||||
log.Info("Deleting %d old avatars ...", deleteCount) | |||||
i := 0 | |||||
for file := range deleteList { | for file := range deleteList { | ||||
if err := os.Remove(file); err != nil { | if err := os.Remove(file); err != nil { | ||||
log.Warn("os.Remove: %v", err) | log.Warn("os.Remove: %v", err) | ||||
} | } | ||||
i++ | |||||
select { | |||||
case <-ticker.C: | |||||
log.Info( | |||||
"%d/%d (%2.0f%%) Old User Avatar(s) deleted. %d Remaining ...", | |||||
i, | |||||
deleteCount, | |||||
float64(i)/float64(deleteCount)*100, | |||||
deleteCount-i) | |||||
default: | |||||
} | |||||
} | } | ||||
log.Info("Completed migrating %d User Avatar(s) and deleting %d Old Avatars", count, deleteCount) | |||||
return nil | return nil | ||||
} | } | ||||
import ( | import ( | ||||
"fmt" | "fmt" | ||||
"math" | |||||
"path/filepath" | "path/filepath" | ||||
"strings" | "strings" | ||||
"time" | |||||
"code.gitea.io/gitea/modules/git" | "code.gitea.io/gitea/modules/git" | ||||
"code.gitea.io/gitea/modules/log" | "code.gitea.io/gitea/modules/log" | ||||
limit = 50 | limit = 50 | ||||
} | } | ||||
ticker := time.NewTicker(5 * time.Second) | |||||
defer ticker.Stop() | |||||
count, err := x.Count(new(PullRequest)) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
log.Info("%d Pull Request(s) to migrate ...", count) | |||||
i := 0 | i := 0 | ||||
start := 0 | |||||
for { | for { | ||||
prs := make([]PullRequest, 0, 50) | prs := make([]PullRequest, 0, 50) | ||||
if err := x.Limit(limit, i).Asc("id").Find(&prs); err != nil { | |||||
if err := x.Limit(limit, start).Asc("id").Find(&prs); err != nil { | |||||
return fmt.Errorf("Find: %v", err) | return fmt.Errorf("Find: %v", err) | ||||
} | } | ||||
if len(prs) == 0 { | if len(prs) == 0 { | ||||
break | break | ||||
} | } | ||||
i += len(prs) | |||||
start += 50 | |||||
for _, pr := range prs { | for _, pr := range prs { | ||||
baseRepo := &Repository{ID: pr.BaseRepoID} | baseRepo := &Repository{ID: pr.BaseRepoID} | ||||
has, err := x.Table("repository").Get(baseRepo) | has, err := x.Table("repository").Get(baseRepo) | ||||
} | } | ||||
pr.MergeBase = strings.TrimSpace(pr.MergeBase) | pr.MergeBase = strings.TrimSpace(pr.MergeBase) | ||||
x.ID(pr.ID).Cols("merge_base").Update(pr) | x.ID(pr.ID).Cols("merge_base").Update(pr) | ||||
i++ | |||||
select { | |||||
case <-ticker.C: | |||||
log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", i, count, float64(i)/float64(count)*100, int(math.Ceil(float64(i)/float64(limit))), count-int64(i)) | |||||
default: | |||||
} | |||||
} | } | ||||
} | } | ||||
log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(i)/float64(limit)))) | |||||
return nil | return nil | ||||
} | } |
import ( | import ( | ||||
"fmt" | "fmt" | ||||
"math" | |||||
"path/filepath" | "path/filepath" | ||||
"strings" | "strings" | ||||
"time" | |||||
"code.gitea.io/gitea/modules/git" | "code.gitea.io/gitea/modules/git" | ||||
"code.gitea.io/gitea/modules/log" | "code.gitea.io/gitea/modules/log" | ||||
limit = 50 | limit = 50 | ||||
} | } | ||||
ticker := time.NewTicker(5 * time.Second) | |||||
defer ticker.Stop() | |||||
count, err := x.Where("has_merged = ?", true).Count(new(PullRequest)) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
log.Info("%d Merged Pull Request(s) to migrate ...", count) | |||||
i := 0 | i := 0 | ||||
start := 0 | |||||
for { | for { | ||||
prs := make([]PullRequest, 0, 50) | prs := make([]PullRequest, 0, 50) | ||||
if err := x.Limit(limit, i).Asc("id").Where("has_merged = ?", true).Find(&prs); err != nil { | |||||
if err := x.Limit(limit, start).Asc("id").Where("has_merged = ?", true).Find(&prs); err != nil { | |||||
return fmt.Errorf("Find: %v", err) | return fmt.Errorf("Find: %v", err) | ||||
} | } | ||||
if len(prs) == 0 { | if len(prs) == 0 { | ||||
break | break | ||||
} | } | ||||
i += len(prs) | |||||
start += 50 | |||||
for _, pr := range prs { | for _, pr := range prs { | ||||
baseRepo := &Repository{ID: pr.BaseRepoID} | baseRepo := &Repository{ID: pr.BaseRepoID} | ||||
has, err := x.Table("repository").Get(baseRepo) | has, err := x.Table("repository").Get(baseRepo) | ||||
} | } | ||||
pr.MergeBase = strings.TrimSpace(pr.MergeBase) | pr.MergeBase = strings.TrimSpace(pr.MergeBase) | ||||
x.ID(pr.ID).Cols("merge_base").Update(pr) | x.ID(pr.ID).Cols("merge_base").Update(pr) | ||||
i++ | |||||
select { | |||||
case <-ticker.C: | |||||
log.Info("%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", i, count, float64(i)/float64(count)*100, int(math.Ceil(float64(i)/float64(limit))), count-int64(i)) | |||||
default: | |||||
} | |||||
} | } | ||||
} | } | ||||
log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(i)/float64(limit)))) | |||||
return nil | return nil | ||||
} | } |
import ( | import ( | ||||
"fmt" | "fmt" | ||||
"math" | |||||
"path/filepath" | |||||
"strings" | |||||
"time" | |||||
"code.gitea.io/gitea/models" | "code.gitea.io/gitea/models" | ||||
"code.gitea.io/gitea/modules/git" | |||||
"code.gitea.io/gitea/modules/log" | "code.gitea.io/gitea/modules/log" | ||||
"code.gitea.io/gitea/modules/setting" | "code.gitea.io/gitea/modules/setting" | ||||
pull_service "code.gitea.io/gitea/services/pull" | |||||
"xorm.io/xorm" | "xorm.io/xorm" | ||||
) | ) | ||||
func addCommitDivergenceToPulls(x *xorm.Engine) error { | func addCommitDivergenceToPulls(x *xorm.Engine) error { | ||||
type Repository struct { | |||||
ID int64 `xorm:"pk autoincr"` | |||||
OwnerID int64 `xorm:"UNIQUE(s) index"` | |||||
OwnerName string | |||||
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` | |||||
Name string `xorm:"INDEX NOT NULL"` | |||||
} | |||||
type PullRequest struct { | |||||
ID int64 `xorm:"pk autoincr"` | |||||
CommitsAhead int | |||||
CommitsBehind int | |||||
BaseRepoID int64 `xorm:"INDEX"` | |||||
BaseBranch string | |||||
HasMerged bool `xorm:"INDEX"` | |||||
MergedCommitID string `xorm:"VARCHAR(40)"` | |||||
} | |||||
if err := x.Sync2(new(models.PullRequest)); err != nil { | if err := x.Sync2(new(models.PullRequest)); err != nil { | ||||
return fmt.Errorf("Sync2: %v", err) | return fmt.Errorf("Sync2: %v", err) | ||||
} | } | ||||
var last int | |||||
last := 0 | |||||
migrated := 0 | |||||
batchSize := setting.Database.IterateBufferSize | batchSize := setting.Database.IterateBufferSize | ||||
sess := x.NewSession() | sess := x.NewSession() | ||||
defer sess.Close() | defer sess.Close() | ||||
ticker := time.NewTicker(5 * time.Second) | |||||
defer ticker.Stop() | |||||
count, err := sess.Where("has_merged = ?", false).Count(new(PullRequest)) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
log.Info("%d Unmerged Pull Request(s) to migrate ...", count) | |||||
for { | for { | ||||
if err := sess.Begin(); err != nil { | if err := sess.Begin(); err != nil { | ||||
return err | return err | ||||
if len(results) == 0 { | if len(results) == 0 { | ||||
break | break | ||||
} | } | ||||
last += len(results) | |||||
last += batchSize | |||||
for _, pr := range results { | for _, pr := range results { | ||||
divergence, err := pull_service.GetDiverging(pr) | |||||
baseRepo := &Repository{ID: pr.BaseRepoID} | |||||
has, err := x.Table("repository").Get(baseRepo) | |||||
if err != nil { | |||||
return fmt.Errorf("Unable to get base repo %d %v", pr.BaseRepoID, err) | |||||
} | |||||
if !has { | |||||
log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID) | |||||
continue | |||||
} | |||||
userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName)) | |||||
repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git") | |||||
gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index) | |||||
divergence, err := git.GetDivergingCommits(repoPath, pr.BaseBranch, gitRefName) | |||||
if err != nil { | if err != nil { | ||||
log.Warn("Could not recalculate Divergence for pull: %d", pr.ID) | log.Warn("Could not recalculate Divergence for pull: %d", pr.ID) | ||||
pr.CommitsAhead = 0 | pr.CommitsAhead = 0 | ||||
pr.CommitsBehind = 0 | pr.CommitsBehind = 0 | ||||
} | } | ||||
if divergence != nil { | |||||
pr.CommitsAhead = divergence.Ahead | |||||
pr.CommitsBehind = divergence.Behind | |||||
} | |||||
pr.CommitsAhead = divergence.Ahead | |||||
pr.CommitsBehind = divergence.Behind | |||||
if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil { | if _, err = sess.ID(pr.ID).Cols("commits_ahead", "commits_behind").Update(pr); err != nil { | ||||
return fmt.Errorf("Update Cols: %v", err) | return fmt.Errorf("Update Cols: %v", err) | ||||
} | } | ||||
migrated++ | |||||
} | } | ||||
if err := sess.Commit(); err != nil { | if err := sess.Commit(); err != nil { | ||||
return err | return err | ||||
} | } | ||||
select { | |||||
case <-ticker.C: | |||||
log.Info( | |||||
"%d/%d (%2.0f%%) Pull Request(s) migrated in %d batches. %d PRs Remaining ...", | |||||
migrated, | |||||
count, | |||||
float64(migrated)/float64(count)*100, | |||||
int(math.Ceil(float64(migrated)/float64(batchSize))), | |||||
count-int64(migrated)) | |||||
default: | |||||
} | |||||
} | } | ||||
log.Info("Completed migrating %d Pull Request(s) in: %d batches", count, int(math.Ceil(float64(migrated)/float64(batchSize)))) | |||||
return nil | return nil | ||||
} | } |