diff options
author | Lunny Xiao <xiaolunwen@gmail.com> | 2019-10-13 21:23:14 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-10-13 21:23:14 +0800 |
commit | f2a3abc683ad4b2177b7c7c6160a2c0b4316120a (patch) | |
tree | 3b92f34b9bb9a015072f511dc5cf6340af18eda5 /models | |
parent | 0a96e59884ca5c4fedc8c3d166d97f35b245ad6e (diff) | |
download | gitea-f2a3abc683ad4b2177b7c7c6160a2c0b4316120a.tar.gz gitea-f2a3abc683ad4b2177b7c7c6160a2c0b4316120a.zip |
Move migrating repository from frontend to backend (#6200)
* move migrating to backend
* add loading image when migrating and fix tests
* fix format
* fix lint
* add redis task queue support and improve docs
* add redis vendor
* fix vet
* add database migrations and fix app.ini sample
* add comments for task section on app.ini.sample
* Update models/migrations/v84.go
Co-Authored-By: lunny <xiaolunwen@gmail.com>
* Update models/repo.go
Co-Authored-By: lunny <xiaolunwen@gmail.com>
* move migrating to backend
* add loading image when migrating and fix tests
* fix fmt
* add redis task queue support and improve docs
* fix fixtures
* fix fixtures
* fix duplicate function on index.js
* fix tests
* rename repository statuses
* check if repository is being create when SSH request
* fix lint
* fix template
* some improvements
* fix template
* unified migrate options
* fix lint
* fix loading page
* refactor
* When gitea restart, don't restart the running tasks because we may have servel gitea instances, that may break the migration
* fix js
* Update models/repo.go
Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>
* Update docs/content/doc/advanced/config-cheat-sheet.en-us.md
Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>
* fix tests
* rename ErrTaskIsNotExist to ErrTaskDoesNotExist
* delete release after add one on tests to make it run happy
* fix tests
* fix tests
* improve codes
* fix lint
* fix lint
* fix migrations
Diffstat (limited to 'models')
-rw-r--r-- | models/fixtures/repository.yml | 43 | ||||
-rw-r--r-- | models/migrations/migrations.go | 2 | ||||
-rw-r--r-- | models/migrations/v99.go | 34 | ||||
-rw-r--r-- | models/models.go | 1 | ||||
-rw-r--r-- | models/repo.go | 83 | ||||
-rw-r--r-- | models/task.go | 240 |
6 files changed, 370 insertions, 33 deletions
diff --git a/models/fixtures/repository.yml b/models/fixtures/repository.yml index 2e38c5e1dd..cf7d24c6cd 100644 --- a/models/fixtures/repository.yml +++ b/models/fixtures/repository.yml @@ -11,6 +11,7 @@ num_milestones: 3 num_closed_milestones: 1 num_watches: 3 + status: 0 - id: 2 @@ -24,6 +25,7 @@ num_closed_pulls: 0 num_stars: 1 close_issues_via_commit_in_any_branch: true + status: 0 - id: 3 @@ -36,6 +38,7 @@ num_pulls: 0 num_closed_pulls: 0 num_watches: 0 + status: 0 - id: 4 @@ -48,6 +51,7 @@ num_pulls: 0 num_closed_pulls: 0 num_stars: 1 + status: 0 - id: 5 @@ -61,6 +65,7 @@ num_closed_pulls: 0 num_watches: 0 is_mirror: true + status: 0 - id: 6 @@ -73,6 +78,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 7 @@ -85,6 +91,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 8 @@ -97,6 +104,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 9 @@ -109,6 +117,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 10 @@ -122,6 +131,7 @@ num_closed_pulls: 0 is_mirror: false num_forks: 1 + status: 0 - id: 11 @@ -135,6 +145,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 12 @@ -147,6 +158,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 13 @@ -159,6 +171,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 14 @@ -172,6 +185,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 15 @@ -179,6 +193,7 @@ lower_name: repo15 name: repo15 is_empty: true + status: 0 - id: 16 @@ -191,6 +206,7 @@ num_pulls: 0 num_closed_pulls: 0 num_watches: 0 + status: 0 - id: 17 @@ -205,6 +221,7 @@ num_watches: 0 is_mirror: false is_fork: false + status: 0 - id: 18 @@ -218,6 +235,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 19 @@ -231,6 +249,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 20 @@ -244,6 +263,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 21 @@ -257,6 +277,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 22 @@ -270,6 +291,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 23 @@ -283,6 +305,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 24 @@ -296,6 +319,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 25 @@ -310,6 +334,7 @@ num_watches: 0 is_mirror: true is_fork: false + status: 0 - id: 26 @@ -324,6 +349,7 @@ num_watches: 0 is_mirror: true is_fork: false + status: 0 - id: 27 @@ -339,6 +365,7 @@ is_mirror: true num_forks: 1 is_fork: false + status: 0 - id: 28 @@ -354,6 +381,7 @@ is_mirror: true num_forks: 1 is_fork: false + status: 0 - id: 29 @@ -368,6 +396,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: true + status: 0 - id: 30 @@ -382,6 +411,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: true + status: 0 - id: 31 @@ -392,6 +422,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 32 # org public repo @@ -403,6 +434,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 33 @@ -410,6 +442,7 @@ lower_name: utf8 name: utf8 is_private: false + status: 0 - id: 34 @@ -421,6 +454,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 35 @@ -432,6 +466,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 36 @@ -443,6 +478,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 37 @@ -454,6 +490,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 38 @@ -465,6 +502,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 39 @@ -476,6 +514,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 40 @@ -487,6 +526,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 41 @@ -519,4 +559,5 @@ num_stars: 0 num_forks: 0 num_issues: 0 - is_mirror: false
\ No newline at end of file + is_mirror: false + status: 0 diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index e14437a04b..ef5cd377a6 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -252,6 +252,8 @@ var migrations = []Migration{ NewMigration("add repo_admin_change_team_access to user", addRepoAdminChangeTeamAccessColumnForUser), // v98 -> v99 NewMigration("add original author name and id on migrated release", addOriginalAuthorOnMigratedReleases), + // v99 -> v100 + NewMigration("add task table and status column for repository table", addTaskTable), } // Migrate database to current version diff --git a/models/migrations/v99.go b/models/migrations/v99.go new file mode 100644 index 0000000000..3eb287af6c --- /dev/null +++ b/models/migrations/v99.go @@ -0,0 +1,34 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/go-xorm/xorm" +) + +func addTaskTable(x *xorm.Engine) error { + type Task struct { + ID int64 + DoerID int64 `xorm:"index"` // operator + OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero + RepoID int64 `xorm:"index"` + Type structs.TaskType + Status structs.TaskStatus `xorm:"index"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + PayloadContent string `xorm:"TEXT"` + Errors string `xorm:"TEXT"` // if task failed, saved the error reason + Created timeutil.TimeStamp `xorm:"created"` + } + + type Repository struct { + Status int `xorm:"NOT NULL DEFAULT 0"` + } + + return x.Sync2(new(Task), new(Repository)) +} diff --git a/models/models.go b/models/models.go index e802a35a77..ea550cb839 100644 --- a/models/models.go +++ b/models/models.go @@ -112,6 +112,7 @@ func init() { new(OAuth2Application), new(OAuth2AuthorizationCode), new(OAuth2Grant), + new(Task), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo.go b/models/repo.go index 8db527477b..23b1c2ef52 100644 --- a/models/repo.go +++ b/models/repo.go @@ -126,6 +126,15 @@ func NewRepoContext() { RemoveAllWithNotice("Clean up repository temporary data", filepath.Join(setting.AppDataPath, "tmp")) } +// RepositoryStatus defines the status of repository +type RepositoryStatus int + +// all kinds of RepositoryStatus +const ( + RepositoryReady RepositoryStatus = iota // a normal repository + RepositoryBeingMigrated // repository is migrating +) + // Repository represents a git repository. type Repository struct { ID int64 `xorm:"pk autoincr"` @@ -156,9 +165,9 @@ type Repository struct { IsPrivate bool `xorm:"INDEX"` IsEmpty bool `xorm:"INDEX"` IsArchived bool `xorm:"INDEX"` - - IsMirror bool `xorm:"INDEX"` - *Mirror `xorm:"-"` + IsMirror bool `xorm:"INDEX"` + *Mirror `xorm:"-"` + Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` ExternalMetas map[string]string `xorm:"-"` Units []*RepoUnit `xorm:"-"` @@ -197,6 +206,16 @@ func (repo *Repository) ColorFormat(s fmt.State) { repo.Name) } +// IsBeingMigrated indicates that repository is being migtated +func (repo *Repository) IsBeingMigrated() bool { + return repo.Status == RepositoryBeingMigrated +} + +// IsBeingCreated indicates that repository is being migrated or forked +func (repo *Repository) IsBeingCreated() bool { + return repo.IsBeingMigrated() +} + // AfterLoad is invoked from XORM after setting the values of all fields of this object. func (repo *Repository) AfterLoad() { // FIXME: use models migration to solve all at once. @@ -884,18 +903,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) { return repo.cloneLink(x, false) } -// MigrateRepoOptions contains the repository migrate options -type MigrateRepoOptions struct { - Name string - Description string - OriginalURL string - IsPrivate bool - IsMirror bool - RemoteAddr string - Wiki bool // include wiki repository - SyncReleasesWithTags bool // sync releases from tags -} - /* GitHub, GitLab, Gogs: *.wiki.git BitBucket: *.git/wiki @@ -915,20 +922,28 @@ func wikiRemoteURL(remote string) string { return "" } -// MigrateRepository migrates an existing repository from other project hosting. -func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) { - repo, err := CreateRepository(doer, u, CreateRepoOptions{ - Name: opts.Name, - Description: opts.Description, - OriginalURL: opts.OriginalURL, - IsPrivate: opts.IsPrivate, - IsMirror: opts.IsMirror, - }) +// CheckCreateRepository check if could created a repository +func CheckCreateRepository(doer, u *User, name string) error { + if !doer.CanCreateRepo() { + return ErrReachLimitOfRepo{u.MaxRepoCreation} + } + + if err := IsUsableRepoName(name); err != nil { + return err + } + + has, err := isRepositoryExist(x, u, name) if err != nil { - return nil, err + return fmt.Errorf("IsRepositoryExist: %v", err) + } else if has { + return ErrRepoAlreadyExist{u.Name, name} } + return nil +} - repoPath := RepoPath(u.Name, opts.Name) +// MigrateRepositoryGitData starts migrating git related data after created migrating repository +func MigrateRepositoryGitData(doer, u *User, repo *Repository, opts api.MigrateRepoOption) (*Repository, error) { + repoPath := RepoPath(u.Name, opts.RepoName) if u.IsOrganization() { t, err := u.GetOwnerTeam() @@ -942,11 +957,12 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second - if err := os.RemoveAll(repoPath); err != nil { + var err error + if err = os.RemoveAll(repoPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) } - if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ + if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -955,8 +971,8 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err } if opts.Wiki { - wikiPath := WikiPath(u.Name, opts.Name) - wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) + wikiPath := WikiPath(u.Name, opts.RepoName) + wikiRemotePath := wikiRemoteURL(opts.CloneAddr) if len(wikiRemotePath) > 0 { if err := os.RemoveAll(wikiPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) @@ -986,7 +1002,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err return repo, fmt.Errorf("git.IsEmpty: %v", err) } - if opts.SyncReleasesWithTags && !repo.IsEmpty { + if !opts.Releases && !repo.IsEmpty { // Try to get HEAD branch and set it as default branch. headBranch, err := gitRepo.GetHEADBranch() if err != nil { @@ -1005,7 +1021,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err log.Error("Failed to update size for repository: %v", err) } - if opts.IsMirror { + if opts.Mirror { if _, err = x.InsertOne(&Mirror{ RepoID: repo.ID, Interval: setting.Mirror.DefaultInterval, @@ -1143,6 +1159,7 @@ type CreateRepoOptions struct { IsPrivate bool IsMirror bool AutoInit bool + Status RepositoryStatus } func getRepoInitFile(tp, name string) ([]byte, error) { @@ -1410,6 +1427,7 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err IsPrivate: opts.IsPrivate, IsFsckEnabled: !opts.IsMirror, CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, + Status: opts.Status, } sess := x.NewSession() @@ -1856,6 +1874,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { &CommitStatus{RepoID: repoID}, &RepoIndexerStatus{RepoID: repoID}, &Comment{RefRepoID: repoID}, + &Task{RepoID: repoID}, ); err != nil { return fmt.Errorf("deleteBeans: %v", err) } diff --git a/models/task.go b/models/task.go new file mode 100644 index 0000000000..cb878d387c --- /dev/null +++ b/models/task.go @@ -0,0 +1,240 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "encoding/json" + "fmt" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" +) + +// Task represents a task +type Task struct { + ID int64 + DoerID int64 `xorm:"index"` // operator + Doer *User `xorm:"-"` + OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero + Owner *User `xorm:"-"` + RepoID int64 `xorm:"index"` + Repo *Repository `xorm:"-"` + Type structs.TaskType + Status structs.TaskStatus `xorm:"index"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + PayloadContent string `xorm:"TEXT"` + Errors string `xorm:"TEXT"` // if task failed, saved the error reason + Created timeutil.TimeStamp `xorm:"created"` +} + +// LoadRepo loads repository of the task +func (task *Task) LoadRepo() error { + return task.loadRepo(x) +} + +func (task *Task) loadRepo(e Engine) error { + if task.Repo != nil { + return nil + } + var repo Repository + has, err := e.ID(task.RepoID).Get(&repo) + if err != nil { + return err + } else if !has { + return ErrRepoNotExist{ + ID: task.RepoID, + } + } + task.Repo = &repo + return nil +} + +// LoadDoer loads do user +func (task *Task) LoadDoer() error { + if task.Doer != nil { + return nil + } + + var doer User + has, err := x.ID(task.DoerID).Get(&doer) + if err != nil { + return err + } else if !has { + return ErrUserNotExist{ + UID: task.DoerID, + } + } + task.Doer = &doer + + return nil +} + +// LoadOwner loads owner user +func (task *Task) LoadOwner() error { + if task.Owner != nil { + return nil + } + + var owner User + has, err := x.ID(task.OwnerID).Get(&owner) + if err != nil { + return err + } else if !has { + return ErrUserNotExist{ + UID: task.OwnerID, + } + } + task.Owner = &owner + + return nil +} + +// UpdateCols updates some columns +func (task *Task) UpdateCols(cols ...string) error { + _, err := x.ID(task.ID).Cols(cols...).Update(task) + return err +} + +// MigrateConfig returns task config when migrate repository +func (task *Task) MigrateConfig() (*structs.MigrateRepoOption, error) { + if task.Type == structs.TaskTypeMigrateRepo { + var opts structs.MigrateRepoOption + err := json.Unmarshal([]byte(task.PayloadContent), &opts) + if err != nil { + return nil, err + } + return &opts, nil + } + return nil, fmt.Errorf("Task type is %s, not Migrate Repo", task.Type.Name()) +} + +// ErrTaskDoesNotExist represents a "TaskDoesNotExist" kind of error. +type ErrTaskDoesNotExist struct { + ID int64 + RepoID int64 + Type structs.TaskType +} + +// IsErrTaskDoesNotExist checks if an error is a ErrTaskIsNotExist. +func IsErrTaskDoesNotExist(err error) bool { + _, ok := err.(ErrTaskDoesNotExist) + return ok +} + +func (err ErrTaskDoesNotExist) Error() string { + return fmt.Sprintf("task is not exist [id: %d, repo_id: %d, type: %d]", + err.ID, err.RepoID, err.Type) +} + +// GetMigratingTask returns the migrating task by repo's id +func GetMigratingTask(repoID int64) (*Task, error) { + var task = Task{ + RepoID: repoID, + Type: structs.TaskTypeMigrateRepo, + } + has, err := x.Get(&task) + if err != nil { + return nil, err + } else if !has { + return nil, ErrTaskDoesNotExist{0, repoID, task.Type} + } + return &task, nil +} + +// FindTaskOptions find all tasks +type FindTaskOptions struct { + Status int +} + +// ToConds generates conditions for database operation. +func (opts FindTaskOptions) ToConds() builder.Cond { + var cond = builder.NewCond() + if opts.Status >= 0 { + cond = cond.And(builder.Eq{"status": opts.Status}) + } + return cond +} + +// FindTasks find all tasks +func FindTasks(opts FindTaskOptions) ([]*Task, error) { + var tasks = make([]*Task, 0, 10) + err := x.Where(opts.ToConds()).Find(&tasks) + return tasks, err +} + +func createTask(e Engine, task *Task) error { + _, err := e.Insert(task) + return err +} + +// CreateMigrateTask creates a migrate task +func CreateMigrateTask(doer, u *User, opts base.MigrateOptions) (*Task, error) { + bs, err := json.Marshal(&opts) + if err != nil { + return nil, err + } + + var task = Task{ + DoerID: doer.ID, + OwnerID: u.ID, + Type: structs.TaskTypeMigrateRepo, + Status: structs.TaskStatusQueue, + PayloadContent: string(bs), + } + + if err := createTask(x, &task); err != nil { + return nil, err + } + + repo, err := CreateRepository(doer, u, CreateRepoOptions{ + Name: opts.RepoName, + Description: opts.Description, + OriginalURL: opts.CloneAddr, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: RepositoryBeingMigrated, + }) + if err != nil { + task.EndTime = timeutil.TimeStampNow() + task.Status = structs.TaskStatusFailed + err2 := task.UpdateCols("end_time", "status") + if err2 != nil { + log.Error("UpdateCols Failed: %v", err2.Error()) + } + return nil, err + } + + task.RepoID = repo.ID + if err = task.UpdateCols("repo_id"); err != nil { + return nil, err + } + + return &task, nil +} + +// FinishMigrateTask updates database when migrate task finished +func FinishMigrateTask(task *Task) error { + task.Status = structs.TaskStatusFinished + task.EndTime = timeutil.TimeStampNow() + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil { + return err + } + task.Repo.Status = RepositoryReady + if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil { + return err + } + + return sess.Commit() +} |