diff options
author | John Olheiser <42128690+jolheiser@users.noreply.github.com> | 2019-11-11 09:15:29 -0600 |
---|---|---|
committer | Lunny Xiao <xiaolunwen@gmail.com> | 2019-11-11 23:15:29 +0800 |
commit | 74a6add4d90beb8133bcbf8ca6b43de35e0aa983 (patch) | |
tree | 868e452d41d71094c5b2cccce67f4211fd14e77b /models | |
parent | 74bb292fe3f4c02fc1dc5f32622c74d820cadd78 (diff) | |
download | gitea-74a6add4d90beb8133bcbf8ca6b43de35e0aa983.tar.gz gitea-74a6add4d90beb8133bcbf8ca6b43de35e0aa983.zip |
Template Repositories (#8768)
* Start work on templates
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Continue work
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Fix IsTemplate vs IsGenerated
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Fix tabs vs spaces
* Tabs vs Spaces
* Add templates to API & start adding tests
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Fix integration tests
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Remove unused User
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Move template tests to existing repos
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Minor re-check updates and cleanup
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* make fmt
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Test cleanup
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Fix optionalbool
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* make fmt
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Test fixes and icon change
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Add new user and repo for tests
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Fix tests (finally)
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Update meta repo with env variables
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Move generation to create page
Combine with repo create template
Modify API search to prioritize owner for repo
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Fix tests and coverage
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Fix swagger and JS lint
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Fix API searching for own private repos
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Change wording
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Fix repo search test. User had a private repo that didn't show up
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Another search test fix
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Clarify git content
Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>
* Feedback updates
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Add topics WIP
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Finish adding topics
Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Update locale
Signed-off-by: jolheiser <john.olheiser@gmail.com>
Diffstat (limited to 'models')
-rw-r--r-- | models/fixtures/repo_unit.yml | 14 | ||||
-rw-r--r-- | models/fixtures/repository.yml | 26 | ||||
-rw-r--r-- | models/fixtures/user.yml | 17 | ||||
-rw-r--r-- | models/migrations/migrations.go | 2 | ||||
-rw-r--r-- | models/migrations/v107.go | 19 | ||||
-rw-r--r-- | models/repo.go | 219 | ||||
-rw-r--r-- | models/repo_list.go | 49 | ||||
-rw-r--r-- | models/repo_list_test.go | 9 | ||||
-rw-r--r-- | models/user_test.go | 2 |
9 files changed, 323 insertions, 34 deletions
diff --git a/models/fixtures/repo_unit.yml b/models/fixtures/repo_unit.yml index 014e5155ba..28c606da43 100644 --- a/models/fixtures/repo_unit.yml +++ b/models/fixtures/repo_unit.yml @@ -438,3 +438,17 @@ type: 3 config: "{\"IgnoreWhitespaceConflicts\":false,\"AllowMerge\":true,\"AllowRebase\":true,\"AllowRebaseMerge\":true,\"AllowSquash\":true}" created_unix: 946684810 + +- + id: 64 + repo_id: 44 + type: 1 + config: "{}" + created_unix: 946684810 + +- + id: 65 + repo_id: 45 + type: 1 + config: "{}" + created_unix: 946684810 diff --git a/models/fixtures/repository.yml b/models/fixtures/repository.yml index 32903723ec..4cec6471e9 100644 --- a/models/fixtures/repository.yml +++ b/models/fixtures/repository.yml @@ -561,3 +561,29 @@ num_issues: 0 is_mirror: false status: 0 + +- + id: 44 + owner_id: 27 + lower_name: template1 + name: template1 + is_private: false + is_template: true + num_stars: 0 + num_forks: 0 + num_issues: 0 + is_mirror: false + status: 0 + +- + id: 45 + owner_id: 27 + lower_name: template2 + name: template2 + is_private: false + is_template: true + num_stars: 0 + num_forks: 0 + num_issues: 0 + is_mirror: false + status: 0 diff --git a/models/fixtures/user.yml b/models/fixtures/user.yml index a204241f9c..5a3b04994c 100644 --- a/models/fixtures/user.yml +++ b/models/fixtures/user.yml @@ -427,4 +427,19 @@ num_repos: 1 num_members: 0 num_teams: 1 - repo_admin_change_team_access: true
\ No newline at end of file + repo_admin_change_team_access: true + +- + id: 27 + lower_name: user27 + name: user27 + full_name: User Twenty-Seven + email: user27@example.com + email_notifications_preference: enabled + passwd: 7d93daa0d1e6f2305cc8fa496847d61dc7320bb16262f9c55dd753480207234cdd96a93194e408341971742f4701772a025a # password + type: 0 # individual + salt: ZogKvWdyEx + is_admin: false + avatar: avatar27 + avatar_email: user27@example.com + num_repos: 2 diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 71ffe2edb3..2e289b6f73 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -268,6 +268,8 @@ var migrations = []Migration{ NewMigration("add includes_all_repositories to teams", addTeamIncludesAllRepositories), // v106 -> v107 NewMigration("add column `mode` to table watch", addModeColumnToWatch), + // v107 -> v108 + NewMigration("Add template options to repository", addTemplateToRepo), } // Migrate database to current version diff --git a/models/migrations/v107.go b/models/migrations/v107.go new file mode 100644 index 0000000000..3d6aeebaf0 --- /dev/null +++ b/models/migrations/v107.go @@ -0,0 +1,19 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "xorm.io/xorm" +) + +func addTemplateToRepo(x *xorm.Engine) error { + + type Repository struct { + IsTemplate bool `xorm:"INDEX NOT NULL DEFAULT false"` + TemplateID int64 `xorm:"INDEX"` + } + + return x.Sync2(new(Repository)) +} diff --git a/models/repo.go b/models/repo.go index 176182e67d..ccecfe2fdf 100644 --- a/models/repo.go +++ b/models/repo.go @@ -179,6 +179,9 @@ type Repository struct { IsFork bool `xorm:"INDEX NOT NULL DEFAULT false"` ForkID int64 `xorm:"INDEX"` BaseRepo *Repository `xorm:"-"` + IsTemplate bool `xorm:"INDEX NOT NULL DEFAULT false"` + TemplateID int64 `xorm:"INDEX"` + TemplateRepo *Repository `xorm:"-"` Size int64 `xorm:"NOT NULL DEFAULT 0"` IndexerStatus *RepoIndexerStatus `xorm:"-"` IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` @@ -351,6 +354,7 @@ func (repo *Repository) innerAPIFormat(e Engine, mode AccessMode, isParent bool) FullName: repo.FullName(), Description: repo.Description, Private: repo.IsPrivate, + Template: repo.IsTemplate, Empty: repo.IsEmpty, Archived: repo.IsArchived, Size: int(repo.Size / 1024), @@ -663,6 +667,27 @@ func (repo *Repository) getBaseRepo(e Engine) (err error) { return err } +// IsGenerated returns whether _this_ repository was generated from a template +func (repo *Repository) IsGenerated() bool { + return repo.TemplateID != 0 +} + +// GetTemplateRepo populates repo.TemplateRepo for a generated repository and +// returns an error on failure (NOTE: no error is returned for +// non-generated repositories, and TemplateRepo will be left untouched) +func (repo *Repository) GetTemplateRepo() (err error) { + return repo.getTemplateRepo(x) +} + +func (repo *Repository) getTemplateRepo(e Engine) (err error) { + if !repo.IsGenerated() { + return nil + } + + repo.TemplateRepo, err = getRepositoryByID(e, repo.TemplateID) + return err +} + func (repo *Repository) repoPath(e Engine) string { return RepoPath(repo.mustOwnerName(e), repo.Name) } @@ -1220,6 +1245,20 @@ type CreateRepoOptions struct { Status RepositoryStatus } +// GenerateRepoOptions contains the template units to generate +type GenerateRepoOptions struct { + Name string + Description string + Private bool + GitContent bool + Topics bool +} + +// IsValid checks whether at least one option is chosen for generation +func (gro GenerateRepoOptions) IsValid() bool { + return gro.GitContent || gro.Topics // or other items as they are added +} + func getRepoInitFile(tp, name string) ([]byte, error) { cleanedName := strings.TrimLeft(path.Clean("/"+name), "/") relPath := path.Join("options", tp, cleanedName) @@ -1323,8 +1362,55 @@ func prepareRepoCommit(e Engine, repo *Repository, tmpDir, repoPath string, opts return nil } -// InitRepository initializes README and .gitignore if needed. -func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts CreateRepoOptions) (err error) { +func generateRepoCommit(e Engine, repo, templateRepo *Repository, tmpDir string) error { + commitTimeStr := time.Now().Format(time.RFC3339) + authorSig := repo.Owner.NewGitSig() + + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+authorSig.Name, + "GIT_COMMITTER_EMAIL="+authorSig.Email, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + + // Clone to temporary path and do the init commit. + templateRepoPath := templateRepo.repoPath(e) + _, stderr, err := process.GetManager().ExecDirEnv( + -1, "", + fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath), + env, + git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir, + ) + if err != nil { + return fmt.Errorf("git clone: %v - %s", err, stderr) + } + + if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil { + return fmt.Errorf("remove git dir: %v", err) + } + + if err := git.InitRepository(tmpDir, false); err != nil { + return err + } + + repoPath := repo.repoPath(e) + _, stderr, err = process.GetManager().ExecDirEnv( + -1, tmpDir, + fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath), + env, + git.GitExecutable, "remote", "add", "origin", repoPath, + ) + if err != nil { + return fmt.Errorf("git remote add: %v - %s", err, stderr) + } + + return initRepoCommit(tmpDir, repo.Owner) +} + +func checkInitRepository(repoPath string) (err error) { // Somehow the directory could exist. if com.IsExist(repoPath) { return fmt.Errorf("initRepository: path already exists: %s", repoPath) @@ -1336,6 +1422,14 @@ func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts C } else if err = createDelegateHooks(repoPath); err != nil { return fmt.Errorf("createDelegateHooks: %v", err) } + return nil +} + +// InitRepository initializes README and .gitignore if needed. +func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts CreateRepoOptions) (err error) { + if err = checkInitRepository(repoPath); err != nil { + return err + } tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond())) @@ -1376,6 +1470,37 @@ func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts C return nil } +// generateRepository initializes repository from template +func generateRepository(e Engine, repo, templateRepo *Repository) (err error) { + tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond())) + + if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil { + return fmt.Errorf("Failed to create dir %s: %v", tmpDir, err) + } + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + log.Error("RemoveAll: %v", err) + } + }() + + if err = generateRepoCommit(e, repo, templateRepo, tmpDir); err != nil { + return fmt.Errorf("generateRepoCommit: %v", err) + } + + // re-fetch repo + if repo, err = getRepositoryByID(e, repo.ID); err != nil { + return fmt.Errorf("getRepositoryByID: %v", err) + } + + repo.DefaultBranch = "master" + if err = updateRepository(e, repo, false); err != nil { + return fmt.Errorf("updateRepository: %v", err) + } + + return nil +} + var ( reservedRepoNames = []string{".", ".."} reservedRepoPatterns = []string{"*.git", "*.wiki"} @@ -2523,6 +2648,28 @@ func HasForkedRepo(ownerID, repoID int64) (*Repository, bool) { return repo, has } +// CopyLFS copies LFS data from one repo to another +func CopyLFS(newRepo, oldRepo *Repository) error { + return copyLFS(x, newRepo, oldRepo) +} + +func copyLFS(e Engine, newRepo, oldRepo *Repository) error { + var lfsObjects []*LFSMetaObject + if err := e.Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil { + return err + } + + for _, v := range lfsObjects { + v.ID = 0 + v.RepositoryID = newRepo.ID + if _, err := e.Insert(v); err != nil { + return err + } + } + + return nil +} + // ForkRepository forks a repository func ForkRepository(doer, owner *User, oldRepo *Repository, name, desc string) (_ *Repository, err error) { forkedRepo, err := oldRepo.GetUserFork(owner.ID) @@ -2593,27 +2740,73 @@ func ForkRepository(doer, owner *User, oldRepo *Repository, name, desc string) ( log.Error("Failed to update size for repository: %v", err) } - // Copy LFS meta objects in new session - sess2 := x.NewSession() - defer sess2.Close() - if err = sess2.Begin(); err != nil { + return repo, CopyLFS(repo, oldRepo) +} + +// GenerateRepository generates a repository from a template +func GenerateRepository(doer, owner *User, templateRepo *Repository, opts GenerateRepoOptions) (_ *Repository, err error) { + repo := &Repository{ + OwnerID: owner.ID, + Owner: owner, + Name: opts.Name, + LowerName: strings.ToLower(opts.Name), + Description: opts.Description, + IsPrivate: opts.Private, + IsEmpty: !opts.GitContent || templateRepo.IsEmpty, + IsFsckEnabled: templateRepo.IsFsckEnabled, + TemplateID: templateRepo.ID, + } + + createSess := x.NewSession() + defer createSess.Close() + if err = createSess.Begin(); err != nil { + return nil, err + } + + if err = createRepository(createSess, doer, owner, repo); err != nil { + return nil, err + } + + //Commit repo to get created repo ID + err = createSess.Commit() + if err != nil { + return nil, err + } + + sess := x.NewSession() + defer sess.Close() + if err = sess.Begin(); err != nil { return repo, err } - var lfsObjects []*LFSMetaObject - if err = sess2.Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil { + repoPath := RepoPath(owner.Name, repo.Name) + if err = checkInitRepository(repoPath); err != nil { return repo, err } - for _, v := range lfsObjects { - v.ID = 0 - v.RepositoryID = repo.ID - if _, err = sess2.Insert(v); err != nil { + if opts.GitContent && !templateRepo.IsEmpty { + if err = generateRepository(sess, repo, templateRepo); err != nil { return repo, err } + + if err = repo.updateSize(sess); err != nil { + return repo, fmt.Errorf("failed to update size for repository: %v", err) + } + + if err = copyLFS(sess, repo, templateRepo); err != nil { + return repo, fmt.Errorf("failed to copy LFS: %v", err) + } + } + + if opts.Topics { + for _, topic := range templateRepo.Topics { + if _, err = addTopicByNameToRepo(sess, repo.ID, topic); err != nil { + return repo, err + } + } } - return repo, sess2.Commit() + return repo, sess.Commit() } // GetForks returns all the forks of the repository diff --git a/models/repo_list.go b/models/repo_list.go index c823647eba..34fac8b055 100644 --- a/models/repo_list.go +++ b/models/repo_list.go @@ -111,17 +111,18 @@ func (repos MirrorRepositoryList) LoadAttributes() error { // SearchRepoOptions holds the search options type SearchRepoOptions struct { - UserID int64 - UserIsAdmin bool - Keyword string - OwnerID int64 - OrderBy SearchOrderBy - Private bool // Include private repositories in results - StarredByID int64 - Page int - IsProfile bool - AllPublic bool // Include also all public repositories - PageSize int // Can be smaller than or equal to setting.ExplorePagingNum + UserID int64 + UserIsAdmin bool + Keyword string + OwnerID int64 + PriorityOwnerID int64 + OrderBy SearchOrderBy + Private bool // Include private repositories in results + StarredByID int64 + Page int + IsProfile bool + AllPublic bool // Include also all public repositories + PageSize int // Can be smaller than or equal to setting.ExplorePagingNum // None -> include collaborative AND non-collaborative // True -> include just collaborative // False -> incude just non-collaborative @@ -130,6 +131,10 @@ type SearchRepoOptions struct { // True -> include just forks // False -> include just non-forks Fork util.OptionalBool + // None -> include templates AND non-templates + // True -> include just templates + // False -> include just non-templates + Template util.OptionalBool // None -> include mirrors AND non-mirrors // True -> include just mirrors // False -> include just non-mirrors @@ -190,6 +195,10 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) { cond = cond.And(accessCond) } + if opts.Template != util.OptionalBoolNone { + cond = cond.And(builder.Eq{"is_template": opts.Template == util.OptionalBoolTrue}) + } + // Restrict to starred repositories if opts.StarredByID > 0 { cond = cond.And(builder.In("id", builder.Select("repo_id").From("star").Where(builder.Eq{"uid": opts.StarredByID}))) @@ -266,6 +275,10 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) { opts.OrderBy = SearchOrderByAlphabetically } + if opts.PriorityOwnerID > 0 { + opts.OrderBy = SearchOrderBy(fmt.Sprintf("CASE WHEN owner_id = %d THEN 0 ELSE owner_id END, %s", opts.PriorityOwnerID, opts.OrderBy)) + } + sess := x.NewSession() defer sess.Close() @@ -308,11 +321,15 @@ func accessibleRepositoryCondition(userID int64) builder.Cond { builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))), ), // 2. Be able to see all repositories that we have access to - builder.In("`repository`.id", builder.Select("repo_id"). - From("`access`"). - Where(builder.And( - builder.Eq{"user_id": userID}, - builder.Gt{"mode": int(AccessModeNone)}))), + builder.Or( + builder.In("`repository`.id", builder.Select("repo_id"). + From("`access`"). + Where(builder.And( + builder.Eq{"user_id": userID}, + builder.Gt{"mode": int(AccessModeNone)}))), + builder.In("`repository`.id", builder.Select("id"). + From("`repository`"). + Where(builder.Eq{"owner_id": userID}))), // 3. Be able to see all repositories that we are in a team builder.In("`repository`.id", builder.Select("`team_repo`.repo_id"). From("team_repo"). diff --git a/models/repo_list_test.go b/models/repo_list_test.go index e3a7acd4a4..b1dbf46af0 100644 --- a/models/repo_list_test.go +++ b/models/repo_list_test.go @@ -174,10 +174,10 @@ func TestSearchRepository(t *testing.T) { opts: &SearchRepoOptions{Keyword: "big_test_", Page: 1, PageSize: 10, Private: true, AllPublic: true, Collaborate: util.OptionalBoolFalse}, count: 14}, {name: "AllPublic/PublicRepositoriesOfUserIncludingCollaborative", - opts: &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, AllPublic: true}, + opts: &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, AllPublic: true, Template: util.OptionalBoolFalse}, count: 22}, {name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborative", - opts: &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, Private: true, AllPublic: true}, + opts: &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, Private: true, AllPublic: true, Template: util.OptionalBoolFalse}, count: 28}, {name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborativeByName", opts: &SearchRepoOptions{Keyword: "test", Page: 1, PageSize: 10, OwnerID: 15, Private: true, AllPublic: true}, @@ -186,8 +186,11 @@ func TestSearchRepository(t *testing.T) { opts: &SearchRepoOptions{Keyword: "test", Page: 1, PageSize: 10, OwnerID: 18, Private: true, AllPublic: true}, count: 13}, {name: "AllPublic/PublicRepositoriesOfOrganization", - opts: &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 17, AllPublic: true, Collaborate: util.OptionalBoolFalse}, + opts: &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 17, AllPublic: true, Collaborate: util.OptionalBoolFalse, Template: util.OptionalBoolFalse}, count: 22}, + {name: "AllTemplates", + opts: &SearchRepoOptions{Page: 1, PageSize: 10, Template: util.OptionalBoolTrue}, + count: 2}, } for _, testCase := range testCases { diff --git a/models/user_test.go b/models/user_test.go index f3952422af..2969e34a76 100644 --- a/models/user_test.go +++ b/models/user_test.go @@ -153,7 +153,7 @@ func TestSearchUsers(t *testing.T) { } testUserSuccess(&SearchUserOptions{OrderBy: "id ASC", Page: 1}, - []int64{1, 2, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 24}) + []int64{1, 2, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 21, 24, 27}) testUserSuccess(&SearchUserOptions{Page: 1, IsActive: util.OptionalBoolFalse}, []int64{9}) |