diff options
author | Richard Mahn <richmahn@users.noreply.github.com> | 2019-04-17 10:06:35 -0600 |
---|---|---|
committer | techknowlogick <matti@mdranta.net> | 2019-04-17 12:06:35 -0400 |
commit | 2262811e407facea09047e94aa1850c192511587 (patch) | |
tree | a478624613dc6cf095784d629f9627b197de15e8 /modules/repofiles | |
parent | 059195b127848d96a4690257af19d8c97c6d2363 (diff) | |
download | gitea-2262811e407facea09047e94aa1850c192511587.tar.gz gitea-2262811e407facea09047e94aa1850c192511587.zip |
Fixes 4762 - Content API for Creating, Updating, Deleting Files (#6314)
Diffstat (limited to 'modules/repofiles')
-rw-r--r-- | modules/repofiles/blob.go | 38 | ||||
-rw-r--r-- | modules/repofiles/blob_test.go | 38 | ||||
-rw-r--r-- | modules/repofiles/content.go | 73 | ||||
-rw-r--r-- | modules/repofiles/content_test.go | 90 | ||||
-rw-r--r-- | modules/repofiles/delete.go | 209 | ||||
-rw-r--r-- | modules/repofiles/delete_test.go | 183 | ||||
-rw-r--r-- | modules/repofiles/diff.go | 41 | ||||
-rw-r--r-- | modules/repofiles/diff_test.go | 143 | ||||
-rw-r--r-- | modules/repofiles/file.go | 125 | ||||
-rw-r--r-- | modules/repofiles/file_test.go | 90 | ||||
-rw-r--r-- | modules/repofiles/repofiles.go | 23 | ||||
-rw-r--r-- | modules/repofiles/repofiles_test.go | 27 | ||||
-rw-r--r-- | modules/repofiles/temp_repo.go | 425 | ||||
-rw-r--r-- | modules/repofiles/tree.go | 92 | ||||
-rw-r--r-- | modules/repofiles/tree_test.go | 50 | ||||
-rw-r--r-- | modules/repofiles/update.go | 331 | ||||
-rw-r--r-- | modules/repofiles/update_test.go | 357 | ||||
-rw-r--r-- | modules/repofiles/upload.go | 209 | ||||
-rw-r--r-- | modules/repofiles/verification.go | 29 |
19 files changed, 2573 insertions, 0 deletions
diff --git a/modules/repofiles/blob.go b/modules/repofiles/blob.go new file mode 100644 index 0000000000..2f9ca72bd4 --- /dev/null +++ b/modules/repofiles/blob.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/sdk/gitea" +) + +// GetBlobBySHA get the GitBlobResponse of a repository using a sha hash. +func GetBlobBySHA(repo *models.Repository, sha string) (*api.GitBlobResponse, error) { + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { + return nil, err + } + gitBlob, err := gitRepo.GetBlob(sha) + if err != nil { + return nil, err + } + content := "" + if gitBlob.Size() <= setting.API.DefaultMaxBlobSize { + content, err = gitBlob.GetBlobContentBase64() + if err != nil { + return nil, err + } + } + return &api.GitBlobResponse{ + SHA: gitBlob.ID.String(), + URL: repo.APIURL() + "/git/blobs/" + gitBlob.ID.String(), + Size: gitBlob.Size(), + Encoding: "base64", + Content: content, + }, nil +} diff --git a/modules/repofiles/blob_test.go b/modules/repofiles/blob_test.go new file mode 100644 index 0000000000..260b775fc4 --- /dev/null +++ b/modules/repofiles/blob_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/test" + api "code.gitea.io/sdk/gitea" + + "github.com/stretchr/testify/assert" +) + +func TestGetBlobBySHA(t *testing.T) { + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" + ctx.SetParams(":id", "1") + ctx.SetParams(":sha", sha) + + gbr, err := GetBlobBySHA(ctx.Repo.Repository, ctx.Params(":sha")) + expectedGBR := &api.GitBlobResponse{ + Content: "Y29tbWl0IDY1ZjFiZjI3YmMzYmY3MGY2NDY1NzY1ODYzNWU2NjA5NGVkYmNiNGQKQXV0aG9yOiB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+CkRhdGU6ICAgU3VuIE1hciAxOSAxNjo0Nzo1OSAyMDE3IC0wNDAwCgogICAgSW5pdGlhbCBjb21taXQKCmRpZmYgLS1naXQgYS9SRUFETUUubWQgYi9SRUFETUUubWQKbmV3IGZpbGUgbW9kZSAxMDA2NDQKaW5kZXggMDAwMDAwMC4uNGI0ODUxYQotLS0gL2Rldi9udWxsCisrKyBiL1JFQURNRS5tZApAQCAtMCwwICsxLDMgQEAKKyMgcmVwbzEKKworRGVzY3JpcHRpb24gZm9yIHJlcG8xClwgTm8gbmV3bGluZSBhdCBlbmQgb2YgZmlsZQo=", + Encoding: "base64", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Size: 180, + } + assert.Nil(t, err) + assert.Equal(t, expectedGBR, gbr) +} diff --git a/modules/repofiles/content.go b/modules/repofiles/content.go new file mode 100644 index 0000000000..d55ca497cd --- /dev/null +++ b/modules/repofiles/content.go @@ -0,0 +1,73 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "net/url" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + api "code.gitea.io/sdk/gitea" +) + +// GetFileContents gets the meta data on a file's contents +func GetFileContents(repo *models.Repository, treePath, ref string) (*api.FileContentResponse, error) { + if ref == "" { + ref = repo.DefaultBranch + } + + // Check that the path given in opts.treePath is valid (not a git path) + treePath = CleanUploadFileName(treePath) + if treePath == "" { + return nil, models.ErrFilenameInvalid{ + Path: treePath, + } + } + + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { + return nil, err + } + + // Get the commit object for the ref + commit, err := gitRepo.GetCommit(ref) + if err != nil { + return nil, err + } + + entry, err := commit.GetTreeEntryByPath(treePath) + if err != nil { + return nil, err + } + + urlRef := ref + if _, err := gitRepo.GetBranchCommit(ref); err == nil { + urlRef = "branch/" + ref + } + + selfURL, _ := url.Parse(repo.APIURL() + "/contents/" + treePath) + gitURL, _ := url.Parse(repo.APIURL() + "/git/blobs/" + entry.ID.String()) + downloadURL, _ := url.Parse(repo.HTMLURL() + "/raw/" + urlRef + "/" + treePath) + htmlURL, _ := url.Parse(repo.HTMLURL() + "/blob/" + ref + "/" + treePath) + + fileContent := &api.FileContentResponse{ + Name: entry.Name(), + Path: treePath, + SHA: entry.ID.String(), + Size: entry.Size(), + URL: selfURL.String(), + HTMLURL: htmlURL.String(), + GitURL: gitURL.String(), + DownloadURL: downloadURL.String(), + Type: string(entry.Type), + Links: &api.FileLinksResponse{ + Self: selfURL.String(), + GitURL: gitURL.String(), + HTMLURL: htmlURL.String(), + }, + } + + return fileContent, nil +} diff --git a/modules/repofiles/content_test.go b/modules/repofiles/content_test.go new file mode 100644 index 0000000000..0257284a53 --- /dev/null +++ b/modules/repofiles/content_test.go @@ -0,0 +1,90 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/sdk/gitea" + + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + models.MainTest(m, filepath.Join("..", "..")) +} + +func TestGetFileContents(t *testing.T) { + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + treePath := "README.md" + ref := ctx.Repo.Repository.DefaultBranch + + expectedFileContentResponse := &gitea.FileContentResponse{ + Name: treePath, + Path: treePath, + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + Size: 30, + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/contents/README.md", + HTMLURL: "https://try.gitea.io/user2/repo1/blob/master/README.md", + GitURL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f", + DownloadURL: "https://try.gitea.io/user2/repo1/raw/branch/master/README.md", + Type: "blob", + Links: &gitea.FileLinksResponse{ + Self: "https://try.gitea.io/api/v1/repos/user2/repo1/contents/README.md", + GitURL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f", + HTMLURL: "https://try.gitea.io/user2/repo1/blob/master/README.md", + }, + } + + t.Run("Get README.md contents", func(t *testing.T) { + fileContentResponse, err := GetFileContents(ctx.Repo.Repository, treePath, ref) + assert.EqualValues(t, expectedFileContentResponse, fileContentResponse) + assert.Nil(t, err) + }) + + t.Run("Get REAMDE.md contents with ref as empty string (should then use the repo's default branch)", func(t *testing.T) { + fileContentResponse, err := GetFileContents(ctx.Repo.Repository, treePath, "") + assert.EqualValues(t, expectedFileContentResponse, fileContentResponse) + assert.Nil(t, err) + }) +} + +func TestGetFileContentsErrors(t *testing.T) { + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + treePath := "README.md" + ref := repo.DefaultBranch + + t.Run("bad treePath", func(t *testing.T) { + badTreePath := "bad/tree.md" + fileContentResponse, err := GetFileContents(repo, badTreePath, ref) + assert.Error(t, err) + assert.EqualError(t, err, "object does not exist [id: , rel_path: bad]") + assert.Nil(t, fileContentResponse) + }) + + t.Run("bad ref", func(t *testing.T) { + badRef := "bad_ref" + fileContentResponse, err := GetFileContents(repo, treePath, badRef) + assert.Error(t, err) + assert.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") + assert.Nil(t, fileContentResponse) + }) +} diff --git a/modules/repofiles/delete.go b/modules/repofiles/delete.go new file mode 100644 index 0000000000..ce7993dc54 --- /dev/null +++ b/modules/repofiles/delete.go @@ -0,0 +1,209 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "fmt" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + api "code.gitea.io/sdk/gitea" +) + +// DeleteRepoFileOptions holds the repository delete file options +type DeleteRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + TreePath string + Message string + SHA string + Author *IdentityOptions + Committer *IdentityOptions +} + +// DeleteRepoFile deletes a file in the given repository +func DeleteRepoFile(repo *models.Repository, doer *models.User, opts *DeleteRepoFileOptions) (*api.FileResponse, error) { + // If no branch name is set, assume the repo's default branch + if opts.OldBranch == "" { + opts.OldBranch = repo.DefaultBranch + } + if opts.NewBranch == "" { + opts.NewBranch = opts.OldBranch + } + + // oldBranch must exist for this operation + if _, err := repo.GetBranch(opts.OldBranch); err != nil { + return nil, err + } + + // A NewBranch can be specified for the file to be created/updated in a new branch. + // Check to make sure the branch does not already exist, otherwise we can't proceed. + // If we aren't branching to a new branch, make sure user can commit to the given branch + if opts.NewBranch != opts.OldBranch { + newBranch, err := repo.GetBranch(opts.NewBranch) + if git.IsErrNotExist(err) { + return nil, err + } + if newBranch != nil { + return nil, models.ErrBranchAlreadyExists{ + BranchName: opts.NewBranch, + } + } + } else { + if protected, _ := repo.IsProtectedBranchForPush(opts.OldBranch, doer); protected { + return nil, models.ErrUserCannotCommit{ + UserName: doer.LowerName, + } + } + } + + // Check that the path given in opts.treeName is valid (not a git path) + treePath := CleanUploadFileName(opts.TreePath) + if treePath == "" { + return nil, models.ErrFilenameInvalid{ + Path: opts.TreePath, + } + } + + message := strings.TrimSpace(opts.Message) + + author, committer := GetAuthorAndCommitterUsers(opts.Committer, opts.Author, doer) + + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return nil, err + } + if err := t.Clone(opts.OldBranch); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + + // Get the commit of the original branch + commit, err := t.GetBranchCommit(opts.OldBranch) + if err != nil { + return nil, err // Couldn't get a commit for the branch + } + + // Assigned LastCommitID in opts if it hasn't been set + if opts.LastCommitID == "" { + opts.LastCommitID = commit.ID.String() + } + + // Get the files in the index + filesInIndex, err := t.LsFiles(opts.TreePath) + if err != nil { + return nil, fmt.Errorf("DeleteRepoFile: %v", err) + } + + // Find the file we want to delete in the index + inFilelist := false + for _, file := range filesInIndex { + if file == opts.TreePath { + inFilelist = true + break + } + } + if !inFilelist { + return nil, models.ErrRepoFileDoesNotExist{ + Path: opts.TreePath, + } + } + + // Get the entry of treePath and check if the SHA given is the same as the file + entry, err := commit.GetTreeEntryByPath(treePath) + if err != nil { + return nil, err + } + if opts.SHA != "" { + // If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error + if opts.SHA != entry.ID.String() { + return nil, models.ErrSHADoesNotMatch{ + Path: treePath, + GivenSHA: opts.SHA, + CurrentSHA: entry.ID.String(), + } + } + } else if opts.LastCommitID != "" { + // If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw + // an error, but only if we aren't creating a new branch. + if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch { + // CommitIDs don't match, but we don't want to throw a ErrCommitIDDoesNotMatch unless + // this specific file has been edited since opts.LastCommitID + if changed, err := commit.FileChangedSinceCommit(treePath, opts.LastCommitID); err != nil { + return nil, err + } else if changed { + return nil, models.ErrCommitIDDoesNotMatch{ + GivenCommitID: opts.LastCommitID, + CurrentCommitID: opts.LastCommitID, + } + } + // The file wasn't modified, so we are good to delete it + } + } else { + // When deleting a file, a lastCommitID or SHA needs to be given to make sure other commits haven't been + // made. We throw an error if one wasn't provided. + return nil, models.ErrSHAOrCommitIDNotProvided{} + } + + // Remove the file from the index + if err := t.RemoveFilesFromIndex(opts.TreePath); err != nil { + return nil, err + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return nil, err + } + + // Now commit the tree + commitHash, err := t.CommitTree(author, committer, treeHash, message) + if err != nil { + return nil, err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return nil, err + } + + // Simulate push event. + oldCommitID := opts.LastCommitID + if opts.NewBranch != opts.OldBranch { + oldCommitID = git.EmptySHA + } + + if err = repo.GetOwner(); err != nil { + return nil, fmt.Errorf("GetOwner: %v", err) + } + err = models.PushUpdate( + opts.NewBranch, + models.PushUpdateOptions{ + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.Owner.Name, + RepoName: repo.Name, + RefFullName: git.BranchPrefix + opts.NewBranch, + OldCommitID: oldCommitID, + NewCommitID: commitHash, + }, + ) + if err != nil { + return nil, fmt.Errorf("PushUpdate: %v", err) + } + + // FIXME: Should we UpdateRepoIndexer(repo) here? + + file, err := GetFileResponseFromCommit(repo, commit, opts.NewBranch, treePath) + if err != nil { + return nil, err + } + return file, nil +} diff --git a/modules/repofiles/delete_test.go b/modules/repofiles/delete_test.go new file mode 100644 index 0000000000..0b0558e766 --- /dev/null +++ b/modules/repofiles/delete_test.go @@ -0,0 +1,183 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/test" + api "code.gitea.io/sdk/gitea" + + "github.com/stretchr/testify/assert" +) + +func getDeleteRepoFileOptions(repo *models.Repository) *DeleteRepoFileOptions { + return &DeleteRepoFileOptions{ + LastCommitID: "", + OldBranch: repo.DefaultBranch, + NewBranch: repo.DefaultBranch, + TreePath: "README.md", + Message: "Deletes README.md", + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + Author: nil, + Committer: nil, + } +} + +func getExpectedDeleteFileResponse() *api.FileResponse { + return &api.FileResponse{ + Content: nil, + Commit: &api.FileCommitResponse{ + CommitMeta: api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + }, + HTMLURL: "https://try.gitea.io/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d", + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: "user1", + Email: "address1@example.com", + }, + Date: "2017-03-19T20:47:59Z", + }, + Committer: &api.CommitUser{ + Identity: api.Identity{ + Name: "Ethan Koenig", + Email: "ethantkoenig@gmail.com", + }, + Date: "2017-03-19T20:47:59Z", + }, + Parents: []*api.CommitMeta{}, + Message: "Initial commit\n", + Tree: &api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/2a2f1d4670728a2e10049e345bd7a276468beab6", + SHA: "2a2f1d4670728a2e10049e345bd7a276468beab6", + }, + }, + Verification: &api.PayloadCommitVerification{ + Verified: false, + Reason: "", + Signature: "", + Payload: "", + }, + } +} + +func TestDeleteRepoFile(t *testing.T) { + // setup + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + doer := ctx.User + opts := getDeleteRepoFileOptions(repo) + + t.Run("Delete README.md file", func(t *testing.T) { + fileResponse, err := DeleteRepoFile(repo, doer, opts) + assert.Nil(t, err) + expectedFileResponse := getExpectedDeleteFileResponse() + assert.EqualValues(t, expectedFileResponse, fileResponse) + }) + + t.Run("Verify README.md has been deleted", func(t *testing.T) { + fileResponse, err := DeleteRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + expectedError := "repository file does not exist [path: " + opts.TreePath + "]" + assert.EqualError(t, err, expectedError) + }) +} + +// Test opts with branch names removed, same results +func TestDeleteRepoFileWithoutBranchNames(t *testing.T) { + // setup + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + doer := ctx.User + opts := getDeleteRepoFileOptions(repo) + opts.OldBranch = "" + opts.NewBranch = "" + + t.Run("Delete README.md without Branch Name", func(t *testing.T) { + fileResponse, err := DeleteRepoFile(repo, doer, opts) + assert.Nil(t, err) + expectedFileResponse := getExpectedDeleteFileResponse() + assert.EqualValues(t, expectedFileResponse, fileResponse) + }) +} + +func TestDeleteRepoFileErrors(t *testing.T) { + // setup + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + doer := ctx.User + + t.Run("Bad branch", func(t *testing.T) { + opts := getDeleteRepoFileOptions(repo) + opts.OldBranch = "bad_branch" + fileResponse, err := DeleteRepoFile(repo, doer, opts) + assert.Error(t, err) + assert.Nil(t, fileResponse) + expectedError := "branch does not exist [name: " + opts.OldBranch + "]" + assert.EqualError(t, err, expectedError) + }) + + t.Run("Bad SHA", func(t *testing.T) { + opts := getDeleteRepoFileOptions(repo) + origSHA := opts.SHA + opts.SHA = "bad_sha" + fileResponse, err := DeleteRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + assert.Error(t, err) + expectedError := "sha does not match [given: " + opts.SHA + ", expected: " + origSHA + "]" + assert.EqualError(t, err, expectedError) + }) + + t.Run("New branch already exists", func(t *testing.T) { + opts := getDeleteRepoFileOptions(repo) + opts.NewBranch = "develop" + fileResponse, err := DeleteRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + assert.Error(t, err) + expectedError := "branch already exists [name: " + opts.NewBranch + "]" + assert.EqualError(t, err, expectedError) + }) + + t.Run("TreePath is empty:", func(t *testing.T) { + opts := getDeleteRepoFileOptions(repo) + opts.TreePath = "" + fileResponse, err := DeleteRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + assert.Error(t, err) + expectedError := "path contains a malformed path component [path: ]" + assert.EqualError(t, err, expectedError) + }) + + t.Run("TreePath is a git directory:", func(t *testing.T) { + opts := getDeleteRepoFileOptions(repo) + opts.TreePath = ".git" + fileResponse, err := DeleteRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + assert.Error(t, err) + expectedError := "path contains a malformed path component [path: " + opts.TreePath + "]" + assert.EqualError(t, err, expectedError) + }) +} diff --git a/modules/repofiles/diff.go b/modules/repofiles/diff.go new file mode 100644 index 0000000000..3b5de5fa6f --- /dev/null +++ b/modules/repofiles/diff.go @@ -0,0 +1,41 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "strings" + + "code.gitea.io/gitea/models" +) + +// GetDiffPreview produces and returns diff result of a file which is not yet committed. +func GetDiffPreview(repo *models.Repository, branch, treePath, content string) (*models.Diff, error) { + if branch == "" { + branch = repo.DefaultBranch + } + t, err := NewTemporaryUploadRepository(repo) + if err != nil { + return nil, err + } + defer t.Close() + if err := t.Clone(branch); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + + // Add the object to the database + objectHash, err := t.HashObject(strings.NewReader(content)) + if err != nil { + return nil, err + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil { + return nil, err + } + return t.DiffIndex() +} diff --git a/modules/repofiles/diff_test.go b/modules/repofiles/diff_test.go new file mode 100644 index 0000000000..bc7d4ebad6 --- /dev/null +++ b/modules/repofiles/diff_test.go @@ -0,0 +1,143 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" +) + +func TestGetDiffPreview(t *testing.T) { + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + branch := ctx.Repo.Repository.DefaultBranch + treePath := "README.md" + content := "# repo1\n\nDescription for repo1\nthis is a new line" + + expectedDiff := &models.Diff{ + TotalAddition: 2, + TotalDeletion: 1, + Files: []*models.DiffFile{ + { + Name: "README.md", + OldName: "README.md", + Index: 1, + Addition: 2, + Deletion: 1, + Type: 2, + IsCreated: false, + IsDeleted: false, + IsBin: false, + IsLFSFile: false, + IsRenamed: false, + IsSubmodule: false, + Sections: []*models.DiffSection{ + { + Name: "", + Lines: []*models.DiffLine{ + { + LeftIdx: 0, + RightIdx: 0, + Type: 4, + Content: "@@ -1,3 +1,4 @@", + Comments: nil, + }, + { + LeftIdx: 1, + RightIdx: 1, + Type: 1, + Content: " # repo1", + Comments: nil, + }, + { + LeftIdx: 2, + RightIdx: 2, + Type: 1, + Content: " ", + Comments: nil, + }, + { + LeftIdx: 3, + RightIdx: 0, + Type: 3, + Content: "-Description for repo1", + Comments: nil, + }, + { + LeftIdx: 0, + RightIdx: 3, + Type: 2, + Content: "+Description for repo1", + Comments: nil, + }, + { + LeftIdx: 0, + RightIdx: 4, + Type: 2, + Content: "+this is a new line", + Comments: nil, + }, + }, + }, + }, + IsIncomplete: false, + }, + }, + IsIncomplete: false, + } + + t.Run("with given branch", func(t *testing.T) { + diff, err := GetDiffPreview(ctx.Repo.Repository, branch, treePath, content) + assert.Nil(t, err) + assert.EqualValues(t, expectedDiff, diff) + }) + + t.Run("empty branch, same results", func(t *testing.T) { + diff, err := GetDiffPreview(ctx.Repo.Repository, "", treePath, content) + assert.Nil(t, err) + assert.EqualValues(t, expectedDiff, diff) + }) +} + +func TestGetDiffPreviewErrors(t *testing.T) { + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + branch := ctx.Repo.Repository.DefaultBranch + treePath := "README.md" + content := "# repo1\n\nDescription for repo1\nthis is a new line" + + t.Run("empty repo", func(t *testing.T) { + diff, err := GetDiffPreview(&models.Repository{}, branch, treePath, content) + assert.Nil(t, diff) + assert.EqualError(t, err, "repository does not exist [id: 0, uid: 0, owner_name: , name: ]") + }) + + t.Run("bad branch", func(t *testing.T) { + badBranch := "bad_branch" + diff, err := GetDiffPreview(ctx.Repo.Repository, badBranch, treePath, content) + assert.Nil(t, diff) + assert.EqualError(t, err, "branch does not exist [name: "+badBranch+"]") + }) + + t.Run("empty treePath", func(t *testing.T) { + diff, err := GetDiffPreview(ctx.Repo.Repository, branch, "", content) + assert.Nil(t, diff) + assert.EqualError(t, err, "path is invalid [path: ]") + }) +} diff --git a/modules/repofiles/file.go b/modules/repofiles/file.go new file mode 100644 index 0000000000..913a9ed535 --- /dev/null +++ b/modules/repofiles/file.go @@ -0,0 +1,125 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "fmt" + "net/url" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + api "code.gitea.io/sdk/gitea" +) + +// GetFileResponseFromCommit Constructs a FileResponse from a Commit object +func GetFileResponseFromCommit(repo *models.Repository, commit *git.Commit, branch, treeName string) (*api.FileResponse, error) { + fileContents, _ := GetFileContents(repo, treeName, branch) // ok if fails, then will be nil + fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(commit) + fileResponse := &api.FileResponse{ + Content: fileContents, + Commit: fileCommitResponse, + Verification: verification, + } + return fileResponse, nil +} + +// GetFileCommitResponse Constructs a FileCommitResponse from a Commit object +func GetFileCommitResponse(repo *models.Repository, commit *git.Commit) (*api.FileCommitResponse, error) { + if repo == nil { + return nil, fmt.Errorf("repo cannot be nil") + } + if commit == nil { + return nil, fmt.Errorf("commit cannot be nil") + } + commitURL, _ := url.Parse(repo.APIURL() + "/git/commits/" + commit.ID.String()) + commitTreeURL, _ := url.Parse(repo.APIURL() + "/git/trees/" + commit.Tree.ID.String()) + parents := make([]*api.CommitMeta, commit.ParentCount()) + for i := 0; i <= commit.ParentCount(); i++ { + if parent, err := commit.Parent(i); err == nil && parent != nil { + parentCommitURL, _ := url.Parse(repo.APIURL() + "/git/commits/" + parent.ID.String()) + parents[i] = &api.CommitMeta{ + SHA: parent.ID.String(), + URL: parentCommitURL.String(), + } + } + } + commitHTMLURL, _ := url.Parse(repo.HTMLURL() + "/commit/" + commit.ID.String()) + fileCommit := &api.FileCommitResponse{ + CommitMeta: api.CommitMeta{ + SHA: commit.ID.String(), + URL: commitURL.String(), + }, + HTMLURL: commitHTMLURL.String(), + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: commit.Author.Name, + Email: commit.Author.Email, + }, + Date: commit.Author.When.UTC().Format(time.RFC3339), + }, + Committer: &api.CommitUser{ + Identity: api.Identity{ + Name: commit.Committer.Name, + Email: commit.Committer.Email, + }, + Date: commit.Committer.When.UTC().Format(time.RFC3339), + }, + Message: commit.Message(), + Tree: &api.CommitMeta{ + URL: commitTreeURL.String(), + SHA: commit.Tree.ID.String(), + }, + Parents: parents, + } + return fileCommit, nil +} + +// GetAuthorAndCommitterUsers Gets the author and committer user objects from the IdentityOptions +func GetAuthorAndCommitterUsers(author, committer *IdentityOptions, doer *models.User) (committerUser, authorUser *models.User) { + // Committer and author are optional. If they are not the doer (not same email address) + // then we use bogus User objects for them to store their FullName and Email. + // If only one of the two are provided, we set both of them to it. + // If neither are provided, both are the doer. + if committer != nil && committer.Email != "" { + if doer != nil && strings.ToLower(doer.Email) == strings.ToLower(committer.Email) { + committerUser = doer // the committer is the doer, so will use their user object + if committer.Name != "" { + committerUser.FullName = committer.Name + } + } else { + committerUser = &models.User{ + FullName: committer.Name, + Email: committer.Email, + } + } + } + if author != nil && author.Email != "" { + if doer != nil && strings.ToLower(doer.Email) == strings.ToLower(author.Email) { + authorUser = doer // the author is the doer, so will use their user object + if authorUser.Name != "" { + authorUser.FullName = author.Name + } + } else { + authorUser = &models.User{ + FullName: author.Name, + Email: author.Email, + } + } + } + if authorUser == nil { + if committerUser != nil { + authorUser = committerUser // No valid author was given so use the committer + } else if doer != nil { + authorUser = doer // No valid author was given and no valid committer so use the doer + } + } + if committerUser == nil { + committerUser = authorUser // No valid committer so use the author as the committer (was set to a valid user above) + } + return authorUser, committerUser +} diff --git a/modules/repofiles/file_test.go b/modules/repofiles/file_test.go new file mode 100644 index 0000000000..c9ee7f21e2 --- /dev/null +++ b/modules/repofiles/file_test.go @@ -0,0 +1,90 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/test" + api "code.gitea.io/sdk/gitea" + + "github.com/stretchr/testify/assert" +) + +func getExpectedFileResponse() *api.FileResponse { + return &api.FileResponse{ + Content: &api.FileContentResponse{ + Name: "README.md", + Path: "README.md", + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + Size: 30, + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/contents/README.md", + HTMLURL: "https://try.gitea.io/user2/repo1/blob/master/README.md", + GitURL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f", + DownloadURL: "https://try.gitea.io/user2/repo1/raw/branch/master/README.md", + Type: "blob", + Links: &api.FileLinksResponse{ + Self: "https://try.gitea.io/api/v1/repos/user2/repo1/contents/README.md", + GitURL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f", + HTMLURL: "https://try.gitea.io/user2/repo1/blob/master/README.md", + }, + }, + Commit: &api.FileCommitResponse{ + CommitMeta: api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + }, + HTMLURL: "https://try.gitea.io/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d", + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: "user1", + Email: "address1@example.com", + }, + Date: "2017-03-19T20:47:59Z", + }, + Committer: &api.CommitUser{ + Identity: api.Identity{ + Name: "Ethan Koenig", + Email: "ethantkoenig@gmail.com", + }, + Date: "2017-03-19T20:47:59Z", + }, + Parents: []*api.CommitMeta{}, + Message: "Initial commit\n", + Tree: &api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/2a2f1d4670728a2e10049e345bd7a276468beab6", + SHA: "2a2f1d4670728a2e10049e345bd7a276468beab6", + }, + }, + Verification: &api.PayloadCommitVerification{ + Verified: false, + Reason: "", + Signature: "", + Payload: "", + }, + } +} + +func TestGetFileResponseFromCommit(t *testing.T) { + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + branch := repo.DefaultBranch + treePath := "README.md" + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commit, _ := gitRepo.GetBranchCommit(branch) + expectedFileResponse := getExpectedFileResponse() + + fileResponse, err := GetFileResponseFromCommit(repo, commit, branch, treePath) + assert.Nil(t, err) + assert.EqualValues(t, expectedFileResponse, fileResponse) +} diff --git a/modules/repofiles/repofiles.go b/modules/repofiles/repofiles.go new file mode 100644 index 0000000000..1fc900490c --- /dev/null +++ b/modules/repofiles/repofiles.go @@ -0,0 +1,23 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file.package repofiles + +package repofiles + +import ( + "path" + "strings" +) + +// CleanUploadFileName Trims a filename and returns empty string if it is a .git directory +func CleanUploadFileName(name string) string { + // Rebase the filename + name = strings.Trim(path.Clean("/"+name), " /") + // Git disallows any filenames to have a .git directory in them. + for _, part := range strings.Split(name, "/") { + if strings.ToLower(part) == ".git" { + return "" + } + } + return name +} diff --git a/modules/repofiles/repofiles_test.go b/modules/repofiles/repofiles_test.go new file mode 100644 index 0000000000..1686378a4a --- /dev/null +++ b/modules/repofiles/repofiles_test.go @@ -0,0 +1,27 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCleanUploadFileName(t *testing.T) { + t.Run("Clean regular file", func(t *testing.T) { + name := "this/is/test" + cleanName := CleanUploadFileName(name) + expectedCleanName := name + assert.EqualValues(t, expectedCleanName, cleanName) + }) + + t.Run("Clean a .git path", func(t *testing.T) { + name := "this/is/test/.git" + cleanName := CleanUploadFileName(name) + expectedCleanName := "" + assert.EqualValues(t, expectedCleanName, cleanName) + }) +} diff --git a/modules/repofiles/temp_repo.go b/modules/repofiles/temp_repo.go new file mode 100644 index 0000000000..5bf64d52a8 --- /dev/null +++ b/modules/repofiles/temp_repo.go @@ -0,0 +1,425 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path" + "regexp" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" + + "github.com/Unknwon/com" +) + +// TemporaryUploadRepository is a type to wrap our upload repositories as a shallow clone +type TemporaryUploadRepository struct { + repo *models.Repository + gitRepo *git.Repository + basePath string +} + +// NewTemporaryUploadRepository creates a new temporary upload repository +func NewTemporaryUploadRepository(repo *models.Repository) (*TemporaryUploadRepository, error) { + timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE + basePath := path.Join(models.LocalCopyPath(), "upload-"+timeStr+".git") + if err := os.MkdirAll(path.Dir(basePath), os.ModePerm); err != nil { + return nil, fmt.Errorf("failed to create dir %s: %v", basePath, err) + } + if repo.RepoPath() == "" { + return nil, fmt.Errorf("no path to repository on system") + } + t := &TemporaryUploadRepository{repo: repo, basePath: basePath} + return t, nil +} + +// Close the repository cleaning up all files +func (t *TemporaryUploadRepository) Close() { + if _, err := os.Stat(t.basePath); !os.IsNotExist(err) { + os.RemoveAll(t.basePath) + } +} + +// Clone the base repository to our path and set branch as the HEAD +func (t *TemporaryUploadRepository) Clone(branch string) error { + if _, stderr, err := process.GetManager().ExecTimeout(5*time.Minute, + fmt.Sprintf("Clone (git clone -s --bare): %s", t.basePath), + "git", "clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath); err != nil { + if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched { + return models.ErrBranchNotExist{ + Name: branch, + } + } else if matched, _ := regexp.MatchString(".* repository .* does not exist.*", stderr); matched { + return models.ErrRepoNotExist{ + ID: t.repo.ID, + UID: t.repo.OwnerID, + OwnerName: t.repo.OwnerName, + Name: t.repo.Name, + } + } else { + return fmt.Errorf("Clone: %v %s", err, stderr) + } + } + gitRepo, err := git.OpenRepository(t.basePath) + if err != nil { + return err + } + t.gitRepo = gitRepo + return nil +} + +// SetDefaultIndex sets the git index to our HEAD +func (t *TemporaryUploadRepository) SetDefaultIndex() error { + if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("SetDefaultIndex (git read-tree HEAD): %s", t.basePath), + "git", "read-tree", "HEAD"); err != nil { + return fmt.Errorf("SetDefaultIndex: %v %s", err, stderr) + } + return nil +} + +// LsFiles checks if the given filename arguments are in the index +func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"ls-files", "-z", "--"} + for _, arg := range filenames { + if arg != "" { + cmdArgs = append(cmdArgs, arg) + } + } + + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("lsFiles: (git ls-files) %v", cmdArgs) + cmd.Dir = t.basePath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + return nil, err + } + + filelist := make([]string, len(filenames)) + for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { + filelist = append(filelist, string(line)) + } + + return filelist, err +} + +// RemoveFilesFromIndex removes the given files from the index +func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) error { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + stdIn := new(bytes.Buffer) + for _, file := range filenames { + if file != "" { + stdIn.WriteString("0 0000000000000000000000000000000000000000\t") + stdIn.WriteString(file) + stdIn.WriteByte('\000') + } + } + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"update-index", "--remove", "-z", "--index-info"} + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("removeFilesFromIndex: (git update-index) %v", filenames) + cmd.Dir = t.basePath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + cmd.Stdin = bytes.NewReader(stdIn.Bytes()) + + if err := cmd.Start(); err != nil { + return fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + } + + return err +} + +// HashObject writes the provided content to the object db and returns its hash +func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) { + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + hashCmd := exec.CommandContext(ctx, "git", "hash-object", "-w", "--stdin") + hashCmd.Dir = t.basePath + hashCmd.Stdin = content + stdOutBuffer := new(bytes.Buffer) + stdErrBuffer := new(bytes.Buffer) + hashCmd.Stdout = stdOutBuffer + hashCmd.Stderr = stdErrBuffer + desc := fmt.Sprintf("hashObject: (git hash-object)") + if err := hashCmd.Start(); err != nil { + return "", fmt.Errorf("git hash-object: %s", err) + } + + pid := process.GetManager().Add(desc, hashCmd) + err := hashCmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOutBuffer, stdErrBuffer) + return "", err + } + + return strings.TrimSpace(stdOutBuffer.String()), nil +} + +// AddObjectToIndex adds the provided object hash to the index with the provided mode and path +func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error { + if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("addObjectToIndex (git update-index): %s", t.basePath), + "git", "update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath); err != nil { + if matched, _ := regexp.MatchString(".*Invalid path '.*", stderr); matched { + return models.ErrFilePathInvalid{ + Message: objectPath, + Path: objectPath, + } + } + return fmt.Errorf("git update-index: %s", stderr) + } + return nil +} + +// WriteTree writes the current index as a tree to the object db and returns its hash +func (t *TemporaryUploadRepository) WriteTree() (string, error) { + treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("WriteTree (git write-tree): %s", t.basePath), + "git", "write-tree") + if err != nil { + return "", fmt.Errorf("git write-tree: %s", stderr) + } + return strings.TrimSpace(treeHash), nil +} + +// GetLastCommit gets the last commit ID SHA of the repo +func (t *TemporaryUploadRepository) GetLastCommit() (string, error) { + return t.GetLastCommitByRef("HEAD") +} + +// GetLastCommitByRef gets the last commit ID SHA of the repo by ref +func (t *TemporaryUploadRepository) GetLastCommitByRef(ref string) (string, error) { + if ref == "" { + ref = "HEAD" + } + treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("GetLastCommit (git rev-parse %s): %s", ref, t.basePath), + "git", "rev-parse", ref) + if err != nil { + return "", fmt.Errorf("git rev-parse %s: %s", ref, stderr) + } + return strings.TrimSpace(treeHash), nil +} + +// CommitTree creates a commit from a given tree for the user with provided message +func (t *TemporaryUploadRepository) CommitTree(author, committer *models.User, treeHash string, message string) (string, error) { + commitTimeStr := time.Now().Format(time.UnixDate) + authorSig := author.NewGitSig() + committerSig := committer.NewGitSig() + + // FIXME: Should we add SSH_ORIGINAL_COMMAND to this + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+committerSig.Name, + "GIT_COMMITTER_EMAIL="+committerSig.Email, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + commitHash, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, + t.basePath, + fmt.Sprintf("commitTree (git commit-tree): %s", t.basePath), + env, + "git", "commit-tree", treeHash, "-p", "HEAD", "-m", message) + if err != nil { + return "", fmt.Errorf("git commit-tree: %s", stderr) + } + return strings.TrimSpace(commitHash), nil +} + +// Push the provided commitHash to the repository branch by the provided user +func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, branch string) error { + isWiki := "false" + if strings.HasSuffix(t.repo.Name, ".wiki") { + isWiki = "true" + } + + sig := doer.NewGitSig() + + // FIXME: Should we add SSH_ORIGINAL_COMMAND to this + // Because calls hooks we need to pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+sig.Name, + "GIT_AUTHOR_EMAIL="+sig.Email, + "GIT_COMMITTER_NAME="+sig.Name, + "GIT_COMMITTER_EMAIL="+sig.Email, + models.EnvRepoName+"="+t.repo.Name, + models.EnvRepoUsername+"="+t.repo.OwnerName, + models.EnvRepoIsWiki+"="+isWiki, + models.EnvPusherName+"="+doer.Name, + models.EnvPusherID+"="+fmt.Sprintf("%d", doer.ID), + models.ProtectedBranchRepoID+"="+fmt.Sprintf("%d", t.repo.ID), + ) + + if _, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, + t.basePath, + fmt.Sprintf("actuallyPush (git push): %s", t.basePath), + env, + "git", "push", t.repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)); err != nil { + return fmt.Errorf("git push: %s", stderr) + } + return nil +} + +// DiffIndex returns a Diff of the current index to the head +func (t *TemporaryUploadRepository) DiffIndex() (diff *models.Diff, err error) { + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + stdErr := new(bytes.Buffer) + + cmd := exec.CommandContext(ctx, "git", "diff-index", "--cached", "-p", "HEAD") + cmd.Dir = t.basePath + cmd.Stderr = stdErr + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("StdoutPipe: %v stderr %s", err, stdErr.String()) + } + + if err = cmd.Start(); err != nil { + return nil, fmt.Errorf("Start: %v stderr %s", err, stdErr.String()) + } + + pid := process.GetManager().Add(fmt.Sprintf("diffIndex [repo_path: %s]", t.repo.RepoPath()), cmd) + defer process.GetManager().Remove(pid) + + diff, err = models.ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout) + if err != nil { + return nil, fmt.Errorf("ParsePatch: %v", err) + } + + if err = cmd.Wait(); err != nil { + return nil, fmt.Errorf("Wait: %v", err) + } + + return diff, nil +} + +// CheckAttribute checks the given attribute of the provided files +func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...string) (map[string]map[string]string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"check-attr", "-z", attribute, "--cached", "--"} + for _, arg := range args { + if arg != "" { + cmdArgs = append(cmdArgs, arg) + } + } + + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("checkAttr: (git check-attr) %s %v", attribute, cmdArgs) + cmd.Dir = t.basePath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + return nil, err + } + + fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) + + if len(fields)%3 != 1 { + return nil, fmt.Errorf("Wrong number of fields in return from check-attr") + } + + var name2attribute2info = make(map[string]map[string]string) + + for i := 0; i < (len(fields) / 3); i++ { + filename := string(fields[3*i]) + attribute := string(fields[3*i+1]) + info := string(fields[3*i+2]) + attribute2info := name2attribute2info[filename] + if attribute2info == nil { + attribute2info = make(map[string]string) + } + attribute2info[attribute] = info + name2attribute2info[filename] = attribute2info + } + + return name2attribute2info, err +} + +// GetBranchCommit Gets the commit object of the given branch +func (t *TemporaryUploadRepository) GetBranchCommit(branch string) (*git.Commit, error) { + if t.gitRepo == nil { + return nil, fmt.Errorf("repository has not been cloned") + } + return t.gitRepo.GetBranchCommit(branch) +} + +// GetCommit Gets the commit object of the given commit ID +func (t *TemporaryUploadRepository) GetCommit(commitID string) (*git.Commit, error) { + if t.gitRepo == nil { + return nil, fmt.Errorf("repository has not been cloned") + } + return t.gitRepo.GetCommit(commitID) +} diff --git a/modules/repofiles/tree.go b/modules/repofiles/tree.go new file mode 100644 index 0000000000..8766ed36d0 --- /dev/null +++ b/modules/repofiles/tree.go @@ -0,0 +1,92 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/sdk/gitea" +) + +// GetTreeBySHA get the GitTreeResponse of a repository using a sha hash. +func GetTreeBySHA(repo *models.Repository, sha string, page, perPage int, recursive bool) (*api.GitTreeResponse, error) { + gitRepo, err := git.OpenRepository(repo.RepoPath()) + gitTree, err := gitRepo.GetTree(sha) + if err != nil || gitTree == nil { + return nil, models.ErrSHANotFound{ + SHA: sha, + } + } + tree := new(api.GitTreeResponse) + tree.SHA = gitTree.ID.String() + tree.URL = repo.APIURL() + "/git/trees/" + tree.SHA + var entries git.Entries + if recursive { + entries, err = gitTree.ListEntriesRecursive() + } else { + entries, err = gitTree.ListEntries() + } + if err != nil { + return nil, err + } + apiURL := repo.APIURL() + apiURLLen := len(apiURL) + + // 51 is len(sha1) + len("/git/blobs/"). 40 + 11. + blobURL := make([]byte, apiURLLen+51) + copy(blobURL[:], apiURL) + copy(blobURL[apiURLLen:], "/git/blobs/") + + // 51 is len(sha1) + len("/git/trees/"). 40 + 11. + treeURL := make([]byte, apiURLLen+51) + copy(treeURL[:], apiURL) + copy(treeURL[apiURLLen:], "/git/trees/") + + // 40 is the size of the sha1 hash in hexadecimal format. + copyPos := len(treeURL) - 40 + + if perPage <= 0 || perPage > setting.API.DefaultGitTreesPerPage { + perPage = setting.API.DefaultGitTreesPerPage + } + if page <= 0 { + page = 1 + } + tree.Page = page + tree.TotalCount = len(entries) + rangeStart := perPage * (page - 1) + if rangeStart >= len(entries) { + return tree, nil + } + var rangeEnd int + if len(entries) > perPage { + tree.Truncated = true + } + if rangeStart+perPage < len(entries) { + rangeEnd = rangeStart + perPage + } else { + rangeEnd = len(entries) + } + tree.Entries = make([]api.GitEntry, rangeEnd-rangeStart) + for e := rangeStart; e < rangeEnd; e++ { + i := e - rangeStart + tree.Entries[i].Path = entries[e].Name() + tree.Entries[i].Mode = fmt.Sprintf("%06x", entries[e].Mode()) + tree.Entries[i].Type = string(entries[e].Type) + tree.Entries[i].Size = entries[e].Size() + tree.Entries[i].SHA = entries[e].ID.String() + + if entries[e].IsDir() { + copy(treeURL[copyPos:], entries[e].ID.String()) + tree.Entries[i].URL = string(treeURL[:]) + } else { + copy(blobURL[copyPos:], entries[e].ID.String()) + tree.Entries[i].URL = string(blobURL[:]) + } + } + return tree, nil +} diff --git a/modules/repofiles/tree_test.go b/modules/repofiles/tree_test.go new file mode 100644 index 0000000000..266dc91670 --- /dev/null +++ b/modules/repofiles/tree_test.go @@ -0,0 +1,50 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/test" + api "code.gitea.io/sdk/gitea" + + "github.com/stretchr/testify/assert" +) + +func TestGetTreeBySHA(t *testing.T) { + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + sha := ctx.Repo.Repository.DefaultBranch + page := 1 + perPage := 10 + ctx.SetParams(":id", "1") + ctx.SetParams(":sha", sha) + + tree, err := GetTreeBySHA(ctx.Repo.Repository, ctx.Params(":sha"), page, perPage, true) + assert.Nil(t, err) + expectedTree := &api.GitTreeResponse{ + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/65f1bf27bc3bf70f64657658635e66094edbcb4d", + Entries: []api.GitEntry{ + { + Path: "README.md", + Mode: "100644", + Type: "blob", + Size: 30, + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f", + }, + }, + Truncated: false, + Page: 1, + TotalCount: 1, + } + assert.EqualValues(t, tree, expectedTree) +} diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go new file mode 100644 index 0000000000..216df18cd3 --- /dev/null +++ b/modules/repofiles/update.go @@ -0,0 +1,331 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "fmt" + "path" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/sdk/gitea" +) + +// IdentityOptions for a person's identity like an author or committer +type IdentityOptions struct { + Name string + Email string +} + +// UpdateRepoFileOptions holds the repository file update options +type UpdateRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + TreePath string + FromTreePath string + Message string + Content string + SHA string + IsNewFile bool + Author *IdentityOptions + Committer *IdentityOptions +} + +// CreateOrUpdateRepoFile adds or updates a file in the given repository +func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *UpdateRepoFileOptions) (*gitea.FileResponse, error) { + // If no branch name is set, assume master + if opts.OldBranch == "" { + opts.OldBranch = repo.DefaultBranch + } + if opts.NewBranch == "" { + opts.NewBranch = opts.OldBranch + } + + // oldBranch must exist for this operation + if _, err := repo.GetBranch(opts.OldBranch); err != nil { + return nil, err + } + + // A NewBranch can be specified for the file to be created/updated in a new branch. + // Check to make sure the branch does not already exist, otherwise we can't proceed. + // If we aren't branching to a new branch, make sure user can commit to the given branch + if opts.NewBranch != opts.OldBranch { + existingBranch, err := repo.GetBranch(opts.NewBranch) + if existingBranch != nil { + return nil, models.ErrBranchAlreadyExists{ + BranchName: opts.NewBranch, + } + } + if err != nil && !models.IsErrBranchNotExist(err) { + return nil, err + } + } else { + if protected, _ := repo.IsProtectedBranchForPush(opts.OldBranch, doer); protected { + return nil, models.ErrUserCannotCommit{UserName: doer.LowerName} + } + } + + // If FromTreePath is not set, set it to the opts.TreePath + if opts.TreePath != "" && opts.FromTreePath == "" { + opts.FromTreePath = opts.TreePath + } + + // Check that the path given in opts.treePath is valid (not a git path) + treePath := CleanUploadFileName(opts.TreePath) + if treePath == "" { + return nil, models.ErrFilenameInvalid{ + Path: opts.TreePath, + } + } + // If there is a fromTreePath (we are copying it), also clean it up + fromTreePath := CleanUploadFileName(opts.FromTreePath) + if fromTreePath == "" && opts.FromTreePath != "" { + return nil, models.ErrFilenameInvalid{ + Path: opts.FromTreePath, + } + } + + message := strings.TrimSpace(opts.Message) + + author, committer := GetAuthorAndCommitterUsers(opts.Committer, opts.Author, doer) + + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return nil, err + } + if err := t.Clone(opts.OldBranch); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + + // Get the commit of the original branch + commit, err := t.GetBranchCommit(opts.OldBranch) + if err != nil { + return nil, err // Couldn't get a commit for the branch + } + + // Assigned LastCommitID in opts if it hasn't been set + if opts.LastCommitID == "" { + opts.LastCommitID = commit.ID.String() + } + + if !opts.IsNewFile { + fromEntry, err := commit.GetTreeEntryByPath(fromTreePath) + if err != nil { + return nil, err + } + if opts.SHA != "" { + // If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error + if opts.SHA != fromEntry.ID.String() { + return nil, models.ErrSHADoesNotMatch{ + Path: treePath, + GivenSHA: opts.SHA, + CurrentSHA: fromEntry.ID.String(), + } + } + } else if opts.LastCommitID != "" { + // If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw + // an error, but only if we aren't creating a new branch. + if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch { + if changed, err := commit.FileChangedSinceCommit(treePath, opts.LastCommitID); err != nil { + return nil, err + } else if changed { + return nil, models.ErrCommitIDDoesNotMatch{ + GivenCommitID: opts.LastCommitID, + CurrentCommitID: opts.LastCommitID, + } + } + // The file wasn't modified, so we are good to delete it + } + } else { + // When updating a file, a lastCommitID or SHA needs to be given to make sure other commits + // haven't been made. We throw an error if one wasn't provided. + return nil, models.ErrSHAOrCommitIDNotProvided{} + } + } + + // For the path where this file will be created/updated, we need to make + // sure no parts of the path are existing files or links except for the last + // item in the path which is the file name, and that shouldn't exist IF it is + // a new file OR is being moved to a new path. + treePathParts := strings.Split(treePath, "/") + subTreePath := "" + for index, part := range treePathParts { + subTreePath = path.Join(subTreePath, part) + entry, err := commit.GetTreeEntryByPath(subTreePath) + if err != nil { + if git.IsErrNotExist(err) { + // Means there is no item with that name, so we're good + break + } + return nil, err + } + if index < len(treePathParts)-1 { + if !entry.IsDir() { + return nil, models.ErrFilePathInvalid{ + Message: fmt.Sprintf("a file exists where you’re trying to create a subdirectory [path: %s]", subTreePath), + Path: subTreePath, + Name: part, + Type: git.EntryModeBlob, + } + } + } else if entry.IsLink() { + return nil, models.ErrFilePathInvalid{ + Message: fmt.Sprintf("a symbolic link exists where you’re trying to create a subdirectory [path: %s]", subTreePath), + Path: subTreePath, + Name: part, + Type: git.EntryModeSymlink, + } + } else if entry.IsDir() { + return nil, models.ErrFilePathInvalid{ + Message: fmt.Sprintf("a directory exists where you’re trying to create a file [path: %s]", subTreePath), + Path: subTreePath, + Name: part, + Type: git.EntryModeTree, + } + } else if fromTreePath != treePath || opts.IsNewFile { + // The entry shouldn't exist if we are creating new file or moving to a new path + return nil, models.ErrRepoFileAlreadyExists{ + Path: treePath, + } + } + + } + + // Get the two paths (might be the same if not moving) from the index if they exist + filesInIndex, err := t.LsFiles(opts.TreePath, opts.FromTreePath) + if err != nil { + return nil, fmt.Errorf("UpdateRepoFile: %v", err) + } + // If is a new file (not updating) then the given path shouldn't exist + if opts.IsNewFile { + for _, file := range filesInIndex { + if file == opts.TreePath { + return nil, models.ErrRepoFileAlreadyExists{ + Path: opts.TreePath, + } + } + } + } + + // Remove the old path from the tree + if fromTreePath != treePath && len(filesInIndex) > 0 { + for _, file := range filesInIndex { + if file == fromTreePath { + if err := t.RemoveFilesFromIndex(opts.FromTreePath); err != nil { + return nil, err + } + } + } + } + + // Check there is no way this can return multiple infos + filename2attribute2info, err := t.CheckAttribute("filter", treePath) + if err != nil { + return nil, err + } + + content := opts.Content + var lfsMetaObject *models.LFSMetaObject + + if filename2attribute2info[treePath] != nil && filename2attribute2info[treePath]["filter"] == "lfs" { + // OK so we are supposed to LFS this data! + oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content)) + if err != nil { + return nil, err + } + lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID} + content = lfsMetaObject.Pointer() + } + + // Add the object to the database + objectHash, err := t.HashObject(strings.NewReader(content)) + if err != nil { + return nil, err + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil { + return nil, err + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return nil, err + } + + // Now commit the tree + commitHash, err := t.CommitTree(author, committer, treeHash, message) + if err != nil { + return nil, err + } + + if lfsMetaObject != nil { + // We have an LFS object - create it + lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) + if err != nil { + return nil, err + } + contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} + if !contentStore.Exists(lfsMetaObject) { + if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil { + if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { + return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) + } + return nil, err + } + } + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return nil, err + } + + // Simulate push event. + oldCommitID := opts.LastCommitID + if opts.NewBranch != opts.OldBranch || oldCommitID == "" { + oldCommitID = git.EmptySHA + } + + if err = repo.GetOwner(); err != nil { + return nil, fmt.Errorf("GetOwner: %v", err) + } + err = models.PushUpdate( + opts.NewBranch, + models.PushUpdateOptions{ + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.Owner.Name, + RepoName: repo.Name, + RefFullName: git.BranchPrefix + opts.NewBranch, + OldCommitID: oldCommitID, + NewCommitID: commitHash, + }, + ) + if err != nil { + return nil, fmt.Errorf("PushUpdate: %v", err) + } + models.UpdateRepoIndexer(repo) + + commit, err = t.GetCommit(commitHash) + if err != nil { + return nil, err + } + + file, err := GetFileResponseFromCommit(repo, commit, opts.NewBranch, treePath) + if err != nil { + return nil, err + } + return file, nil +} diff --git a/modules/repofiles/update_test.go b/modules/repofiles/update_test.go new file mode 100644 index 0000000000..bf28021793 --- /dev/null +++ b/modules/repofiles/update_test.go @@ -0,0 +1,357 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "testing" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/test" + api "code.gitea.io/sdk/gitea" + + "github.com/stretchr/testify/assert" +) + +func getCreateRepoFileOptions(repo *models.Repository) *UpdateRepoFileOptions { + return &UpdateRepoFileOptions{ + OldBranch: repo.DefaultBranch, + NewBranch: repo.DefaultBranch, + TreePath: "new/file.txt", + Message: "Creates new/file.txt", + Content: "This is a NEW file", + IsNewFile: true, + Author: nil, + Committer: nil, + } +} + +func getUpdateRepoFileOptions(repo *models.Repository) *UpdateRepoFileOptions { + return &UpdateRepoFileOptions{ + OldBranch: repo.DefaultBranch, + NewBranch: repo.DefaultBranch, + TreePath: "README.md", + Message: "Updates README.md", + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + Content: "This is UPDATED content for the README file", + IsNewFile: false, + Author: nil, + Committer: nil, + } +} + +func getExpectedFileResponseForCreate(commitID string) *api.FileResponse { + return &api.FileResponse{ + Content: &api.FileContentResponse{ + Name: "file.txt", + Path: "new/file.txt", + SHA: "103ff9234cefeee5ec5361d22b49fbb04d385885", + Size: 18, + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/contents/new/file.txt", + HTMLURL: "https://try.gitea.io/user2/repo1/blob/master/new/file.txt", + GitURL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/103ff9234cefeee5ec5361d22b49fbb04d385885", + DownloadURL: "https://try.gitea.io/user2/repo1/raw/branch/master/new/file.txt", + Type: "blob", + Links: &api.FileLinksResponse{ + Self: "https://try.gitea.io/api/v1/repos/user2/repo1/contents/new/file.txt", + GitURL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/103ff9234cefeee5ec5361d22b49fbb04d385885", + HTMLURL: "https://try.gitea.io/user2/repo1/blob/master/new/file.txt", + }, + }, + Commit: &api.FileCommitResponse{ + CommitMeta: api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/" + commitID, + SHA: commitID, + }, + HTMLURL: "https://try.gitea.io/user2/repo1/commit/" + commitID, + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: "User Two", + Email: "user2@", + }, + Date: time.Now().UTC().Format(time.RFC3339), + }, + Committer: &api.CommitUser{ + Identity: api.Identity{ + Name: "User Two", + Email: "user2@", + }, + Date: time.Now().UTC().Format(time.RFC3339), + }, + Parents: []*api.CommitMeta{ + { + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + }, + }, + Message: "Updates README.md\n", + Tree: &api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/f93e3a1a1525fb5b91020da86e44810c87a2d7bc", + SHA: "f93e3a1a1525fb5b91020git dda86e44810c87a2d7bc", + }, + }, + Verification: &api.PayloadCommitVerification{ + Verified: false, + Reason: "unsigned", + Signature: "", + Payload: "", + }, + } +} + +func getExpectedFileResponseForUpdate(commitID string) *api.FileResponse { + return &api.FileResponse{ + Content: &api.FileContentResponse{ + Name: "README.md", + Path: "README.md", + SHA: "dbf8d00e022e05b7e5cf7e535de857de57925647", + Size: 43, + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/contents/README.md", + HTMLURL: "https://try.gitea.io/user2/repo1/blob/master/README.md", + GitURL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/dbf8d00e022e05b7e5cf7e535de857de57925647", + DownloadURL: "https://try.gitea.io/user2/repo1/raw/branch/master/README.md", + Type: "blob", + Links: &api.FileLinksResponse{ + Self: "https://try.gitea.io/api/v1/repos/user2/repo1/contents/README.md", + GitURL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/dbf8d00e022e05b7e5cf7e535de857de57925647", + HTMLURL: "https://try.gitea.io/user2/repo1/blob/master/README.md", + }, + }, + Commit: &api.FileCommitResponse{ + CommitMeta: api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/" + commitID, + SHA: commitID, + }, + HTMLURL: "https://try.gitea.io/user2/repo1/commit/" + commitID, + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: "User Two", + Email: "user2@", + }, + Date: time.Now().UTC().Format(time.RFC3339), + }, + Committer: &api.CommitUser{ + Identity: api.Identity{ + Name: "User Two", + Email: "user2@", + }, + Date: time.Now().UTC().Format(time.RFC3339), + }, + Parents: []*api.CommitMeta{ + { + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + }, + }, + Message: "Updates README.md\n", + Tree: &api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/f93e3a1a1525fb5b91020da86e44810c87a2d7bc", + SHA: "f93e3a1a1525fb5b91020da86e44810c87a2d7bc", + }, + }, + Verification: &api.PayloadCommitVerification{ + Verified: false, + Reason: "unsigned", + Signature: "", + Payload: "", + }, + } +} + +func TestCreateOrUpdateRepoFileForCreate(t *testing.T) { + // setup + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + doer := ctx.User + opts := getCreateRepoFileOptions(repo) + + // test + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + + // asserts + assert.Nil(t, err) + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commitID, _ := gitRepo.GetBranchCommitID(opts.NewBranch) + expectedFileResponse := getExpectedFileResponseForCreate(commitID) + assert.EqualValues(t, expectedFileResponse.Content, fileResponse.Content) + assert.EqualValues(t, expectedFileResponse.Commit.SHA, fileResponse.Commit.SHA) + assert.EqualValues(t, expectedFileResponse.Commit.HTMLURL, fileResponse.Commit.HTMLURL) + assert.EqualValues(t, expectedFileResponse.Commit.Author.Email, fileResponse.Commit.Author.Email) + assert.EqualValues(t, expectedFileResponse.Commit.Author.Name, fileResponse.Commit.Author.Name) +} + +func TestCreateOrUpdateRepoFileForUpdate(t *testing.T) { + // setup + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + doer := ctx.User + opts := getUpdateRepoFileOptions(repo) + + // test + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + + // asserts + assert.Nil(t, err) + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commitID, _ := gitRepo.GetBranchCommitID(opts.NewBranch) + expectedFileResponse := getExpectedFileResponseForUpdate(commitID) + assert.EqualValues(t, expectedFileResponse.Content, fileResponse.Content) + assert.EqualValues(t, expectedFileResponse.Commit.SHA, fileResponse.Commit.SHA) + assert.EqualValues(t, expectedFileResponse.Commit.HTMLURL, fileResponse.Commit.HTMLURL) + assert.EqualValues(t, expectedFileResponse.Commit.Author.Email, fileResponse.Commit.Author.Email) + assert.EqualValues(t, expectedFileResponse.Commit.Author.Name, fileResponse.Commit.Author.Name) +} + +func TestCreateOrUpdateRepoFileForUpdateWithFileMove(t *testing.T) { + // setup + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + doer := ctx.User + opts := getUpdateRepoFileOptions(repo) + suffix := "_new" + opts.FromTreePath = "README.md" + opts.TreePath = "README.md" + suffix // new file name, README.md_new + + // test + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + + // asserts + assert.Nil(t, err) + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commit, _ := gitRepo.GetBranchCommit(opts.NewBranch) + expectedFileResponse := getExpectedFileResponseForUpdate(commit.ID.String()) + // assert that the old file no longer exists in the last commit of the branch + fromEntry, err := commit.GetTreeEntryByPath(opts.FromTreePath) + toEntry, err := commit.GetTreeEntryByPath(opts.TreePath) + assert.Nil(t, fromEntry) // Should no longer exist here + assert.NotNil(t, toEntry) // Should exist here + // assert SHA has remained the same but paths use the new file name + assert.EqualValues(t, expectedFileResponse.Content.SHA, fileResponse.Content.SHA) + assert.EqualValues(t, expectedFileResponse.Content.Name+suffix, fileResponse.Content.Name) + assert.EqualValues(t, expectedFileResponse.Content.Path+suffix, fileResponse.Content.Path) + assert.EqualValues(t, expectedFileResponse.Content.URL+suffix, fileResponse.Content.URL) + assert.EqualValues(t, expectedFileResponse.Commit.SHA, fileResponse.Commit.SHA) + assert.EqualValues(t, expectedFileResponse.Commit.HTMLURL, fileResponse.Commit.HTMLURL) +} + +// Test opts with branch names removed, should get same results as above test +func TestCreateOrUpdateRepoFileWithoutBranchNames(t *testing.T) { + // setup + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + doer := ctx.User + opts := getUpdateRepoFileOptions(repo) + opts.OldBranch = "" + opts.NewBranch = "" + + // test + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + + // asserts + assert.Nil(t, err) + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commitID, _ := gitRepo.GetBranchCommitID(repo.DefaultBranch) + expectedFileResponse := getExpectedFileResponseForUpdate(commitID) + assert.EqualValues(t, expectedFileResponse.Content, fileResponse.Content) +} + +func TestCreateOrUpdateRepoFileErrors(t *testing.T) { + // setup + models.PrepareTestEnv(t) + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + repo := ctx.Repo.Repository + doer := ctx.User + + t.Run("bad branch", func(t *testing.T) { + opts := getUpdateRepoFileOptions(repo) + opts.OldBranch = "bad_branch" + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + assert.Error(t, err) + assert.Nil(t, fileResponse) + expectedError := "branch does not exist [name: " + opts.OldBranch + "]" + assert.EqualError(t, err, expectedError) + }) + + t.Run("bad SHA", func(t *testing.T) { + opts := getUpdateRepoFileOptions(repo) + origSHA := opts.SHA + opts.SHA = "bad_sha" + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + assert.Error(t, err) + expectedError := "sha does not match [given: " + opts.SHA + ", expected: " + origSHA + "]" + assert.EqualError(t, err, expectedError) + }) + + t.Run("new branch already exists", func(t *testing.T) { + opts := getUpdateRepoFileOptions(repo) + opts.NewBranch = "develop" + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + assert.Error(t, err) + expectedError := "branch already exists [name: " + opts.NewBranch + "]" + assert.EqualError(t, err, expectedError) + }) + + t.Run("treePath is empty:", func(t *testing.T) { + opts := getUpdateRepoFileOptions(repo) + opts.TreePath = "" + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + assert.Error(t, err) + expectedError := "path contains a malformed path component [path: ]" + assert.EqualError(t, err, expectedError) + }) + + t.Run("treePath is a git directory:", func(t *testing.T) { + opts := getUpdateRepoFileOptions(repo) + opts.TreePath = ".git" + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + assert.Error(t, err) + expectedError := "path contains a malformed path component [path: " + opts.TreePath + "]" + assert.EqualError(t, err, expectedError) + }) + + t.Run("create file that already exists", func(t *testing.T) { + opts := getCreateRepoFileOptions(repo) + opts.TreePath = "README.md" //already exists + fileResponse, err := CreateOrUpdateRepoFile(repo, doer, opts) + assert.Nil(t, fileResponse) + assert.Error(t, err) + expectedError := "repository file already exists [path: " + opts.TreePath + "]" + assert.EqualError(t, err, expectedError) + }) +} diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go new file mode 100644 index 0000000000..ed6a9438c7 --- /dev/null +++ b/modules/repofiles/upload.go @@ -0,0 +1,209 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "fmt" + "os" + "path" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" +) + +// UploadRepoFileOptions contains the uploaded repository file options +type UploadRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + TreePath string + Message string + Files []string // In UUID format. +} + +type uploadInfo struct { + upload *models.Upload + lfsMetaObject *models.LFSMetaObject +} + +func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error { + for _, info := range *infos { + if info.lfsMetaObject == nil { + continue + } + if !info.lfsMetaObject.Existing { + if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil { + original = fmt.Errorf("%v, %v", original, err) + } + } + } + return original +} + +// UploadRepoFiles uploads files to the given repository +func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRepoFileOptions) error { + if len(opts.Files) == 0 { + return nil + } + + uploads, err := models.GetUploadsByUUIDs(opts.Files) + if err != nil { + return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err) + } + + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return err + } + if err := t.Clone(opts.OldBranch); err != nil { + return err + } + if err := t.SetDefaultIndex(); err != nil { + return err + } + + names := make([]string, len(uploads)) + infos := make([]uploadInfo, len(uploads)) + for i, upload := range uploads { + names[i] = upload.Name + infos[i] = uploadInfo{upload: upload} + } + + filename2attribute2info, err := t.CheckAttribute("filter", names...) + if err != nil { + return err + } + + // Copy uploaded files into repository. + for i, uploadInfo := range infos { + file, err := os.Open(uploadInfo.upload.LocalPath()) + if err != nil { + return err + } + defer file.Close() + + var objectHash string + if filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" { + // Handle LFS + // FIXME: Inefficient! this should probably happen in models.Upload + oid, err := models.GenerateLFSOid(file) + if err != nil { + return err + } + fileInfo, err := file.Stat() + if err != nil { + return err + } + + uploadInfo.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID} + + if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer())); err != nil { + return err + } + infos[i] = uploadInfo + + } else { + if objectHash, err = t.HashObject(file); err != nil { + return err + } + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, uploadInfo.upload.Name)); err != nil { + return err + + } + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + // make author and committer the doer + author := doer + committer := doer + + // Now commit the tree + commitHash, err := t.CommitTree(author, committer, treeHash, opts.Message) + if err != nil { + return err + } + + // Now deal with LFS objects + for _, uploadInfo := range infos { + if uploadInfo.lfsMetaObject == nil { + continue + } + uploadInfo.lfsMetaObject, err = models.NewLFSMetaObject(uploadInfo.lfsMetaObject) + if err != nil { + // OK Now we need to cleanup + return cleanUpAfterFailure(&infos, t, err) + } + // Don't move the files yet - we need to ensure that + // everything can be inserted first + } + + // OK now we can insert the data into the store - there's no way to clean up the store + // once it's in there, it's in there. + contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} + for _, uploadInfo := range infos { + if uploadInfo.lfsMetaObject == nil { + continue + } + if !contentStore.Exists(uploadInfo.lfsMetaObject) { + file, err := os.Open(uploadInfo.upload.LocalPath()) + if err != nil { + return cleanUpAfterFailure(&infos, t, err) + } + defer file.Close() + // FIXME: Put regenerates the hash and copies the file over. + // I guess this strictly ensures the soundness of the store but this is inefficient. + if err := contentStore.Put(uploadInfo.lfsMetaObject, file); err != nil { + // OK Now we need to cleanup + // Can't clean up the store, once uploaded there they're there. + return cleanUpAfterFailure(&infos, t, err) + } + } + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return err + } + + // Simulate push event. + oldCommitID := opts.LastCommitID + if opts.NewBranch != opts.OldBranch { + oldCommitID = git.EmptySHA + } + + if err = repo.GetOwner(); err != nil { + return fmt.Errorf("GetOwner: %v", err) + } + err = models.PushUpdate( + opts.NewBranch, + models.PushUpdateOptions{ + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.Owner.Name, + RepoName: repo.Name, + RefFullName: git.BranchPrefix + opts.NewBranch, + OldCommitID: oldCommitID, + NewCommitID: commitHash, + }, + ) + if err != nil { + return fmt.Errorf("PushUpdate: %v", err) + } + // FIXME: Should we models.UpdateRepoIndexer(repo) here? + + return models.DeleteUploads(uploads...) +} diff --git a/modules/repofiles/verification.go b/modules/repofiles/verification.go new file mode 100644 index 0000000000..75ead92d0f --- /dev/null +++ b/modules/repofiles/verification.go @@ -0,0 +1,29 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repofiles + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/sdk/gitea" +) + +// GetPayloadCommitVerification returns the verification information of a commit +func GetPayloadCommitVerification(commit *git.Commit) *gitea.PayloadCommitVerification { + verification := &gitea.PayloadCommitVerification{} + commitVerification := models.ParseCommitWithSignature(commit) + if commit.Signature != nil { + verification.Signature = commit.Signature.Signature + verification.Payload = commit.Signature.Payload + } + if verification.Reason != "" { + verification.Reason = commitVerification.Reason + } else { + if verification.Verified { + verification.Reason = "unsigned" + } + } + return verification +} |