diff options
Diffstat (limited to 'services')
323 files changed, 12695 insertions, 6184 deletions
diff --git a/services/actions/auth.go b/services/actions/auth.go index 1ef21f6e0e..12a8fba53f 100644 --- a/services/actions/auth.go +++ b/services/actions/auth.go @@ -4,6 +4,7 @@ package actions import ( + "errors" "fmt" "net/http" "strings" @@ -80,7 +81,7 @@ func ParseAuthorizationToken(req *http.Request) (int64, error) { parts := strings.SplitN(h, " ", 2) if len(parts) != 2 { log.Error("split token failed: %s", h) - return 0, fmt.Errorf("split token failed") + return 0, errors.New("split token failed") } return TokenToTaskID(parts[1]) @@ -100,7 +101,7 @@ func TokenToTaskID(token string) (int64, error) { c, ok := parsedToken.Claims.(*actionsClaims) if !parsedToken.Valid || !ok { - return 0, fmt.Errorf("invalid token claim") + return 0, errors.New("invalid token claim") } return c.TaskID, nil diff --git a/services/actions/auth_test.go b/services/actions/auth_test.go index 85e7409105..38d0ba7f82 100644 --- a/services/actions/auth_test.go +++ b/services/actions/auth_test.go @@ -18,7 +18,7 @@ func TestCreateAuthorizationToken(t *testing.T) { var taskID int64 = 23 token, err := CreateAuthorizationToken(taskID, 1, 2) assert.NoError(t, err) - assert.NotEqual(t, "", token) + assert.NotEmpty(t, token) claims := jwt.MapClaims{} _, err = jwt.ParseWithClaims(token, claims, func(t *jwt.Token) (any, error) { return setting.GetGeneralTokenSigningSecret(), nil @@ -44,7 +44,7 @@ func TestParseAuthorizationToken(t *testing.T) { var taskID int64 = 23 token, err := CreateAuthorizationToken(taskID, 1, 2) assert.NoError(t, err) - assert.NotEqual(t, "", token) + assert.NotEmpty(t, token) headers := http.Header{} headers.Set("Authorization", "Bearer "+token) rTaskID, err := ParseAuthorizationToken(&http.Request{ diff --git a/services/actions/cleanup.go b/services/actions/cleanup.go index 1223ebcab6..d0cc63e538 100644 --- a/services/actions/cleanup.go +++ b/services/actions/cleanup.go @@ -5,18 +5,23 @@ package actions import ( "context" + "errors" "fmt" "time" actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" actions_module "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" ) -// Cleanup removes expired actions logs, data and artifacts +// Cleanup removes expired actions logs, data, artifacts and used ephemeral runners func Cleanup(ctx context.Context) error { // clean up expired artifacts if err := CleanupArtifacts(ctx); err != nil { @@ -24,10 +29,15 @@ func Cleanup(ctx context.Context) error { } // clean up old logs - if err := CleanupLogs(ctx); err != nil { + if err := CleanupExpiredLogs(ctx); err != nil { return fmt.Errorf("cleanup logs: %w", err) } + // clean up old ephemeral runners + if err := CleanupEphemeralRunners(ctx); err != nil { + return fmt.Errorf("cleanup old ephemeral runners: %w", err) + } + return nil } @@ -52,9 +62,9 @@ func cleanExpiredArtifacts(taskCtx context.Context) error { } if err := storage.ActionsArtifacts.Delete(artifact.StoragePath); err != nil { log.Error("Cannot delete artifact %d: %v", artifact.ID, err) - continue + // go on } - log.Info("Artifact %d set expired", artifact.ID) + log.Info("Artifact %d is deleted (due to expiration)", artifact.ID) } return nil } @@ -76,9 +86,9 @@ func cleanNeedDeleteArtifacts(taskCtx context.Context) error { } if err := storage.ActionsArtifacts.Delete(artifact.StoragePath); err != nil { log.Error("Cannot delete artifact %d: %v", artifact.ID, err) - continue + // go on } - log.Info("Artifact %d set deleted", artifact.ID) + log.Info("Artifact %d is deleted (due to pending deletion)", artifact.ID) } if len(artifacts) < deleteArtifactBatchSize { log.Debug("No more artifacts pending deletion") @@ -90,8 +100,15 @@ func cleanNeedDeleteArtifacts(taskCtx context.Context) error { const deleteLogBatchSize = 100 -// CleanupLogs removes logs which are older than the configured retention time -func CleanupLogs(ctx context.Context) error { +func removeTaskLog(ctx context.Context, task *actions_model.ActionTask) { + if err := actions_module.RemoveLogs(ctx, task.LogInStorage, task.LogFilename); err != nil { + log.Error("Failed to remove log %s (in storage %v) of task %v: %v", task.LogFilename, task.LogInStorage, task.ID, err) + // do not return error here, go on + } +} + +// CleanupExpiredLogs removes logs which are older than the configured retention time +func CleanupExpiredLogs(ctx context.Context) error { olderThan := timeutil.TimeStampNow().AddDuration(-time.Duration(setting.Actions.LogRetentionDays) * 24 * time.Hour) count := 0 @@ -101,11 +118,7 @@ func CleanupLogs(ctx context.Context) error { return fmt.Errorf("find old tasks: %w", err) } for _, task := range tasks { - if err := actions_module.RemoveLogs(ctx, task.LogInStorage, task.LogFilename); err != nil { - log.Error("Failed to remove log %s (in storage %v) of task %v: %v", task.LogFilename, task.LogInStorage, task.ID, err) - // do not return error here, continue to next task - continue - } + removeTaskLog(ctx, task) task.LogIndexes = nil // clear log indexes since it's a heavy field task.LogExpired = true if err := actions_model.UpdateTask(ctx, task, "log_indexes", "log_expired"); err != nil { @@ -124,3 +137,124 @@ func CleanupLogs(ctx context.Context) error { log.Info("Removed %d logs", count) return nil } + +// CleanupEphemeralRunners removes used ephemeral runners which are no longer able to process jobs +func CleanupEphemeralRunners(ctx context.Context) error { + subQuery := builder.Select("`action_runner`.id"). + From(builder.Select("*").From("`action_runner`"), "`action_runner`"). // mysql needs this redundant subquery + Join("INNER", "`action_task`", "`action_task`.`runner_id` = `action_runner`.`id`"). + Where(builder.Eq{"`action_runner`.`ephemeral`": true}). + And(builder.NotIn("`action_task`.`status`", actions_model.StatusWaiting, actions_model.StatusRunning, actions_model.StatusBlocked)) + b := builder.Delete(builder.In("id", subQuery)).From("`action_runner`") + res, err := db.GetEngine(ctx).Exec(b) + if err != nil { + return fmt.Errorf("find runners: %w", err) + } + affected, _ := res.RowsAffected() + log.Info("Removed %d runners", affected) + return nil +} + +// CleanupEphemeralRunnersByPickedTaskOfRepo removes all ephemeral runners that have active/finished tasks on the given repository +func CleanupEphemeralRunnersByPickedTaskOfRepo(ctx context.Context, repoID int64) error { + subQuery := builder.Select("`action_runner`.id"). + From(builder.Select("*").From("`action_runner`"), "`action_runner`"). // mysql needs this redundant subquery + Join("INNER", "`action_task`", "`action_task`.`runner_id` = `action_runner`.`id`"). + Where(builder.And(builder.Eq{"`action_runner`.`ephemeral`": true}, builder.Eq{"`action_task`.`repo_id`": repoID})) + b := builder.Delete(builder.In("id", subQuery)).From("`action_runner`") + res, err := db.GetEngine(ctx).Exec(b) + if err != nil { + return fmt.Errorf("find runners: %w", err) + } + affected, _ := res.RowsAffected() + log.Info("Removed %d runners", affected) + return nil +} + +// DeleteRun deletes workflow run, including all logs and artifacts. +func DeleteRun(ctx context.Context, run *actions_model.ActionRun) error { + if !run.Status.IsDone() { + return errors.New("run is not done") + } + + repoID := run.RepoID + + jobs, err := actions_model.GetRunJobsByRunID(ctx, run.ID) + if err != nil { + return err + } + jobIDs := container.FilterSlice(jobs, func(j *actions_model.ActionRunJob) (int64, bool) { + return j.ID, true + }) + tasks := make(actions_model.TaskList, 0) + if len(jobIDs) > 0 { + if err := db.GetEngine(ctx).Where("repo_id = ?", repoID).In("job_id", jobIDs).Find(&tasks); err != nil { + return err + } + } + + artifacts, err := db.Find[actions_model.ActionArtifact](ctx, actions_model.FindArtifactsOptions{ + RepoID: repoID, + RunID: run.ID, + }) + if err != nil { + return err + } + + var recordsToDelete []any + + recordsToDelete = append(recordsToDelete, &actions_model.ActionRun{ + RepoID: repoID, + ID: run.ID, + }) + recordsToDelete = append(recordsToDelete, &actions_model.ActionRunJob{ + RepoID: repoID, + RunID: run.ID, + }) + for _, tas := range tasks { + recordsToDelete = append(recordsToDelete, &actions_model.ActionTask{ + RepoID: repoID, + ID: tas.ID, + }) + recordsToDelete = append(recordsToDelete, &actions_model.ActionTaskStep{ + RepoID: repoID, + TaskID: tas.ID, + }) + recordsToDelete = append(recordsToDelete, &actions_model.ActionTaskOutput{ + TaskID: tas.ID, + }) + } + recordsToDelete = append(recordsToDelete, &actions_model.ActionArtifact{ + RepoID: repoID, + RunID: run.ID, + }) + + if err := db.WithTx(ctx, func(ctx context.Context) error { + // TODO: Deleting task records could break current ephemeral runner implementation. This is a temporary workaround suggested by ChristopherHX. + // Since you delete potentially the only task an ephemeral act_runner has ever run, please delete the affected runners first. + // one of + // call cleanup ephemeral runners first + // delete affected ephemeral act_runners + // I would make ephemeral runners fully delete directly before formally finishing the task + // + // See also: https://github.com/go-gitea/gitea/pull/34337#issuecomment-2862222788 + if err := CleanupEphemeralRunners(ctx); err != nil { + return err + } + return db.DeleteBeans(ctx, recordsToDelete...) + }); err != nil { + return err + } + + // Delete files on storage + for _, tas := range tasks { + removeTaskLog(ctx, tas) + } + for _, art := range artifacts { + if err := storage.ActionsArtifacts.Delete(art.StoragePath); err != nil { + log.Error("remove artifact file %q: %v", art.StoragePath, err) + } + } + + return nil +} diff --git a/services/actions/clear_tasks.go b/services/actions/clear_tasks.go index 67373782d5..274c04aa57 100644 --- a/services/actions/clear_tasks.go +++ b/services/actions/clear_tasks.go @@ -10,10 +10,13 @@ import ( actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/actions" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + webhook_module "code.gitea.io/gitea/modules/webhook" + notify_service "code.gitea.io/gitea/services/notify" ) // StopZombieTasks stops the task which have running status, but haven't been updated for a long time @@ -32,6 +35,32 @@ func StopEndlessTasks(ctx context.Context) error { }) } +func notifyWorkflowJobStatusUpdate(ctx context.Context, jobs []*actions_model.ActionRunJob) { + if len(jobs) > 0 { + CreateCommitStatus(ctx, jobs...) + for _, job := range jobs { + _ = job.LoadAttributes(ctx) + notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) + } + if len(jobs) > 0 { + job := jobs[0] + notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) + } + } +} + +func CancelPreviousJobs(ctx context.Context, repoID int64, ref, workflowID string, event webhook_module.HookEventType) error { + jobs, err := actions_model.CancelPreviousJobs(ctx, repoID, ref, workflowID, event) + notifyWorkflowJobStatusUpdate(ctx, jobs) + return err +} + +func CleanRepoScheduleTasks(ctx context.Context, repo *repo_model.Repository) error { + jobs, err := actions_model.CleanRepoScheduleTasks(ctx, repo) + notifyWorkflowJobStatusUpdate(ctx, jobs) + return err +} + func stopTasks(ctx context.Context, opts actions_model.FindTaskOptions) error { tasks, err := db.Find[actions_model.ActionTask](ctx, opts) if err != nil { @@ -67,7 +96,7 @@ func stopTasks(ctx context.Context, opts actions_model.FindTaskOptions) error { remove() } - CreateCommitStatus(ctx, jobs...) + notifyWorkflowJobStatusUpdate(ctx, jobs) return nil } @@ -87,14 +116,20 @@ func CancelAbandonedJobs(ctx context.Context) error { for _, job := range jobs { job.Status = actions_model.StatusCancelled job.Stopped = now + updated := false if err := db.WithTx(ctx, func(ctx context.Context) error { - _, err := actions_model.UpdateRunJob(ctx, job, nil, "status", "stopped") + n, err := actions_model.UpdateRunJob(ctx, job, nil, "status", "stopped") + updated = err == nil && n > 0 return err }); err != nil { log.Warn("cancel abandoned job %v: %v", job.ID, err) // go on } CreateCommitStatus(ctx, job) + if updated { + NotifyWorkflowRunStatusUpdateWithReload(ctx, job) + notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) + } } return nil diff --git a/services/actions/commit_status.go b/services/actions/commit_status.go index 94ab89a3b7..ef241e5091 100644 --- a/services/actions/commit_status.go +++ b/services/actions/commit_status.go @@ -5,6 +5,7 @@ package actions import ( "context" + "errors" "fmt" "path" @@ -13,9 +14,9 @@ import ( git_model "code.gitea.io/gitea/models/git" user_model "code.gitea.io/gitea/models/user" actions_module "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/commitstatus" git "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" - api "code.gitea.io/gitea/modules/structs" webhook_module "code.gitea.io/gitea/modules/webhook" commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus" @@ -51,7 +52,7 @@ func createCommitStatus(ctx context.Context, job *actions_model.ActionRunJob) er return fmt.Errorf("GetPushEventPayload: %w", err) } if payload.HeadCommit == nil { - return fmt.Errorf("head commit is missing in event payload") + return errors.New("head commit is missing in event payload") } sha = payload.HeadCommit.ID case // pull_request @@ -71,9 +72,9 @@ func createCommitStatus(ctx context.Context, job *actions_model.ActionRunJob) er return fmt.Errorf("GetPullRequestEventPayload: %w", err) } if payload.PullRequest == nil { - return fmt.Errorf("pull request is missing in event payload") + return errors.New("pull request is missing in event payload") } else if payload.PullRequest.Head == nil { - return fmt.Errorf("head of pull request is missing in event payload") + return errors.New("head of pull request is missing in event payload") } sha = payload.PullRequest.Head.Sha case webhook_module.HookEventRelease: @@ -91,7 +92,7 @@ func createCommitStatus(ctx context.Context, job *actions_model.ActionRunJob) er } ctxname := fmt.Sprintf("%s / %s (%s)", runName, job.Name, event) state := toCommitStatus(job.Status) - if statuses, _, err := git_model.GetLatestCommitStatus(ctx, repo.ID, sha, db.ListOptionsAll); err == nil { + if statuses, err := git_model.GetLatestCommitStatus(ctx, repo.ID, sha, db.ListOptionsAll); err == nil { for _, v := range statuses { if v.Context == ctxname { if v.State == state { @@ -146,16 +147,18 @@ func createCommitStatus(ctx context.Context, job *actions_model.ActionRunJob) er return commitstatus_service.CreateCommitStatus(ctx, repo, creator, commitID.String(), &status) } -func toCommitStatus(status actions_model.Status) api.CommitStatusState { +func toCommitStatus(status actions_model.Status) commitstatus.CommitStatusState { switch status { - case actions_model.StatusSuccess, actions_model.StatusSkipped: - return api.CommitStatusSuccess + case actions_model.StatusSuccess: + return commitstatus.CommitStatusSuccess case actions_model.StatusFailure, actions_model.StatusCancelled: - return api.CommitStatusFailure + return commitstatus.CommitStatusFailure case actions_model.StatusWaiting, actions_model.StatusBlocked, actions_model.StatusRunning: - return api.CommitStatusPending + return commitstatus.CommitStatusPending + case actions_model.StatusSkipped: + return commitstatus.CommitStatusSkipped default: - return api.CommitStatusError + return commitstatus.CommitStatusError } } diff --git a/services/actions/context.go b/services/actions/context.go new file mode 100644 index 0000000000..b6de429ccf --- /dev/null +++ b/services/actions/context.go @@ -0,0 +1,201 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "context" + "fmt" + "strconv" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" + actions_module "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + + "github.com/nektos/act/pkg/model" +) + +type GiteaContext map[string]any + +// GenerateGiteaContext generate the gitea context without token and gitea_runtime_token +// job can be nil when generating a context for parsing workflow-level expressions +func GenerateGiteaContext(run *actions_model.ActionRun, job *actions_model.ActionRunJob) GiteaContext { + event := map[string]any{} + _ = json.Unmarshal([]byte(run.EventPayload), &event) + + baseRef := "" + headRef := "" + ref := run.Ref + sha := run.CommitSHA + if pullPayload, err := run.GetPullRequestEventPayload(); err == nil && pullPayload.PullRequest != nil && pullPayload.PullRequest.Base != nil && pullPayload.PullRequest.Head != nil { + baseRef = pullPayload.PullRequest.Base.Ref + headRef = pullPayload.PullRequest.Head.Ref + + // if the TriggerEvent is pull_request_target, ref and sha need to be set according to the base of pull request + // In GitHub's documentation, ref should be the branch or tag that triggered workflow. But when the TriggerEvent is pull_request_target, + // the ref will be the base branch. + if run.TriggerEvent == actions_module.GithubEventPullRequestTarget { + ref = git.BranchPrefix + pullPayload.PullRequest.Base.Name + sha = pullPayload.PullRequest.Base.Sha + } + } + + refName := git.RefName(ref) + + gitContext := GiteaContext{ + // standard contexts, see https://docs.github.com/en/actions/learn-github-actions/contexts#github-context + "action": "", // string, The name of the action currently running, or the id of a step. GitHub removes special characters, and uses the name __run when the current step runs a script without an id. If you use the same action more than once in the same job, the name will include a suffix with the sequence number with underscore before it. For example, the first script you run will have the name __run, and the second script will be named __run_2. Similarly, the second invocation of actions/checkout will be actionscheckout2. + "action_path": "", // string, The path where an action is located. This property is only supported in composite actions. You can use this path to access files located in the same repository as the action. + "action_ref": "", // string, For a step executing an action, this is the ref of the action being executed. For example, v2. + "action_repository": "", // string, For a step executing an action, this is the owner and repository name of the action. For example, actions/checkout. + "action_status": "", // string, For a composite action, the current result of the composite action. + "actor": run.TriggerUser.Name, // string, The username of the user that triggered the initial workflow run. If the workflow run is a re-run, this value may differ from github.triggering_actor. Any workflow re-runs will use the privileges of github.actor, even if the actor initiating the re-run (github.triggering_actor) has different privileges. + "api_url": setting.AppURL + "api/v1", // string, The URL of the GitHub REST API. + "base_ref": baseRef, // string, The base_ref or target branch of the pull request in a workflow run. This property is only available when the event that triggers a workflow run is either pull_request or pull_request_target. + "env": "", // string, Path on the runner to the file that sets environment variables from workflow commands. This file is unique to the current step and is a different file for each step in a job. For more information, see "Workflow commands for GitHub Actions." + "event": event, // object, The full event webhook payload. You can access individual properties of the event using this context. This object is identical to the webhook payload of the event that triggered the workflow run, and is different for each event. The webhooks for each GitHub Actions event is linked in "Events that trigger workflows." For example, for a workflow run triggered by the push event, this object contains the contents of the push webhook payload. + "event_name": run.TriggerEvent, // string, The name of the event that triggered the workflow run. + "event_path": "", // string, The path to the file on the runner that contains the full event webhook payload. + "graphql_url": "", // string, The URL of the GitHub GraphQL API. + "head_ref": headRef, // string, The head_ref or source branch of the pull request in a workflow run. This property is only available when the event that triggers a workflow run is either pull_request or pull_request_target. + "job": "", // string, The job_id of the current job. + "ref": ref, // string, The fully-formed ref of the branch or tag that triggered the workflow run. For workflows triggered by push, this is the branch or tag ref that was pushed. For workflows triggered by pull_request, this is the pull request merge branch. For workflows triggered by release, this is the release tag created. For other triggers, this is the branch or tag ref that triggered the workflow run. This is only set if a branch or tag is available for the event type. The ref given is fully-formed, meaning that for branches the format is refs/heads/<branch_name>, for pull requests it is refs/pull/<pr_number>/merge, and for tags it is refs/tags/<tag_name>. For example, refs/heads/feature-branch-1. + "ref_name": refName.ShortName(), // string, The short ref name of the branch or tag that triggered the workflow run. This value matches the branch or tag name shown on GitHub. For example, feature-branch-1. + "ref_protected": false, // boolean, true if branch protections are configured for the ref that triggered the workflow run. + "ref_type": string(refName.RefType()), // string, The type of ref that triggered the workflow run. Valid values are branch or tag. + "path": "", // string, Path on the runner to the file that sets system PATH variables from workflow commands. This file is unique to the current step and is a different file for each step in a job. For more information, see "Workflow commands for GitHub Actions." + "repository": run.Repo.OwnerName + "/" + run.Repo.Name, // string, The owner and repository name. For example, Codertocat/Hello-World. + "repository_owner": run.Repo.OwnerName, // string, The repository owner's name. For example, Codertocat. + "repositoryUrl": run.Repo.HTMLURL(), // string, The Git URL to the repository. For example, git://github.com/codertocat/hello-world.git. + "retention_days": "", // string, The number of days that workflow run logs and artifacts are kept. + "run_id": "", // string, A unique number for each workflow run within a repository. This number does not change if you re-run the workflow run. + "run_number": strconv.FormatInt(run.Index, 10), // string, A unique number for each run of a particular workflow in a repository. This number begins at 1 for the workflow's first run, and increments with each new run. This number does not change if you re-run the workflow run. + "run_attempt": "", // string, A unique number for each attempt of a particular workflow run in a repository. This number begins at 1 for the workflow run's first attempt, and increments with each re-run. + "secret_source": "Actions", // string, The source of a secret used in a workflow. Possible values are None, Actions, Dependabot, or Codespaces. + "server_url": setting.AppURL, // string, The URL of the GitHub server. For example: https://github.com. + "sha": sha, // string, The commit SHA that triggered the workflow. The value of this commit SHA depends on the event that triggered the workflow. For more information, see "Events that trigger workflows." For example, ffac537e6cbbf934b08745a378932722df287a53. + "triggering_actor": "", // string, The username of the user that initiated the workflow run. If the workflow run is a re-run, this value may differ from github.actor. Any workflow re-runs will use the privileges of github.actor, even if the actor initiating the re-run (github.triggering_actor) has different privileges. + "workflow": run.WorkflowID, // string, The name of the workflow. If the workflow file doesn't specify a name, the value of this property is the full path of the workflow file in the repository. + "workspace": "", // string, The default working directory on the runner for steps, and the default location of your repository when using the checkout action. + + // additional contexts + "gitea_default_actions_url": setting.Actions.DefaultActionsURL.URL(), + } + + if job != nil { + gitContext["job"] = job.JobID + gitContext["run_id"] = strconv.FormatInt(job.RunID, 10) + gitContext["run_attempt"] = strconv.FormatInt(job.Attempt, 10) + } + + return gitContext +} + +type TaskNeed struct { + Result actions_model.Status + Outputs map[string]string +} + +// FindTaskNeeds finds the `needs` for the task by the task's job +func FindTaskNeeds(ctx context.Context, job *actions_model.ActionRunJob) (map[string]*TaskNeed, error) { + if len(job.Needs) == 0 { + return nil, nil + } + needs := container.SetOf(job.Needs...) + + jobs, err := db.Find[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{RunID: job.RunID}) + if err != nil { + return nil, fmt.Errorf("FindRunJobs: %w", err) + } + + jobIDJobs := make(map[string][]*actions_model.ActionRunJob) + for _, job := range jobs { + jobIDJobs[job.JobID] = append(jobIDJobs[job.JobID], job) + } + + ret := make(map[string]*TaskNeed, len(needs)) + for jobID, jobsWithSameID := range jobIDJobs { + if !needs.Contains(jobID) { + continue + } + var jobOutputs map[string]string + for _, job := range jobsWithSameID { + if job.TaskID == 0 || !job.Status.IsDone() { + // it shouldn't happen, or the job has been rerun + continue + } + got, err := actions_model.FindTaskOutputByTaskID(ctx, job.TaskID) + if err != nil { + return nil, fmt.Errorf("FindTaskOutputByTaskID: %w", err) + } + outputs := make(map[string]string, len(got)) + for _, v := range got { + outputs[v.OutputKey] = v.OutputValue + } + if len(jobOutputs) == 0 { + jobOutputs = outputs + } else { + jobOutputs = mergeTwoOutputs(outputs, jobOutputs) + } + } + ret[jobID] = &TaskNeed{ + Outputs: jobOutputs, + Result: actions_model.AggregateJobStatus(jobsWithSameID), + } + } + return ret, nil +} + +// mergeTwoOutputs merges two outputs from two different ActionRunJobs +// Values with the same output name may be overridden. The user should ensure the output names are unique. +// See https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#using-job-outputs-in-a-matrix-job +func mergeTwoOutputs(o1, o2 map[string]string) map[string]string { + ret := make(map[string]string, len(o1)) + for k1, v1 := range o1 { + if len(v1) > 0 { + ret[k1] = v1 + } else { + ret[k1] = o2[k1] + } + } + return ret +} + +func (g *GiteaContext) ToGitHubContext() *model.GithubContext { + return &model.GithubContext{ + Event: util.GetMapValueOrDefault(*g, "event", map[string]any(nil)), + EventPath: util.GetMapValueOrDefault(*g, "event_path", ""), + Workflow: util.GetMapValueOrDefault(*g, "workflow", ""), + RunID: util.GetMapValueOrDefault(*g, "run_id", ""), + RunNumber: util.GetMapValueOrDefault(*g, "run_number", ""), + Actor: util.GetMapValueOrDefault(*g, "actor", ""), + Repository: util.GetMapValueOrDefault(*g, "repository", ""), + EventName: util.GetMapValueOrDefault(*g, "event_name", ""), + Sha: util.GetMapValueOrDefault(*g, "sha", ""), + Ref: util.GetMapValueOrDefault(*g, "ref", ""), + RefName: util.GetMapValueOrDefault(*g, "ref_name", ""), + RefType: util.GetMapValueOrDefault(*g, "ref_type", ""), + HeadRef: util.GetMapValueOrDefault(*g, "head_ref", ""), + BaseRef: util.GetMapValueOrDefault(*g, "base_ref", ""), + Token: "", // deliberately omitted for security + Workspace: util.GetMapValueOrDefault(*g, "workspace", ""), + Action: util.GetMapValueOrDefault(*g, "action", ""), + ActionPath: util.GetMapValueOrDefault(*g, "action_path", ""), + ActionRef: util.GetMapValueOrDefault(*g, "action_ref", ""), + ActionRepository: util.GetMapValueOrDefault(*g, "action_repository", ""), + Job: util.GetMapValueOrDefault(*g, "job", ""), + JobName: "", // not present in GiteaContext + RepositoryOwner: util.GetMapValueOrDefault(*g, "repository_owner", ""), + RetentionDays: util.GetMapValueOrDefault(*g, "retention_days", ""), + RunnerPerflog: "", // not present in GiteaContext + RunnerTrackingID: "", // not present in GiteaContext + ServerURL: util.GetMapValueOrDefault(*g, "server_url", ""), + APIURL: util.GetMapValueOrDefault(*g, "api_url", ""), + GraphQLURL: util.GetMapValueOrDefault(*g, "graphql_url", ""), + } +} diff --git a/services/actions/context_test.go b/services/actions/context_test.go new file mode 100644 index 0000000000..74ef694021 --- /dev/null +++ b/services/actions/context_test.go @@ -0,0 +1,28 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "testing" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/unittest" + + "github.com/stretchr/testify/assert" +) + +func TestFindTaskNeeds(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + task := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionTask{ID: 51}) + job := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRunJob{ID: task.JobID}) + + ret, err := FindTaskNeeds(t.Context(), job) + assert.NoError(t, err) + assert.Len(t, ret, 1) + assert.Contains(t, ret, "job1") + assert.Len(t, ret["job1"].Outputs, 2) + assert.Equal(t, "abc", ret["job1"].Outputs["output_a"]) + assert.Equal(t, "bbb", ret["job1"].Outputs["output_b"]) +} diff --git a/services/actions/init_test.go b/services/actions/init_test.go index 59c321ccd7..7ef0702204 100644 --- a/services/actions/init_test.go +++ b/services/actions/init_test.go @@ -17,9 +17,7 @@ import ( ) func TestMain(m *testing.M) { - unittest.MainTest(m, &unittest.TestOptions{ - FixtureFiles: []string{"action_runner_token.yml"}, - }) + unittest.MainTest(m) os.Exit(m.Run()) } diff --git a/services/actions/interface.go b/services/actions/interface.go index d4fa782fec..a054c38e4f 100644 --- a/services/actions/interface.go +++ b/services/actions/interface.go @@ -25,4 +25,16 @@ type API interface { UpdateVariable(*context.APIContext) // GetRegistrationToken get registration token GetRegistrationToken(*context.APIContext) + // CreateRegistrationToken get registration token + CreateRegistrationToken(*context.APIContext) + // ListRunners list runners + ListRunners(*context.APIContext) + // GetRunner get a runner + GetRunner(*context.APIContext) + // DeleteRunner delete runner + DeleteRunner(*context.APIContext) + // ListWorkflowJobs list jobs + ListWorkflowJobs(*context.APIContext) + // ListWorkflowRuns list runs + ListWorkflowRuns(*context.APIContext) } diff --git a/services/actions/job_emitter.go b/services/actions/job_emitter.go index 1f859fcf70..47c9f59094 100644 --- a/services/actions/job_emitter.go +++ b/services/actions/job_emitter.go @@ -11,7 +11,9 @@ import ( actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/queue" + notify_service "code.gitea.io/gitea/services/notify" "github.com/nektos/act/pkg/jobparser" "xorm.io/builder" @@ -49,6 +51,7 @@ func checkJobsOfRun(ctx context.Context, runID int64) error { if err != nil { return err } + var updatedjobs []*actions_model.ActionRunJob if err := db.WithTx(ctx, func(ctx context.Context) error { idToJobs := make(map[string][]*actions_model.ActionRunJob, len(jobs)) for _, job := range jobs { @@ -64,6 +67,7 @@ func checkJobsOfRun(ctx context.Context, runID int64) error { } else if n != 1 { return fmt.Errorf("no affected for updating blocked job %v", job.ID) } + updatedjobs = append(updatedjobs, job) } } return nil @@ -71,9 +75,34 @@ func checkJobsOfRun(ctx context.Context, runID int64) error { return err } CreateCommitStatus(ctx, jobs...) + for _, job := range updatedjobs { + _ = job.LoadAttributes(ctx) + notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, nil) + } + if len(jobs) > 0 { + runUpdated := true + for _, job := range jobs { + if !job.Status.IsDone() { + runUpdated = false + break + } + } + if runUpdated { + NotifyWorkflowRunStatusUpdateWithReload(ctx, jobs[0]) + } + } return nil } +func NotifyWorkflowRunStatusUpdateWithReload(ctx context.Context, job *actions_model.ActionRunJob) { + job.Run = nil + if err := job.LoadAttributes(ctx); err != nil { + log.Error("LoadAttributes: %v", err) + return + } + notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) +} + type jobStatusResolver struct { statuses map[int64]actions_model.Status needs map[int64][]int64 diff --git a/services/actions/notifier.go b/services/actions/notifier.go index 67e33e7cce..d10cc0ab34 100644 --- a/services/actions/notifier.go +++ b/services/actions/notifier.go @@ -6,13 +6,16 @@ package actions import ( "context" + actions_model "code.gitea.io/gitea/models/actions" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/organization" packages_model "code.gitea.io/gitea/models/packages" perm_model "code.gitea.io/gitea/models/perm" access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -532,7 +535,7 @@ func (n *actionsNotifier) PushCommits(ctx context.Context, pusher *user_model.Us ctx = withMethod(ctx, "PushCommits") apiPusher := convert.ToUser(ctx, pusher, nil) - apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo.RepoPath(), repo.HTMLURL()) + apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo) if err != nil { log.Error("commits.ToAPIPayloadCommits failed: %v", err) return @@ -563,9 +566,9 @@ func (n *actionsNotifier) CreateRef(ctx context.Context, pusher *user_model.User newNotifyInput(repo, pusher, webhook_module.HookEventCreate). WithRef(refFullName.String()). WithPayload(&api.CreatePayload{ - Ref: refFullName.String(), + Ref: refFullName.String(), // HINT: here is inconsistent with the Webhook's payload: webhook uses ShortName Sha: refID, - RefType: refFullName.RefType(), + RefType: string(refFullName.RefType()), Repo: apiRepo, Sender: apiPusher, }). @@ -580,8 +583,8 @@ func (n *actionsNotifier) DeleteRef(ctx context.Context, pusher *user_model.User newNotifyInput(repo, pusher, webhook_module.HookEventDelete). WithPayload(&api.DeletePayload{ - Ref: refFullName.String(), - RefType: refFullName.RefType(), + Ref: refFullName.String(), // HINT: here is inconsistent with the Webhook's payload: webhook uses ShortName + RefType: string(refFullName.RefType()), PusherType: api.PusherTypeUser, Repo: apiRepo, Sender: apiPusher, @@ -593,7 +596,7 @@ func (n *actionsNotifier) SyncPushCommits(ctx context.Context, pusher *user_mode ctx = withMethod(ctx, "SyncPushCommits") apiPusher := convert.ToUser(ctx, pusher, nil) - apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo.RepoPath(), repo.HTMLURL()) + apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo) if err != nil { log.Error("commits.ToAPIPayloadCommits failed: %v", err) return @@ -762,3 +765,41 @@ func (n *actionsNotifier) MigrateRepository(ctx context.Context, doer, u *user_m Sender: convert.ToUser(ctx, doer, nil), }).Notify(ctx) } + +func (n *actionsNotifier) WorkflowRunStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, run *actions_model.ActionRun) { + ctx = withMethod(ctx, "WorkflowRunStatusUpdate") + + var org *api.Organization + if repo.Owner.IsOrganization() { + org = convert.ToOrganization(ctx, organization.OrgFromUser(repo.Owner)) + } + + status := convert.ToWorkflowRunAction(run.Status) + + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + log.Error("OpenRepository: %v", err) + return + } + defer gitRepo.Close() + + convertedWorkflow, err := convert.GetActionWorkflow(ctx, gitRepo, repo, run.WorkflowID) + if err != nil { + log.Error("GetActionWorkflow: %v", err) + return + } + convertedRun, err := convert.ToActionWorkflowRun(ctx, repo, run) + if err != nil { + log.Error("ToActionWorkflowRun: %v", err) + return + } + + newNotifyInput(repo, sender, webhook_module.HookEventWorkflowRun).WithPayload(&api.WorkflowRunPayload{ + Action: status, + Workflow: convertedWorkflow, + WorkflowRun: convertedRun, + Organization: org, + Repo: convert.ToRepo(ctx, repo, access_model.Permission{AccessMode: perm_model.AccessModeOwner}), + Sender: convert.ToUser(ctx, sender, nil), + }).Notify(ctx) +} diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 323c6a76e4..8010f51a86 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -27,12 +27,15 @@ import ( api "code.gitea.io/gitea/modules/structs" webhook_module "code.gitea.io/gitea/modules/webhook" "code.gitea.io/gitea/services/convert" + notify_service "code.gitea.io/gitea/services/notify" "github.com/nektos/act/pkg/jobparser" "github.com/nektos/act/pkg/model" ) -var methodCtxKey struct{} +type methodCtxKeyType struct{} + +var methodCtxKey methodCtxKeyType // withMethod sets the notification method that this context currently executes. // Used for debugging/ troubleshooting purposes. @@ -43,8 +46,7 @@ func withMethod(ctx context.Context, method string) context.Context { return ctx } } - // FIXME: review the use of this nolint directive - return context.WithValue(ctx, methodCtxKey, method) //nolint:staticcheck + return context.WithValue(ctx, methodCtxKey, method) } // getMethod gets the notification method that this context currently executes. @@ -117,7 +119,7 @@ func (input *notifyInput) Notify(ctx context.Context) { func notify(ctx context.Context, input *notifyInput) error { shouldDetectSchedules := input.Event == webhook_module.HookEventPush && input.Ref.BranchName() == input.Repo.DefaultBranch - if input.Doer.IsActions() { + if input.Doer.IsGiteaActions() { // avoiding triggering cyclically, for example: // a comment of an issue will trigger the runner to add a new comment as reply, // and the new comment will trigger the runner again. @@ -136,7 +138,7 @@ func notify(ctx context.Context, input *notifyInput) error { return nil } if unit_model.TypeActions.UnitGlobalDisabled() { - if err := actions_model.CleanRepoScheduleTasks(ctx, input.Repo); err != nil { + if err := CleanRepoScheduleTasks(ctx, input.Repo); err != nil { log.Error("CleanRepoScheduleTasks: %v", err) } return nil @@ -177,7 +179,7 @@ func notify(ctx context.Context, input *notifyInput) error { return fmt.Errorf("gitRepo.GetCommit: %w", err) } - if skipWorkflows(input, commit) { + if skipWorkflows(ctx, input, commit) { return nil } @@ -242,7 +244,7 @@ func notify(ctx context.Context, input *notifyInput) error { return handleWorkflows(ctx, detectedWorkflows, commit, input, ref.String()) } -func skipWorkflows(input *notifyInput, commit *git.Commit) bool { +func skipWorkflows(ctx context.Context, input *notifyInput, commit *git.Commit) bool { // skip workflow runs with a configured skip-ci string in commit message or pr title if the event is push or pull_request(_sync) // https://docs.github.com/en/actions/managing-workflow-runs/skipping-workflow-runs skipWorkflowEvents := []webhook_module.HookEventType{ @@ -262,6 +264,27 @@ func skipWorkflows(input *notifyInput, commit *git.Commit) bool { } } } + if input.Event == webhook_module.HookEventWorkflowRun { + wrun, ok := input.Payload.(*api.WorkflowRunPayload) + for i := 0; i < 5 && ok && wrun.WorkflowRun != nil; i++ { + if wrun.WorkflowRun.Event != "workflow_run" { + return false + } + r, err := actions_model.GetRunByRepoAndID(ctx, input.Repo.ID, wrun.WorkflowRun.ID) + if err != nil { + log.Error("GetRunByRepoAndID: %v", err) + return true + } + wrun, err = r.GetWorkflowRunEventPayload() + if err != nil { + log.Error("GetWorkflowRunEventPayload: %v", err) + return true + } + } + // skip workflow runs events exceeding the maxiumum of 5 recursive events + log.Debug("repo %s: skipped workflow_run because of recursive event of 5", input.Repo.RepoPath()) + return true + } return false } @@ -301,9 +324,11 @@ func handleWorkflows( run := &actions_model.ActionRun{ Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], RepoID: input.Repo.ID, + Repo: input.Repo, OwnerID: input.Repo.OwnerID, WorkflowID: dwf.EntryName, TriggerUserID: input.Doer.ID, + TriggerUser: input.Doer, Ref: ref, CommitSHA: commit.ID.String(), IsForkPullRequest: isForkPullRequest, @@ -332,16 +357,22 @@ func handleWorkflows( continue } - jobs, err := jobparser.Parse(dwf.Content, jobparser.WithVars(vars)) + giteaCtx := GenerateGiteaContext(run, nil) + + jobs, err := jobparser.Parse(dwf.Content, jobparser.WithVars(vars), jobparser.WithGitContext(giteaCtx.ToGitHubContext())) if err != nil { log.Error("jobparser.Parse: %v", err) continue } + if len(jobs) > 0 && jobs[0].RunName != "" { + run.Title = jobs[0].RunName + } + // cancel running jobs if the event is push or pull_request_sync if run.Event == webhook_module.HookEventPush || run.Event == webhook_module.HookEventPullRequestSync { - if err := actions_model.CancelPreviousJobs( + if err := CancelPreviousJobs( ctx, run.RepoID, run.Ref, @@ -363,6 +394,18 @@ func handleWorkflows( continue } CreateCommitStatus(ctx, alljobs...) + if len(alljobs) > 0 { + job := alljobs[0] + err := job.LoadRun(ctx) + if err != nil { + log.Error("LoadRun: %v", err) + continue + } + notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) + } + for _, job := range alljobs { + notify_service.WorkflowJobStatusUpdate(ctx, input.Repo, input.Doer, job, nil) + } } return nil } @@ -472,7 +515,7 @@ func handleSchedules( log.Error("CountSchedules: %v", err) return err } else if count > 0 { - if err := actions_model.CleanRepoScheduleTasks(ctx, input.Repo); err != nil { + if err := CleanRepoScheduleTasks(ctx, input.Repo); err != nil { log.Error("CleanRepoScheduleTasks: %v", err) } } @@ -504,9 +547,11 @@ func handleSchedules( run := &actions_model.ActionSchedule{ Title: strings.SplitN(commit.CommitMessage, "\n", 2)[0], RepoID: input.Repo.ID, + Repo: input.Repo, OwnerID: input.Repo.OwnerID, WorkflowID: dwf.EntryName, TriggerUserID: user_model.ActionsUserID, + TriggerUser: user_model.NewActionsUser(), Ref: ref, CommitSHA: commit.ID.String(), Event: input.Event, @@ -514,6 +559,25 @@ func handleSchedules( Specs: schedules, Content: dwf.Content, } + + vars, err := actions_model.GetVariablesOfRun(ctx, run.ToActionRun()) + if err != nil { + log.Error("GetVariablesOfRun: %v", err) + continue + } + + giteaCtx := GenerateGiteaContext(run.ToActionRun(), nil) + + jobs, err := jobparser.Parse(dwf.Content, jobparser.WithVars(vars), jobparser.WithGitContext(giteaCtx.ToGitHubContext())) + if err != nil { + log.Error("jobparser.Parse: %v", err) + continue + } + + if len(jobs) > 0 && jobs[0].RunName != "" { + run.Title = jobs[0].RunName + } + crons = append(crons, run) } diff --git a/services/actions/schedule_tasks.go b/services/actions/schedule_tasks.go index 18f3324fd2..c029c5a1a2 100644 --- a/services/actions/schedule_tasks.go +++ b/services/actions/schedule_tasks.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" webhook_module "code.gitea.io/gitea/modules/webhook" + notify_service "code.gitea.io/gitea/services/notify" "github.com/nektos/act/pkg/jobparser" ) @@ -55,7 +56,7 @@ func startTasks(ctx context.Context) error { // cancel running jobs if the event is push if row.Schedule.Event == webhook_module.HookEventPush { // cancel running jobs of the same workflow - if err := actions_model.CancelPreviousJobs( + if err := CancelPreviousJobs( ctx, row.RepoID, row.Schedule.Ref, @@ -148,6 +149,18 @@ func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule) if err := actions_model.InsertRun(ctx, run, workflows); err != nil { return err } + allJobs, err := db.Find[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{RunID: run.ID}) + if err != nil { + log.Error("FindRunJobs: %v", err) + } + err = run.LoadAttributes(ctx) + if err != nil { + log.Error("LoadAttributes: %v", err) + } + notify_service.WorkflowRunStatusUpdate(ctx, run.Repo, run.TriggerUser, run) + for _, job := range allJobs { + notify_service.WorkflowJobStatusUpdate(ctx, run.Repo, run.TriggerUser, job, nil) + } // Return nil if no errors occurred return nil diff --git a/services/actions/task.go b/services/actions/task.go new file mode 100644 index 0000000000..6a547c1c12 --- /dev/null +++ b/services/actions/task.go @@ -0,0 +1,132 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "context" + "errors" + "fmt" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" + secret_model "code.gitea.io/gitea/models/secret" + notify_service "code.gitea.io/gitea/services/notify" + + runnerv1 "code.gitea.io/actions-proto-go/runner/v1" + "google.golang.org/protobuf/types/known/structpb" +) + +func PickTask(ctx context.Context, runner *actions_model.ActionRunner) (*runnerv1.Task, bool, error) { + var ( + task *runnerv1.Task + job *actions_model.ActionRunJob + actionTask *actions_model.ActionTask + ) + + if runner.Ephemeral { + var task actions_model.ActionTask + has, err := db.GetEngine(ctx).Where("runner_id = ?", runner.ID).Get(&task) + // Let the runner retry the request, do not allow to proceed + if err != nil { + return nil, false, err + } + if has { + if task.Status == actions_model.StatusWaiting || task.Status == actions_model.StatusRunning || task.Status == actions_model.StatusBlocked { + return nil, false, nil + } + // task has been finished, remove it + _, err = db.DeleteByID[actions_model.ActionRunner](ctx, runner.ID) + if err != nil { + return nil, false, err + } + return nil, false, errors.New("runner has been removed") + } + } + + if err := db.WithTx(ctx, func(ctx context.Context) error { + t, ok, err := actions_model.CreateTaskForRunner(ctx, runner) + if err != nil { + return fmt.Errorf("CreateTaskForRunner: %w", err) + } + if !ok { + return nil + } + + if err := t.LoadAttributes(ctx); err != nil { + return fmt.Errorf("task LoadAttributes: %w", err) + } + job = t.Job + actionTask = t + + secrets, err := secret_model.GetSecretsOfTask(ctx, t) + if err != nil { + return fmt.Errorf("GetSecretsOfTask: %w", err) + } + + vars, err := actions_model.GetVariablesOfRun(ctx, t.Job.Run) + if err != nil { + return fmt.Errorf("GetVariablesOfRun: %w", err) + } + + needs, err := findTaskNeeds(ctx, job) + if err != nil { + return fmt.Errorf("findTaskNeeds: %w", err) + } + + taskContext, err := generateTaskContext(t) + if err != nil { + return fmt.Errorf("generateTaskContext: %w", err) + } + + task = &runnerv1.Task{ + Id: t.ID, + WorkflowPayload: t.Job.WorkflowPayload, + Context: taskContext, + Secrets: secrets, + Vars: vars, + Needs: needs, + } + + return nil + }); err != nil { + return nil, false, err + } + + if task == nil { + return nil, false, nil + } + + CreateCommitStatus(ctx, job) + notify_service.WorkflowJobStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job, actionTask) + + return task, true, nil +} + +func generateTaskContext(t *actions_model.ActionTask) (*structpb.Struct, error) { + giteaRuntimeToken, err := CreateAuthorizationToken(t.ID, t.Job.RunID, t.JobID) + if err != nil { + return nil, err + } + + gitCtx := GenerateGiteaContext(t.Job.Run, t.Job) + gitCtx["token"] = t.Token + gitCtx["gitea_runtime_token"] = giteaRuntimeToken + + return structpb.NewStruct(gitCtx) +} + +func findTaskNeeds(ctx context.Context, taskJob *actions_model.ActionRunJob) (map[string]*runnerv1.TaskNeed, error) { + taskNeeds, err := FindTaskNeeds(ctx, taskJob) + if err != nil { + return nil, err + } + ret := make(map[string]*runnerv1.TaskNeed, len(taskNeeds)) + for jobID, taskNeed := range taskNeeds { + ret[jobID] = &runnerv1.TaskNeed{ + Outputs: taskNeed.Outputs, + Result: runnerv1.Result(taskNeed.Result), + } + } + return ret, nil +} diff --git a/services/actions/variables.go b/services/actions/variables.go index 8dde9c4af5..2603f1d461 100644 --- a/services/actions/variables.go +++ b/services/actions/variables.go @@ -6,7 +6,6 @@ package actions import ( "context" "regexp" - "strings" actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/modules/log" @@ -14,7 +13,7 @@ import ( secret_service "code.gitea.io/gitea/services/secrets" ) -func CreateVariable(ctx context.Context, ownerID, repoID int64, name, data string) (*actions_model.ActionVariable, error) { +func CreateVariable(ctx context.Context, ownerID, repoID int64, name, data, description string) (*actions_model.ActionVariable, error) { if err := secret_service.ValidateName(name); err != nil { return nil, err } @@ -23,7 +22,7 @@ func CreateVariable(ctx context.Context, ownerID, repoID int64, name, data strin return nil, err } - v, err := actions_model.InsertVariable(ctx, ownerID, repoID, name, util.ReserveLineBreakForTextarea(data)) + v, err := actions_model.InsertVariable(ctx, ownerID, repoID, name, util.ReserveLineBreakForTextarea(data), description) if err != nil { return nil, err } @@ -31,20 +30,18 @@ func CreateVariable(ctx context.Context, ownerID, repoID int64, name, data strin return v, nil } -func UpdateVariable(ctx context.Context, variableID int64, name, data string) (bool, error) { - if err := secret_service.ValidateName(name); err != nil { +func UpdateVariableNameData(ctx context.Context, variable *actions_model.ActionVariable) (bool, error) { + if err := secret_service.ValidateName(variable.Name); err != nil { return false, err } - if err := envNameCIRegexMatch(name); err != nil { + if err := envNameCIRegexMatch(variable.Name); err != nil { return false, err } - return actions_model.UpdateVariable(ctx, &actions_model.ActionVariable{ - ID: variableID, - Name: strings.ToUpper(name), - Data: util.ReserveLineBreakForTextarea(data), - }) + variable.Data = util.ReserveLineBreakForTextarea(variable.Data) + + return actions_model.UpdateVariableCols(ctx, variable, "name", "data", "description") } func DeleteVariableByID(ctx context.Context, variableID int64) error { diff --git a/services/actions/workflow.go b/services/actions/workflow.go new file mode 100644 index 0000000000..233e22b5dd --- /dev/null +++ b/services/actions/workflow.go @@ -0,0 +1,219 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "fmt" + "strings" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/perm" + access_model "code.gitea.io/gitea/models/perm/access" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unit" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/reqctx" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/context" + "code.gitea.io/gitea/services/convert" + notify_service "code.gitea.io/gitea/services/notify" + + "github.com/nektos/act/pkg/jobparser" + "github.com/nektos/act/pkg/model" +) + +func EnableOrDisableWorkflow(ctx *context.APIContext, workflowID string, isEnable bool) error { + workflow, err := convert.GetActionWorkflow(ctx, ctx.Repo.GitRepo, ctx.Repo.Repository, workflowID) + if err != nil { + return err + } + + cfgUnit := ctx.Repo.Repository.MustGetUnit(ctx, unit.TypeActions) + cfg := cfgUnit.ActionsConfig() + + if isEnable { + cfg.EnableWorkflow(workflow.ID) + } else { + cfg.DisableWorkflow(workflow.ID) + } + + return repo_model.UpdateRepoUnit(ctx, cfgUnit) +} + +func DispatchActionWorkflow(ctx reqctx.RequestContext, doer *user_model.User, repo *repo_model.Repository, gitRepo *git.Repository, workflowID, ref string, processInputs func(model *model.WorkflowDispatch, inputs map[string]any) error) error { + if workflowID == "" { + return util.ErrorWrapLocale( + util.NewNotExistErrorf("workflowID is empty"), + "actions.workflow.not_found", workflowID, + ) + } + + if ref == "" { + return util.ErrorWrapLocale( + util.NewNotExistErrorf("ref is empty"), + "form.target_ref_not_exist", ref, + ) + } + + // can not rerun job when workflow is disabled + cfgUnit := repo.MustGetUnit(ctx, unit.TypeActions) + cfg := cfgUnit.ActionsConfig() + if cfg.IsWorkflowDisabled(workflowID) { + return util.ErrorWrapLocale( + util.NewPermissionDeniedErrorf("workflow is disabled"), + "actions.workflow.disabled", + ) + } + + // get target commit of run from specified ref + refName := git.RefName(ref) + var runTargetCommit *git.Commit + var err error + if refName.IsTag() { + runTargetCommit, err = gitRepo.GetTagCommit(refName.TagName()) + } else if refName.IsBranch() { + runTargetCommit, err = gitRepo.GetBranchCommit(refName.BranchName()) + } else { + refName = git.RefNameFromBranch(ref) + runTargetCommit, err = gitRepo.GetBranchCommit(ref) + } + if err != nil { + return util.ErrorWrapLocale( + util.NewNotExistErrorf("ref %q doesn't exist", ref), + "form.target_ref_not_exist", ref, + ) + } + + // get workflow entry from runTargetCommit + _, entries, err := actions.ListWorkflows(runTargetCommit) + if err != nil { + return err + } + + // find workflow from commit + var workflows []*jobparser.SingleWorkflow + var entry *git.TreeEntry + + run := &actions_model.ActionRun{ + Title: strings.SplitN(runTargetCommit.CommitMessage, "\n", 2)[0], + RepoID: repo.ID, + Repo: repo, + OwnerID: repo.OwnerID, + WorkflowID: workflowID, + TriggerUserID: doer.ID, + TriggerUser: doer, + Ref: string(refName), + CommitSHA: runTargetCommit.ID.String(), + IsForkPullRequest: false, + Event: "workflow_dispatch", + TriggerEvent: "workflow_dispatch", + Status: actions_model.StatusWaiting, + } + + for _, e := range entries { + if e.Name() != workflowID { + continue + } + entry = e + break + } + + if entry == nil { + return util.ErrorWrapLocale( + util.NewNotExistErrorf("workflow %q doesn't exist", workflowID), + "actions.workflow.not_found", workflowID, + ) + } + + content, err := actions.GetContentFromEntry(entry) + if err != nil { + return err + } + + giteaCtx := GenerateGiteaContext(run, nil) + + workflows, err = jobparser.Parse(content, jobparser.WithGitContext(giteaCtx.ToGitHubContext())) + if err != nil { + return err + } + + if len(workflows) > 0 && workflows[0].RunName != "" { + run.Title = workflows[0].RunName + } + + if len(workflows) == 0 { + return util.ErrorWrapLocale( + util.NewNotExistErrorf("workflow %q doesn't exist", workflowID), + "actions.workflow.not_found", workflowID, + ) + } + + // get inputs from post + workflow := &model.Workflow{ + RawOn: workflows[0].RawOn, + } + inputsWithDefaults := make(map[string]any) + if workflowDispatch := workflow.WorkflowDispatchConfig(); workflowDispatch != nil { + if err = processInputs(workflowDispatch, inputsWithDefaults); err != nil { + return err + } + } + + // ctx.Req.PostForm -> WorkflowDispatchPayload.Inputs -> ActionRun.EventPayload -> runner: ghc.Event + // https://docs.github.com/en/actions/learn-github-actions/contexts#github-context + // https://docs.github.com/en/webhooks/webhook-events-and-payloads#workflow_dispatch + workflowDispatchPayload := &api.WorkflowDispatchPayload{ + Workflow: workflowID, + Ref: ref, + Repository: convert.ToRepo(ctx, repo, access_model.Permission{AccessMode: perm.AccessModeNone}), + Inputs: inputsWithDefaults, + Sender: convert.ToUserWithAccessMode(ctx, doer, perm.AccessModeNone), + } + + var eventPayload []byte + if eventPayload, err = workflowDispatchPayload.JSONPayload(); err != nil { + return fmt.Errorf("JSONPayload: %w", err) + } + run.EventPayload = string(eventPayload) + + // cancel running jobs of the same workflow + if err := CancelPreviousJobs( + ctx, + run.RepoID, + run.Ref, + run.WorkflowID, + run.Event, + ); err != nil { + log.Error("CancelRunningJobs: %v", err) + } + + // Insert the action run and its associated jobs into the database + if err := actions_model.InsertRun(ctx, run, workflows); err != nil { + return fmt.Errorf("InsertRun: %w", err) + } + + allJobs, err := db.Find[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{RunID: run.ID}) + if err != nil { + log.Error("FindRunJobs: %v", err) + } + CreateCommitStatus(ctx, allJobs...) + if len(allJobs) > 0 { + job := allJobs[0] + err := job.LoadRun(ctx) + if err != nil { + log.Error("LoadRun: %v", err) + } else { + notify_service.WorkflowRunStatusUpdate(ctx, job.Run.Repo, job.Run.TriggerUser, job.Run) + } + } + for _, job := range allJobs { + notify_service.WorkflowJobStatusUpdate(ctx, repo, doer, job, nil) + } + return nil +} diff --git a/services/agit/agit.go b/services/agit/agit.go index 83b12dfcdb..0fe28c5d66 100644 --- a/services/agit/agit.go +++ b/services/agit/agit.go @@ -13,6 +13,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" @@ -56,10 +57,10 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. baseBranchName := opts.RefFullNames[i].ForBranchName() currentTopicBranch := "" - if !gitRepo.IsBranchExist(baseBranchName) { + if !gitrepo.IsBranchExist(ctx, repo, baseBranchName) { // try match refs/for/<target-branch>/<topic-branch> for p, v := range baseBranchName { - if v == '/' && gitRepo.IsBranchExist(baseBranchName[:p]) && p != len(baseBranchName)-1 { + if v == '/' && gitrepo.IsBranchExist(ctx, repo, baseBranchName[:p]) && p != len(baseBranchName)-1 { currentTopicBranch = baseBranchName[p+1:] baseBranchName = baseBranchName[:p] break @@ -153,7 +154,7 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. OriginalRef: opts.RefFullNames[i], OldOID: objectFormat.EmptyObjectID().String(), NewOID: opts.NewCommitIDs[i], - IsCreatePR: true, + IsCreatePR: false, // AGit always creates a pull request so there is no point in prompting user to create one URL: fmt.Sprintf("%s/pulls/%d", repo.HTMLURL(), pr.Index), ShouldShowMessage: setting.Git.PullRequestPushMessage && repo.AllowsPulls(ctx), HeadBranch: headBranch, @@ -182,9 +183,9 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. } if !forcePush.Value() { - output, _, err := git.NewCommand(ctx, "rev-list", "--max-count=1"). + output, _, err := git.NewCommand("rev-list", "--max-count=1"). AddDynamicArguments(oldCommitID, "^"+opts.NewCommitIDs[i]). - RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: os.Environ()}) + RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: os.Environ()}) if err != nil { return nil, fmt.Errorf("failed to detect force push: %w", err) } else if len(output) > 0 { @@ -203,7 +204,7 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. return nil, fmt.Errorf("failed to update pull ref. Error: %w", err) } - pull_service.AddToTaskQueue(ctx, pr) + pull_service.StartPullRequestCheckImmediately(ctx, pr) err = pr.LoadIssue(ctx) if err != nil { return nil, fmt.Errorf("failed to load pull issue. Error: %w", err) diff --git a/services/asymkey/commit.go b/services/asymkey/commit.go new file mode 100644 index 0000000000..148f51fd10 --- /dev/null +++ b/services/asymkey/commit.go @@ -0,0 +1,475 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package asymkey + +import ( + "context" + "fmt" + "slices" + "strings" + + asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/db" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "github.com/42wim/sshsig" + "github.com/ProtonMail/go-crypto/openpgp/packet" +) + +// ParseCommitWithSignature check if signature is good against keystore. +func ParseCommitWithSignature(ctx context.Context, c *git.Commit) *asymkey_model.CommitVerification { + var committer *user_model.User + if c.Committer != nil { + var err error + // Find Committer account + committer, err = user_model.GetUserByEmail(ctx, c.Committer.Email) // This finds the user by primary email or activated email so commit will not be valid if email is not + if err != nil { // Skipping not user for committer + committer = &user_model.User{ + Name: c.Committer.Name, + Email: c.Committer.Email, + } + // We can expect this to often be an ErrUserNotExist. in the case + // it is not, however, it is important to log it. + if !user_model.IsErrUserNotExist(err) { + log.Error("GetUserByEmail: %v", err) + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.no_committer_account", + } + } + } + } + + return ParseCommitWithSignatureCommitter(ctx, c, committer) +} + +func ParseCommitWithSignatureCommitter(ctx context.Context, c *git.Commit, committer *user_model.User) *asymkey_model.CommitVerification { + // If no signature just report the committer + if c.Signature == nil { + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, // Default value + Reason: "gpg.error.not_signed_commit", // Default value + } + } + + // If this a SSH signature handle it differently + if strings.HasPrefix(c.Signature.Signature, "-----BEGIN SSH SIGNATURE-----") { + return ParseCommitWithSSHSignature(ctx, c, committer) + } + + // Parsing signature + sig, err := asymkey_model.ExtractSignature(c.Signature.Signature) + if err != nil { // Skipping failed to extract sign + log.Error("SignatureRead err: %v", err) + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.extract_sign", + } + } + + keyID := asymkey_model.TryGetKeyIDFromSignature(sig) + defaultReason := asymkey_model.NoKeyFound + + // First check if the sig has a keyID and if so just look at that + if commitVerification := HashAndVerifyForKeyID( + ctx, + sig, + c.Signature.Payload, + committer, + keyID, + setting.AppName, + ""); commitVerification != nil { + if commitVerification.Reason == asymkey_model.BadSignature { + defaultReason = asymkey_model.BadSignature + } else { + return commitVerification + } + } + + // Now try to associate the signature with the committer, if present + if committer.ID != 0 { + keys, err := db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{ + OwnerID: committer.ID, + }) + if err != nil { // Skipping failed to get gpg keys of user + log.Error("ListGPGKeys: %v", err) + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.failed_retrieval_gpg_keys", + } + } + + if err := asymkey_model.GPGKeyList(keys).LoadSubKeys(ctx); err != nil { + log.Error("LoadSubKeys: %v", err) + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.failed_retrieval_gpg_keys", + } + } + + committerEmailAddresses, _ := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committer.ID, user_model.GetEmailAddresses) + activated := false + for _, e := range committerEmailAddresses { + if e.IsActivated && strings.EqualFold(e.Email, c.Committer.Email) { + activated = true + break + } + } + + for _, k := range keys { + // Pre-check (& optimization) that emails attached to key can be attached to the committer email and can validate + canValidate := false + email := "" + if k.Verified && activated { + canValidate = true + email = c.Committer.Email + } + if !canValidate { + for _, e := range k.Emails { + if e.IsActivated && strings.EqualFold(e.Email, c.Committer.Email) { + canValidate = true + email = e.Email + break + } + } + } + if !canValidate { + continue // Skip this key + } + + commitVerification := asymkey_model.HashAndVerifyWithSubKeysCommitVerification(sig, c.Signature.Payload, k, committer, committer, email) + if commitVerification != nil { + return commitVerification + } + } + } + + if setting.Repository.Signing.SigningKey != "" && setting.Repository.Signing.SigningKey != "default" && setting.Repository.Signing.SigningKey != "none" { + // OK we should try the default key + gpgSettings := git.GPGSettings{ + Sign: true, + KeyID: setting.Repository.Signing.SigningKey, + Name: setting.Repository.Signing.SigningName, + Email: setting.Repository.Signing.SigningEmail, + } + if err := gpgSettings.LoadPublicKeyContent(); err != nil { + log.Error("Error getting default signing key: %s %v", gpgSettings.KeyID, err) + } else if commitVerification := VerifyWithGPGSettings(ctx, &gpgSettings, sig, c.Signature.Payload, committer, keyID); commitVerification != nil { + if commitVerification.Reason == asymkey_model.BadSignature { + defaultReason = asymkey_model.BadSignature + } else { + return commitVerification + } + } + } + + defaultGPGSettings, err := c.GetRepositoryDefaultPublicGPGKey(false) + if err != nil { + log.Error("Error getting default public gpg key: %v", err) + } else if defaultGPGSettings == nil { + log.Warn("Unable to get defaultGPGSettings for unattached commit: %s", c.ID.String()) + } else if defaultGPGSettings.Sign { + if commitVerification := VerifyWithGPGSettings(ctx, defaultGPGSettings, sig, c.Signature.Payload, committer, keyID); commitVerification != nil { + if commitVerification.Reason == asymkey_model.BadSignature { + defaultReason = asymkey_model.BadSignature + } else { + return commitVerification + } + } + } + + return &asymkey_model.CommitVerification{ // Default at this stage + CommittingUser: committer, + Verified: false, + Warning: defaultReason != asymkey_model.NoKeyFound, + Reason: defaultReason, + SigningKey: &asymkey_model.GPGKey{ + KeyID: keyID, + }, + } +} + +func checkKeyEmails(ctx context.Context, email string, keys ...*asymkey_model.GPGKey) (bool, string) { + uid := int64(0) + var userEmails []*user_model.EmailAddress + var user *user_model.User + for _, key := range keys { + for _, e := range key.Emails { + if e.IsActivated && (email == "" || strings.EqualFold(e.Email, email)) { + return true, e.Email + } + } + if key.Verified && key.OwnerID != 0 { + if uid != key.OwnerID { + userEmails, _ = cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, key.OwnerID, user_model.GetEmailAddresses) + uid = key.OwnerID + user, _ = cache.GetWithContextCache(ctx, cachegroup.User, uid, user_model.GetUserByID) + } + for _, e := range userEmails { + if e.IsActivated && (email == "" || strings.EqualFold(e.Email, email)) { + return true, e.Email + } + } + if user.KeepEmailPrivate && strings.EqualFold(email, user.GetEmail()) { + return true, user.GetEmail() + } + } + } + return false, email +} + +func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload string, committer *user_model.User, keyID, name, email string) *asymkey_model.CommitVerification { + if keyID == "" { + return nil + } + keys, err := cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, keyID, asymkey_model.FindGPGKeyWithSubKeys) + if err != nil { + log.Error("GetGPGKeysByKeyID: %v", err) + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.failed_retrieval_gpg_keys", + } + } + if len(keys) == 0 { + return nil + } + for _, key := range keys { + var primaryKeys []*asymkey_model.GPGKey + if key.PrimaryKeyID != "" { + primaryKeys, err = cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, key.PrimaryKeyID, asymkey_model.FindGPGKeyWithSubKeys) + if err != nil { + log.Error("GetGPGKeysByKeyID: %v", err) + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.failed_retrieval_gpg_keys", + } + } + } + + activated, email := checkKeyEmails(ctx, email, append([]*asymkey_model.GPGKey{key}, primaryKeys...)...) + if !activated { + continue + } + + signer := &user_model.User{ + Name: name, + Email: email, + } + if key.OwnerID > 0 { + owner, err := cache.GetWithContextCache(ctx, cachegroup.User, key.OwnerID, user_model.GetUserByID) + if err == nil { + signer = owner + } else if !user_model.IsErrUserNotExist(err) { + log.Error("Failed to user_model.GetUserByID: %d for key ID: %d (%s) %v", key.OwnerID, key.ID, key.KeyID, err) + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.no_committer_account", + } + } + } + commitVerification := asymkey_model.HashAndVerifyWithSubKeysCommitVerification(sig, payload, key, committer, signer, email) + if commitVerification != nil { + return commitVerification + } + } + // This is a bad situation ... We have a key id that is in our database but the signature doesn't match. + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Warning: true, + Reason: asymkey_model.BadSignature, + } +} + +func VerifyWithGPGSettings(ctx context.Context, gpgSettings *git.GPGSettings, sig *packet.Signature, payload string, committer *user_model.User, keyID string) *asymkey_model.CommitVerification { + // First try to find the key in the db + if commitVerification := HashAndVerifyForKeyID(ctx, sig, payload, committer, gpgSettings.KeyID, gpgSettings.Name, gpgSettings.Email); commitVerification != nil { + return commitVerification + } + + // Otherwise we have to parse the key + ekeys, err := asymkey_model.CheckArmoredGPGKeyString(gpgSettings.PublicKeyContent) + if err != nil { + log.Error("Unable to get default signing key: %v", err) + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.generate_hash", + } + } + for _, ekey := range ekeys { + pubkey := ekey.PrimaryKey + content, err := asymkey_model.Base64EncPubKey(pubkey) + if err != nil { + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.generate_hash", + } + } + k := &asymkey_model.GPGKey{ + Content: content, + CanSign: pubkey.CanSign(), + KeyID: pubkey.KeyIdString(), + } + for _, subKey := range ekey.Subkeys { + content, err := asymkey_model.Base64EncPubKey(subKey.PublicKey) + if err != nil { + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Reason: "gpg.error.generate_hash", + } + } + k.SubsKey = append(k.SubsKey, &asymkey_model.GPGKey{ + Content: content, + CanSign: subKey.PublicKey.CanSign(), + KeyID: subKey.PublicKey.KeyIdString(), + }) + } + if commitVerification := asymkey_model.HashAndVerifyWithSubKeysCommitVerification(sig, payload, k, committer, &user_model.User{ + Name: gpgSettings.Name, + Email: gpgSettings.Email, + }, gpgSettings.Email); commitVerification != nil { + return commitVerification + } + if keyID == k.KeyID { + // This is a bad situation ... We have a key id that matches our default key but the signature doesn't match. + return &asymkey_model.CommitVerification{ + CommittingUser: committer, + Verified: false, + Warning: true, + Reason: asymkey_model.BadSignature, + } + } + } + return nil +} + +func verifySSHCommitVerificationByInstanceKey(c *git.Commit, committerUser, signerUser *user_model.User, committerGitEmail, publicKeyContent string) *asymkey_model.CommitVerification { + fingerprint, err := asymkey_model.CalcFingerprint(publicKeyContent) + if err != nil { + log.Error("Error calculating the fingerprint public key %q, err: %v", publicKeyContent, err) + return nil + } + sshPubKey := &asymkey_model.PublicKey{ + Verified: true, + Content: publicKeyContent, + Fingerprint: fingerprint, + HasUsed: true, + } + return verifySSHCommitVerification(c.Signature.Signature, c.Signature.Payload, sshPubKey, committerUser, signerUser, committerGitEmail) +} + +// ParseCommitWithSSHSignature check if signature is good against keystore. +func ParseCommitWithSSHSignature(ctx context.Context, c *git.Commit, committerUser *user_model.User) *asymkey_model.CommitVerification { + // Now try to associate the signature with the committer, if present + if committerUser.ID != 0 { + keys, err := db.Find[asymkey_model.PublicKey](ctx, asymkey_model.FindPublicKeyOptions{ + OwnerID: committerUser.ID, + NotKeytype: asymkey_model.KeyTypePrincipal, + }) + if err != nil { // Skipping failed to get ssh keys of user + log.Error("ListPublicKeys: %v", err) + return &asymkey_model.CommitVerification{ + CommittingUser: committerUser, + Verified: false, + Reason: "gpg.error.failed_retrieval_gpg_keys", + } + } + + committerEmailAddresses, err := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committerUser.ID, user_model.GetEmailAddresses) + if err != nil { + log.Error("GetEmailAddresses: %v", err) + } + + activated := false + for _, e := range committerEmailAddresses { + if e.IsActivated && strings.EqualFold(e.Email, c.Committer.Email) { + activated = true + break + } + } + + for _, k := range keys { + if k.Verified && activated { + commitVerification := verifySSHCommitVerification(c.Signature.Signature, c.Signature.Payload, k, committerUser, committerUser, c.Committer.Email) + if commitVerification != nil { + return commitVerification + } + } + } + } + + // Try the pre-set trusted keys (for key-rotation purpose) + // At the moment, we still use the SigningName&SigningEmail for the rotated keys. + // Maybe in the future we can extend the key format to "ssh-xxx .... old-user@example.com" to support different signer emails. + for _, k := range setting.Repository.Signing.TrustedSSHKeys { + signerUser := &user_model.User{ + Name: setting.Repository.Signing.SigningName, + Email: setting.Repository.Signing.SigningEmail, + } + commitVerification := verifySSHCommitVerificationByInstanceKey(c, committerUser, signerUser, c.Committer.Email, k) + if commitVerification != nil && commitVerification.Verified { + return commitVerification + } + } + + // Try the configured instance-wide SSH public key + if setting.Repository.Signing.SigningFormat == git.SigningKeyFormatSSH && !slices.Contains([]string{"", "default", "none"}, setting.Repository.Signing.SigningKey) { + gpgSettings := git.GPGSettings{ + Sign: true, + KeyID: setting.Repository.Signing.SigningKey, + Name: setting.Repository.Signing.SigningName, + Email: setting.Repository.Signing.SigningEmail, + Format: setting.Repository.Signing.SigningFormat, + } + signerUser := &user_model.User{ + Name: gpgSettings.Name, + Email: gpgSettings.Email, + } + if err := gpgSettings.LoadPublicKeyContent(); err != nil { + log.Error("Error getting instance-wide SSH signing key %q, err: %v", gpgSettings.KeyID, err) + } else { + commitVerification := verifySSHCommitVerificationByInstanceKey(c, committerUser, signerUser, gpgSettings.Email, gpgSettings.PublicKeyContent) + if commitVerification != nil && commitVerification.Verified { + return commitVerification + } + } + } + + return &asymkey_model.CommitVerification{ + CommittingUser: committerUser, + Verified: false, + Reason: asymkey_model.NoKeyFound, + } +} + +func verifySSHCommitVerification(sig, payload string, k *asymkey_model.PublicKey, committer, signer *user_model.User, email string) *asymkey_model.CommitVerification { + if err := sshsig.Verify(strings.NewReader(payload), []byte(sig), []byte(k.Content), "git"); err != nil { + return nil + } + + return &asymkey_model.CommitVerification{ // Everything is ok + CommittingUser: committer, + Verified: true, + Reason: fmt.Sprintf("%s / %s", signer.Name, k.Fingerprint), + SigningUser: signer, + SigningSSHKey: k, + SigningEmail: email, + } +} diff --git a/services/asymkey/commit_test.go b/services/asymkey/commit_test.go new file mode 100644 index 0000000000..0438209a61 --- /dev/null +++ b/services/asymkey/commit_test.go @@ -0,0 +1,54 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package asymkey + +import ( + "strings" + "testing" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseCommitWithSSHSignature(t *testing.T) { + // Here we only test the TrustedSSHKeys. The complete signing test is in tests/integration/gpg_ssh_git_test.go + t.Run("TrustedSSHKey", func(t *testing.T) { + defer test.MockVariableValue(&setting.Repository.Signing.SigningName, "gitea")() + defer test.MockVariableValue(&setting.Repository.Signing.SigningEmail, "gitea@fake.local")() + defer test.MockVariableValue(&setting.Repository.Signing.TrustedSSHKeys, []string{"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH6Y4idVaW3E+bLw1uqoAfJD7o5Siu+HqS51E9oQLPE9"})() + + commit, err := git.CommitFromReader(nil, git.Sha1ObjectFormat.EmptyObjectID(), strings.NewReader(`tree 9a93ffa76e8b72bdb6431910b3a506fa2b39f42e +author User Two <user2@example.com> 1749230009 +0200 +committer User Two <user2@example.com> 1749230009 +0200 +gpgsig -----BEGIN SSH SIGNATURE----- + U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgfpjiJ1VpbcT5svDW6qgB8kPujl + KK74epLnUT2hAs8T0AAAADZ2l0AAAAAAAAAAZzaGE1MTIAAABTAAAAC3NzaC1lZDI1NTE5 + AAAAQDX2t2iHuuLxEWHLJetYXKsgayv3c43r0pJNfAzdLN55Q65pC5M7rG6++gT2bxcpOu + Y6EXbpLqia9sunEF3+LQY= + -----END SSH SIGNATURE----- + +Initial commit with signed file +`)) + require.NoError(t, err) + committingUser := &user_model.User{ + ID: 2, + Name: "User Two", + Email: "user2@example.com", + } + ret := ParseCommitWithSSHSignature(t.Context(), commit, committingUser) + require.NotNil(t, ret) + assert.True(t, ret.Verified) + assert.False(t, ret.Warning) + assert.Equal(t, committingUser, ret.CommittingUser) + if assert.NotNil(t, ret.SigningUser) { + assert.Equal(t, "gitea", ret.SigningUser.Name) + assert.Equal(t, "gitea@fake.local", ret.SigningUser.Email) + } + }) +} diff --git a/services/asymkey/sign.go b/services/asymkey/sign.go index 2f5d76a293..f94462ea46 100644 --- a/services/asymkey/sign.go +++ b/services/asymkey/sign.go @@ -6,6 +6,7 @@ package asymkey import ( "context" "fmt" + "os" "strings" asymkey_model "code.gitea.io/gitea/models/asymkey" @@ -85,63 +86,87 @@ func IsErrWontSign(err error) bool { } // SigningKey returns the KeyID and git Signature for the repo -func SigningKey(ctx context.Context, repoPath string) (string, *git.Signature) { +func SigningKey(ctx context.Context, repoPath string) (*git.SigningKey, *git.Signature) { if setting.Repository.Signing.SigningKey == "none" { - return "", nil + return nil, nil } if setting.Repository.Signing.SigningKey == "default" || setting.Repository.Signing.SigningKey == "" { // Can ignore the error here as it means that commit.gpgsign is not set - value, _, _ := git.NewCommand(ctx, "config", "--get", "commit.gpgsign").RunStdString(&git.RunOpts{Dir: repoPath}) + value, _, _ := git.NewCommand("config", "--get", "commit.gpgsign").RunStdString(ctx, &git.RunOpts{Dir: repoPath}) sign, valid := git.ParseBool(strings.TrimSpace(value)) if !sign || !valid { - return "", nil + return nil, nil } - signingKey, _, _ := git.NewCommand(ctx, "config", "--get", "user.signingkey").RunStdString(&git.RunOpts{Dir: repoPath}) - signingName, _, _ := git.NewCommand(ctx, "config", "--get", "user.name").RunStdString(&git.RunOpts{Dir: repoPath}) - signingEmail, _, _ := git.NewCommand(ctx, "config", "--get", "user.email").RunStdString(&git.RunOpts{Dir: repoPath}) - return strings.TrimSpace(signingKey), &git.Signature{ - Name: strings.TrimSpace(signingName), - Email: strings.TrimSpace(signingEmail), + format, _, _ := git.NewCommand("config", "--default", git.SigningKeyFormatOpenPGP, "--get", "gpg.format").RunStdString(ctx, &git.RunOpts{Dir: repoPath}) + signingKey, _, _ := git.NewCommand("config", "--get", "user.signingkey").RunStdString(ctx, &git.RunOpts{Dir: repoPath}) + signingName, _, _ := git.NewCommand("config", "--get", "user.name").RunStdString(ctx, &git.RunOpts{Dir: repoPath}) + signingEmail, _, _ := git.NewCommand("config", "--get", "user.email").RunStdString(ctx, &git.RunOpts{Dir: repoPath}) + + if strings.TrimSpace(signingKey) == "" { + return nil, nil } + + return &git.SigningKey{ + KeyID: strings.TrimSpace(signingKey), + Format: strings.TrimSpace(format), + }, &git.Signature{ + Name: strings.TrimSpace(signingName), + Email: strings.TrimSpace(signingEmail), + } } - return setting.Repository.Signing.SigningKey, &git.Signature{ - Name: setting.Repository.Signing.SigningName, - Email: setting.Repository.Signing.SigningEmail, + if setting.Repository.Signing.SigningKey == "" { + return nil, nil } + + return &git.SigningKey{ + KeyID: setting.Repository.Signing.SigningKey, + Format: setting.Repository.Signing.SigningFormat, + }, &git.Signature{ + Name: setting.Repository.Signing.SigningName, + Email: setting.Repository.Signing.SigningEmail, + } } // PublicSigningKey gets the public signing key within a provided repository directory -func PublicSigningKey(ctx context.Context, repoPath string) (string, error) { +func PublicSigningKey(ctx context.Context, repoPath string) (content, format string, err error) { signingKey, _ := SigningKey(ctx, repoPath) - if signingKey == "" { - return "", nil + if signingKey == nil { + return "", "", nil + } + if signingKey.Format == git.SigningKeyFormatSSH { + content, err := os.ReadFile(signingKey.KeyID) + if err != nil { + log.Error("Unable to read SSH public key file in %s: %s, %v", repoPath, signingKey, err) + return "", signingKey.Format, err + } + return string(content), signingKey.Format, nil } content, stderr, err := process.GetManager().ExecDir(ctx, -1, repoPath, - "gpg --export -a", "gpg", "--export", "-a", signingKey) + "gpg --export -a", "gpg", "--export", "-a", signingKey.KeyID) if err != nil { log.Error("Unable to get default signing key in %s: %s, %s, %v", repoPath, signingKey, stderr, err) - return "", err + return "", signingKey.Format, err } - return content, nil + return content, signingKey.Format, nil } // SignInitialCommit determines if we should sign the initial commit to this repository -func SignInitialCommit(ctx context.Context, repoPath string, u *user_model.User) (bool, string, *git.Signature, error) { +func SignInitialCommit(ctx context.Context, repoPath string, u *user_model.User) (bool, *git.SigningKey, *git.Signature, error) { rules := signingModeFromStrings(setting.Repository.Signing.InitialCommit) signingKey, sig := SigningKey(ctx, repoPath) - if signingKey == "" { - return false, "", nil, &ErrWontSign{noKey} + if signingKey == nil { + return false, nil, nil, &ErrWontSign{noKey} } Loop: for _, rule := range rules { switch rule { case never: - return false, "", nil, &ErrWontSign{never} + return false, nil, nil, &ErrWontSign{never} case always: break Loop case pubkey: @@ -150,18 +175,18 @@ Loop: IncludeSubKeys: true, }) if err != nil { - return false, "", nil, err + return false, nil, nil, err } if len(keys) == 0 { - return false, "", nil, &ErrWontSign{pubkey} + return false, nil, nil, &ErrWontSign{pubkey} } case twofa: twofaModel, err := auth.GetTwoFactorByUID(ctx, u.ID) if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { - return false, "", nil, err + return false, nil, nil, err } if twofaModel == nil { - return false, "", nil, &ErrWontSign{twofa} + return false, nil, nil, &ErrWontSign{twofa} } } } @@ -169,19 +194,19 @@ Loop: } // SignWikiCommit determines if we should sign the commits to this repository wiki -func SignWikiCommit(ctx context.Context, repo *repo_model.Repository, u *user_model.User) (bool, string, *git.Signature, error) { +func SignWikiCommit(ctx context.Context, repo *repo_model.Repository, u *user_model.User) (bool, *git.SigningKey, *git.Signature, error) { repoWikiPath := repo.WikiPath() rules := signingModeFromStrings(setting.Repository.Signing.Wiki) signingKey, sig := SigningKey(ctx, repoWikiPath) - if signingKey == "" { - return false, "", nil, &ErrWontSign{noKey} + if signingKey == nil { + return false, nil, nil, &ErrWontSign{noKey} } Loop: for _, rule := range rules { switch rule { case never: - return false, "", nil, &ErrWontSign{never} + return false, nil, nil, &ErrWontSign{never} case always: break Loop case pubkey: @@ -190,35 +215,35 @@ Loop: IncludeSubKeys: true, }) if err != nil { - return false, "", nil, err + return false, nil, nil, err } if len(keys) == 0 { - return false, "", nil, &ErrWontSign{pubkey} + return false, nil, nil, &ErrWontSign{pubkey} } case twofa: twofaModel, err := auth.GetTwoFactorByUID(ctx, u.ID) if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { - return false, "", nil, err + return false, nil, nil, err } if twofaModel == nil { - return false, "", nil, &ErrWontSign{twofa} + return false, nil, nil, &ErrWontSign{twofa} } case parentSigned: - gitRepo, err := gitrepo.OpenWikiRepository(ctx, repo) + gitRepo, err := gitrepo.OpenRepository(ctx, repo.WikiStorageRepo()) if err != nil { - return false, "", nil, err + return false, nil, nil, err } defer gitRepo.Close() commit, err := gitRepo.GetCommit("HEAD") if err != nil { - return false, "", nil, err + return false, nil, nil, err } if commit.Signature == nil { - return false, "", nil, &ErrWontSign{parentSigned} + return false, nil, nil, &ErrWontSign{parentSigned} } - verification := asymkey_model.ParseCommitWithSignature(ctx, commit) + verification := ParseCommitWithSignature(ctx, commit) if !verification.Verified { - return false, "", nil, &ErrWontSign{parentSigned} + return false, nil, nil, &ErrWontSign{parentSigned} } } } @@ -226,18 +251,18 @@ Loop: } // SignCRUDAction determines if we should sign a CRUD commit to this repository -func SignCRUDAction(ctx context.Context, repoPath string, u *user_model.User, tmpBasePath, parentCommit string) (bool, string, *git.Signature, error) { +func SignCRUDAction(ctx context.Context, repoPath string, u *user_model.User, tmpBasePath, parentCommit string) (bool, *git.SigningKey, *git.Signature, error) { rules := signingModeFromStrings(setting.Repository.Signing.CRUDActions) signingKey, sig := SigningKey(ctx, repoPath) - if signingKey == "" { - return false, "", nil, &ErrWontSign{noKey} + if signingKey == nil { + return false, nil, nil, &ErrWontSign{noKey} } Loop: for _, rule := range rules { switch rule { case never: - return false, "", nil, &ErrWontSign{never} + return false, nil, nil, &ErrWontSign{never} case always: break Loop case pubkey: @@ -246,35 +271,35 @@ Loop: IncludeSubKeys: true, }) if err != nil { - return false, "", nil, err + return false, nil, nil, err } if len(keys) == 0 { - return false, "", nil, &ErrWontSign{pubkey} + return false, nil, nil, &ErrWontSign{pubkey} } case twofa: twofaModel, err := auth.GetTwoFactorByUID(ctx, u.ID) if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { - return false, "", nil, err + return false, nil, nil, err } if twofaModel == nil { - return false, "", nil, &ErrWontSign{twofa} + return false, nil, nil, &ErrWontSign{twofa} } case parentSigned: gitRepo, err := git.OpenRepository(ctx, tmpBasePath) if err != nil { - return false, "", nil, err + return false, nil, nil, err } defer gitRepo.Close() commit, err := gitRepo.GetCommit(parentCommit) if err != nil { - return false, "", nil, err + return false, nil, nil, err } if commit.Signature == nil { - return false, "", nil, &ErrWontSign{parentSigned} + return false, nil, nil, &ErrWontSign{parentSigned} } - verification := asymkey_model.ParseCommitWithSignature(ctx, commit) + verification := ParseCommitWithSignature(ctx, commit) if !verification.Verified { - return false, "", nil, &ErrWontSign{parentSigned} + return false, nil, nil, &ErrWontSign{parentSigned} } } } @@ -282,16 +307,16 @@ Loop: } // SignMerge determines if we should sign a PR merge commit to the base repository -func SignMerge(ctx context.Context, pr *issues_model.PullRequest, u *user_model.User, tmpBasePath, baseCommit, headCommit string) (bool, string, *git.Signature, error) { +func SignMerge(ctx context.Context, pr *issues_model.PullRequest, u *user_model.User, tmpBasePath, baseCommit, headCommit string) (bool, *git.SigningKey, *git.Signature, error) { if err := pr.LoadBaseRepo(ctx); err != nil { log.Error("Unable to get Base Repo for pull request") - return false, "", nil, err + return false, nil, nil, err } repo := pr.BaseRepo signingKey, signer := SigningKey(ctx, repo.RepoPath()) - if signingKey == "" { - return false, "", nil, &ErrWontSign{noKey} + if signingKey == nil { + return false, nil, nil, &ErrWontSign{noKey} } rules := signingModeFromStrings(setting.Repository.Signing.Merges) @@ -302,7 +327,7 @@ Loop: for _, rule := range rules { switch rule { case never: - return false, "", nil, &ErrWontSign{never} + return false, nil, nil, &ErrWontSign{never} case always: break Loop case pubkey: @@ -311,91 +336,91 @@ Loop: IncludeSubKeys: true, }) if err != nil { - return false, "", nil, err + return false, nil, nil, err } if len(keys) == 0 { - return false, "", nil, &ErrWontSign{pubkey} + return false, nil, nil, &ErrWontSign{pubkey} } case twofa: twofaModel, err := auth.GetTwoFactorByUID(ctx, u.ID) if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { - return false, "", nil, err + return false, nil, nil, err } if twofaModel == nil { - return false, "", nil, &ErrWontSign{twofa} + return false, nil, nil, &ErrWontSign{twofa} } case approved: protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, pr.BaseBranch) if err != nil { - return false, "", nil, err + return false, nil, nil, err } if protectedBranch == nil { - return false, "", nil, &ErrWontSign{approved} + return false, nil, nil, &ErrWontSign{approved} } if issues_model.GetGrantedApprovalsCount(ctx, protectedBranch, pr) < 1 { - return false, "", nil, &ErrWontSign{approved} + return false, nil, nil, &ErrWontSign{approved} } case baseSigned: if gitRepo == nil { gitRepo, err = git.OpenRepository(ctx, tmpBasePath) if err != nil { - return false, "", nil, err + return false, nil, nil, err } defer gitRepo.Close() } commit, err := gitRepo.GetCommit(baseCommit) if err != nil { - return false, "", nil, err + return false, nil, nil, err } - verification := asymkey_model.ParseCommitWithSignature(ctx, commit) + verification := ParseCommitWithSignature(ctx, commit) if !verification.Verified { - return false, "", nil, &ErrWontSign{baseSigned} + return false, nil, nil, &ErrWontSign{baseSigned} } case headSigned: if gitRepo == nil { gitRepo, err = git.OpenRepository(ctx, tmpBasePath) if err != nil { - return false, "", nil, err + return false, nil, nil, err } defer gitRepo.Close() } commit, err := gitRepo.GetCommit(headCommit) if err != nil { - return false, "", nil, err + return false, nil, nil, err } - verification := asymkey_model.ParseCommitWithSignature(ctx, commit) + verification := ParseCommitWithSignature(ctx, commit) if !verification.Verified { - return false, "", nil, &ErrWontSign{headSigned} + return false, nil, nil, &ErrWontSign{headSigned} } case commitsSigned: if gitRepo == nil { gitRepo, err = git.OpenRepository(ctx, tmpBasePath) if err != nil { - return false, "", nil, err + return false, nil, nil, err } defer gitRepo.Close() } commit, err := gitRepo.GetCommit(headCommit) if err != nil { - return false, "", nil, err + return false, nil, nil, err } - verification := asymkey_model.ParseCommitWithSignature(ctx, commit) + verification := ParseCommitWithSignature(ctx, commit) if !verification.Verified { - return false, "", nil, &ErrWontSign{commitsSigned} + return false, nil, nil, &ErrWontSign{commitsSigned} } // need to work out merge-base mergeBaseCommit, _, err := gitRepo.GetMergeBase("", baseCommit, headCommit) if err != nil { - return false, "", nil, err + return false, nil, nil, err } commitList, err := commit.CommitsBeforeUntil(mergeBaseCommit) if err != nil { - return false, "", nil, err + return false, nil, nil, err } for _, commit := range commitList { - verification := asymkey_model.ParseCommitWithSignature(ctx, commit) + verification := ParseCommitWithSignature(ctx, commit) if !verification.Verified { - return false, "", nil, &ErrWontSign{commitsSigned} + return false, nil, nil, &ErrWontSign{commitsSigned} } } } diff --git a/services/attachment/attachment_test.go b/services/attachment/attachment_test.go index 142bcfe629..65475836be 100644 --- a/services/attachment/attachment_test.go +++ b/services/attachment/attachment_test.go @@ -41,6 +41,6 @@ func TestUploadAttachment(t *testing.T) { attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attach.UUID) assert.NoError(t, err) - assert.EqualValues(t, user.ID, attachment.UploaderID) + assert.Equal(t, user.ID, attachment.UploaderID) assert.Equal(t, int64(0), attachment.DownloadCount) } diff --git a/services/auth/auth.go b/services/auth/auth.go index 43ff95f053..fb6612290b 100644 --- a/services/auth/auth.go +++ b/services/auth/auth.go @@ -9,6 +9,7 @@ import ( "net/http" "regexp" "strings" + "sync" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/auth/webauthn" @@ -21,44 +22,88 @@ import ( user_service "code.gitea.io/gitea/services/user" ) +type globalVarsStruct struct { + gitRawOrAttachPathRe *regexp.Regexp + lfsPathRe *regexp.Regexp + archivePathRe *regexp.Regexp + feedPathRe *regexp.Regexp + feedRefPathRe *regexp.Regexp +} + +var globalVars = sync.OnceValue(func() *globalVarsStruct { + return &globalVarsStruct{ + gitRawOrAttachPathRe: regexp.MustCompile(`^/[-.\w]+/[-.\w]+/(?:(?:git-(?:(?:upload)|(?:receive))-pack$)|(?:info/refs$)|(?:HEAD$)|(?:objects/)|(?:raw/)|(?:releases/download/)|(?:attachments/))`), + lfsPathRe: regexp.MustCompile(`^/[-.\w]+/[-.\w]+/info/lfs/`), + archivePathRe: regexp.MustCompile(`^/[-.\w]+/[-.\w]+/archive/`), + feedPathRe: regexp.MustCompile(`^/[-.\w]+(/[-.\w]+)?\.(rss|atom)$`), // "/owner.rss" or "/owner/repo.atom" + feedRefPathRe: regexp.MustCompile(`^/[-.\w]+/[-.\w]+/(rss|atom)/`), // "/owner/repo/rss/branch/..." + } +}) + // Init should be called exactly once when the application starts to allow plugins // to allocate necessary resources func Init() { webauthn.Init() } +type authPathDetector struct { + req *http.Request + vars *globalVarsStruct +} + +func newAuthPathDetector(req *http.Request) *authPathDetector { + return &authPathDetector{req: req, vars: globalVars()} +} + +// isAPIPath returns true if the specified URL is an API path +func (a *authPathDetector) isAPIPath() bool { + return strings.HasPrefix(a.req.URL.Path, "/api/") +} + // isAttachmentDownload check if request is a file download (GET) with URL to an attachment -func isAttachmentDownload(req *http.Request) bool { - return strings.HasPrefix(req.URL.Path, "/attachments/") && req.Method == "GET" +func (a *authPathDetector) isAttachmentDownload() bool { + return strings.HasPrefix(a.req.URL.Path, "/attachments/") && a.req.Method == http.MethodGet } -// isContainerPath checks if the request targets the container endpoint -func isContainerPath(req *http.Request) bool { - return strings.HasPrefix(req.URL.Path, "/v2/") +func (a *authPathDetector) isFeedRequest(req *http.Request) bool { + if !setting.Other.EnableFeed { + return false + } + if req.Method != http.MethodGet { + return false + } + return a.vars.feedPathRe.MatchString(req.URL.Path) || a.vars.feedRefPathRe.MatchString(req.URL.Path) } -var ( - gitRawOrAttachPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/(?:(?:git-(?:(?:upload)|(?:receive))-pack$)|(?:info/refs$)|(?:HEAD$)|(?:objects/)|(?:raw/)|(?:releases/download/)|(?:attachments/))`) - lfsPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/info/lfs/`) - archivePathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/archive/`) -) +// isContainerPath checks if the request targets the container endpoint +func (a *authPathDetector) isContainerPath() bool { + return strings.HasPrefix(a.req.URL.Path, "/v2/") +} -func isGitRawOrAttachPath(req *http.Request) bool { - return gitRawOrAttachPathRe.MatchString(req.URL.Path) +func (a *authPathDetector) isGitRawOrAttachPath() bool { + return a.vars.gitRawOrAttachPathRe.MatchString(a.req.URL.Path) } -func isGitRawOrAttachOrLFSPath(req *http.Request) bool { - if isGitRawOrAttachPath(req) { +func (a *authPathDetector) isGitRawOrAttachOrLFSPath() bool { + if a.isGitRawOrAttachPath() { return true } if setting.LFS.StartServer { - return lfsPathRe.MatchString(req.URL.Path) + return a.vars.lfsPathRe.MatchString(a.req.URL.Path) } return false } -func isArchivePath(req *http.Request) bool { - return archivePathRe.MatchString(req.URL.Path) +func (a *authPathDetector) isArchivePath() bool { + return a.vars.archivePathRe.MatchString(a.req.URL.Path) +} + +func (a *authPathDetector) isAuthenticatedTokenRequest() bool { + switch a.req.URL.Path { + case "/login/oauth/userinfo", "/login/oauth/introspect": + return true + } + return false } // handleSignIn clears existing session variables and stores new ones for the specified user object @@ -104,7 +149,7 @@ func handleSignIn(resp http.ResponseWriter, req *http.Request, sess SessionStore middleware.SetLocaleCookie(resp, user.Language, 0) // force to generate a new CSRF token - if ctx := gitea_context.GetWebContext(req); ctx != nil { + if ctx := gitea_context.GetWebContext(req.Context()); ctx != nil { ctx.Csrf.PrepareForSessionUser(ctx) } } diff --git a/services/auth/auth_test.go b/services/auth/auth_test.go index 3adaa28664..c45f312c90 100644 --- a/services/auth/auth_test.go +++ b/services/auth/auth_test.go @@ -9,6 +9,9 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" ) func Test_isGitRawOrLFSPath(t *testing.T) { @@ -90,6 +93,19 @@ func Test_isGitRawOrLFSPath(t *testing.T) { true, }, } + + defer test.MockVariableValue(&setting.LFS.StartServer)() + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodPost, "http://localhost"+tt.path, nil) + setting.LFS.StartServer = false + assert.Equal(t, tt.want, newAuthPathDetector(req).isGitRawOrAttachOrLFSPath()) + + setting.LFS.StartServer = true + assert.Equal(t, tt.want, newAuthPathDetector(req).isGitRawOrAttachOrLFSPath()) + }) + } + lfsTests := []string{ "/owner/repo/info/lfs/", "/owner/repo/info/lfs/objects/batch", @@ -101,34 +117,39 @@ func Test_isGitRawOrLFSPath(t *testing.T) { "/owner/repo/info/lfs/locks/verify", "/owner/repo/info/lfs/locks/123/unlock", } - - origLFSStartServer := setting.LFS.StartServer - - for _, tt := range tests { - t.Run(tt.path, func(t *testing.T) { - req, _ := http.NewRequest("POST", "http://localhost"+tt.path, nil) - setting.LFS.StartServer = false - if got := isGitRawOrAttachOrLFSPath(req); got != tt.want { - t.Errorf("isGitOrLFSPath() = %v, want %v", got, tt.want) - } - setting.LFS.StartServer = true - if got := isGitRawOrAttachOrLFSPath(req); got != tt.want { - t.Errorf("isGitOrLFSPath() = %v, want %v", got, tt.want) - } - }) - } for _, tt := range lfsTests { t.Run(tt, func(t *testing.T) { - req, _ := http.NewRequest("POST", tt, nil) + req, _ := http.NewRequest(http.MethodPost, tt, nil) setting.LFS.StartServer = false - if got := isGitRawOrAttachOrLFSPath(req); got != setting.LFS.StartServer { - t.Errorf("isGitOrLFSPath(%q) = %v, want %v, %v", tt, got, setting.LFS.StartServer, gitRawOrAttachPathRe.MatchString(tt)) - } + got := newAuthPathDetector(req).isGitRawOrAttachOrLFSPath() + assert.Equalf(t, setting.LFS.StartServer, got, "isGitOrLFSPath(%q) = %v, want %v, %v", tt, got, setting.LFS.StartServer, globalVars().gitRawOrAttachPathRe.MatchString(tt)) + setting.LFS.StartServer = true - if got := isGitRawOrAttachOrLFSPath(req); got != setting.LFS.StartServer { - t.Errorf("isGitOrLFSPath(%q) = %v, want %v", tt, got, setting.LFS.StartServer) - } + got = newAuthPathDetector(req).isGitRawOrAttachOrLFSPath() + assert.Equalf(t, setting.LFS.StartServer, got, "isGitOrLFSPath(%q) = %v, want %v", tt, got, setting.LFS.StartServer) + }) + } +} + +func Test_isFeedRequest(t *testing.T) { + tests := []struct { + want bool + path string + }{ + {true, "/user.rss"}, + {true, "/user/repo.atom"}, + {false, "/user/repo"}, + {false, "/use/repo/file.rss"}, + + {true, "/org/repo/rss/branch/xxx"}, + {true, "/org/repo/atom/tag/xxx"}, + {false, "/org/repo/branch/main/rss/any"}, + {false, "/org/atom/any"}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, "http://localhost"+tt.path, nil) + assert.Equal(t, tt.want, newAuthPathDetector(req).isFeedRequest(req)) }) } - setting.LFS.StartServer = origLFSStartServer } diff --git a/services/auth/basic.go b/services/auth/basic.go index 6a05b2fe53..b2bd14ef5d 100644 --- a/services/auth/basic.go +++ b/services/auth/basic.go @@ -7,17 +7,15 @@ package auth import ( "errors" "net/http" - "strings" actions_model "code.gitea.io/gitea/models/actions" auth_model "code.gitea.io/gitea/models/auth" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/auth/httpauth" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/modules/web/middleware" ) // Ensure the struct implements the interface. @@ -48,22 +46,22 @@ func (b *Basic) Name() string { // name/token on successful validation. // Returns nil if header is empty or validation fails. func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { - // Basic authentication should only fire on API, Download or on Git or LFSPaths - if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) { + // Basic authentication should only fire on API, Feed, Download or on Git or LFSPaths + // Not all feed (rss/atom) clients feature the ability to add cookies or headers, so we need to allow basic auth for feeds + detector := newAuthPathDetector(req) + if !detector.isAPIPath() && !detector.isFeedRequest(req) && !detector.isContainerPath() && !detector.isAttachmentDownload() && !detector.isGitRawOrAttachOrLFSPath() { return nil, nil } - baHead := req.Header.Get("Authorization") - if len(baHead) == 0 { + authHeader := req.Header.Get("Authorization") + if authHeader == "" { return nil, nil } - - auths := strings.SplitN(baHead, " ", 2) - if len(auths) != 2 || (strings.ToLower(auths[0]) != "basic") { + parsed, ok := httpauth.ParseAuthorizationHeader(authHeader) + if !ok || parsed.BasicAuth == nil { return nil, nil } - - uname, passwd, _ := base.BasicAuthDecode(auths[1]) + uname, passwd := parsed.BasicAuth.Username, parsed.BasicAuth.Password // Check if username or password is a token isUsernameToken := len(passwd) == 0 || passwd == "x-oauth-basic" @@ -141,14 +139,14 @@ func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore return nil, err } - if skipper, ok := source.Cfg.(LocalTwoFASkipper); !ok || !skipper.IsSkipLocalTwoFA() { - // Check if the user has webAuthn registration + if !source.TwoFactorShouldSkip() { + // Check if the user has WebAuthn registration hasWebAuthn, err := auth_model.HasWebAuthnRegistrationsByUID(req.Context(), u.ID) if err != nil { return nil, err } if hasWebAuthn { - return nil, errors.New("Basic authorization is not allowed while webAuthn enrolled") + return nil, errors.New("basic authorization is not allowed while WebAuthn enrolled") } if err := validateTOTP(req, u); err != nil { diff --git a/services/auth/httpsign.go b/services/auth/httpsign.go index 83a36bef23..25e96ff32d 100644 --- a/services/auth/httpsign.go +++ b/services/auth/httpsign.go @@ -134,7 +134,7 @@ func VerifyCert(r *http.Request) (*asymkey_model.PublicKey, error) { // Check if it's really a ssh certificate cert, ok := pk.(*ssh.Certificate) if !ok { - return nil, fmt.Errorf("no certificate found") + return nil, errors.New("no certificate found") } c := &ssh.CertChecker{ @@ -153,7 +153,7 @@ func VerifyCert(r *http.Request) (*asymkey_model.PublicKey, error) { // check the CA of the cert if !c.IsUserAuthority(cert.SignatureKey) { - return nil, fmt.Errorf("CA check failed") + return nil, errors.New("CA check failed") } // Create a verifier @@ -191,7 +191,7 @@ func VerifyCert(r *http.Request) (*asymkey_model.PublicKey, error) { } // No public key matching a principal in the certificate is registered in gitea - return nil, fmt.Errorf("no valid principal found") + return nil, errors.New("no valid principal found") } // doVerify iterates across the provided public keys attempting the verify the current request against each key in turn diff --git a/services/auth/interface.go b/services/auth/interface.go index 275b4dd56c..e7eccecea0 100644 --- a/services/auth/interface.go +++ b/services/auth/interface.go @@ -35,11 +35,6 @@ type PasswordAuthenticator interface { Authenticate(ctx context.Context, user *user_model.User, login, password string) (*user_model.User, error) } -// LocalTwoFASkipper represents a source of authentication that can skip local 2fa -type LocalTwoFASkipper interface { - IsSkipLocalTwoFA() bool -} - // SynchronizableSource represents a source that can synchronize users type SynchronizableSource interface { Sync(ctx context.Context, updateExisting bool) error diff --git a/services/auth/oauth2.go b/services/auth/oauth2.go index 6f2cadd4ab..7df6f4638e 100644 --- a/services/auth/oauth2.go +++ b/services/auth/oauth2.go @@ -13,10 +13,10 @@ import ( actions_model "code.gitea.io/gitea/models/actions" auth_model "code.gitea.io/gitea/models/auth" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/auth/httpauth" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/web/middleware" "code.gitea.io/gitea/services/actions" "code.gitea.io/gitea/services/oauth2_provider" ) @@ -98,9 +98,9 @@ func parseToken(req *http.Request) (string, bool) { // check header token if auHead := req.Header.Get("Authorization"); auHead != "" { - auths := strings.Fields(auHead) - if len(auths) == 2 && (auths[0] == "token" || strings.ToLower(auths[0]) == "bearer") { - return auths[1], true + parsed, ok := httpauth.ParseAuthorizationHeader(auHead) + if ok && parsed.BearerToken != nil { + return parsed.BearerToken.Token, true } } return "", false @@ -162,8 +162,9 @@ func (o *OAuth2) userIDFromToken(ctx context.Context, tokenSHA string, store Dat // Returns nil if verification fails. func (o *OAuth2) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { // These paths are not API paths, but we still want to check for tokens because they maybe in the API returned URLs - if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isAuthenticatedTokenRequest(req) && - !isGitRawOrAttachPath(req) && !isArchivePath(req) { + detector := newAuthPathDetector(req) + if !detector.isAPIPath() && !detector.isAttachmentDownload() && !detector.isAuthenticatedTokenRequest() && + !detector.isGitRawOrAttachPath() && !detector.isArchivePath() { return nil, nil } @@ -190,13 +191,3 @@ func (o *OAuth2) Verify(req *http.Request, w http.ResponseWriter, store DataStor log.Trace("OAuth2 Authorization: Logged in user %-v", user) return user, nil } - -func isAuthenticatedTokenRequest(req *http.Request) bool { - switch req.URL.Path { - case "/login/oauth/userinfo": - fallthrough - case "/login/oauth/introspect": - return true - } - return false -} diff --git a/services/auth/oauth2_test.go b/services/auth/oauth2_test.go index 0d9e793cf3..f003742a94 100644 --- a/services/auth/oauth2_test.go +++ b/services/auth/oauth2_test.go @@ -4,7 +4,6 @@ package auth import ( - "context" "testing" "code.gitea.io/gitea/models/unittest" @@ -26,8 +25,8 @@ func TestUserIDFromToken(t *testing.T) { ds := make(reqctx.ContextData) o := OAuth2{} - uid := o.userIDFromToken(context.Background(), token, ds) - assert.Equal(t, int64(user_model.ActionsUserID), uid) + uid := o.userIDFromToken(t.Context(), token, ds) + assert.Equal(t, user_model.ActionsUserID, uid) assert.Equal(t, true, ds["IsActionsToken"]) assert.Equal(t, ds["ActionsTaskID"], int64(RunningTaskID)) }) @@ -48,7 +47,7 @@ func TestCheckTaskIsRunning(t *testing.T) { for name := range cases { c := cases[name] t.Run(name, func(t *testing.T) { - actual := CheckTaskIsRunning(context.Background(), c.TaskID) + actual := CheckTaskIsRunning(t.Context(), c.TaskID) assert.Equal(t, c.Expected, actual) }) } diff --git a/services/auth/reverseproxy.go b/services/auth/reverseproxy.go index 36b4ef68f4..d6664d738d 100644 --- a/services/auth/reverseproxy.go +++ b/services/auth/reverseproxy.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/web/middleware" gouuid "github.com/google/uuid" ) @@ -117,7 +116,8 @@ func (r *ReverseProxy) Verify(req *http.Request, w http.ResponseWriter, store Da } // Make sure requests to API paths, attachment downloads, git and LFS do not create a new session - if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) { + detector := newAuthPathDetector(req) + if !detector.isAPIPath() && !detector.isAttachmentDownload() && !detector.isGitRawOrAttachOrLFSPath() { if sess != nil && (sess.Get("uid") == nil || sess.Get("uid").(int64) != user.ID) { handleSignIn(w, req, sess, user) } diff --git a/services/auth/source/db/source.go b/services/auth/source/db/source.go index bb2270cbd6..90baa61f5b 100644 --- a/services/auth/source/db/source.go +++ b/services/auth/source/db/source.go @@ -11,7 +11,9 @@ import ( ) // Source is a password authentication service -type Source struct{} +type Source struct { + auth.ConfigBase `json:"-"` +} // FromDB fills up an OAuth2Config from serialized format. func (source *Source) FromDB(bs []byte) error { diff --git a/services/auth/source/ldap/assert_interface_test.go b/services/auth/source/ldap/assert_interface_test.go index 33347687dc..8e8accd668 100644 --- a/services/auth/source/ldap/assert_interface_test.go +++ b/services/auth/source/ldap/assert_interface_test.go @@ -15,13 +15,11 @@ import ( type sourceInterface interface { auth.PasswordAuthenticator auth.SynchronizableSource - auth.LocalTwoFASkipper auth_model.SSHKeyProvider auth_model.Config auth_model.SkipVerifiable auth_model.HasTLSer auth_model.UseTLSer - auth_model.SourceSettable } var _ (sourceInterface) = &ldap.Source{} diff --git a/services/auth/source/ldap/source.go b/services/auth/source/ldap/source.go index 963cdba7c2..d49dbc45ce 100644 --- a/services/auth/source/ldap/source.go +++ b/services/auth/source/ldap/source.go @@ -24,6 +24,8 @@ import ( // Source Basic LDAP authentication service type Source struct { + auth.ConfigBase `json:"-"` + Name string // canonical name (ie. corporate.ad) Host string // LDAP host Port int // port number @@ -54,9 +56,6 @@ type Source struct { GroupTeamMap string // Map LDAP groups to teams GroupTeamMapRemoval bool // Remove user from teams which are synchronized and user is not a member of the corresponding LDAP group UserUID string // User Attribute listed in Group - SkipLocalTwoFA bool `json:",omitempty"` // Skip Local 2fa for users authenticated with this source - - authSource *auth.Source // reference to the authSource } // FromDB fills up a LDAPConfig from serialized format. @@ -109,11 +108,6 @@ func (source *Source) ProvidesSSHKeys() bool { return strings.TrimSpace(source.AttributeSSHPublicKey) != "" } -// SetAuthSource sets the related AuthSource -func (source *Source) SetAuthSource(authSource *auth.Source) { - source.authSource = authSource -} - func init() { auth.RegisterTypeConfig(auth.LDAP, &Source{}) auth.RegisterTypeConfig(auth.DLDAP, &Source{}) diff --git a/services/auth/source/ldap/source_authenticate.go b/services/auth/source/ldap/source_authenticate.go index 6a6c60cd40..6005a4744a 100644 --- a/services/auth/source/ldap/source_authenticate.go +++ b/services/auth/source/ldap/source_authenticate.go @@ -5,7 +5,6 @@ package ldap import ( "context" - "fmt" "strings" asymkey_model "code.gitea.io/gitea/models/asymkey" @@ -26,7 +25,7 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u if user != nil { loginName = user.LoginName } - sr := source.SearchEntry(loginName, password, source.authSource.Type == auth.DLDAP) + sr := source.SearchEntry(loginName, password, source.AuthSource.Type == auth.DLDAP) if sr == nil { // User not in LDAP, do nothing return nil, user_model.ErrUserNotExist{Name: loginName} @@ -41,7 +40,7 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u sr.Username = userName } if sr.Mail == "" { - sr.Mail = fmt.Sprintf("%s@localhost.local", sr.Username) + sr.Mail = sr.Username + "@localhost.local" } isAttributeSSHPublicKeySet := strings.TrimSpace(source.AttributeSSHPublicKey) != "" @@ -59,7 +58,7 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u opts := &user_service.UpdateOptions{} if source.AdminFilter != "" && user.IsAdmin != sr.IsAdmin { // Change existing admin flag only if AdminFilter option is set - opts.IsAdmin = optional.Some(sr.IsAdmin) + opts.IsAdmin = user_service.UpdateOptionFieldFromSync(sr.IsAdmin) } if !sr.IsAdmin && source.RestrictedFilter != "" && user.IsRestricted != sr.IsRestricted { // Change existing restricted flag only if RestrictedFilter option is set @@ -74,7 +73,7 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u } if user != nil { - if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(ctx, user, source.authSource, sr.SSHPublicKey) { + if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(ctx, user, source.AuthSource, sr.SSHPublicKey) { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { return user, err } @@ -85,8 +84,8 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u Name: sr.Username, FullName: composeFullName(sr.Name, sr.Surname, sr.Username), Email: sr.Mail, - LoginType: source.authSource.Type, - LoginSource: source.authSource.ID, + LoginType: source.AuthSource.Type, + LoginSource: source.AuthSource.ID, LoginName: userName, IsAdmin: sr.IsAdmin, } @@ -100,7 +99,7 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u return user, err } - if isAttributeSSHPublicKeySet && asymkey_model.AddPublicKeysBySource(ctx, user, source.authSource, sr.SSHPublicKey) { + if isAttributeSSHPublicKeySet && asymkey_model.AddPublicKeysBySource(ctx, user, source.AuthSource, sr.SSHPublicKey) { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { return user, err } @@ -124,8 +123,3 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u return user, nil } - -// IsSkipLocalTwoFA returns if this source should skip local 2fa for password authentication -func (source *Source) IsSkipLocalTwoFA() bool { - return source.SkipLocalTwoFA -} diff --git a/services/auth/source/ldap/source_search.go b/services/auth/source/ldap/source_search.go index fa2c45ce4a..e6bce04a83 100644 --- a/services/auth/source/ldap/source_search.go +++ b/services/auth/source/ldap/source_search.go @@ -117,10 +117,10 @@ func dial(source *Source) (*ldap.Conn, error) { } if source.SecurityProtocol == SecurityProtocolLDAPS { - return ldap.DialTLS("tcp", net.JoinHostPort(source.Host, strconv.Itoa(source.Port)), tlsConfig) //nolint:staticcheck + return ldap.DialTLS("tcp", net.JoinHostPort(source.Host, strconv.Itoa(source.Port)), tlsConfig) //nolint:staticcheck // DialTLS is deprecated } - conn, err := ldap.Dial("tcp", net.JoinHostPort(source.Host, strconv.Itoa(source.Port))) //nolint:staticcheck + conn, err := ldap.Dial("tcp", net.JoinHostPort(source.Host, strconv.Itoa(source.Port))) //nolint:staticcheck // Dial is deprecated if err != nil { return nil, fmt.Errorf("error during Dial: %w", err) } diff --git a/services/auth/source/ldap/source_sync.go b/services/auth/source/ldap/source_sync.go index e817bf1fa9..7b401c5c96 100644 --- a/services/auth/source/ldap/source_sync.go +++ b/services/auth/source/ldap/source_sync.go @@ -5,7 +5,6 @@ package ldap import ( "context" - "fmt" "strings" asymkey_model "code.gitea.io/gitea/models/asymkey" @@ -23,21 +22,21 @@ import ( // Sync causes this ldap source to synchronize its users with the db func (source *Source) Sync(ctx context.Context, updateExisting bool) error { - log.Trace("Doing: SyncExternalUsers[%s]", source.authSource.Name) + log.Trace("Doing: SyncExternalUsers[%s]", source.AuthSource.Name) isAttributeSSHPublicKeySet := strings.TrimSpace(source.AttributeSSHPublicKey) != "" var sshKeysNeedUpdate bool // Find all users with this login type - FIXME: Should this be an iterator? - users, err := user_model.GetUsersBySource(ctx, source.authSource) + users, err := user_model.GetUsersBySource(ctx, source.AuthSource) if err != nil { log.Error("SyncExternalUsers: %v", err) return err } select { case <-ctx.Done(): - log.Warn("SyncExternalUsers: Cancelled before update of %s", source.authSource.Name) - return db.ErrCancelledf("Before update of %s", source.authSource.Name) + log.Warn("SyncExternalUsers: Cancelled before update of %s", source.AuthSource.Name) + return db.ErrCancelledf("Before update of %s", source.AuthSource.Name) default: } @@ -52,7 +51,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { sr, err := source.SearchEntries() if err != nil { - log.Error("SyncExternalUsers LDAP source failure [%s], skipped", source.authSource.Name) + log.Error("SyncExternalUsers LDAP source failure [%s], skipped", source.AuthSource.Name) return nil } @@ -75,7 +74,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { for _, su := range sr { select { case <-ctx.Done(): - log.Warn("SyncExternalUsers: Cancelled at update of %s before completed update of users", source.authSource.Name) + log.Warn("SyncExternalUsers: Cancelled at update of %s before completed update of users", source.AuthSource.Name) // Rewrite authorized_keys file if LDAP Public SSH Key attribute is set and any key was added or removed if sshKeysNeedUpdate { err = asymkey_service.RewriteAllPublicKeys(ctx) @@ -83,7 +82,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { log.Error("RewriteAllPublicKeys: %v", err) } } - return db.ErrCancelledf("During update of %s before completed update of users", source.authSource.Name) + return db.ErrCancelledf("During update of %s before completed update of users", source.AuthSource.Name) default: } if su.Username == "" && su.Mail == "" { @@ -106,20 +105,20 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { } if su.Mail == "" { - su.Mail = fmt.Sprintf("%s@localhost.local", su.Username) + su.Mail = su.Username + "@localhost.local" } fullName := composeFullName(su.Name, su.Surname, su.Username) // If no existing user found, create one if usr == nil { - log.Trace("SyncExternalUsers[%s]: Creating user %s", source.authSource.Name, su.Username) + log.Trace("SyncExternalUsers[%s]: Creating user %s", source.AuthSource.Name, su.Username) usr = &user_model.User{ LowerName: su.LowerName, Name: su.Username, FullName: fullName, - LoginType: source.authSource.Type, - LoginSource: source.authSource.ID, + LoginType: source.AuthSource.Type, + LoginSource: source.AuthSource.ID, LoginName: su.Username, Email: su.Mail, IsAdmin: su.IsAdmin, @@ -131,12 +130,12 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { err = user_model.CreateUser(ctx, usr, &user_model.Meta{}, overwriteDefault) if err != nil { - log.Error("SyncExternalUsers[%s]: Error creating user %s: %v", source.authSource.Name, su.Username, err) + log.Error("SyncExternalUsers[%s]: Error creating user %s: %v", source.AuthSource.Name, su.Username, err) } if err == nil && isAttributeSSHPublicKeySet { - log.Trace("SyncExternalUsers[%s]: Adding LDAP Public SSH Keys for user %s", source.authSource.Name, usr.Name) - if asymkey_model.AddPublicKeysBySource(ctx, usr, source.authSource, su.SSHPublicKey) { + log.Trace("SyncExternalUsers[%s]: Adding LDAP Public SSH Keys for user %s", source.AuthSource.Name, usr.Name) + if asymkey_model.AddPublicKeysBySource(ctx, usr, source.AuthSource, su.SSHPublicKey) { sshKeysNeedUpdate = true } } @@ -146,7 +145,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { } } else if updateExisting { // Synchronize SSH Public Key if that attribute is set - if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(ctx, usr, source.authSource, su.SSHPublicKey) { + if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(ctx, usr, source.AuthSource, su.SSHPublicKey) { sshKeysNeedUpdate = true } @@ -156,14 +155,14 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { !strings.EqualFold(usr.Email, su.Mail) || usr.FullName != fullName || !usr.IsActive { - log.Trace("SyncExternalUsers[%s]: Updating user %s", source.authSource.Name, usr.Name) + log.Trace("SyncExternalUsers[%s]: Updating user %s", source.AuthSource.Name, usr.Name) opts := &user_service.UpdateOptions{ FullName: optional.Some(fullName), IsActive: optional.Some(true), } if source.AdminFilter != "" { - opts.IsAdmin = optional.Some(su.IsAdmin) + opts.IsAdmin = user_service.UpdateOptionFieldFromSync(su.IsAdmin) } // Change existing restricted flag only if RestrictedFilter option is set if !su.IsAdmin && source.RestrictedFilter != "" { @@ -171,16 +170,17 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { } if err := user_service.UpdateUser(ctx, usr, opts); err != nil { - log.Error("SyncExternalUsers[%s]: Error updating user %s: %v", source.authSource.Name, usr.Name, err) + log.Error("SyncExternalUsers[%s]: Error updating user %s: %v", source.AuthSource.Name, usr.Name, err) } if err := user_service.ReplacePrimaryEmailAddress(ctx, usr, su.Mail); err != nil { - log.Error("SyncExternalUsers[%s]: Error updating user %s primary email %s: %v", source.authSource.Name, usr.Name, su.Mail, err) + log.Error("SyncExternalUsers[%s]: Error updating user %s primary email %s: %v", source.AuthSource.Name, usr.Name, su.Mail, err) } } - if usr.IsUploadAvatarChanged(su.Avatar) { - if err == nil && source.AttributeAvatar != "" { + if source.AttributeAvatar != "" { + if len(su.Avatar) > 0 && usr.IsUploadAvatarChanged(su.Avatar) { + log.Trace("SyncExternalUsers[%s]: Uploading new avatar for %s", source.AuthSource.Name, usr.Name) _ = user_service.UploadAvatar(ctx, usr, su.Avatar) } } @@ -203,8 +203,8 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { select { case <-ctx.Done(): - log.Warn("SyncExternalUsers: Cancelled during update of %s before delete users", source.authSource.Name) - return db.ErrCancelledf("During update of %s before delete users", source.authSource.Name) + log.Warn("SyncExternalUsers: Cancelled during update of %s before delete users", source.AuthSource.Name) + return db.ErrCancelledf("During update of %s before delete users", source.AuthSource.Name) default: } @@ -215,13 +215,13 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { continue } - log.Trace("SyncExternalUsers[%s]: Deactivating user %s", source.authSource.Name, usr.Name) + log.Trace("SyncExternalUsers[%s]: Deactivating user %s", source.AuthSource.Name, usr.Name) opts := &user_service.UpdateOptions{ IsActive: optional.Some(false), } if err := user_service.UpdateUser(ctx, usr, opts); err != nil { - log.Error("SyncExternalUsers[%s]: Error deactivating user %s: %v", source.authSource.Name, usr.Name, err) + log.Error("SyncExternalUsers[%s]: Error deactivating user %s: %v", source.AuthSource.Name, usr.Name, err) } } } diff --git a/services/auth/source/oauth2/assert_interface_test.go b/services/auth/source/oauth2/assert_interface_test.go index 56fe0e4aa8..d870ac1dcd 100644 --- a/services/auth/source/oauth2/assert_interface_test.go +++ b/services/auth/source/oauth2/assert_interface_test.go @@ -14,7 +14,6 @@ import ( type sourceInterface interface { auth_model.Config - auth_model.SourceSettable auth_model.RegisterableSource auth.PasswordAuthenticator } diff --git a/services/auth/source/oauth2/source.go b/services/auth/source/oauth2/source.go index 3454c9ad55..08837de377 100644 --- a/services/auth/source/oauth2/source.go +++ b/services/auth/source/oauth2/source.go @@ -10,6 +10,8 @@ import ( // Source holds configuration for the OAuth2 login source. type Source struct { + auth.ConfigBase `json:"-"` + Provider string ClientID string ClientSecret string @@ -25,10 +27,6 @@ type Source struct { GroupTeamMap string GroupTeamMapRemoval bool RestrictedGroup string - SkipLocalTwoFA bool `json:",omitempty"` - - // reference to the authSource - authSource *auth.Source } // FromDB fills up an OAuth2Config from serialized format. @@ -41,11 +39,6 @@ func (source *Source) ToDB() ([]byte, error) { return json.Marshal(source) } -// SetAuthSource sets the related AuthSource -func (source *Source) SetAuthSource(authSource *auth.Source) { - source.authSource = authSource -} - func init() { auth.RegisterTypeConfig(auth.OAuth2, &Source{}) } diff --git a/services/auth/source/oauth2/source_callout.go b/services/auth/source/oauth2/source_callout.go index 8d70bee248..f09d25c772 100644 --- a/services/auth/source/oauth2/source_callout.go +++ b/services/auth/source/oauth2/source_callout.go @@ -13,7 +13,7 @@ import ( // Callout redirects request/response pair to authenticate against the provider func (source *Source) Callout(request *http.Request, response http.ResponseWriter) error { // not sure if goth is thread safe (?) when using multiple providers - request.Header.Set(ProviderHeaderKey, source.authSource.Name) + request.Header.Set(ProviderHeaderKey, source.AuthSource.Name) // don't use the default gothic begin handler to prevent issues when some error occurs // normally the gothic library will write some custom stuff to the response instead of our own nice error page @@ -33,7 +33,7 @@ func (source *Source) Callout(request *http.Request, response http.ResponseWrite // this will trigger a new authentication request, but because we save it in the session we can use that func (source *Source) Callback(request *http.Request, response http.ResponseWriter) (goth.User, error) { // not sure if goth is thread safe (?) when using multiple providers - request.Header.Set(ProviderHeaderKey, source.authSource.Name) + request.Header.Set(ProviderHeaderKey, source.AuthSource.Name) gothRWMutex.RLock() defer gothRWMutex.RUnlock() diff --git a/services/auth/source/oauth2/source_register.go b/services/auth/source/oauth2/source_register.go index 82a36acaa6..12da56c11b 100644 --- a/services/auth/source/oauth2/source_register.go +++ b/services/auth/source/oauth2/source_register.go @@ -9,13 +9,13 @@ import ( // RegisterSource causes an OAuth2 configuration to be registered func (source *Source) RegisterSource() error { - err := RegisterProviderWithGothic(source.authSource.Name, source) - return wrapOpenIDConnectInitializeError(err, source.authSource.Name, source) + err := RegisterProviderWithGothic(source.AuthSource.Name, source) + return wrapOpenIDConnectInitializeError(err, source.AuthSource.Name, source) } // UnregisterSource causes an OAuth2 configuration to be unregistered func (source *Source) UnregisterSource() error { - RemoveProviderFromGothic(source.authSource.Name) + RemoveProviderFromGothic(source.AuthSource.Name) return nil } diff --git a/services/auth/source/oauth2/source_sync.go b/services/auth/source/oauth2/source_sync.go index 5e30313c8f..c2e3dfb1a8 100644 --- a/services/auth/source/oauth2/source_sync.go +++ b/services/auth/source/oauth2/source_sync.go @@ -18,27 +18,27 @@ import ( // Sync causes this OAuth2 source to synchronize its users with the db. func (source *Source) Sync(ctx context.Context, updateExisting bool) error { - log.Trace("Doing: SyncExternalUsers[%s] %d", source.authSource.Name, source.authSource.ID) + log.Trace("Doing: SyncExternalUsers[%s] %d", source.AuthSource.Name, source.AuthSource.ID) if !updateExisting { - log.Info("SyncExternalUsers[%s] not running since updateExisting is false", source.authSource.Name) + log.Info("SyncExternalUsers[%s] not running since updateExisting is false", source.AuthSource.Name) return nil } - provider, err := createProvider(source.authSource.Name, source) + provider, err := createProvider(source.AuthSource.Name, source) if err != nil { return err } if !provider.RefreshTokenAvailable() { - log.Trace("SyncExternalUsers[%s] provider doesn't support refresh tokens, can't synchronize", source.authSource.Name) + log.Trace("SyncExternalUsers[%s] provider doesn't support refresh tokens, can't synchronize", source.AuthSource.Name) return nil } opts := user_model.FindExternalUserOptions{ HasRefreshToken: true, Expired: true, - LoginSourceID: source.authSource.ID, + LoginSourceID: source.AuthSource.ID, } return user_model.IterateExternalLogin(ctx, opts, func(ctx context.Context, u *user_model.ExternalLoginUser) error { @@ -77,7 +77,7 @@ func (source *Source) refresh(ctx context.Context, provider goth.Provider, u *us // recognizes them as a valid user, they will be able to login // via their provider and reactivate their account. if shouldDisable { - log.Info("SyncExternalUsers[%s] disabling user %d", source.authSource.Name, user.ID) + log.Info("SyncExternalUsers[%s] disabling user %d", source.AuthSource.Name, user.ID) return db.WithTx(ctx, func(ctx context.Context) error { if hasUser { diff --git a/services/auth/source/oauth2/source_sync_test.go b/services/auth/source/oauth2/source_sync_test.go index 893ed62502..2927f3634b 100644 --- a/services/auth/source/oauth2/source_sync_test.go +++ b/services/auth/source/oauth2/source_sync_test.go @@ -4,7 +4,6 @@ package oauth2 import ( - "context" "testing" "code.gitea.io/gitea/models/auth" @@ -19,24 +18,26 @@ func TestSource(t *testing.T) { source := &Source{ Provider: "fake", - authSource: &auth.Source{ - ID: 12, - Type: auth.OAuth2, - Name: "fake", - IsActive: true, - IsSyncEnabled: true, + ConfigBase: auth.ConfigBase{ + AuthSource: &auth.Source{ + ID: 12, + Type: auth.OAuth2, + Name: "fake", + IsActive: true, + IsSyncEnabled: true, + }, }, } user := &user_model.User{ LoginName: "external", LoginType: auth.OAuth2, - LoginSource: source.authSource.ID, + LoginSource: source.AuthSource.ID, Name: "test", Email: "external@example.com", } - err := user_model.CreateUser(context.Background(), user, &user_model.Meta{}, &user_model.CreateUserOverwriteOptions{}) + err := user_model.CreateUser(t.Context(), user, &user_model.Meta{}, &user_model.CreateUserOverwriteOptions{}) assert.NoError(t, err) e := &user_model.ExternalLoginUser{ @@ -45,15 +46,15 @@ func TestSource(t *testing.T) { LoginSourceID: user.LoginSource, RefreshToken: "valid", } - err = user_model.LinkExternalToUser(context.Background(), user, e) + err = user_model.LinkExternalToUser(t.Context(), user, e) assert.NoError(t, err) - provider, err := createProvider(source.authSource.Name, source) + provider, err := createProvider(source.AuthSource.Name, source) assert.NoError(t, err) t.Run("refresh", func(t *testing.T) { t.Run("valid", func(t *testing.T) { - err := source.refresh(context.Background(), provider, e) + err := source.refresh(t.Context(), provider, e) assert.NoError(t, err) e := &user_model.ExternalLoginUser{ @@ -61,19 +62,19 @@ func TestSource(t *testing.T) { LoginSourceID: e.LoginSourceID, } - ok, err := user_model.GetExternalLogin(context.Background(), e) + ok, err := user_model.GetExternalLogin(t.Context(), e) assert.NoError(t, err) assert.True(t, ok) assert.Equal(t, "refresh", e.RefreshToken) assert.Equal(t, "token", e.AccessToken) - u, err := user_model.GetUserByID(context.Background(), user.ID) + u, err := user_model.GetUserByID(t.Context(), user.ID) assert.NoError(t, err) assert.True(t, u.IsActive) }) t.Run("expired", func(t *testing.T) { - err := source.refresh(context.Background(), provider, &user_model.ExternalLoginUser{ + err := source.refresh(t.Context(), provider, &user_model.ExternalLoginUser{ ExternalID: "external", UserID: user.ID, LoginSourceID: user.LoginSource, @@ -86,13 +87,13 @@ func TestSource(t *testing.T) { LoginSourceID: e.LoginSourceID, } - ok, err := user_model.GetExternalLogin(context.Background(), e) + ok, err := user_model.GetExternalLogin(t.Context(), e) assert.NoError(t, err) assert.True(t, ok) - assert.Equal(t, "", e.RefreshToken) - assert.Equal(t, "", e.AccessToken) + assert.Empty(t, e.RefreshToken) + assert.Empty(t, e.AccessToken) - u, err := user_model.GetUserByID(context.Background(), user.ID) + u, err := user_model.GetUserByID(t.Context(), user.ID) assert.NoError(t, err) assert.False(t, u.IsActive) }) diff --git a/services/auth/source/oauth2/urlmapping.go b/services/auth/source/oauth2/urlmapping.go index d0442d58a8..b9f445caa7 100644 --- a/services/auth/source/oauth2/urlmapping.go +++ b/services/auth/source/oauth2/urlmapping.go @@ -14,11 +14,11 @@ type CustomURLMapping struct { // CustomURLSettings describes the urls values and availability to use when customizing OAuth2 provider URLs type CustomURLSettings struct { - AuthURL Attribute `json:",omitempty"` - TokenURL Attribute `json:",omitempty"` - ProfileURL Attribute `json:",omitempty"` - EmailURL Attribute `json:",omitempty"` - Tenant Attribute `json:",omitempty"` + AuthURL Attribute + TokenURL Attribute + ProfileURL Attribute + EmailURL Attribute + Tenant Attribute } // Attribute describes the availability, and required status for a custom url configuration diff --git a/services/auth/source/pam/assert_interface_test.go b/services/auth/source/pam/assert_interface_test.go index 8e7648b8d3..908d097d96 100644 --- a/services/auth/source/pam/assert_interface_test.go +++ b/services/auth/source/pam/assert_interface_test.go @@ -15,7 +15,6 @@ import ( type sourceInterface interface { auth.PasswordAuthenticator auth_model.Config - auth_model.SourceSettable } var _ (sourceInterface) = &pam.Source{} diff --git a/services/auth/source/pam/source.go b/services/auth/source/pam/source.go index 96b182e185..d1db6db9b7 100644 --- a/services/auth/source/pam/source.go +++ b/services/auth/source/pam/source.go @@ -17,12 +17,10 @@ import ( // Source holds configuration for the PAM login source. type Source struct { - ServiceName string // pam service (e.g. system-auth) - EmailDomain string - SkipLocalTwoFA bool `json:",omitempty"` // Skip Local 2fa for users authenticated with this source + auth.ConfigBase `json:"-"` - // reference to the authSource - authSource *auth.Source + ServiceName string // pam service (e.g. system-auth) + EmailDomain string } // FromDB fills up a PAMConfig from serialized format. @@ -35,11 +33,6 @@ func (source *Source) ToDB() ([]byte, error) { return json.Marshal(source) } -// SetAuthSource sets the related AuthSource -func (source *Source) SetAuthSource(authSource *auth.Source) { - source.authSource = authSource -} - func init() { auth.RegisterTypeConfig(auth.PAM, &Source{}) } diff --git a/services/auth/source/pam/source_authenticate.go b/services/auth/source/pam/source_authenticate.go index 6fd02dc29f..db7c6aab96 100644 --- a/services/auth/source/pam/source_authenticate.go +++ b/services/auth/source/pam/source_authenticate.go @@ -56,7 +56,7 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u Email: email, Passwd: password, LoginType: auth.PAM, - LoginSource: source.authSource.ID, + LoginSource: source.AuthSource.ID, LoginName: userName, // This is what the user typed in } overwriteDefault := &user_model.CreateUserOverwriteOptions{ @@ -69,8 +69,3 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u return user, nil } - -// IsSkipLocalTwoFA returns if this source should skip local 2fa for password authentication -func (source *Source) IsSkipLocalTwoFA() bool { - return source.SkipLocalTwoFA -} diff --git a/services/auth/source/smtp/assert_interface_test.go b/services/auth/source/smtp/assert_interface_test.go index 6c9cde66e1..56edad0c71 100644 --- a/services/auth/source/smtp/assert_interface_test.go +++ b/services/auth/source/smtp/assert_interface_test.go @@ -18,7 +18,6 @@ type sourceInterface interface { auth_model.SkipVerifiable auth_model.HasTLSer auth_model.UseTLSer - auth_model.SourceSettable } var _ (sourceInterface) = &smtp.Source{} diff --git a/services/auth/source/smtp/source.go b/services/auth/source/smtp/source.go index 2a648e421e..2ae81ad4f2 100644 --- a/services/auth/source/smtp/source.go +++ b/services/auth/source/smtp/source.go @@ -17,6 +17,8 @@ import ( // Source holds configuration for the SMTP login source. type Source struct { + auth.ConfigBase `json:"-"` + Auth string Host string Port int @@ -25,10 +27,6 @@ type Source struct { SkipVerify bool HeloHostname string DisableHelo bool - SkipLocalTwoFA bool `json:",omitempty"` - - // reference to the authSource - authSource *auth.Source } // FromDB fills up an SMTPConfig from serialized format. @@ -56,11 +54,6 @@ func (source *Source) UseTLS() bool { return source.ForceSMTPS || source.Port == 465 } -// SetAuthSource sets the related AuthSource -func (source *Source) SetAuthSource(authSource *auth.Source) { - source.authSource = authSource -} - func init() { auth.RegisterTypeConfig(auth.SMTP, &Source{}) } diff --git a/services/auth/source/smtp/source_authenticate.go b/services/auth/source/smtp/source_authenticate.go index b2e94933a6..b8e668f5f9 100644 --- a/services/auth/source/smtp/source_authenticate.go +++ b/services/auth/source/smtp/source_authenticate.go @@ -72,7 +72,7 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u Email: userName, Passwd: password, LoginType: auth_model.SMTP, - LoginSource: source.authSource.ID, + LoginSource: source.AuthSource.ID, LoginName: userName, } overwriteDefault := &user_model.CreateUserOverwriteOptions{ @@ -85,8 +85,3 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u return user, nil } - -// IsSkipLocalTwoFA returns if this source should skip local 2fa for password authentication -func (source *Source) IsSkipLocalTwoFA() bool { - return source.SkipLocalTwoFA -} diff --git a/services/auth/source/sspi/source.go b/services/auth/source/sspi/source.go index bdd6ef451c..3b7b5cb033 100644 --- a/services/auth/source/sspi/source.go +++ b/services/auth/source/sspi/source.go @@ -17,6 +17,8 @@ import ( // Source holds configuration for SSPI single sign-on. type Source struct { + auth.ConfigBase `json:"-"` + AutoCreateUsers bool AutoActivateUsers bool StripDomainNames bool diff --git a/services/auth/sspi.go b/services/auth/sspi.go index 8ac83dcb04..8cb39886c4 100644 --- a/services/auth/sspi.go +++ b/services/auth/sspi.go @@ -17,7 +17,6 @@ import ( "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" - "code.gitea.io/gitea/modules/web/middleware" "code.gitea.io/gitea/services/auth/source/sspi" gitea_context "code.gitea.io/gitea/services/context" @@ -89,7 +88,7 @@ func (s *SSPI) Verify(req *http.Request, w http.ResponseWriter, store DataStore, store.GetData()["EnableSSPI"] = true // in this case, the Verify function is called in Gitea's web context // FIXME: it doesn't look good to render the page here, why not redirect? - gitea_context.GetWebContext(req).HTML(http.StatusUnauthorized, tplSignIn) + gitea_context.GetWebContext(req.Context()).HTML(http.StatusUnauthorized, tplSignIn) return nil, err } if outToken != "" { @@ -120,7 +119,8 @@ func (s *SSPI) Verify(req *http.Request, w http.ResponseWriter, store DataStore, } // Make sure requests to API paths and PWA resources do not create a new session - if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) { + detector := newAuthPathDetector(req) + if !detector.isAPIPath() && !detector.isAttachmentDownload() { handleSignIn(w, req, sess, user) } @@ -155,8 +155,9 @@ func (s *SSPI) shouldAuthenticate(req *http.Request) (shouldAuth bool) { } else if req.FormValue("auth_with_sspi") == "1" { shouldAuth = true } - } else if middleware.IsAPIPath(req) || isAttachmentDownload(req) { - shouldAuth = true + } else { + detector := newAuthPathDetector(req) + shouldAuth = detector.isAPIPath() || detector.isAttachmentDownload() } return shouldAuth } diff --git a/services/automerge/automerge.go b/services/automerge/automerge.go index a1ee204882..0520a097d3 100644 --- a/services/automerge/automerge.go +++ b/services/automerge/automerge.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/queue" notify_service "code.gitea.io/gitea/services/notify" pull_service "code.gitea.io/gitea/services/pull" + repo_service "code.gitea.io/gitea/services/repository" ) // prAutoMergeQueue represents a queue to handle update pull request tests @@ -35,7 +36,7 @@ func Init() error { prAutoMergeQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "pr_auto_merge", handler) if prAutoMergeQueue == nil { - return fmt.Errorf("unable to create pr_auto_merge queue") + return errors.New("unable to create pr_auto_merge queue") } go graceful.GetManager().RunWithCancel(prAutoMergeQueue) return nil @@ -63,9 +64,9 @@ func addToQueue(pr *issues_model.PullRequest, sha string) { } // ScheduleAutoMerge if schedule is false and no error, pull can be merged directly -func ScheduleAutoMerge(ctx context.Context, doer *user_model.User, pull *issues_model.PullRequest, style repo_model.MergeStyle, message string) (scheduled bool, err error) { +func ScheduleAutoMerge(ctx context.Context, doer *user_model.User, pull *issues_model.PullRequest, style repo_model.MergeStyle, message string, deleteBranchAfterMerge bool) (scheduled bool, err error) { err = db.WithTx(ctx, func(ctx context.Context) error { - if err := pull_model.ScheduleAutoMerge(ctx, doer, pull.ID, style, message); err != nil { + if err := pull_model.ScheduleAutoMerge(ctx, doer, pull.ID, style, message, deleteBranchAfterMerge); err != nil { return err } scheduled = true @@ -247,13 +248,13 @@ func handlePullRequestAutoMerge(pullID int64, sha string) { switch pr.Flow { case issues_model.PullRequestFlowGithub: - headBranchExist := headGitRepo.IsBranchExist(pr.HeadBranch) - if pr.HeadRepo == nil || !headBranchExist { + headBranchExist := pr.HeadRepo != nil && gitrepo.IsBranchExist(ctx, pr.HeadRepo, pr.HeadBranch) + if !headBranchExist { log.Warn("Head branch of auto merge %-v does not exist [HeadRepoID: %d, Branch: %s]", pr, pr.HeadRepoID, pr.HeadBranch) return } case issues_model.PullRequestFlowAGit: - headBranchExist := git.IsReferenceExist(ctx, baseGitRepo.Path, pr.GetGitRefName()) + headBranchExist := gitrepo.IsReferenceExist(ctx, pr.BaseRepo, pr.GetGitRefName()) if !headBranchExist { log.Warn("Head branch of auto merge %-v does not exist [HeadRepoID: %d, Branch(Agit): %s]", pr, pr.HeadRepoID, pr.HeadBranch) return @@ -288,7 +289,7 @@ func handlePullRequestAutoMerge(pullID int64, sha string) { } if err := pull_service.CheckPullMergeable(ctx, doer, &perm, pr, pull_service.MergeCheckTypeGeneral, false); err != nil { - if errors.Is(err, pull_service.ErrUserNotAllowedToMerge) { + if errors.Is(err, pull_service.ErrNotReadyToMerge) { log.Info("%-v was scheduled to automerge by an unauthorized user", pr) return } @@ -303,4 +304,10 @@ func handlePullRequestAutoMerge(pullID int64, sha string) { // on the pull request page. But this should not be finished in a bug fix PR which will be backport to release branch. return } + + if pr.Flow == issues_model.PullRequestFlowGithub && scheduledPRM.DeleteBranchAfterMerge { + if err := repo_service.DeleteBranch(ctx, doer, pr.HeadRepo, headGitRepo, pr.HeadBranch, pr); err != nil { + log.Error("DeletePullRequestHeadBranch: %v", err) + } + } } diff --git a/services/context/access_log.go b/services/context/access_log.go index 0926748ac5..caade113a7 100644 --- a/services/context/access_log.go +++ b/services/context/access_log.go @@ -5,7 +5,6 @@ package context import ( "bytes" - "fmt" "net" "net/http" "strings" @@ -18,13 +17,14 @@ import ( "code.gitea.io/gitea/modules/web/middleware" ) -type routerLoggerOptions struct { - req *http.Request +type accessLoggerTmplData struct { Identity *string Start *time.Time - ResponseWriter http.ResponseWriter - Ctx map[string]any - RequestID *string + ResponseWriter struct { + Status, Size int + } + Ctx map[string]any + RequestID *string } const keyOfRequestIDInTemplate = ".RequestID" @@ -46,56 +46,70 @@ func parseRequestIDFromRequestHeader(req *http.Request) string { } } if len(requestID) > maxRequestIDByteLength { - requestID = fmt.Sprintf("%s...", requestID[:maxRequestIDByteLength]) + requestID = requestID[:maxRequestIDByteLength] + "..." } return requestID } +type accessLogRecorder struct { + logger log.BaseLogger + logTemplate *template.Template + needRequestID bool +} + +func (lr *accessLogRecorder) record(start time.Time, respWriter ResponseWriter, req *http.Request) { + var requestID string + if lr.needRequestID { + requestID = parseRequestIDFromRequestHeader(req) + } + + reqHost, _, err := net.SplitHostPort(req.RemoteAddr) + if err != nil { + reqHost = req.RemoteAddr + } + + identity := "-" + data := middleware.GetContextData(req.Context()) + if signedUser, ok := data[middleware.ContextDataKeySignedUser].(*user_model.User); ok { + identity = signedUser.Name + } + buf := bytes.NewBuffer([]byte{}) + tmplData := accessLoggerTmplData{ + Identity: &identity, + Start: &start, + Ctx: map[string]any{ + "RemoteAddr": req.RemoteAddr, + "RemoteHost": reqHost, + "Req": req, + }, + RequestID: &requestID, + } + tmplData.ResponseWriter.Status = respWriter.WrittenStatus() + tmplData.ResponseWriter.Size = respWriter.WrittenSize() + err = lr.logTemplate.Execute(buf, tmplData) + if err != nil { + log.Error("Could not execute access logger template: %v", err.Error()) + } + + lr.logger.Log(1, &log.Event{Level: log.INFO}, "%s", buf.String()) +} + +func newAccessLogRecorder() *accessLogRecorder { + return &accessLogRecorder{ + logger: log.GetLogger("access"), + logTemplate: template.Must(template.New("log").Parse(setting.Log.AccessLogTemplate)), + needRequestID: len(setting.Log.RequestIDHeaders) > 0 && strings.Contains(setting.Log.AccessLogTemplate, keyOfRequestIDInTemplate), + } +} + // AccessLogger returns a middleware to log access logger func AccessLogger() func(http.Handler) http.Handler { - logger := log.GetLogger("access") - needRequestID := len(setting.Log.RequestIDHeaders) > 0 && strings.Contains(setting.Log.AccessLogTemplate, keyOfRequestIDInTemplate) - logTemplate, _ := template.New("log").Parse(setting.Log.AccessLogTemplate) + recorder := newAccessLogRecorder() return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { start := time.Now() - - var requestID string - if needRequestID { - requestID = parseRequestIDFromRequestHeader(req) - } - - reqHost, _, err := net.SplitHostPort(req.RemoteAddr) - if err != nil { - reqHost = req.RemoteAddr - } - next.ServeHTTP(w, req) - rw := w.(ResponseWriter) - - identity := "-" - data := middleware.GetContextData(req.Context()) - if signedUser, ok := data[middleware.ContextDataKeySignedUser].(*user_model.User); ok { - identity = signedUser.Name - } - buf := bytes.NewBuffer([]byte{}) - err = logTemplate.Execute(buf, routerLoggerOptions{ - req: req, - Identity: &identity, - Start: &start, - ResponseWriter: rw, - Ctx: map[string]any{ - "RemoteAddr": req.RemoteAddr, - "RemoteHost": reqHost, - "Req": req, - }, - RequestID: &requestID, - }) - if err != nil { - log.Error("Could not execute access logger template: %v", err.Error()) - } - - logger.Info("%s", buf.String()) + recorder.record(start, w.(ResponseWriter), req) }) } } diff --git a/services/context/access_log_test.go b/services/context/access_log_test.go new file mode 100644 index 0000000000..139a6eb217 --- /dev/null +++ b/services/context/access_log_test.go @@ -0,0 +1,71 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package context + +import ( + "fmt" + "net/http" + "net/url" + "testing" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +type testAccessLoggerMock struct { + logs []string +} + +func (t *testAccessLoggerMock) Log(skip int, event *log.Event, format string, v ...any) { + t.logs = append(t.logs, fmt.Sprintf(format, v...)) +} + +func (t *testAccessLoggerMock) GetLevel() log.Level { + return log.INFO +} + +type testAccessLoggerResponseWriterMock struct{} + +func (t testAccessLoggerResponseWriterMock) Header() http.Header { + return nil +} + +func (t testAccessLoggerResponseWriterMock) Before(f func(ResponseWriter)) {} + +func (t testAccessLoggerResponseWriterMock) WriteHeader(statusCode int) {} + +func (t testAccessLoggerResponseWriterMock) Write(bytes []byte) (int, error) { + return 0, nil +} + +func (t testAccessLoggerResponseWriterMock) Flush() {} + +func (t testAccessLoggerResponseWriterMock) WrittenStatus() int { + return http.StatusOK +} + +func (t testAccessLoggerResponseWriterMock) WrittenSize() int { + return 123123 +} + +func TestAccessLogger(t *testing.T) { + setting.Log.AccessLogTemplate = `{{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"` + recorder := newAccessLogRecorder() + mockLogger := &testAccessLoggerMock{} + recorder.logger = mockLogger + req := &http.Request{ + RemoteAddr: "remote-addr", + Method: http.MethodGet, + Proto: "https", + URL: &url.URL{Path: "/path"}, + } + req.Header = http.Header{} + req.Header.Add("Referer", "referer") + req.Header.Add("User-Agent", "user-agent") + recorder.record(time.Date(2000, 1, 2, 3, 4, 5, 0, time.UTC), &testAccessLoggerResponseWriterMock{}, req) + assert.Equal(t, []string{`remote-addr - - [02/Jan/2000:03:04:05 +0000] "GET /path https" 200 123123 "referer" "user-agent"`}, mockLogger.logs) +} diff --git a/services/context/api.go b/services/context/api.go index bda705cb48..ab50a360f4 100644 --- a/services/context/api.go +++ b/services/context/api.go @@ -5,23 +5,31 @@ package context import ( + "errors" "fmt" "net/http" "net/url" + "slices" + "strconv" "strings" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" web_types "code.gitea.io/gitea/modules/web/types" ) // APIContext is a specific context for API service +// ATTENTION: This struct should never be manually constructed in routes/services, +// it has many internal details which should be carefully prepared by the framework. +// If it is abused, it would cause strange bugs like panic/resource-leak. type APIContext struct { *Base @@ -103,14 +111,28 @@ type APIRepoArchivedError struct { APIError } -// ServerError responds with error message, status is 500 -func (ctx *APIContext) ServerError(title string, err error) { - ctx.Error(http.StatusInternalServerError, title, err) +// APIErrorInternal responds with error message, status is 500 +func (ctx *APIContext) APIErrorInternal(err error) { + ctx.apiErrorInternal(1, err) } -// Error responds with an error message to client with given obj as the message. +func (ctx *APIContext) apiErrorInternal(skip int, err error) { + log.ErrorWithSkip(skip+1, "InternalServerError: %v", err) + + var message string + if !setting.IsProd || (ctx.Doer != nil && ctx.Doer.IsAdmin) { + message = err.Error() + } + + ctx.JSON(http.StatusInternalServerError, APIError{ + Message: message, + URL: setting.API.SwaggerURL, + }) +} + +// APIError responds with an error message to client with given obj as the message. // If status is 500, also it prints error to log. -func (ctx *APIContext) Error(status int, title string, obj any) { +func (ctx *APIContext) APIError(status int, obj any) { var message string if err, ok := obj.(error); ok { message = err.Error() @@ -119,7 +141,7 @@ func (ctx *APIContext) Error(status int, title string, obj any) { } if status == http.StatusInternalServerError { - log.ErrorWithSkip(1, "%s: %s", title, message) + log.ErrorWithSkip(1, "APIError: %s", message) if setting.IsProd && !(ctx.Doer != nil && ctx.Doer.IsAdmin) { message = "" @@ -132,22 +154,6 @@ func (ctx *APIContext) Error(status int, title string, obj any) { }) } -// InternalServerError responds with an error message to the client with the error as a message -// and the file and line of the caller. -func (ctx *APIContext) InternalServerError(err error) { - log.ErrorWithSkip(1, "InternalServerError: %v", err) - - var message string - if !setting.IsProd || (ctx.Doer != nil && ctx.Doer.IsAdmin) { - message = err.Error() - } - - ctx.JSON(http.StatusInternalServerError, APIError{ - Message: message, - URL: setting.API.SwaggerURL, - }) -} - type apiContextKeyType struct{} var apiContextKey = apiContextKeyType{} @@ -165,7 +171,7 @@ func genAPILinks(curURL *url.URL, total, pageSize, curPage int) []string { if paginater.HasNext() { u := *curURL queries := u.Query() - queries.Set("page", fmt.Sprintf("%d", paginater.Next())) + queries.Set("page", strconv.Itoa(paginater.Next())) u.RawQuery = queries.Encode() links = append(links, fmt.Sprintf("<%s%s>; rel=\"next\"", setting.AppURL, u.RequestURI()[1:])) @@ -173,7 +179,7 @@ func genAPILinks(curURL *url.URL, total, pageSize, curPage int) []string { if !paginater.IsLast() { u := *curURL queries := u.Query() - queries.Set("page", fmt.Sprintf("%d", paginater.TotalPages())) + queries.Set("page", strconv.Itoa(paginater.TotalPages())) u.RawQuery = queries.Encode() links = append(links, fmt.Sprintf("<%s%s>; rel=\"last\"", setting.AppURL, u.RequestURI()[1:])) @@ -189,7 +195,7 @@ func genAPILinks(curURL *url.URL, total, pageSize, curPage int) []string { if paginater.HasPrevious() { u := *curURL queries := u.Query() - queries.Set("page", fmt.Sprintf("%d", paginater.Previous())) + queries.Set("page", strconv.Itoa(paginater.Previous())) u.RawQuery = queries.Encode() links = append(links, fmt.Sprintf("<%s%s>; rel=\"prev\"", setting.AppURL, u.RequestURI()[1:])) @@ -207,7 +213,7 @@ func (ctx *APIContext) SetLinkHeader(total, pageSize int) { } } -// APIContexter returns apicontext as middleware +// APIContexter returns APIContext middleware func APIContexter() func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -222,14 +228,14 @@ func APIContexter() func(http.Handler) http.Handler { ctx.SetContextValue(apiContextKey, ctx) // If request sends files, parse them here otherwise the Query() can't be parsed and the CsrfToken will be invalid. - if ctx.Req.Method == "POST" && strings.Contains(ctx.Req.Header.Get("Content-Type"), "multipart/form-data") { + if ctx.Req.Method == http.MethodPost && strings.Contains(ctx.Req.Header.Get("Content-Type"), "multipart/form-data") { if err := ctx.Req.ParseMultipartForm(setting.Attachment.MaxSize << 20); err != nil && !strings.Contains(err.Error(), "EOF") { // 32MB max size - ctx.InternalServerError(err) + ctx.APIErrorInternal(err) return } } - httpcache.SetCacheControlInHeader(ctx.Resp.Header(), 0, "no-transform") + httpcache.SetCacheControlInHeader(ctx.Resp.Header(), &httpcache.CacheControlOptions{NoTransform: true}) ctx.Resp.Header().Set(`X-Frame-Options`, setting.CORSConfig.XFrameOptions) next.ServeHTTP(ctx.Resp, ctx.Req) @@ -237,11 +243,11 @@ func APIContexter() func(http.Handler) http.Handler { } } -// NotFound handles 404s for APIContext +// APIErrorNotFound handles 404s for APIContext // String will replace message, errors will be added to a slice -func (ctx *APIContext) NotFound(objs ...any) { - message := ctx.Locale.TrString("error.not_found") - var errors []string +func (ctx *APIContext) APIErrorNotFound(objs ...any) { + var message string + var errs []string for _, obj := range objs { // Ignore nil if obj == nil { @@ -249,16 +255,15 @@ func (ctx *APIContext) NotFound(objs ...any) { } if err, ok := obj.(error); ok { - errors = append(errors, err.Error()) + errs = append(errs, err.Error()) } else { message = obj.(string) } } - ctx.JSON(http.StatusNotFound, map[string]any{ - "message": message, + "message": util.IfZero(message, "not found"), // do not use locale in API "url": setting.API.SwaggerURL, - "errors": errors, + "errors": errs, }) } @@ -276,7 +281,7 @@ func ReferencesGitRepo(allowEmpty ...bool) func(ctx *APIContext) { var err error ctx.Repo.GitRepo, err = gitrepo.RepositoryFromRequestContextOrOpen(ctx, ctx.Repo.Repository) if err != nil { - ctx.Error(http.StatusInternalServerError, fmt.Sprintf("Open Repository %v failed", ctx.Repo.Repository.FullName()), err) + ctx.APIErrorInternal(err) return } } @@ -288,41 +293,33 @@ func RepoRefForAPI(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { ctx := GetAPIContext(req) - if ctx.Repo.GitRepo == nil { - ctx.InternalServerError(fmt.Errorf("no open git repo")) + if ctx.Repo.Repository.IsEmpty { + ctx.APIErrorNotFound("repository is empty") return } - // NOTICE: the "ref" here for internal usage only (e.g. woodpecker) - refName, _ := getRefNameLegacy(ctx.Base, ctx.Repo, ctx.FormTrim("ref")) - var err error + if ctx.Repo.GitRepo == nil { + panic("no GitRepo, forgot to call the middleware?") // it is a programming error + } - if ctx.Repo.GitRepo.IsBranchExist(refName) { + refName, refType, _ := getRefNameLegacy(ctx.Base, ctx.Repo, ctx.PathParam("*"), ctx.FormTrim("ref")) + var err error + switch refType { + case git.RefTypeBranch: ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refName) - if err != nil { - ctx.InternalServerError(err) - return - } - ctx.Repo.CommitID = ctx.Repo.Commit.ID.String() - } else if ctx.Repo.GitRepo.IsTagExist(refName) { + case git.RefTypeTag: ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetTagCommit(refName) - if err != nil { - ctx.InternalServerError(err) - return - } - ctx.Repo.CommitID = ctx.Repo.Commit.ID.String() - } else if len(refName) == ctx.Repo.GetObjectFormat().FullLength() { - ctx.Repo.CommitID = refName + case git.RefTypeCommit: ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetCommit(refName) - if err != nil { - ctx.NotFound("GetCommit", err) - return - } - } else { - ctx.NotFound(fmt.Errorf("not exist: '%s'", ctx.PathParam("*"))) + } + if ctx.Repo.Commit == nil || errors.Is(err, util.ErrNotExist) { + ctx.APIErrorNotFound("unable to find a git ref") + return + } else if err != nil { + ctx.APIErrorInternal(err) return } - + ctx.Repo.CommitID = ctx.Repo.Commit.ID.String() next.ServeHTTP(w, req) }) } @@ -348,12 +345,12 @@ func (ctx *APIContext) GetErrMsg() string { // NotFoundOrServerError use error check function to determine if the error // is about not found. It responds with 404 status code for not found error, // or error context description for logging purpose of 500 server error. -func (ctx *APIContext) NotFoundOrServerError(logMsg string, errCheck func(error) bool, logErr error) { - if errCheck(logErr) { +func (ctx *APIContext) NotFoundOrServerError(err error) { + if errors.Is(err, util.ErrNotExist) { ctx.JSON(http.StatusNotFound, nil) return } - ctx.Error(http.StatusInternalServerError, "NotFoundOrServerError", logMsg) + ctx.APIErrorInternal(err) } // IsUserSiteAdmin returns true if current user is a site admin @@ -366,13 +363,7 @@ func (ctx *APIContext) IsUserRepoAdmin() bool { return ctx.Repo.IsAdmin() } -// IsUserRepoWriter returns true if current user has write privilege in current repo +// IsUserRepoWriter returns true if current user has "write" privilege in current repo func (ctx *APIContext) IsUserRepoWriter(unitTypes []unit.Type) bool { - for _, unitType := range unitTypes { - if ctx.Repo.CanWrite(unitType) { - return true - } - } - - return false + return slices.ContainsFunc(unitTypes, ctx.Repo.CanWrite) } diff --git a/services/context/api_test.go b/services/context/api_test.go index 911a49949e..87d74004db 100644 --- a/services/context/api_test.go +++ b/services/context/api_test.go @@ -45,6 +45,6 @@ func TestGenAPILinks(t *testing.T) { links := genAPILinks(u, 100, 20, curPage) - assert.EqualValues(t, links, response) + assert.Equal(t, links, response) } } diff --git a/services/context/base.go b/services/context/base.go index 7a39353e09..f3f92b7eeb 100644 --- a/services/context/base.go +++ b/services/context/base.go @@ -4,11 +4,11 @@ package context import ( - "context" "fmt" "html/template" "io" "net/http" + "strconv" "strings" "code.gitea.io/gitea/modules/httplib" @@ -24,9 +24,12 @@ type BaseContextKeyType struct{} var BaseContextKey BaseContextKeyType +// Base is the base context for all web handlers +// ATTENTION: This struct should never be manually constructed in routes/services, +// it has many internal details which should be carefully prepared by the framework. +// If it is abused, it would cause strange bugs like panic/resource-leak. type Base struct { - context.Context - reqctx.RequestDataStore + reqctx.RequestContext Resp ResponseWriter Req *http.Request @@ -51,7 +54,7 @@ func (b *Base) AppendAccessControlExposeHeaders(names ...string) { // SetTotalCountHeader set "X-Total-Count" header func (b *Base) SetTotalCountHeader(total int64) { - b.RespHeader().Set("X-Total-Count", fmt.Sprint(total)) + b.RespHeader().Set("X-Total-Count", strconv.FormatInt(total, 10)) b.AppendAccessControlExposeHeaders("X-Total-Count") } @@ -79,8 +82,8 @@ func (b *Base) RespHeader() http.Header { return b.Resp.Header() } -// Error returned an error to web browser -func (b *Base) Error(status int, contents ...string) { +// HTTPError returned an error to web browser +func (b *Base) HTTPError(status int, contents ...string) { v := http.StatusText(status) if len(contents) > 0 { v = contents[0] @@ -172,19 +175,19 @@ func (b *Base) TrN(cnt any, key1, keyN string, args ...any) template.HTML { } func NewBaseContext(resp http.ResponseWriter, req *http.Request) *Base { - ds := reqctx.GetRequestDataStore(req.Context()) + reqCtx := reqctx.FromContext(req.Context()) b := &Base{ - Context: req.Context(), - RequestDataStore: ds, - Req: req, - Resp: WrapResponseWriter(resp), - Locale: middleware.Locale(resp, req), - Data: ds.GetData(), + RequestContext: reqCtx, + + Req: req, + Resp: WrapResponseWriter(resp), + Locale: middleware.Locale(resp, req), + Data: reqCtx.GetData(), } b.Req = b.Req.WithContext(b) - ds.SetContextValue(BaseContextKey, b) - ds.SetContextValue(translation.ContextKey, b.Locale) - ds.SetContextValue(httplib.RequestContextKey, b.Req) + reqCtx.SetContextValue(BaseContextKey, b) + reqCtx.SetContextValue(translation.ContextKey, b.Locale) + reqCtx.SetContextValue(httplib.RequestContextKey, b.Req) return b } diff --git a/services/context/base_form.go b/services/context/base_form.go index 5b8cae9e99..81fd7cd328 100644 --- a/services/context/base_form.go +++ b/services/context/base_form.go @@ -12,6 +12,8 @@ import ( ) // FormString returns the first value matching the provided key in the form as a string +// It works the same as http.Request.FormValue: +// try urlencoded request body first, then query string, then multipart form body func (b *Base) FormString(key string, def ...string) string { s := b.Req.FormValue(key) if s == "" { @@ -20,7 +22,7 @@ func (b *Base) FormString(key string, def ...string) string { return s } -// FormStrings returns a string slice for the provided key from the form +// FormStrings returns a values for the key in the form (including query parameters), similar to FormString func (b *Base) FormStrings(key string) []string { if b.Req.Form == nil { if err := b.Req.ParseMultipartForm(32 << 20); err != nil { diff --git a/services/context/base_test.go b/services/context/base_test.go index b936b76f58..2a4f86dddf 100644 --- a/services/context/base_test.go +++ b/services/context/base_test.go @@ -15,7 +15,7 @@ import ( func TestRedirect(t *testing.T) { setting.IsInTesting = true - req, _ := http.NewRequest("GET", "/", nil) + req, _ := http.NewRequest(http.MethodGet, "/", nil) cases := []struct { url string @@ -36,7 +36,7 @@ func TestRedirect(t *testing.T) { assert.Equal(t, c.keep, has, "url = %q", c.url) } - req, _ = http.NewRequest("GET", "/", nil) + req, _ = http.NewRequest(http.MethodGet, "/", nil) resp := httptest.NewRecorder() req.Header.Add("HX-Request", "true") b := NewBaseContextForTest(resp, req) diff --git a/services/context/context.go b/services/context/context.go index 6715c5663d..32ec260aab 100644 --- a/services/context/context.go +++ b/services/context/context.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/modules/web/middleware" web_types "code.gitea.io/gitea/modules/web/types" @@ -34,7 +35,10 @@ type Render interface { HTML(w io.Writer, status int, name templates.TplName, data any, templateCtx context.Context) error } -// Context represents context of a request. +// Context represents context of a web request. +// ATTENTION: This struct should never be manually constructed in routes/services, +// it has many internal details which should be carefully prepared by the framework. +// If it is abused, it would cause strange bugs like panic/resource-leak. type Context struct { *Base @@ -76,9 +80,9 @@ type webContextKeyType struct{} var WebContextKey = webContextKeyType{} -func GetWebContext(req *http.Request) *Context { - ctx, _ := req.Context().Value(WebContextKey).(*Context) - return ctx +func GetWebContext(ctx context.Context) *Context { + webCtx, _ := ctx.Value(WebContextKey).(*Context) + return webCtx } // ValidateContext is a special context for form validation middleware. It may be different from other contexts. @@ -132,6 +136,7 @@ func NewWebContext(base *Base, render Render, session session.Store) *Context { } ctx.TemplateContext = NewTemplateContextForWeb(ctx) ctx.Flash = &middleware.Flash{DataStore: ctx, Values: url.Values{}} + ctx.SetContextValue(WebContextKey, ctx) return ctx } @@ -162,21 +167,12 @@ func Contexter() func(next http.Handler) http.Handler { ctx.PageData = map[string]any{} ctx.Data["PageData"] = ctx.PageData - ctx.Base.SetContextValue(WebContextKey, ctx) ctx.Csrf = NewCSRFProtector(csrfOpts) - // Get the last flash message from cookie - lastFlashCookie := middleware.GetSiteCookie(ctx.Req, CookieNameFlash) + // get the last flash message from cookie + lastFlashCookie, lastFlashMsg := middleware.GetSiteCookieFlashMessage(ctx, ctx.Req, CookieNameFlash) if vals, _ := url.ParseQuery(lastFlashCookie); len(vals) > 0 { - // store last Flash message into the template data, to render it - ctx.Data["Flash"] = &middleware.Flash{ - DataStore: ctx, - Values: vals, - ErrorMsg: vals.Get("error"), - SuccessMsg: vals.Get("success"), - InfoMsg: vals.Get("info"), - WarningMsg: vals.Get("warning"), - } + ctx.Data["Flash"] = lastFlashMsg // store last Flash message into the template data, to render it } // if there are new messages in the ctx.Flash, write them into cookie @@ -189,18 +185,20 @@ func Contexter() func(next http.Handler) http.Handler { }) // If request sends files, parse them here otherwise the Query() can't be parsed and the CsrfToken will be invalid. - if ctx.Req.Method == "POST" && strings.Contains(ctx.Req.Header.Get("Content-Type"), "multipart/form-data") { + if ctx.Req.Method == http.MethodPost && strings.Contains(ctx.Req.Header.Get("Content-Type"), "multipart/form-data") { if err := ctx.Req.ParseMultipartForm(setting.Attachment.MaxSize << 20); err != nil && !strings.Contains(err.Error(), "EOF") { // 32MB max size ctx.ServerError("ParseMultipartForm", err) return } } - httpcache.SetCacheControlInHeader(ctx.Resp.Header(), 0, "no-transform") + httpcache.SetCacheControlInHeader(ctx.Resp.Header(), &httpcache.CacheControlOptions{NoTransform: true}) ctx.Resp.Header().Set(`X-Frame-Options`, setting.CORSConfig.XFrameOptions) ctx.Data["SystemConfig"] = setting.Config() + ctx.Data["ShowTwoFactorRequiredMessage"] = ctx.DoerNeedTwoFactorAuth() + // FIXME: do we really always need these setting? There should be someway to have to avoid having to always set these ctx.Data["DisableMigrations"] = setting.Repository.DisableMigrations ctx.Data["DisableStars"] = setting.Repository.DisableStars @@ -214,17 +212,27 @@ func Contexter() func(next http.Handler) http.Handler { } } +func (ctx *Context) DoerNeedTwoFactorAuth() bool { + if !setting.TwoFactorAuthEnforced { + return false + } + return ctx.Session.Get(session.KeyUserHasTwoFactorAuth) == false +} + // HasError returns true if error occurs in form validation. // Attention: this function changes ctx.Data and ctx.Flash // If HasError is called, then before Redirect, the error message should be stored by ctx.Flash.Error(ctx.GetErrMsg()) again. func (ctx *Context) HasError() bool { - hasErr, ok := ctx.Data["HasError"] - if !ok { + hasErr, _ := ctx.Data["HasError"].(bool) + hasErr = hasErr || ctx.Flash.ErrorMsg != "" + if !hasErr { return false } - ctx.Flash.ErrorMsg = ctx.GetErrMsg() + if ctx.Flash.ErrorMsg == "" { + ctx.Flash.ErrorMsg = ctx.GetErrMsg() + } ctx.Data["Flash"] = ctx.Flash - return hasErr.(bool) + return hasErr } // GetErrMsg returns error message in form validation. @@ -254,3 +262,11 @@ func (ctx *Context) JSONError(msg any) { panic(fmt.Sprintf("unsupported type: %T", msg)) } } + +func (ctx *Context) JSONErrorNotFound(optMsg ...string) { + msg := util.OptionalArg(optMsg) + if msg == "" { + msg = ctx.Locale.TrString("error.not_found") + } + ctx.JSON(http.StatusNotFound, map[string]any{"errorMessage": msg, "renderFormat": "text"}) +} diff --git a/services/context/context_model.go b/services/context/context_model.go index 4f70aac516..3a1776102f 100644 --- a/services/context/context_model.go +++ b/services/context/context_model.go @@ -3,27 +3,7 @@ package context -import ( - "code.gitea.io/gitea/models/unit" -) - // IsUserSiteAdmin returns true if current user is a site admin func (ctx *Context) IsUserSiteAdmin() bool { return ctx.IsSigned && ctx.Doer.IsAdmin } - -// IsUserRepoAdmin returns true if current user is admin in current repo -func (ctx *Context) IsUserRepoAdmin() bool { - return ctx.Repo.IsAdmin() -} - -// IsUserRepoWriter returns true if current user has write privilege in current repo -func (ctx *Context) IsUserRepoWriter(unitTypes []unit.Type) bool { - for _, unitType := range unitTypes { - if ctx.Repo.CanWrite(unitType) { - return true - } - } - - return false -} diff --git a/services/context/context_response.go b/services/context/context_response.go index c7044791eb..4e11e29b69 100644 --- a/services/context/context_response.go +++ b/services/context/context_response.go @@ -28,7 +28,7 @@ import ( func RedirectToUser(ctx *Base, userName string, redirectUserID int64) { user, err := user_model.GetUserByID(ctx, redirectUserID) if err != nil { - ctx.Error(http.StatusInternalServerError, "unable to get user") + ctx.HTTPError(http.StatusInternalServerError, "unable to get user") return } @@ -122,8 +122,8 @@ func (ctx *Context) RenderWithErr(msg any, tpl templates.TplName, form any) { } // NotFound displays a 404 (Not Found) page and prints the given error, if any. -func (ctx *Context) NotFound(logMsg string, logErr error) { - ctx.notFoundInternal(logMsg, logErr) +func (ctx *Context) NotFound(logErr error) { + ctx.notFoundInternal("", logErr) } func (ctx *Context) notFoundInternal(logMsg string, logErr error) { @@ -150,7 +150,7 @@ func (ctx *Context) notFoundInternal(logMsg string, logErr error) { ctx.Data["IsRepo"] = ctx.Repo.Repository != nil ctx.Data["Title"] = "Page Not Found" - ctx.HTML(http.StatusNotFound, templates.TplName("status/404")) + ctx.HTML(http.StatusNotFound, "status/404") } // ServerError displays a 500 (Internal Server Error) page and prints the given error, if any. diff --git a/services/context/org.go b/services/context/org.go index be87cef7a3..c8b6ed09b7 100644 --- a/services/context/org.go +++ b/services/context/org.go @@ -51,7 +51,7 @@ func GetOrganizationByParams(ctx *Context) { if err == nil { RedirectToUser(ctx.Base, orgName, redirectUserID) } else if user_model.IsErrUserRedirectNotExist(err) { - ctx.NotFound("GetUserByName", err) + ctx.NotFound(err) } else { ctx.ServerError("LookupUserRedirect", err) } @@ -62,215 +62,193 @@ func GetOrganizationByParams(ctx *Context) { } } -// HandleOrgAssignment handles organization assignment -func HandleOrgAssignment(ctx *Context, args ...bool) { - var ( - requireMember bool - requireOwner bool - requireTeamMember bool - requireTeamAdmin bool - ) - if len(args) >= 1 { - requireMember = args[0] - } - if len(args) >= 2 { - requireOwner = args[1] - } - if len(args) >= 3 { - requireTeamMember = args[2] - } - if len(args) >= 4 { - requireTeamAdmin = args[3] - } - - var err error +type OrgAssignmentOptions struct { + RequireMember bool + RequireOwner bool + RequireTeamMember bool + RequireTeamAdmin bool +} - if ctx.ContextUser == nil { - // if Organization is not defined, get it from params - if ctx.Org.Organization == nil { - GetOrganizationByParams(ctx) - if ctx.Written() { - return +// OrgAssignment returns a middleware to handle organization assignment +func OrgAssignment(opts OrgAssignmentOptions) func(ctx *Context) { + return func(ctx *Context) { + var err error + if ctx.ContextUser == nil { + // if Organization is not defined, get it from params + if ctx.Org.Organization == nil { + GetOrganizationByParams(ctx) + if ctx.Written() { + return + } } + } else if ctx.ContextUser.IsOrganization() { + ctx.Org.Organization = (*organization.Organization)(ctx.ContextUser) + } else { + // ContextUser is an individual User + return } - } else if ctx.ContextUser.IsOrganization() { - if ctx.Org == nil { - ctx.Org = &Organization{} - } - ctx.Org.Organization = (*organization.Organization)(ctx.ContextUser) - } else { - // ContextUser is an individual User - return - } - org := ctx.Org.Organization + org := ctx.Org.Organization - // Handle Visibility - if org.Visibility != structs.VisibleTypePublic && !ctx.IsSigned { - // We must be signed in to see limited or private organizations - ctx.NotFound("OrgAssignment", err) - return - } - - if org.Visibility == structs.VisibleTypePrivate { - requireMember = true - } else if ctx.IsSigned && ctx.Doer.IsRestricted { - requireMember = true - } - - ctx.ContextUser = org.AsUser() - ctx.Data["Org"] = org - - // Admin has super access. - if ctx.IsSigned && ctx.Doer.IsAdmin { - ctx.Org.IsOwner = true - ctx.Org.IsMember = true - ctx.Org.IsTeamMember = true - ctx.Org.IsTeamAdmin = true - ctx.Org.CanCreateOrgRepo = true - } else if ctx.IsSigned { - ctx.Org.IsOwner, err = org.IsOwnedBy(ctx, ctx.Doer.ID) - if err != nil { - ctx.ServerError("IsOwnedBy", err) + // Handle Visibility + if org.Visibility != structs.VisibleTypePublic && !ctx.IsSigned { + // We must be signed in to see limited or private organizations + ctx.NotFound(err) return } - if ctx.Org.IsOwner { + if org.Visibility == structs.VisibleTypePrivate { + opts.RequireMember = true + } else if ctx.IsSigned && ctx.Doer.IsRestricted { + opts.RequireMember = true + } + + ctx.ContextUser = org.AsUser() + ctx.Data["Org"] = org + + // Admin has super access. + if ctx.IsSigned && ctx.Doer.IsAdmin { + ctx.Org.IsOwner = true ctx.Org.IsMember = true ctx.Org.IsTeamMember = true ctx.Org.IsTeamAdmin = true ctx.Org.CanCreateOrgRepo = true - } else { - ctx.Org.IsMember, err = org.IsOrgMember(ctx, ctx.Doer.ID) + } else if ctx.IsSigned { + ctx.Org.IsOwner, err = org.IsOwnedBy(ctx, ctx.Doer.ID) if err != nil { - ctx.ServerError("IsOrgMember", err) + ctx.ServerError("IsOwnedBy", err) return } - ctx.Org.CanCreateOrgRepo, err = org.CanCreateOrgRepo(ctx, ctx.Doer.ID) - if err != nil { - ctx.ServerError("CanCreateOrgRepo", err) - return + + if ctx.Org.IsOwner { + ctx.Org.IsMember = true + ctx.Org.IsTeamMember = true + ctx.Org.IsTeamAdmin = true + ctx.Org.CanCreateOrgRepo = true + } else { + ctx.Org.IsMember, err = org.IsOrgMember(ctx, ctx.Doer.ID) + if err != nil { + ctx.ServerError("IsOrgMember", err) + return + } + ctx.Org.CanCreateOrgRepo, err = org.CanCreateOrgRepo(ctx, ctx.Doer.ID) + if err != nil { + ctx.ServerError("CanCreateOrgRepo", err) + return + } } + } else { + // Fake data. + ctx.Data["SignedUser"] = &user_model.User{} } - } else { - // Fake data. - ctx.Data["SignedUser"] = &user_model.User{} - } - if (requireMember && !ctx.Org.IsMember) || - (requireOwner && !ctx.Org.IsOwner) { - ctx.NotFound("OrgAssignment", err) - return - } - ctx.Data["IsOrganizationOwner"] = ctx.Org.IsOwner - ctx.Data["IsOrganizationMember"] = ctx.Org.IsMember - ctx.Data["IsPackageEnabled"] = setting.Packages.Enabled - ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled - ctx.Data["IsPublicMember"] = func(uid int64) bool { - is, _ := organization.IsPublicMembership(ctx, ctx.Org.Organization.ID, uid) - return is - } - ctx.Data["CanCreateOrgRepo"] = ctx.Org.CanCreateOrgRepo + if (opts.RequireMember && !ctx.Org.IsMember) || (opts.RequireOwner && !ctx.Org.IsOwner) { + ctx.NotFound(err) + return + } + ctx.Data["IsOrganizationOwner"] = ctx.Org.IsOwner + ctx.Data["IsOrganizationMember"] = ctx.Org.IsMember + ctx.Data["IsPackageEnabled"] = setting.Packages.Enabled + ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled + ctx.Data["IsPublicMember"] = func(uid int64) bool { + is, _ := organization.IsPublicMembership(ctx, ctx.Org.Organization.ID, uid) + return is + } + ctx.Data["CanCreateOrgRepo"] = ctx.Org.CanCreateOrgRepo - ctx.Org.OrgLink = org.AsUser().OrganisationLink() - ctx.Data["OrgLink"] = ctx.Org.OrgLink + ctx.Org.OrgLink = org.AsUser().OrganisationLink() + ctx.Data["OrgLink"] = ctx.Org.OrgLink - // Member - opts := &organization.FindOrgMembersOpts{ - Doer: ctx.Doer, - OrgID: org.ID, - IsDoerMember: ctx.Org.IsMember, - } - ctx.Data["NumMembers"], err = organization.CountOrgMembers(ctx, opts) - if err != nil { - ctx.ServerError("CountOrgMembers", err) - return - } + // Member + findMembersOpts := &organization.FindOrgMembersOpts{ + Doer: ctx.Doer, + OrgID: org.ID, + IsDoerMember: ctx.Org.IsMember, + } + ctx.Data["NumMembers"], err = organization.CountOrgMembers(ctx, findMembersOpts) + if err != nil { + ctx.ServerError("CountOrgMembers", err) + return + } - // Team. - if ctx.Org.IsMember { - shouldSeeAllTeams := false - if ctx.Org.IsOwner { - shouldSeeAllTeams = true - } else { - teams, err := org.GetUserTeams(ctx, ctx.Doer.ID) - if err != nil { - ctx.ServerError("GetUserTeams", err) - return + // Team. + if ctx.Org.IsMember { + shouldSeeAllTeams := false + if ctx.Org.IsOwner { + shouldSeeAllTeams = true + } else { + teams, err := org.GetUserTeams(ctx, ctx.Doer.ID) + if err != nil { + ctx.ServerError("GetUserTeams", err) + return + } + for _, team := range teams { + if team.IncludesAllRepositories && team.HasAdminAccess() { + shouldSeeAllTeams = true + break + } + } } - for _, team := range teams { - if team.IncludesAllRepositories && team.AccessMode >= perm.AccessModeAdmin { - shouldSeeAllTeams = true - break + if shouldSeeAllTeams { + ctx.Org.Teams, err = org.LoadTeams(ctx) + if err != nil { + ctx.ServerError("LoadTeams", err) + return + } + } else { + ctx.Org.Teams, err = org.GetUserTeams(ctx, ctx.Doer.ID) + if err != nil { + ctx.ServerError("GetUserTeams", err) + return } } + ctx.Data["NumTeams"] = len(ctx.Org.Teams) } - if shouldSeeAllTeams { - ctx.Org.Teams, err = org.LoadTeams(ctx) - if err != nil { - ctx.ServerError("LoadTeams", err) - return + + teamName := ctx.PathParam("team") + if len(teamName) > 0 { + teamExists := false + for _, team := range ctx.Org.Teams { + if team.LowerName == strings.ToLower(teamName) { + teamExists = true + ctx.Org.Team = team + ctx.Org.IsTeamMember = true + ctx.Data["Team"] = ctx.Org.Team + break + } } - } else { - ctx.Org.Teams, err = org.GetUserTeams(ctx, ctx.Doer.ID) - if err != nil { - ctx.ServerError("GetUserTeams", err) + + if !teamExists { + ctx.NotFound(err) return } - } - ctx.Data["NumTeams"] = len(ctx.Org.Teams) - } - teamName := ctx.PathParam("team") - if len(teamName) > 0 { - teamExists := false - for _, team := range ctx.Org.Teams { - if team.LowerName == strings.ToLower(teamName) { - teamExists = true - ctx.Org.Team = team - ctx.Org.IsTeamMember = true - ctx.Data["Team"] = ctx.Org.Team - break + ctx.Data["IsTeamMember"] = ctx.Org.IsTeamMember + if opts.RequireTeamMember && !ctx.Org.IsTeamMember { + ctx.NotFound(err) + return } - } - - if !teamExists { - ctx.NotFound("OrgAssignment", err) - return - } - ctx.Data["IsTeamMember"] = ctx.Org.IsTeamMember - if requireTeamMember && !ctx.Org.IsTeamMember { - ctx.NotFound("OrgAssignment", err) - return + ctx.Org.IsTeamAdmin = ctx.Org.Team.IsOwnerTeam() || ctx.Org.Team.HasAdminAccess() + ctx.Data["IsTeamAdmin"] = ctx.Org.IsTeamAdmin + if opts.RequireTeamAdmin && !ctx.Org.IsTeamAdmin { + ctx.NotFound(err) + return + } } + ctx.Data["ContextUser"] = ctx.ContextUser - ctx.Org.IsTeamAdmin = ctx.Org.Team.IsOwnerTeam() || ctx.Org.Team.AccessMode >= perm.AccessModeAdmin - ctx.Data["IsTeamAdmin"] = ctx.Org.IsTeamAdmin - if requireTeamAdmin && !ctx.Org.IsTeamAdmin { - ctx.NotFound("OrgAssignment", err) - return - } - } - ctx.Data["ContextUser"] = ctx.ContextUser + ctx.Data["CanReadProjects"] = ctx.Org.CanReadUnit(ctx, unit.TypeProjects) + ctx.Data["CanReadPackages"] = ctx.Org.CanReadUnit(ctx, unit.TypePackages) + ctx.Data["CanReadCode"] = ctx.Org.CanReadUnit(ctx, unit.TypeCode) - ctx.Data["CanReadProjects"] = ctx.Org.CanReadUnit(ctx, unit.TypeProjects) - ctx.Data["CanReadPackages"] = ctx.Org.CanReadUnit(ctx, unit.TypePackages) - ctx.Data["CanReadCode"] = ctx.Org.CanReadUnit(ctx, unit.TypeCode) - - ctx.Data["IsFollowing"] = ctx.Doer != nil && user_model.IsFollowing(ctx, ctx.Doer.ID, ctx.ContextUser.ID) - if len(ctx.ContextUser.Description) != 0 { - content, err := markdown.RenderString(markup.NewRenderContext(ctx), ctx.ContextUser.Description) - if err != nil { - ctx.ServerError("RenderString", err) - return + ctx.Data["IsFollowing"] = ctx.Doer != nil && user_model.IsFollowing(ctx, ctx.Doer.ID, ctx.ContextUser.ID) + if len(ctx.ContextUser.Description) != 0 { + content, err := markdown.RenderString(markup.NewRenderContext(ctx), ctx.ContextUser.Description) + if err != nil { + ctx.ServerError("RenderString", err) + return + } + ctx.Data["RenderedDescription"] = content } - ctx.Data["RenderedDescription"] = content - } -} - -// OrgAssignment returns a middleware to handle organization assignment -func OrgAssignment(args ...bool) func(ctx *Context) { - return func(ctx *Context) { - HandleOrgAssignment(ctx, args...) } } diff --git a/services/context/package.go b/services/context/package.go index e98e01acbb..8b722932b1 100644 --- a/services/context/package.go +++ b/services/context/package.go @@ -33,15 +33,15 @@ type packageAssignmentCtx struct { // PackageAssignment returns a middleware to handle Context.Package assignment func PackageAssignment() func(ctx *Context) { return func(ctx *Context) { - errorFn := func(status int, title string, obj any) { + errorFn := func(status int, obj any) { err, ok := obj.(error) if !ok { err = fmt.Errorf("%s", obj) } if status == http.StatusNotFound { - ctx.NotFound(title, err) + ctx.NotFound(err) } else { - ctx.ServerError(title, err) + ctx.ServerError("PackageAssignment", err) } } paCtx := &packageAssignmentCtx{Base: ctx.Base, Doer: ctx.Doer, ContextUser: ctx.ContextUser} @@ -53,18 +53,18 @@ func PackageAssignment() func(ctx *Context) { func PackageAssignmentAPI() func(ctx *APIContext) { return func(ctx *APIContext) { paCtx := &packageAssignmentCtx{Base: ctx.Base, Doer: ctx.Doer, ContextUser: ctx.ContextUser} - ctx.Package = packageAssignment(paCtx, ctx.Error) + ctx.Package = packageAssignment(paCtx, ctx.APIError) } } -func packageAssignment(ctx *packageAssignmentCtx, errCb func(int, string, any)) *Package { +func packageAssignment(ctx *packageAssignmentCtx, errCb func(int, any)) *Package { pkg := &Package{ Owner: ctx.ContextUser, } var err error pkg.AccessMode, err = determineAccessMode(ctx.Base, pkg, ctx.Doer) if err != nil { - errCb(http.StatusInternalServerError, "determineAccessMode", err) + errCb(http.StatusInternalServerError, fmt.Errorf("determineAccessMode: %w", err)) return pkg } @@ -75,16 +75,16 @@ func packageAssignment(ctx *packageAssignmentCtx, errCb func(int, string, any)) pv, err := packages_model.GetVersionByNameAndVersion(ctx, pkg.Owner.ID, packages_model.Type(packageType), name, version) if err != nil { if err == packages_model.ErrPackageNotExist { - errCb(http.StatusNotFound, "GetVersionByNameAndVersion", err) + errCb(http.StatusNotFound, fmt.Errorf("GetVersionByNameAndVersion: %w", err)) } else { - errCb(http.StatusInternalServerError, "GetVersionByNameAndVersion", err) + errCb(http.StatusInternalServerError, fmt.Errorf("GetVersionByNameAndVersion: %w", err)) } return pkg } pkg.Descriptor, err = packages_model.GetPackageDescriptor(ctx, pv) if err != nil { - errCb(http.StatusInternalServerError, "GetPackageDescriptor", err) + errCb(http.StatusInternalServerError, fmt.Errorf("GetPackageDescriptor: %w", err)) return pkg } } @@ -93,7 +93,7 @@ func packageAssignment(ctx *packageAssignmentCtx, errCb func(int, string, any)) } func determineAccessMode(ctx *Base, pkg *Package, doer *user_model.User) (perm.AccessMode, error) { - if setting.Service.RequireSignInView && (doer == nil || doer.IsGhost()) { + if setting.Service.RequireSignInViewStrict && (doer == nil || doer.IsGhost()) { return perm.AccessModeNone, nil } @@ -154,9 +154,9 @@ func PackageContexter() func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { base := NewBaseContext(resp, req) - // it is still needed when rendering 500 page in a package handler + // FIXME: web Context is still needed when rendering 500 page in a package handler + // It should be refactored to use new error handling mechanisms ctx := NewWebContext(base, renderer, nil) - ctx.SetContextValue(WebContextKey, ctx) next.ServeHTTP(ctx.Resp, ctx.Req) }) } diff --git a/services/context/pagination.go b/services/context/pagination.go index d33dd217d0..2a9805db05 100644 --- a/services/context/pagination.go +++ b/services/context/pagination.go @@ -21,14 +21,20 @@ type Pagination struct { // NewPagination creates a new instance of the Pagination struct. // "pagingNum" is "page size" or "limit", "current" is "page" +// total=-1 means only showing prev/next func NewPagination(total, pagingNum, current, numPages int) *Pagination { p := &Pagination{} p.Paginater = paginator.New(total, pagingNum, current, numPages) return p } -func (p *Pagination) AddParamFromRequest(req *http.Request) { - for key, values := range req.URL.Query() { +func (p *Pagination) WithCurRows(n int) *Pagination { + p.Paginater.SetCurRows(n) + return p +} + +func (p *Pagination) AddParamFromQuery(q url.Values) { + for key, values := range q { if key == "page" || len(values) == 0 || (len(values) == 1 && values[0] == "") { continue } @@ -39,6 +45,10 @@ func (p *Pagination) AddParamFromRequest(req *http.Request) { } } +func (p *Pagination) AddParamFromRequest(req *http.Request) { + p.AddParamFromQuery(req.URL.Query()) +} + // GetParams returns the configured URL params func (p *Pagination) GetParams() template.URL { return template.URL(strings.Join(p.urlParams, "&")) diff --git a/services/context/permission.go b/services/context/permission.go index 9338587257..c0a5a98724 100644 --- a/services/context/permission.go +++ b/services/context/permission.go @@ -5,112 +5,55 @@ package context import ( "net/http" + "slices" auth_model "code.gitea.io/gitea/models/auth" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" - "code.gitea.io/gitea/modules/log" ) // RequireRepoAdmin returns a middleware for requiring repository admin permission func RequireRepoAdmin() func(ctx *Context) { return func(ctx *Context) { if !ctx.IsSigned || !ctx.Repo.IsAdmin() { - ctx.NotFound(ctx.Req.URL.RequestURI(), nil) + ctx.NotFound(nil) return } } } -// RequireRepoWriter returns a middleware for requiring repository write to the specify unitType -func RequireRepoWriter(unitType unit.Type) func(ctx *Context) { - return func(ctx *Context) { - if !ctx.Repo.CanWrite(unitType) { - ctx.NotFound(ctx.Req.URL.RequestURI(), nil) - return - } - } -} - -// CanEnableEditor checks if the user is allowed to write to the branch of the repo -func CanEnableEditor() func(ctx *Context) { +// CanWriteToBranch checks if the user is allowed to write to the branch of the repo +func CanWriteToBranch() func(ctx *Context) { return func(ctx *Context) { if !ctx.Repo.CanWriteToBranch(ctx, ctx.Doer, ctx.Repo.BranchName) { - ctx.NotFound("CanWriteToBranch denies permission", nil) + ctx.NotFound(nil) return } } } -// RequireRepoWriterOr returns a middleware for requiring repository write to one of the unit permission -func RequireRepoWriterOr(unitTypes ...unit.Type) func(ctx *Context) { +// RequireUnitWriter returns a middleware for requiring repository write to one of the unit permission +func RequireUnitWriter(unitTypes ...unit.Type) func(ctx *Context) { return func(ctx *Context) { - for _, unitType := range unitTypes { - if ctx.Repo.CanWrite(unitType) { - return - } - } - ctx.NotFound(ctx.Req.URL.RequestURI(), nil) - } -} - -// RequireRepoReader returns a middleware for requiring repository read to the specify unitType -func RequireRepoReader(unitType unit.Type) func(ctx *Context) { - return func(ctx *Context) { - if !ctx.Repo.CanRead(unitType) { - if unitType == unit.TypeCode && canWriteAsMaintainer(ctx) { - return - } - if log.IsTrace() { - if ctx.IsSigned { - log.Trace("Permission Denied: User %-v cannot read %-v in Repo %-v\n"+ - "User in Repo has Permissions: %-+v", - ctx.Doer, - unitType, - ctx.Repo.Repository, - ctx.Repo.Permission) - } else { - log.Trace("Permission Denied: Anonymous user cannot read %-v in Repo %-v\n"+ - "Anonymous user in Repo has Permissions: %-+v", - unitType, - ctx.Repo.Repository, - ctx.Repo.Permission) - } - } - ctx.NotFound(ctx.Req.URL.RequestURI(), nil) + if slices.ContainsFunc(unitTypes, ctx.Repo.CanWrite) { return } + ctx.NotFound(nil) } } -// RequireRepoReaderOr returns a middleware for requiring repository write to one of the unit permission -func RequireRepoReaderOr(unitTypes ...unit.Type) func(ctx *Context) { +// RequireUnitReader returns a middleware for requiring repository write to one of the unit permission +func RequireUnitReader(unitTypes ...unit.Type) func(ctx *Context) { return func(ctx *Context) { for _, unitType := range unitTypes { if ctx.Repo.CanRead(unitType) { return } - } - if log.IsTrace() { - var format string - var args []any - if ctx.IsSigned { - format = "Permission Denied: User %-v cannot read [" - args = append(args, ctx.Doer) - } else { - format = "Permission Denied: Anonymous user cannot read [" - } - for _, unit := range unitTypes { - format += "%-v, " - args = append(args, unit) + if unitType == unit.TypeCode && canWriteAsMaintainer(ctx) { + return } - - format = format[:len(format)-2] + "] in Repo %-v\n" + - "User in Repo has Permissions: %-+v" - args = append(args, ctx.Repo.Repository, ctx.Repo.Permission) - log.Trace(format, args...) } - ctx.NotFound(ctx.Req.URL.RequestURI(), nil) + ctx.NotFound(nil) } } @@ -134,7 +77,7 @@ func CheckRepoScopedToken(ctx *Context, repo *repo_model.Repository, level auth_ } if publicOnly && repo.IsPrivate { - ctx.Error(http.StatusForbidden) + ctx.HTTPError(http.StatusForbidden) return } @@ -145,7 +88,7 @@ func CheckRepoScopedToken(ctx *Context, repo *repo_model.Repository, level auth_ } if !scopeMatched { - ctx.Error(http.StatusForbidden) + ctx.HTTPError(http.StatusForbidden) return } } diff --git a/services/context/private.go b/services/context/private.go index 51857da8fe..d20e49f588 100644 --- a/services/context/private.go +++ b/services/context/private.go @@ -5,7 +5,6 @@ package context import ( "context" - "fmt" "net/http" "time" @@ -29,7 +28,6 @@ func init() { }) } -// Deadline is part of the interface for context.Context and we pass this to the request context func (ctx *PrivateContext) Deadline() (deadline time.Time, ok bool) { if ctx.Override != nil { return ctx.Override.Deadline() @@ -37,7 +35,6 @@ func (ctx *PrivateContext) Deadline() (deadline time.Time, ok bool) { return ctx.Base.Deadline() } -// Done is part of the interface for context.Context and we pass this to the request context func (ctx *PrivateContext) Done() <-chan struct{} { if ctx.Override != nil { return ctx.Override.Done() @@ -45,7 +42,6 @@ func (ctx *PrivateContext) Done() <-chan struct{} { return ctx.Base.Done() } -// Err is part of the interface for context.Context and we pass this to the request context func (ctx *PrivateContext) Err() error { if ctx.Override != nil { return ctx.Override.Err() @@ -53,14 +49,14 @@ func (ctx *PrivateContext) Err() error { return ctx.Base.Err() } -var privateContextKey any = "default_private_context" +type privateContextKeyType struct{} + +var privateContextKey privateContextKeyType -// GetPrivateContext returns a context for Private routes func GetPrivateContext(req *http.Request) *PrivateContext { return req.Context().Value(privateContextKey).(*PrivateContext) } -// PrivateContexter returns apicontext as middleware func PrivateContexter() func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -82,7 +78,7 @@ func OverrideContext() func(http.Handler) http.Handler { // We now need to override the request context as the base for our work because even if the request is cancelled we have to continue this work ctx := GetPrivateContext(req) var finished func() - ctx.Override, _, finished = process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), fmt.Sprintf("PrivateContext: %s", ctx.Req.RequestURI), process.RequestProcessType, true) + ctx.Override, _, finished = process.GetManager().AddTypedContext(graceful.GetManager().HammerContext(), "PrivateContext: "+ctx.Req.RequestURI, process.RequestProcessType, true) defer finished() next.ServeHTTP(ctx.Resp, ctx.Req) }) diff --git a/services/context/repo.go b/services/context/repo.go index 4de905ef2c..572211712b 100644 --- a/services/context/repo.go +++ b/services/context/repo.go @@ -46,22 +46,21 @@ type PullRequest struct { // Repository contains information to operate a repository type Repository struct { access_model.Permission - IsWatching bool - IsViewBranch bool - IsViewTag bool - IsViewCommit bool - Repository *repo_model.Repository - Owner *user_model.User + + Repository *repo_model.Repository + Owner *user_model.User + + RepoLink string + GitRepo *git.Repository + + // RefFullName is the full ref name that the user is viewing + RefFullName git.RefName + BranchName string // it is the RefFullName's short name if its type is "branch" + TreePath string + + // Commit it is always set to the commit for the branch or tag, or just the commit that the user is viewing Commit *git.Commit - Tag *git.Tag - GitRepo *git.Repository - RefName string - BranchName string - TagName string - TreePath string CommitID string - RepoLink string - CloneLink repo_model.CloneLink CommitsCount int64 PullRequest *PullRequest @@ -72,11 +71,6 @@ func (r *Repository) CanWriteToBranch(ctx context.Context, user *user_model.User return issues_model.CanMaintainerWriteToBranch(ctx, r.Permission, branch, user) } -// CanEnableEditor returns true if repository is editable and user has proper access level. -func (r *Repository) CanEnableEditor(ctx context.Context, user *user_model.User) bool { - return r.IsViewBranch && r.CanWriteToBranch(ctx, user, r.BranchName) && r.Repository.CanEnableEditor() && !r.Repository.IsArchived -} - // CanCreateBranch returns true if repository is editable and user has proper access level. func (r *Repository) CanCreateBranch() bool { return r.Permission.CanWrite(unit_model.TypeCode) && r.Repository.CanCreateBranch() @@ -90,66 +84,108 @@ func (r *Repository) GetObjectFormat() git.ObjectFormat { func RepoMustNotBeArchived() func(ctx *Context) { return func(ctx *Context) { if ctx.Repo.Repository.IsArchived { - ctx.NotFound("IsArchived", errors.New(ctx.Locale.TrString("repo.archive.title"))) + ctx.NotFound(errors.New(ctx.Locale.TrString("repo.archive.title"))) } } } -// CanCommitToBranchResults represents the results of CanCommitToBranch -type CanCommitToBranchResults struct { - CanCommitToBranch bool - EditorEnabled bool - UserCanPush bool - RequireSigned bool - WillSign bool - SigningKey string - WontSignReason string +type CommitFormOptions struct { + NeedFork bool + + TargetRepo *repo_model.Repository + TargetFormAction string + WillSubmitToFork bool + CanCommitToBranch bool + UserCanPush bool + RequireSigned bool + WillSign bool + SigningKey *git.SigningKey + WontSignReason string + CanCreatePullRequest bool + CanCreateBasePullRequest bool } -// CanCommitToBranch returns true if repository is editable and user has proper access level -// -// and branch is not protected for push -func (r *Repository) CanCommitToBranch(ctx context.Context, doer *user_model.User) (CanCommitToBranchResults, error) { - protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, r.Repository.ID, r.BranchName) +func PrepareCommitFormOptions(ctx *Context, doer *user_model.User, targetRepo *repo_model.Repository, doerRepoPerm access_model.Permission, refName git.RefName) (*CommitFormOptions, error) { + if !refName.IsBranch() { + // it shouldn't happen because middleware already checks + return nil, util.NewInvalidArgumentErrorf("ref %q is not a branch", refName) + } + + originRepo := targetRepo + branchName := refName.ShortName() + // TODO: CanMaintainerWriteToBranch is a bad name, but it really does what "CanWriteToBranch" does + if !issues_model.CanMaintainerWriteToBranch(ctx, doerRepoPerm, branchName, doer) { + targetRepo = repo_model.GetForkedRepo(ctx, doer.ID, targetRepo.ID) + if targetRepo == nil { + return &CommitFormOptions{NeedFork: true}, nil + } + // now, we get our own forked repo; it must be writable by us. + } + submitToForkedRepo := targetRepo.ID != originRepo.ID + err := targetRepo.GetBaseRepo(ctx) + if err != nil { + return nil, err + } + + protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, targetRepo.ID, branchName) if err != nil { - return CanCommitToBranchResults{}, err + return nil, err } - userCanPush := true - requireSigned := false + canPushWithProtection := true + protectionRequireSigned := false if protectedBranch != nil { - protectedBranch.Repo = r.Repository - userCanPush = protectedBranch.CanUserPush(ctx, doer) - requireSigned = protectedBranch.RequireSignedCommits + protectedBranch.Repo = targetRepo + canPushWithProtection = protectedBranch.CanUserPush(ctx, doer) + protectionRequireSigned = protectedBranch.RequireSignedCommits } - sign, keyID, _, err := asymkey_service.SignCRUDAction(ctx, r.Repository.RepoPath(), doer, r.Repository.RepoPath(), git.BranchPrefix+r.BranchName) - - canCommit := r.CanEnableEditor(ctx, doer) && userCanPush - if requireSigned { - canCommit = canCommit && sign - } + willSign, signKeyID, _, err := asymkey_service.SignCRUDAction(ctx, targetRepo.RepoPath(), doer, targetRepo.RepoPath(), refName.String()) wontSignReason := "" - if err != nil { - if asymkey_service.IsErrWontSign(err) { - wontSignReason = string(err.(*asymkey_service.ErrWontSign).Reason) - err = nil - } else { - wontSignReason = "error" - } + if asymkey_service.IsErrWontSign(err) { + wontSignReason = string(err.(*asymkey_service.ErrWontSign).Reason) + } else if err != nil { + return nil, err + } + + canCommitToBranch := !submitToForkedRepo /* same repo */ && targetRepo.CanEnableEditor() && canPushWithProtection + if protectionRequireSigned { + canCommitToBranch = canCommitToBranch && willSign } - return CanCommitToBranchResults{ - CanCommitToBranch: canCommit, - EditorEnabled: r.CanEnableEditor(ctx, doer), - UserCanPush: userCanPush, - RequireSigned: requireSigned, - WillSign: sign, - SigningKey: keyID, + canCreateBasePullRequest := targetRepo.BaseRepo != nil && targetRepo.BaseRepo.UnitEnabled(ctx, unit_model.TypePullRequests) + canCreatePullRequest := targetRepo.UnitEnabled(ctx, unit_model.TypePullRequests) || canCreateBasePullRequest + + opts := &CommitFormOptions{ + TargetRepo: targetRepo, + WillSubmitToFork: submitToForkedRepo, + CanCommitToBranch: canCommitToBranch, + UserCanPush: canPushWithProtection, + RequireSigned: protectionRequireSigned, + WillSign: willSign, + SigningKey: signKeyID, WontSignReason: wontSignReason, - }, err + + CanCreatePullRequest: canCreatePullRequest, + CanCreateBasePullRequest: canCreateBasePullRequest, + } + editorAction := ctx.PathParam("editor_action") + editorPathParamRemaining := util.PathEscapeSegments(branchName) + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) + if submitToForkedRepo { + // there is only "default branch" in forked repo, we will use "from_base_branch" to get a new branch from base repo + editorPathParamRemaining = util.PathEscapeSegments(targetRepo.DefaultBranch) + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) + "?from_base_branch=" + url.QueryEscape(branchName) + } + if editorAction == "_cherrypick" { + opts.TargetFormAction = targetRepo.Link() + "/" + editorAction + "/" + ctx.PathParam("sha") + "/" + editorPathParamRemaining + } else { + opts.TargetFormAction = targetRepo.Link() + "/" + editorAction + "/" + editorPathParamRemaining + } + if ctx.Req.URL.RawQuery != "" { + opts.TargetFormAction += util.Iif(strings.Contains(opts.TargetFormAction, "?"), "&", "?") + ctx.Req.URL.RawQuery + } + return opts, nil } -// CanUseTimetracker returns whether or not a user can use the timetracker. +// CanUseTimetracker returns whether a user can use the timetracker. func (r *Repository) CanUseTimetracker(ctx context.Context, issue *issues_model.Issue, user *user_model.User) bool { // Checking for following: // 1. Is timetracker enabled @@ -169,15 +205,9 @@ func (r *Repository) GetCommitsCount() (int64, error) { if r.Commit == nil { return 0, nil } - var contextName string - if r.IsViewBranch { - contextName = r.BranchName - } else if r.IsViewTag { - contextName = r.TagName - } else { - contextName = r.CommitID - } - return cache.GetInt64(r.Repository.GetCommitsCountCacheKey(contextName, r.IsViewBranch || r.IsViewTag), func() (int64, error) { + contextName := r.RefFullName.ShortName() + isRef := r.RefFullName.IsBranch() || r.RefFullName.IsTag() + return cache.GetInt64(r.Repository.GetCommitsCountCacheKey(contextName, isRef), func() (int64, error) { return r.Commit.CommitsCount() }) } @@ -199,33 +229,13 @@ func (r *Repository) GetCommitGraphsCount(ctx context.Context, hidePRRefs bool, }) } -// BranchNameSubURL sub-URL for the BranchName field -func (r *Repository) BranchNameSubURL() string { - switch { - case r.IsViewBranch: - return "branch/" + util.PathEscapeSegments(r.BranchName) - case r.IsViewTag: - return "tag/" + util.PathEscapeSegments(r.TagName) - case r.IsViewCommit: - return "commit/" + util.PathEscapeSegments(r.CommitID) - } - log.Error("Unknown view type for repo: %v", r) - return "" -} - -// FileExists returns true if a file exists in the given repo branch -func (r *Repository) FileExists(path, branch string) (bool, error) { - if branch == "" { - branch = r.Repository.DefaultBranch - } - commit, err := r.GitRepo.GetBranchCommit(branch) - if err != nil { - return false, err - } - if _, err := commit.GetTreeEntryByPath(path); err != nil { - return false, err - } - return true, nil +// RefTypeNameSubURL makes a sub-url for the current ref (branch/tag/commit) field, for example: +// * "branch/master" +// * "tag/v1.0.0" +// * "commit/123456" +// It is usually used to construct a link like ".../src/{{RefTypeNameSubURL}}/{{PathEscapeSegments TreePath}}" +func (r *Repository) RefTypeNameSubURL() string { + return r.RefFullName.RefWebLinkPath() } // GetEditorconfig returns the .editorconfig definition if found in the @@ -342,7 +352,7 @@ func RedirectToRepo(ctx *Base, redirectRepoID int64) { repo, err := repo_model.GetRepositoryByID(ctx, redirectRepoID) if err != nil { log.Error("GetRepositoryByID: %v", err) - ctx.Error(http.StatusInternalServerError, "GetRepositoryByID") + ctx.HTTPError(http.StatusInternalServerError, "GetRepositoryByID") return } @@ -355,7 +365,9 @@ func RedirectToRepo(ctx *Base, redirectRepoID int64) { if ctx.Req.URL.RawQuery != "" { redirectPath += "?" + ctx.Req.URL.RawQuery } - ctx.Redirect(path.Join(setting.AppSubURL, redirectPath), http.StatusTemporaryRedirect) + // Git client needs a 301 redirect by default to follow the new location + // It's not documentated in git documentation, but it's the behavior of git client + ctx.Redirect(path.Join(setting.AppSubURL, redirectPath), http.StatusMovedPermanently) } func repoAssignment(ctx *Context, repo *repo_model.Repository) { @@ -365,18 +377,22 @@ func repoAssignment(ctx *Context, repo *repo_model.Repository) { return } - ctx.Repo.Permission, err = access_model.GetUserRepoPermission(ctx, repo, ctx.Doer) - if err != nil { - ctx.ServerError("GetUserRepoPermission", err) - return + if ctx.DoerNeedTwoFactorAuth() { + ctx.Repo.Permission = access_model.PermissionNoAccess() + } else { + ctx.Repo.Permission, err = access_model.GetUserRepoPermission(ctx, repo, ctx.Doer) + if err != nil { + ctx.ServerError("GetUserRepoPermission", err) + return + } } - if !ctx.Repo.Permission.HasAnyUnitAccessOrEveryoneAccess() && !canWriteAsMaintainer(ctx) { + if !ctx.Repo.Permission.HasAnyUnitAccessOrPublicAccess() && !canWriteAsMaintainer(ctx) { if ctx.FormString("go-get") == "1" { EarlyResponseForGoGetMeta(ctx) return } - ctx.NotFound("no access right", nil) + ctx.NotFound(nil) return } ctx.Data["Permission"] = &ctx.Repo.Permission @@ -398,33 +414,25 @@ func repoAssignment(ctx *Context, repo *repo_model.Repository) { // RepoAssignment returns a middleware to handle repository assignment func RepoAssignment(ctx *Context) { - if _, repoAssignmentOnce := ctx.Data["repoAssignmentExecuted"]; repoAssignmentOnce { - // FIXME: it should panic in dev/test modes to have a clear behavior - if !setting.IsProd || setting.IsInTesting { - panic("RepoAssignment should not be executed twice") - } - return + if ctx.Data["Repository"] != nil { + setting.PanicInDevOrTesting("RepoAssignment should not be executed twice") } - ctx.Data["repoAssignmentExecuted"] = true - - var ( - owner *user_model.User - err error - ) + var err error userName := ctx.PathParam("username") repoName := ctx.PathParam("reponame") repoName = strings.TrimSuffix(repoName, ".git") if setting.Other.EnableFeed { + ctx.Data["EnableFeed"] = true repoName = strings.TrimSuffix(repoName, ".rss") repoName = strings.TrimSuffix(repoName, ".atom") } // Check if the user is the same as the repository owner if ctx.IsSigned && ctx.Doer.LowerName == strings.ToLower(userName) { - owner = ctx.Doer + ctx.Repo.Owner = ctx.Doer } else { - owner, err = user_model.GetUserByName(ctx, userName) + ctx.Repo.Owner, err = user_model.GetUserByName(ctx, userName) if err != nil { if user_model.IsErrUserNotExist(err) { // go-get does not support redirects @@ -437,7 +445,7 @@ func RepoAssignment(ctx *Context) { if redirectUserID, err := user_model.LookupUserRedirect(ctx, userName); err == nil { RedirectToUser(ctx.Base, userName, redirectUserID) } else if user_model.IsErrUserRedirectNotExist(err) { - ctx.NotFound("GetUserByName", nil) + ctx.NotFound(nil) } else { ctx.ServerError("LookupUserRedirect", err) } @@ -447,10 +455,8 @@ func RepoAssignment(ctx *Context) { return } } - ctx.Repo.Owner = owner - ctx.ContextUser = owner + ctx.ContextUser = ctx.Repo.Owner ctx.Data["ContextUser"] = ctx.ContextUser - ctx.Data["Username"] = ctx.Repo.Owner.Name // redirect link to wiki if strings.HasSuffix(repoName, ".wiki") { @@ -473,10 +479,10 @@ func RepoAssignment(ctx *Context) { } // Get repository. - repo, err := repo_model.GetRepositoryByName(ctx, owner.ID, repoName) + repo, err := repo_model.GetRepositoryByName(ctx, ctx.Repo.Owner.ID, repoName) if err != nil { if repo_model.IsErrRepoNotExist(err) { - redirectRepoID, err := repo_model.LookupRedirect(ctx, owner.ID, repoName) + redirectRepoID, err := repo_model.LookupRedirect(ctx, ctx.Repo.Owner.ID, repoName) if err == nil { RedirectToRepo(ctx.Base, redirectRepoID) } else if repo_model.IsErrRedirectNotExist(err) { @@ -484,7 +490,7 @@ func RepoAssignment(ctx *Context) { EarlyResponseForGoGetMeta(ctx) return } - ctx.NotFound("GetRepositoryByName", nil) + ctx.NotFound(nil) } else { ctx.ServerError("LookupRepoRedirect", err) } @@ -493,7 +499,7 @@ func RepoAssignment(ctx *Context) { } return } - repo.Owner = owner + repo.Owner = ctx.Repo.Owner repoAssignment(ctx, repo) if ctx.Written() { @@ -502,12 +508,7 @@ func RepoAssignment(ctx *Context) { ctx.Repo.RepoLink = repo.Link() ctx.Data["RepoLink"] = ctx.Repo.RepoLink - ctx.Data["RepoRelPath"] = ctx.Repo.Owner.Name + "/" + ctx.Repo.Repository.Name - - if setting.Other.EnableFeed { - ctx.Data["EnableFeed"] = true - ctx.Data["FeedURL"] = ctx.Repo.RepoLink - } + ctx.Data["FeedURL"] = ctx.Repo.RepoLink unit, err := ctx.Repo.Repository.GetUnit(ctx, unit_model.TypeExternalTracker) if err == nil { @@ -534,12 +535,9 @@ func RepoAssignment(ctx *Context) { return } - ctx.Data["Title"] = owner.Name + "/" + repo.Name + ctx.Data["Title"] = repo.Owner.Name + "/" + repo.Name ctx.Data["Repository"] = repo ctx.Data["Owner"] = ctx.Repo.Repository.Owner - ctx.Data["IsRepositoryOwner"] = ctx.Repo.IsOwner() - ctx.Data["IsRepositoryAdmin"] = ctx.Repo.IsAdmin() - ctx.Data["RepoOwnerIsOrganization"] = repo.Owner.IsOrganization() ctx.Data["CanWriteCode"] = ctx.Repo.CanWrite(unit_model.TypeCode) ctx.Data["CanWriteIssues"] = ctx.Repo.CanWrite(unit_model.TypeIssues) ctx.Data["CanWritePulls"] = ctx.Repo.CanWrite(unit_model.TypePullRequests) @@ -607,7 +605,6 @@ func RepoAssignment(ctx *Context) { // Disable everything when the repo is being created if ctx.Repo.Repository.IsBeingCreated() || ctx.Repo.Repository.IsBroken() { - ctx.Data["BranchName"] = ctx.Repo.Repository.DefaultBranch if !isHomeOrSettings { ctx.Redirect(ctx.Repo.RepoLink) } @@ -615,9 +612,7 @@ func RepoAssignment(ctx *Context) { } if ctx.Repo.GitRepo != nil { - if !setting.IsProd || setting.IsInTesting { - panic("RepoAssignment: GitRepo should be nil") - } + setting.PanicInDevOrTesting("RepoAssignment: GitRepo should be nil") _ = ctx.Repo.GitRepo.Close() ctx.Repo.GitRepo = nil } @@ -627,7 +622,6 @@ func RepoAssignment(ctx *Context) { if strings.Contains(err.Error(), "repository does not exist") || strings.Contains(err.Error(), "no such file or directory") { log.Error("Repository %-v has a broken repository on the file system: %s Error: %v", ctx.Repo.Repository, ctx.Repo.Repository.RepoPath(), err) ctx.Repo.Repository.MarkAsBrokenEmpty() - ctx.Data["BranchName"] = ctx.Repo.Repository.DefaultBranch // Only allow access to base of repo or settings if !isHomeOrSettings { ctx.Redirect(ctx.Repo.RepoLink) @@ -640,7 +634,6 @@ func RepoAssignment(ctx *Context) { // Stop at this point when the repo is empty. if ctx.Repo.Repository.IsEmpty { - ctx.Data["BranchName"] = ctx.Repo.Repository.DefaultBranch return } @@ -666,22 +659,6 @@ func RepoAssignment(ctx *Context) { ctx.Data["BranchesCount"] = branchesTotal - // If no branch is set in the request URL, try to guess a default one. - if len(ctx.Repo.BranchName) == 0 { - if len(ctx.Repo.Repository.DefaultBranch) > 0 && ctx.Repo.GitRepo.IsBranchExist(ctx.Repo.Repository.DefaultBranch) { - ctx.Repo.BranchName = ctx.Repo.Repository.DefaultBranch - } else { - ctx.Repo.BranchName, _ = gitrepo.GetDefaultBranch(ctx, ctx.Repo.Repository) - if ctx.Repo.BranchName == "" { - // If it still can't get a default branch, fall back to default branch from setting. - // Something might be wrong. Either site admin should fix the repo sync or Gitea should fix a potential bug. - ctx.Repo.BranchName = setting.Repository.DefaultBranch - } - } - ctx.Repo.RefName = ctx.Repo.BranchName - } - ctx.Data["BranchName"] = ctx.Repo.BranchName - // People who have push access or have forked repository can propose a new pull request. canPush := ctx.Repo.CanWrite(unit_model.TypeCode) || (ctx.IsSigned && repo_model.HasForkedRepo(ctx, ctx.Doer.ID, ctx.Repo.Repository.ID)) @@ -719,39 +696,20 @@ func RepoAssignment(ctx *Context) { ctx.Data["RepoTransfer"] = repoTransfer if ctx.Doer != nil { - ctx.Data["CanUserAcceptTransfer"] = repoTransfer.CanUserAcceptTransfer(ctx, ctx.Doer) + ctx.Data["CanUserAcceptOrRejectTransfer"] = repoTransfer.CanUserAcceptOrRejectTransfer(ctx, ctx.Doer) } } if ctx.FormString("go-get") == "1" { - ctx.Data["GoGetImport"] = ComposeGoGetImport(ctx, owner.Name, repo.Name) + ctx.Data["GoGetImport"] = ComposeGoGetImport(ctx, repo.Owner.Name, repo.Name) fullURLPrefix := repo.HTMLURL() + "/src/branch/" + util.PathEscapeSegments(ctx.Repo.BranchName) ctx.Data["GoDocDirectory"] = fullURLPrefix + "{/dir}" ctx.Data["GoDocFile"] = fullURLPrefix + "{/dir}/{file}#L{line}" } } -// RepoRefType type of repo reference -type RepoRefType int - -const ( - // RepoRefUnknown is for legacy support, makes the code to "guess" the ref type - RepoRefUnknown RepoRefType = iota - RepoRefBranch - RepoRefTag - RepoRefCommit - RepoRefBlob -) - const headRefName = "HEAD" -// RepoRef handles repository reference names when the ref name is not -// explicitly given -func RepoRef() func(*Context) { - // since no ref name is explicitly specified, ok to just use branch - return RepoRefByType(RepoRefBranch) -} - func getRefNameFromPath(repo *Repository, path string, isExist func(string) bool) string { refName := "" parts := strings.Split(path, "/") @@ -765,37 +723,29 @@ func getRefNameFromPath(repo *Repository, path string, isExist func(string) bool return "" } -func getRefNameLegacy(ctx *Base, repo *Repository, optionalExtraRef ...string) (string, RepoRefType) { - extraRef := util.OptionalArg(optionalExtraRef) - reqPath := ctx.PathParam("*") - reqPath = path.Join(extraRef, reqPath) - - if refName := getRefName(ctx, repo, RepoRefBranch); refName != "" { - return refName, RepoRefBranch +func getRefNameLegacy(ctx *Base, repo *Repository, reqPath, extraRef string) (refName string, refType git.RefType, fallbackDefaultBranch bool) { + reqRefPath := path.Join(extraRef, reqPath) + reqRefPathParts := strings.Split(reqRefPath, "/") + if refName := getRefName(ctx, repo, reqRefPath, git.RefTypeBranch); refName != "" { + return refName, git.RefTypeBranch, false } - if refName := getRefName(ctx, repo, RepoRefTag); refName != "" { - return refName, RepoRefTag + if refName := getRefName(ctx, repo, reqRefPath, git.RefTypeTag); refName != "" { + return refName, git.RefTypeTag, false } - - // For legacy support only full commit sha - parts := strings.Split(reqPath, "/") - if git.IsStringLikelyCommitID(git.ObjectFormatFromName(repo.Repository.ObjectFormatName), parts[0]) { + if git.IsStringLikelyCommitID(git.ObjectFormatFromName(repo.Repository.ObjectFormatName), reqRefPathParts[0]) { // FIXME: this logic is different from other types. Ideally, it should also try to GetCommit to check if it exists - repo.TreePath = strings.Join(parts[1:], "/") - return parts[0], RepoRefCommit - } - - if refName := getRefName(ctx, repo, RepoRefBlob); len(refName) > 0 { - return refName, RepoRefBlob + repo.TreePath = strings.Join(reqRefPathParts[1:], "/") + return reqRefPathParts[0], git.RefTypeCommit, false } + // FIXME: the old code falls back to default branch if "ref" doesn't exist, there could be an edge case: + // "README?ref=no-such" would read the README file from the default branch, but the user might expect a 404 repo.TreePath = reqPath - return repo.Repository.DefaultBranch, RepoRefBranch + return repo.Repository.DefaultBranch, git.RefTypeBranch, true } -func getRefName(ctx *Base, repo *Repository, pathType RepoRefType) string { - path := ctx.PathParam("*") - switch pathType { - case RepoRefBranch: +func getRefName(ctx *Base, repo *Repository, path string, refType git.RefType) string { + switch refType { + case git.RefTypeBranch: ref := getRefNameFromPath(repo, path, repo.GitRepo.IsBranchExist) if len(ref) == 0 { // check if ref is HEAD @@ -825,9 +775,9 @@ func getRefName(ctx *Base, repo *Repository, pathType RepoRefType) string { } return ref - case RepoRefTag: + case git.RefTypeTag: return getRefNameFromPath(repo, path, repo.GitRepo.IsTagExist) - case RepoRefCommit: + case git.RefTypeCommit: parts := strings.Split(path, "/") if git.IsStringLikelyCommitID(repo.GetObjectFormat(), parts[0], 7) { // FIXME: this logic is different from other types. Ideally, it should also try to GetCommit to check if it exists @@ -844,66 +794,76 @@ func getRefName(ctx *Base, repo *Repository, pathType RepoRefType) string { repo.TreePath = strings.Join(parts[1:], "/") return commit.ID.String() } - case RepoRefBlob: - _, err := repo.GitRepo.GetBlob(path) - if err != nil { - return "" - } - return path default: - panic(fmt.Sprintf("Unrecognized path type: %v", pathType)) + panic(fmt.Sprintf("Unrecognized ref type: %v", refType)) } return "" } -type RepoRefByTypeOptions struct { - IgnoreNotExistErr bool +func repoRefFullName(typ git.RefType, shortName string) git.RefName { + switch typ { + case git.RefTypeBranch: + return git.RefNameFromBranch(shortName) + case git.RefTypeTag: + return git.RefNameFromTag(shortName) + case git.RefTypeCommit: + return git.RefNameFromCommit(shortName) + default: + setting.PanicInDevOrTesting("Unknown RepoRefType: %v", typ) + return git.RefNameFromBranch("main") // just a dummy result, it shouldn't happen + } +} + +func RepoRefByDefaultBranch() func(*Context) { + return func(ctx *Context) { + ctx.Repo.RefFullName = git.RefNameFromBranch(ctx.Repo.Repository.DefaultBranch) + ctx.Repo.BranchName = ctx.Repo.Repository.DefaultBranch + ctx.Repo.Commit, _ = ctx.Repo.GitRepo.GetBranchCommit(ctx.Repo.BranchName) + ctx.Repo.CommitsCount, _ = ctx.Repo.GetCommitsCount() + ctx.Data["RefFullName"] = ctx.Repo.RefFullName + ctx.Data["BranchName"] = ctx.Repo.BranchName + ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount + } } // RepoRefByType handles repository reference name for a specific type // of repository reference -func RepoRefByType(detectRefType RepoRefType, opts ...RepoRefByTypeOptions) func(*Context) { - opt := util.OptionalArg(opts) +func RepoRefByType(detectRefType git.RefType) func(*Context) { return func(ctx *Context) { + var err error refType := detectRefType + if ctx.Repo.Repository.IsBeingCreated() || ctx.Repo.Repository.IsBroken() { + return // no git repo, so do nothing, users will see a "migrating" UI provided by "migrate/migrating.tmpl", or empty repo guide + } // Empty repository does not have reference information. if ctx.Repo.Repository.IsEmpty { // assume the user is viewing the (non-existent) default branch - ctx.Repo.IsViewBranch = true ctx.Repo.BranchName = ctx.Repo.Repository.DefaultBranch + ctx.Repo.RefFullName = git.RefNameFromBranch(ctx.Repo.BranchName) + // these variables are used by the template to "add/upload" new files + ctx.Data["BranchName"] = ctx.Repo.BranchName ctx.Data["TreePath"] = "" return } - var ( - refName string - err error - ) - - if ctx.Repo.GitRepo == nil { - ctx.Repo.GitRepo, err = gitrepo.RepositoryFromRequestContextOrOpen(ctx, ctx.Repo.Repository) - if err != nil { - ctx.ServerError(fmt.Sprintf("Open Repository %v failed", ctx.Repo.Repository.FullName()), err) - return - } - } - // Get default branch. - if len(ctx.PathParam("*")) == 0 { - refName = ctx.Repo.Repository.DefaultBranch - if !ctx.Repo.GitRepo.IsBranchExist(refName) { - brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 1) + var refShortName string + reqPath := ctx.PathParam("*") + if reqPath == "" { + refShortName = ctx.Repo.Repository.DefaultBranch + if !gitrepo.IsBranchExist(ctx, ctx.Repo.Repository, refShortName) { + brs, _, err := ctx.Repo.GitRepo.GetBranchNames(0, 1) if err == nil && len(brs) != 0 { - refName = brs[0].Name + refShortName = brs[0] } else if len(brs) == 0 { log.Error("No branches in non-empty repository %s", ctx.Repo.GitRepo.Path) } else { log.Error("GetBranches error: %v", err) } } - ctx.Repo.RefName = refName - ctx.Repo.BranchName = refName - ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refName) + ctx.Repo.RefFullName = git.RefNameFromBranch(refShortName) + ctx.Repo.BranchName = refShortName + ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refShortName) if err == nil { ctx.Repo.CommitID = ctx.Repo.Commit.ID.String() } else if strings.Contains(err.Error(), "fatal: not a git repository") || strings.Contains(err.Error(), "object does not exist") { @@ -913,92 +873,99 @@ func RepoRefByType(detectRefType RepoRefType, opts ...RepoRefByTypeOptions) func ctx.ServerError("GetBranchCommit", err) return } - ctx.Repo.IsViewBranch = true - } else { - guessLegacyPath := refType == RepoRefUnknown + } else { // there is a path in request + guessLegacyPath := refType == "" + fallbackDefaultBranch := false if guessLegacyPath { - refName, refType = getRefNameLegacy(ctx.Base, ctx.Repo) + refShortName, refType, fallbackDefaultBranch = getRefNameLegacy(ctx.Base, ctx.Repo, reqPath, "") } else { - refName = getRefName(ctx.Base, ctx.Repo, refType) + refShortName = getRefName(ctx.Base, ctx.Repo, reqPath, refType) } - ctx.Repo.RefName = refName + ctx.Repo.RefFullName = repoRefFullName(refType, refShortName) isRenamedBranch, has := ctx.Data["IsRenamedBranch"].(bool) if isRenamedBranch && has { renamedBranchName := ctx.Data["RenamedBranchName"].(string) - ctx.Flash.Info(ctx.Tr("repo.branch.renamed", refName, renamedBranchName)) - link := setting.AppSubURL + strings.Replace(ctx.Req.URL.EscapedPath(), util.PathEscapeSegments(refName), util.PathEscapeSegments(renamedBranchName), 1) + ctx.Flash.Info(ctx.Tr("repo.branch.renamed", refShortName, renamedBranchName)) + link := setting.AppSubURL + strings.Replace(ctx.Req.URL.EscapedPath(), util.PathEscapeSegments(refShortName), util.PathEscapeSegments(renamedBranchName), 1) ctx.Redirect(link) return } - if refType == RepoRefBranch && ctx.Repo.GitRepo.IsBranchExist(refName) { - ctx.Repo.IsViewBranch = true - ctx.Repo.BranchName = refName + if refType == git.RefTypeBranch && gitrepo.IsBranchExist(ctx, ctx.Repo.Repository, refShortName) { + ctx.Repo.BranchName = refShortName + ctx.Repo.RefFullName = git.RefNameFromBranch(refShortName) - ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refName) + ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refShortName) if err != nil { ctx.ServerError("GetBranchCommit", err) return } ctx.Repo.CommitID = ctx.Repo.Commit.ID.String() - } else if refType == RepoRefTag && ctx.Repo.GitRepo.IsTagExist(refName) { - ctx.Repo.IsViewTag = true - ctx.Repo.TagName = refName + } else if refType == git.RefTypeTag && gitrepo.IsTagExist(ctx, ctx.Repo.Repository, refShortName) { + ctx.Repo.RefFullName = git.RefNameFromTag(refShortName) - ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetTagCommit(refName) + ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetTagCommit(refShortName) if err != nil { if git.IsErrNotExist(err) { - ctx.NotFound("GetTagCommit", err) + ctx.NotFound(err) return } ctx.ServerError("GetTagCommit", err) return } ctx.Repo.CommitID = ctx.Repo.Commit.ID.String() - } else if git.IsStringLikelyCommitID(ctx.Repo.GetObjectFormat(), refName, 7) { - ctx.Repo.IsViewCommit = true - ctx.Repo.CommitID = refName + } else if git.IsStringLikelyCommitID(ctx.Repo.GetObjectFormat(), refShortName, 7) { + ctx.Repo.RefFullName = git.RefNameFromCommit(refShortName) + ctx.Repo.CommitID = refShortName - ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetCommit(refName) + ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetCommit(refShortName) if err != nil { - ctx.NotFound("GetCommit", err) + ctx.NotFound(err) return } // If short commit ID add canonical link header - if len(refName) < ctx.Repo.GetObjectFormat().FullLength() { - canonicalURL := util.URLJoin(httplib.GuessCurrentAppURL(ctx), strings.Replace(ctx.Req.URL.RequestURI(), util.PathEscapeSegments(refName), url.PathEscape(ctx.Repo.Commit.ID.String()), 1)) + if len(refShortName) < ctx.Repo.GetObjectFormat().FullLength() { + canonicalURL := util.URLJoin(httplib.GuessCurrentAppURL(ctx), strings.Replace(ctx.Req.URL.RequestURI(), util.PathEscapeSegments(refShortName), url.PathEscape(ctx.Repo.Commit.ID.String()), 1)) ctx.RespHeader().Set("Link", fmt.Sprintf(`<%s>; rel="canonical"`, canonicalURL)) } } else { - if opt.IgnoreNotExistErr { - return - } - ctx.NotFound("RepoRef invalid repo", fmt.Errorf("branch or tag not exist: %s", refName)) + ctx.NotFound(fmt.Errorf("branch or tag not exist: %s", refShortName)) return } if guessLegacyPath { // redirect from old URL scheme to new URL scheme - prefix := strings.TrimPrefix(setting.AppSubURL+strings.ToLower(strings.TrimSuffix(ctx.Req.URL.Path, ctx.PathParam("*"))), strings.ToLower(ctx.Repo.RepoLink)) - redirect := path.Join( - ctx.Repo.RepoLink, - util.PathEscapeSegments(prefix), - ctx.Repo.BranchNameSubURL(), - util.PathEscapeSegments(ctx.Repo.TreePath)) + // * /user2/repo1/commits/master => /user2/repo1/commits/branch/master + // * /user2/repo1/src/master => /user2/repo1/src/branch/master + // * /user2/repo1/src/README.md => /user2/repo1/src/branch/master/README.md (fallback to default branch) + var redirect string + refSubPath := "src" + // remove the "/subpath/owner/repo/" prefix, the names are case-insensitive + remainingLowerPath, cut := strings.CutPrefix(setting.AppSubURL+strings.ToLower(ctx.Req.URL.Path), strings.ToLower(ctx.Repo.RepoLink)+"/") + if cut { + refSubPath, _, _ = strings.Cut(remainingLowerPath, "/") // it could be "src" or "commits" + } + if fallbackDefaultBranch { + redirect = fmt.Sprintf("%s/%s/%s/%s/%s", ctx.Repo.RepoLink, refSubPath, refType, util.PathEscapeSegments(refShortName), ctx.PathParamRaw("*")) + } else { + redirect = fmt.Sprintf("%s/%s/%s/%s", ctx.Repo.RepoLink, refSubPath, refType, ctx.PathParamRaw("*")) + } + if ctx.Req.URL.RawQuery != "" { + redirect += "?" + ctx.Req.URL.RawQuery + } ctx.Redirect(redirect) return } } + ctx.Data["RefFullName"] = ctx.Repo.RefFullName + ctx.Data["RefTypeNameSubURL"] = ctx.Repo.RefTypeNameSubURL() + ctx.Data["TreePath"] = ctx.Repo.TreePath + ctx.Data["BranchName"] = ctx.Repo.BranchName - ctx.Data["RefName"] = ctx.Repo.RefName - ctx.Data["BranchNameSubURL"] = ctx.Repo.BranchNameSubURL() - ctx.Data["TagName"] = ctx.Repo.TagName + ctx.Data["CommitID"] = ctx.Repo.CommitID - ctx.Data["TreePath"] = ctx.Repo.TreePath - ctx.Data["IsViewBranch"] = ctx.Repo.IsViewBranch - ctx.Data["IsViewTag"] = ctx.Repo.IsViewTag - ctx.Data["IsViewCommit"] = ctx.Repo.IsViewCommit + ctx.Data["CanCreateBranch"] = ctx.Repo.CanCreateBranch() // only used by the branch selector dropdown: AllowCreateNewRef ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount() @@ -1006,6 +973,15 @@ func RepoRefByType(detectRefType RepoRefType, opts ...RepoRefByTypeOptions) func ctx.ServerError("GetCommitsCount", err) return } + if ctx.Repo.RefFullName.IsTag() { + rel, err := repo_model.GetRelease(ctx, ctx.Repo.Repository.ID, ctx.Repo.RefFullName.TagName()) + if err == nil && rel.NumCommits <= 0 { + rel.NumCommits = ctx.Repo.CommitsCount + if err := repo_model.UpdateReleaseNumCommits(ctx, rel); err != nil { + log.Error("UpdateReleaseNumCommits", err) + } + } + } ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache()) } @@ -1015,7 +991,7 @@ func RepoRefByType(detectRefType RepoRefType, opts ...RepoRefByTypeOptions) func func GitHookService() func(ctx *Context) { return func(ctx *Context) { if !ctx.Doer.CanEditGitHook() { - ctx.NotFound("GitHookService", nil) + ctx.NotFound(nil) return } } diff --git a/services/context/response.go b/services/context/response.go index 2f271f211b..c7368ebc6f 100644 --- a/services/context/response.go +++ b/services/context/response.go @@ -11,31 +11,29 @@ import ( // ResponseWriter represents a response writer for HTTP type ResponseWriter interface { - http.ResponseWriter - http.Flusher - web_types.ResponseStatusProvider - - Before(func(ResponseWriter)) + http.ResponseWriter // provides Header/Write/WriteHeader + http.Flusher // provides Flush + web_types.ResponseStatusProvider // provides WrittenStatus - Status() int // used by access logger template - Size() int // used by access logger template + Before(fn func(ResponseWriter)) + WrittenSize() int } -var _ ResponseWriter = &Response{} +var _ ResponseWriter = (*Response)(nil) // Response represents a response type Response struct { http.ResponseWriter written int status int - befores []func(ResponseWriter) + beforeFuncs []func(ResponseWriter) beforeExecuted bool } // Write writes bytes to HTTP endpoint func (r *Response) Write(bs []byte) (int, error) { if !r.beforeExecuted { - for _, before := range r.befores { + for _, before := range r.beforeFuncs { before(r) } r.beforeExecuted = true @@ -51,18 +49,14 @@ func (r *Response) Write(bs []byte) (int, error) { return size, nil } -func (r *Response) Status() int { - return r.status -} - -func (r *Response) Size() int { +func (r *Response) WrittenSize() int { return r.written } // WriteHeader write status code func (r *Response) WriteHeader(statusCode int) { if !r.beforeExecuted { - for _, before := range r.befores { + for _, before := range r.beforeFuncs { before(r) } r.beforeExecuted = true @@ -87,17 +81,13 @@ func (r *Response) WrittenStatus() int { // Before allows for a function to be called before the ResponseWriter has been written to. This is // useful for setting headers or any other operations that must happen before a response has been written. -func (r *Response) Before(f func(ResponseWriter)) { - r.befores = append(r.befores, f) +func (r *Response) Before(fn func(ResponseWriter)) { + r.beforeFuncs = append(r.beforeFuncs, fn) } func WrapResponseWriter(resp http.ResponseWriter) *Response { if v, ok := resp.(*Response); ok { return v } - return &Response{ - ResponseWriter: resp, - status: 0, - befores: make([]func(ResponseWriter), 0), - } + return &Response{ResponseWriter: resp} } diff --git a/services/context/upload/upload.go b/services/context/upload/upload.go index da4370a433..23707950d4 100644 --- a/services/context/upload/upload.go +++ b/services/context/upload/upload.go @@ -11,7 +11,9 @@ import ( "regexp" "strings" + repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/context" ) @@ -39,7 +41,7 @@ func Verify(buf []byte, fileName, allowedTypesStr string) error { allowedTypesStr = strings.ReplaceAll(allowedTypesStr, "|", ",") // compat for old config format allowedTypes := []string{} - for _, entry := range strings.Split(allowedTypesStr, ",") { + for entry := range strings.SplitSeq(allowedTypesStr, ",") { entry = strings.ToLower(strings.TrimSpace(entry)) if entry != "" { allowedTypes = append(allowedTypes, entry) @@ -87,14 +89,15 @@ func Verify(buf []byte, fileName, allowedTypesStr string) error { // AddUploadContext renders template values for dropzone func AddUploadContext(ctx *context.Context, uploadType string) { - if uploadType == "release" { + switch uploadType { + case "release": ctx.Data["UploadUrl"] = ctx.Repo.RepoLink + "/releases/attachments" ctx.Data["UploadRemoveUrl"] = ctx.Repo.RepoLink + "/releases/attachments/remove" ctx.Data["UploadLinkUrl"] = ctx.Repo.RepoLink + "/releases/attachments" ctx.Data["UploadAccepts"] = strings.ReplaceAll(setting.Repository.Release.AllowedTypes, "|", ",") ctx.Data["UploadMaxFiles"] = setting.Attachment.MaxFiles ctx.Data["UploadMaxSize"] = setting.Attachment.MaxSize - } else if uploadType == "comment" { + case "comment": ctx.Data["UploadUrl"] = ctx.Repo.RepoLink + "/issues/attachments" ctx.Data["UploadRemoveUrl"] = ctx.Repo.RepoLink + "/issues/attachments/remove" if len(ctx.PathParam("index")) > 0 { @@ -105,12 +108,17 @@ func AddUploadContext(ctx *context.Context, uploadType string) { ctx.Data["UploadAccepts"] = strings.ReplaceAll(setting.Attachment.AllowedTypes, "|", ",") ctx.Data["UploadMaxFiles"] = setting.Attachment.MaxFiles ctx.Data["UploadMaxSize"] = setting.Attachment.MaxSize - } else if uploadType == "repo" { - ctx.Data["UploadUrl"] = ctx.Repo.RepoLink + "/upload-file" - ctx.Data["UploadRemoveUrl"] = ctx.Repo.RepoLink + "/upload-remove" - ctx.Data["UploadLinkUrl"] = ctx.Repo.RepoLink + "/upload-file" - ctx.Data["UploadAccepts"] = strings.ReplaceAll(setting.Repository.Upload.AllowedTypes, "|", ",") - ctx.Data["UploadMaxFiles"] = setting.Repository.Upload.MaxFiles - ctx.Data["UploadMaxSize"] = setting.Repository.Upload.FileMaxSize + default: + setting.PanicInDevOrTesting("Invalid upload type: %s", uploadType) } } + +func AddUploadContextForRepo(ctx reqctx.RequestContext, repo *repo_model.Repository) { + ctxData, repoLink := ctx.GetData(), repo.Link() + ctxData["UploadUrl"] = repoLink + "/upload-file" + ctxData["UploadRemoveUrl"] = repoLink + "/upload-remove" + ctxData["UploadLinkUrl"] = repoLink + "/upload-file" + ctxData["UploadAccepts"] = strings.ReplaceAll(setting.Repository.Upload.AllowedTypes, "|", ",") + ctxData["UploadMaxFiles"] = setting.Repository.Upload.MaxFiles + ctxData["UploadMaxSize"] = setting.Repository.Upload.FileMaxSize +} diff --git a/services/context/user.go b/services/context/user.go index dbc35e198d..c09ded8339 100644 --- a/services/context/user.go +++ b/services/context/user.go @@ -14,15 +14,15 @@ import ( // UserAssignmentWeb returns a middleware to handle context-user assignment for web routes func UserAssignmentWeb() func(ctx *Context) { return func(ctx *Context) { - errorFn := func(status int, title string, obj any) { + errorFn := func(status int, obj any) { err, ok := obj.(error) if !ok { err = fmt.Errorf("%s", obj) } if status == http.StatusNotFound { - ctx.NotFound(title, err) + ctx.NotFound(err) } else { - ctx.ServerError(title, err) + ctx.ServerError("UserAssignmentWeb", err) } } ctx.ContextUser = userAssignment(ctx.Base, ctx.Doer, errorFn) @@ -42,9 +42,9 @@ func UserIDAssignmentAPI() func(ctx *APIContext) { ctx.ContextUser, err = user_model.GetUserByID(ctx, userID) if err != nil { if user_model.IsErrUserNotExist(err) { - ctx.Error(http.StatusNotFound, "GetUserByID", err) + ctx.APIError(http.StatusNotFound, err) } else { - ctx.Error(http.StatusInternalServerError, "GetUserByID", err) + ctx.APIErrorInternal(err) } } } @@ -54,11 +54,11 @@ func UserIDAssignmentAPI() func(ctx *APIContext) { // UserAssignmentAPI returns a middleware to handle context-user assignment for api routes func UserAssignmentAPI() func(ctx *APIContext) { return func(ctx *APIContext) { - ctx.ContextUser = userAssignment(ctx.Base, ctx.Doer, ctx.Error) + ctx.ContextUser = userAssignment(ctx.Base, ctx.Doer, ctx.APIError) } } -func userAssignment(ctx *Base, doer *user_model.User, errCb func(int, string, any)) (contextUser *user_model.User) { +func userAssignment(ctx *Base, doer *user_model.User, errCb func(int, any)) (contextUser *user_model.User) { username := ctx.PathParam("username") if doer != nil && doer.LowerName == strings.ToLower(username) { @@ -71,12 +71,12 @@ func userAssignment(ctx *Base, doer *user_model.User, errCb func(int, string, an if redirectUserID, err := user_model.LookupUserRedirect(ctx, username); err == nil { RedirectToUser(ctx, username, redirectUserID) } else if user_model.IsErrUserRedirectNotExist(err) { - errCb(http.StatusNotFound, "GetUserByName", err) + errCb(http.StatusNotFound, err) } else { - errCb(http.StatusInternalServerError, "LookupUserRedirect", err) + errCb(http.StatusInternalServerError, fmt.Errorf("LookupUserRedirect: %w", err)) } } else { - errCb(http.StatusInternalServerError, "GetUserByName", err) + errCb(http.StatusInternalServerError, fmt.Errorf("GetUserByName: %w", err)) } } } diff --git a/services/contexttest/context_tests.go b/services/contexttest/context_tests.go index b0f71cad20..b54023897b 100644 --- a/services/contexttest/context_tests.go +++ b/services/contexttest/context_tests.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + git_module "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/session" @@ -30,6 +31,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func mockRequest(t *testing.T, reqPath string) *http.Request { @@ -67,7 +69,6 @@ func MockContext(t *testing.T, reqPath string, opts ...MockContextOption) (*cont chiCtx := chi.NewRouteContext() ctx := context.NewWebContext(base, opt.Render, nil) - ctx.SetContextValue(context.WebContextKey, ctx) ctx.SetContextValue(chi.RouteCtxKey, chiCtx) if opt.SessionStore != nil { ctx.SetContextValue(session.MockStoreContextKey, opt.SessionStore) @@ -86,7 +87,7 @@ func MockAPIContext(t *testing.T, reqPath string) (*context.APIContext, *httptes base := context.NewBaseContext(resp, req) base.Data = middleware.GetContextData(req.Context()) base.Locale = &translation.MockLocale{} - ctx := &context.APIContext{Base: base} + ctx := &context.APIContext{Base: base, Repo: &context.Repository{}} chiCtx := chi.NewRouteContext() ctx.SetContextValue(chi.RouteCtxKey, chiCtx) return ctx, resp @@ -107,13 +108,13 @@ func MockPrivateContext(t *testing.T, reqPath string) (*context.PrivateContext, // LoadRepo load a repo into a test context. func LoadRepo(t *testing.T, ctx gocontext.Context, repoID int64) { var doer *user_model.User - repo := &context.Repository{} + var repo *context.Repository switch ctx := ctx.(type) { case *context.Context: - ctx.Repo = repo + repo = ctx.Repo doer = ctx.Doer case *context.APIContext: - ctx.Repo = repo + repo = ctx.Repo doer = ctx.Doer default: assert.FailNow(t, "context is not *context.Context or *context.APIContext") @@ -141,15 +142,17 @@ func LoadRepoCommit(t *testing.T, ctx gocontext.Context) { } gitRepo, err := gitrepo.OpenRepository(ctx, repo.Repository) - assert.NoError(t, err) + require.NoError(t, err) defer gitRepo.Close() - branch, err := gitRepo.GetHEADBranch() - assert.NoError(t, err) - assert.NotNil(t, branch) - if branch != nil { - repo.Commit, err = gitRepo.GetBranchCommit(branch.Name) - assert.NoError(t, err) + + if repo.RefFullName == "" { + repo.RefFullName = git_module.RefNameFromBranch(repo.Repository.DefaultBranch) } + if repo.RefFullName.IsPull() { + repo.BranchName = repo.RefFullName.ShortName() + } + repo.Commit, err = gitRepo.GetCommit(repo.RefFullName.String()) + require.NoError(t, err) } // LoadUser load a user into a test context @@ -167,10 +170,19 @@ func LoadUser(t *testing.T, ctx gocontext.Context, userID int64) { // LoadGitRepo load a git repo into a test context. Requires that ctx.Repo has // already been populated. -func LoadGitRepo(t *testing.T, ctx *context.Context) { - assert.NoError(t, ctx.Repo.Repository.LoadOwner(ctx)) +func LoadGitRepo(t *testing.T, ctx gocontext.Context) { + var repo *context.Repository + switch ctx := any(ctx).(type) { + case *context.Context: + repo = ctx.Repo + case *context.APIContext: + repo = ctx.Repo + default: + assert.FailNow(t, "context is not *context.Context or *context.APIContext") + } + assert.NoError(t, repo.Repository.LoadOwner(ctx)) var err error - ctx.Repo.GitRepo, err = gitrepo.OpenRepository(ctx, ctx.Repo.Repository) + repo.GitRepo, err = gitrepo.OpenRepository(ctx, repo.Repository) assert.NoError(t, err) } diff --git a/services/convert/convert.go b/services/convert/convert.go index c8cad2a2ad..0de3822140 100644 --- a/services/convert/convert.go +++ b/services/convert/convert.go @@ -5,8 +5,11 @@ package convert import ( + "bytes" "context" "fmt" + "net/url" + "path" "strconv" "strings" "time" @@ -14,6 +17,7 @@ import ( actions_model "code.gitea.io/gitea/models/actions" asymkey_model "code.gitea.io/gitea/models/asymkey" "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/organization" @@ -22,13 +26,18 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/actions" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + asymkey_service "code.gitea.io/gitea/services/asymkey" "code.gitea.io/gitea/services/gitdiff" + + runnerv1 "code.gitea.io/actions-proto-go/runner/v1" + "github.com/nektos/act/pkg/model" ) // ToEmail convert models.EmailAddress to api.Email @@ -140,7 +149,7 @@ func ToBranchProtection(ctx context.Context, bp *git_model.ProtectedBranch, repo mergeWhitelistUsernames := getWhitelistEntities(readers, bp.MergeWhitelistUserIDs) approvalsWhitelistUsernames := getWhitelistEntities(readers, bp.ApprovalsWhitelistUserIDs) - teamReaders, err := organization.OrgFromUser(repo.Owner).TeamsWithAccessToRepo(ctx, repo.ID, perm.AccessModeRead) + teamReaders, err := organization.GetTeamsWithAccessToAnyRepoUnit(ctx, repo.Owner.ID, repo.ID, perm.AccessModeRead, unit.TypeCode, unit.TypePullRequests) if err != nil { log.Error("Repo.Owner.TeamsWithAccessToRepo: %v", err) } @@ -194,13 +203,22 @@ func ToBranchProtection(ctx context.Context, bp *git_model.ProtectedBranch, repo // ToTag convert a git.Tag to an api.Tag func ToTag(repo *repo_model.Repository, t *git.Tag) *api.Tag { + tarballURL := util.URLJoin(repo.HTMLURL(), "archive", t.Name+".tar.gz") + zipballURL := util.URLJoin(repo.HTMLURL(), "archive", t.Name+".zip") + + // Archive URLs are "" if the download feature is disabled + if setting.Repository.DisableDownloadSourceArchives { + tarballURL = "" + zipballURL = "" + } + return &api.Tag{ Name: t.Name, Message: strings.TrimSpace(t.Message), ID: t.ID.String(), Commit: ToCommitMeta(repo, t), - ZipballURL: util.URLJoin(repo.HTMLURL(), "archive", t.Name+".zip"), - TarballURL: util.URLJoin(repo.HTMLURL(), "archive", t.Name+".tar.gz"), + ZipballURL: zipballURL, + TarballURL: tarballURL, } } @@ -229,9 +247,291 @@ func ToActionTask(ctx context.Context, t *actions_model.ActionTask) (*api.Action }, nil } +func ToActionWorkflowRun(ctx context.Context, repo *repo_model.Repository, run *actions_model.ActionRun) (*api.ActionWorkflowRun, error) { + err := run.LoadAttributes(ctx) + if err != nil { + return nil, err + } + status, conclusion := ToActionsStatus(run.Status) + return &api.ActionWorkflowRun{ + ID: run.ID, + URL: fmt.Sprintf("%s/actions/runs/%d", repo.APIURL(), run.ID), + HTMLURL: run.HTMLURL(), + RunNumber: run.Index, + StartedAt: run.Started.AsLocalTime(), + CompletedAt: run.Stopped.AsLocalTime(), + Event: string(run.Event), + DisplayTitle: run.Title, + HeadBranch: git.RefName(run.Ref).BranchName(), + HeadSha: run.CommitSHA, + Status: status, + Conclusion: conclusion, + Path: fmt.Sprintf("%s@%s", run.WorkflowID, run.Ref), + Repository: ToRepo(ctx, repo, access_model.Permission{AccessMode: perm.AccessModeNone}), + TriggerActor: ToUser(ctx, run.TriggerUser, nil), + // We do not have a way to get a different User for the actor than the trigger user + Actor: ToUser(ctx, run.TriggerUser, nil), + }, nil +} + +func ToWorkflowRunAction(status actions_model.Status) string { + var action string + switch status { + case actions_model.StatusWaiting, actions_model.StatusBlocked: + action = "requested" + case actions_model.StatusRunning: + action = "in_progress" + } + if status.IsDone() { + action = "completed" + } + return action +} + +func ToActionsStatus(status actions_model.Status) (string, string) { + var action string + var conclusion string + switch status { + // This is a naming conflict of the webhook between Gitea and GitHub Actions + case actions_model.StatusWaiting: + action = "queued" + case actions_model.StatusBlocked: + action = "waiting" + case actions_model.StatusRunning: + action = "in_progress" + } + if status.IsDone() { + action = "completed" + switch status { + case actions_model.StatusSuccess: + conclusion = "success" + case actions_model.StatusCancelled: + conclusion = "cancelled" + case actions_model.StatusFailure: + conclusion = "failure" + case actions_model.StatusSkipped: + conclusion = "skipped" + } + } + return action, conclusion +} + +// ToActionWorkflowJob convert a actions_model.ActionRunJob to an api.ActionWorkflowJob +// task is optional and can be nil +func ToActionWorkflowJob(ctx context.Context, repo *repo_model.Repository, task *actions_model.ActionTask, job *actions_model.ActionRunJob) (*api.ActionWorkflowJob, error) { + err := job.LoadAttributes(ctx) + if err != nil { + return nil, err + } + + jobIndex := 0 + jobs, err := actions_model.GetRunJobsByRunID(ctx, job.RunID) + if err != nil { + return nil, err + } + for i, j := range jobs { + if j.ID == job.ID { + jobIndex = i + break + } + } + + status, conclusion := ToActionsStatus(job.Status) + var runnerID int64 + var runnerName string + var steps []*api.ActionWorkflowStep + + if job.TaskID != 0 { + if task == nil { + task, _, err = db.GetByID[actions_model.ActionTask](ctx, job.TaskID) + if err != nil { + return nil, err + } + } + + runnerID = task.RunnerID + if runner, ok, _ := db.GetByID[actions_model.ActionRunner](ctx, runnerID); ok { + runnerName = runner.Name + } + for i, step := range task.Steps { + stepStatus, stepConclusion := ToActionsStatus(job.Status) + steps = append(steps, &api.ActionWorkflowStep{ + Name: step.Name, + Number: int64(i), + Status: stepStatus, + Conclusion: stepConclusion, + StartedAt: step.Started.AsTime().UTC(), + CompletedAt: step.Stopped.AsTime().UTC(), + }) + } + } + + return &api.ActionWorkflowJob{ + ID: job.ID, + // missing api endpoint for this location + URL: fmt.Sprintf("%s/actions/jobs/%d", repo.APIURL(), job.ID), + HTMLURL: fmt.Sprintf("%s/jobs/%d", job.Run.HTMLURL(), jobIndex), + RunID: job.RunID, + // Missing api endpoint for this location, artifacts are available under a nested url + RunURL: fmt.Sprintf("%s/actions/runs/%d", repo.APIURL(), job.RunID), + Name: job.Name, + Labels: job.RunsOn, + RunAttempt: job.Attempt, + HeadSha: job.Run.CommitSHA, + HeadBranch: git.RefName(job.Run.Ref).BranchName(), + Status: status, + Conclusion: conclusion, + RunnerID: runnerID, + RunnerName: runnerName, + Steps: steps, + CreatedAt: job.Created.AsTime().UTC(), + StartedAt: job.Started.AsTime().UTC(), + CompletedAt: job.Stopped.AsTime().UTC(), + }, nil +} + +func getActionWorkflowEntry(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, folder string, entry *git.TreeEntry) *api.ActionWorkflow { + cfgUnit := repo.MustGetUnit(ctx, unit.TypeActions) + cfg := cfgUnit.ActionsConfig() + + defaultBranch, _ := commit.GetBranchName() + + workflowURL := fmt.Sprintf("%s/actions/workflows/%s", repo.APIURL(), util.PathEscapeSegments(entry.Name())) + workflowRepoURL := fmt.Sprintf("%s/src/branch/%s/%s/%s", repo.HTMLURL(ctx), util.PathEscapeSegments(defaultBranch), util.PathEscapeSegments(folder), util.PathEscapeSegments(entry.Name())) + badgeURL := fmt.Sprintf("%s/actions/workflows/%s/badge.svg?branch=%s", repo.HTMLURL(ctx), util.PathEscapeSegments(entry.Name()), url.QueryEscape(repo.DefaultBranch)) + + // See https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#get-a-workflow + // State types: + // - active + // - deleted + // - disabled_fork + // - disabled_inactivity + // - disabled_manually + state := "active" + if cfg.IsWorkflowDisabled(entry.Name()) { + state = "disabled_manually" + } + + // The CreatedAt and UpdatedAt fields currently reflect the timestamp of the latest commit, which can later be refined + // by retrieving the first and last commits for the file history. The first commit would indicate the creation date, + // while the last commit would represent the modification date. The DeletedAt could be determined by identifying + // the last commit where the file existed. However, this implementation has not been done here yet, as it would likely + // cause a significant performance degradation. + createdAt := commit.Author.When + updatedAt := commit.Author.When + + content, err := actions.GetContentFromEntry(entry) + name := entry.Name() + if err == nil { + workflow, err := model.ReadWorkflow(bytes.NewReader(content)) + if err == nil { + // Only use the name when specified in the workflow file + if workflow.Name != "" { + name = workflow.Name + } + } else { + log.Error("getActionWorkflowEntry: Failed to parse workflow: %v", err) + } + } else { + log.Error("getActionWorkflowEntry: Failed to get content from entry: %v", err) + } + + return &api.ActionWorkflow{ + ID: entry.Name(), + Name: name, + Path: path.Join(folder, entry.Name()), + State: state, + CreatedAt: createdAt, + UpdatedAt: updatedAt, + URL: workflowURL, + HTMLURL: workflowRepoURL, + BadgeURL: badgeURL, + } +} + +func ListActionWorkflows(ctx context.Context, gitrepo *git.Repository, repo *repo_model.Repository) ([]*api.ActionWorkflow, error) { + defaultBranchCommit, err := gitrepo.GetBranchCommit(repo.DefaultBranch) + if err != nil { + return nil, err + } + + folder, entries, err := actions.ListWorkflows(defaultBranchCommit) + if err != nil { + return nil, err + } + + workflows := make([]*api.ActionWorkflow, len(entries)) + for i, entry := range entries { + workflows[i] = getActionWorkflowEntry(ctx, repo, defaultBranchCommit, folder, entry) + } + + return workflows, nil +} + +func GetActionWorkflow(ctx context.Context, gitrepo *git.Repository, repo *repo_model.Repository, workflowID string) (*api.ActionWorkflow, error) { + entries, err := ListActionWorkflows(ctx, gitrepo, repo) + if err != nil { + return nil, err + } + + for _, entry := range entries { + if entry.ID == workflowID { + return entry, nil + } + } + + return nil, util.NewNotExistErrorf("workflow %q not found", workflowID) +} + +// ToActionArtifact convert a actions_model.ActionArtifact to an api.ActionArtifact +func ToActionArtifact(repo *repo_model.Repository, art *actions_model.ActionArtifact) (*api.ActionArtifact, error) { + url := fmt.Sprintf("%s/actions/artifacts/%d", repo.APIURL(), art.ID) + + return &api.ActionArtifact{ + ID: art.ID, + Name: art.ArtifactName, + SizeInBytes: art.FileSize, + Expired: art.Status == actions_model.ArtifactStatusExpired, + URL: url, + ArchiveDownloadURL: url + "/zip", + CreatedAt: art.CreatedUnix.AsLocalTime(), + UpdatedAt: art.UpdatedUnix.AsLocalTime(), + ExpiresAt: art.ExpiredUnix.AsLocalTime(), + WorkflowRun: &api.ActionWorkflowRun{ + ID: art.RunID, + RepositoryID: art.RepoID, + HeadSha: art.CommitSHA, + }, + }, nil +} + +func ToActionRunner(ctx context.Context, runner *actions_model.ActionRunner) *api.ActionRunner { + status := runner.Status() + apiStatus := "offline" + if runner.IsOnline() { + apiStatus = "online" + } + labels := make([]*api.ActionRunnerLabel, len(runner.AgentLabels)) + for i, label := range runner.AgentLabels { + labels[i] = &api.ActionRunnerLabel{ + ID: int64(i), + Name: label, + Type: "custom", + } + } + return &api.ActionRunner{ + ID: runner.ID, + Name: runner.Name, + Status: apiStatus, + Busy: status == runnerv1.RunnerStatus_RUNNER_STATUS_ACTIVE, + Ephemeral: runner.Ephemeral, + Labels: labels, + } +} + // ToVerification convert a git.Commit.Signature to an api.PayloadCommitVerification func ToVerification(ctx context.Context, c *git.Commit) *api.PayloadCommitVerification { - verif := asymkey_model.ParseCommitWithSignature(ctx, c) + verif := asymkey_service.ParseCommitWithSignature(ctx, c) commitVerification := &api.PayloadCommitVerification{ Verified: verif.Verified, Reason: verif.Reason, @@ -258,6 +558,7 @@ func ToPublicKey(apiLink string, key *asymkey_model.PublicKey) *api.PublicKey { Title: key.Name, Fingerprint: key.Fingerprint, Created: key.CreatedUnix.AsTime(), + Updated: key.UpdatedUnix.AsTime(), } } @@ -426,7 +727,7 @@ func ToTagProtection(ctx context.Context, pt *git_model.ProtectedTag, repo *repo whitelistUsernames := getWhitelistEntities(readers, pt.AllowlistUserIDs) - teamReaders, err := organization.OrgFromUser(repo.Owner).TeamsWithAccessToRepo(ctx, repo.ID, perm.AccessModeRead) + teamReaders, err := organization.GetTeamsWithAccessToAnyRepoUnit(ctx, repo.Owner.ID, repo.ID, perm.AccessModeRead, unit.TypeCode, unit.TypePullRequests) if err != nil { log.Error("Repo.Owner.TeamsWithAccessToRepo: %v", err) } diff --git a/services/convert/git_commit.go b/services/convert/git_commit.go index e0efcddbcb..3ec81b52ee 100644 --- a/services/convert/git_commit.go +++ b/services/convert/git_commit.go @@ -210,17 +210,15 @@ func ToCommit(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Rep // Get diff stats for commit if opts.Stat { - diff, err := gitdiff.GetDiff(ctx, gitRepo, &gitdiff.DiffOptions{ - AfterCommitID: commit.ID.String(), - }) + diffShortStat, err := gitdiff.GetDiffShortStat(gitRepo, "", commit.ID.String()) if err != nil { return nil, err } res.Stats = &api.CommitStats{ - Total: diff.TotalAddition + diff.TotalDeletion, - Additions: diff.TotalAddition, - Deletions: diff.TotalDeletion, + Total: diffShortStat.TotalAddition + diffShortStat.TotalDeletion, + Additions: diffShortStat.TotalAddition, + Deletions: diffShortStat.TotalDeletion, } } diff --git a/services/convert/git_commit_test.go b/services/convert/git_commit_test.go index 73cb5e8c71..ad1cc0eca3 100644 --- a/services/convert/git_commit_test.go +++ b/services/convert/git_commit_test.go @@ -33,7 +33,7 @@ func TestToCommitMeta(t *testing.T) { commitMeta := ToCommitMeta(headRepo, tag) assert.NotNil(t, commitMeta) - assert.EqualValues(t, &api.CommitMeta{ + assert.Equal(t, &api.CommitMeta{ SHA: sha1.EmptyObjectID().String(), URL: util.URLJoin(headRepo.APIURL(), "git/commits", sha1.EmptyObjectID().String()), Created: time.Unix(0, 0), diff --git a/services/convert/issue.go b/services/convert/issue.go index e3124efd64..7f386e6293 100644 --- a/services/convert/issue.go +++ b/services/convert/issue.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" ) func ToIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Issue) *api.Issue { @@ -40,6 +41,9 @@ func toIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Iss if err := issue.LoadAttachments(ctx); err != nil { return &api.Issue{} } + if err := issue.LoadPinOrder(ctx); err != nil { + return &api.Issue{} + } apiIssue := &api.Issue{ ID: issue.ID, @@ -54,7 +58,7 @@ func toIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Iss Comments: issue.NumComments, Created: issue.CreatedUnix.AsTime(), Updated: issue.UpdatedUnix.AsTime(), - PinOrder: issue.PinOrder, + PinOrder: util.Iif(issue.PinOrder == -1, 0, issue.PinOrder), // -1 means loaded with no pin order } if issue.Repo != nil { @@ -66,7 +70,7 @@ func toIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Iss if err := issue.LoadLabels(ctx); err != nil { return &api.Issue{} } - apiIssue.Labels = ToLabelList(issue.Labels, issue.Repo, issue.Repo.Owner) + apiIssue.Labels = util.SliceNilAsEmpty(ToLabelList(issue.Labels, issue.Repo, issue.Repo.Owner)) apiIssue.Repo = &api.RepositoryMeta{ ID: issue.Repo.ID, Name: issue.Repo.Name, @@ -121,6 +125,7 @@ func toIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Iss // ToIssueList converts an IssueList to API format func ToIssueList(ctx context.Context, doer *user_model.User, il issues_model.IssueList) []*api.Issue { result := make([]*api.Issue, len(il)) + _ = il.LoadPinOrder(ctx) for i := range il { result[i] = ToIssue(ctx, doer, il[i]) } @@ -130,6 +135,7 @@ func ToIssueList(ctx context.Context, doer *user_model.User, il issues_model.Iss // ToAPIIssueList converts an IssueList to API format func ToAPIIssueList(ctx context.Context, doer *user_model.User, il issues_model.IssueList) []*api.Issue { result := make([]*api.Issue, len(il)) + _ = il.LoadPinOrder(ctx) for i := range il { result[i] = ToAPIIssue(ctx, doer, il[i]) } @@ -186,7 +192,7 @@ func ToStopWatches(ctx context.Context, sws []*issues_model.Stopwatch) (api.Stop result = append(result, api.StopWatch{ Created: sw.CreatedUnix.AsTime(), Seconds: sw.Seconds(), - Duration: sw.Duration(), + Duration: util.SecToHours(sw.Seconds()), IssueIndex: issue.Index, IssueTitle: issue.Title, RepoOwnerName: repo.OwnerName, diff --git a/services/convert/issue_comment.go b/services/convert/issue_comment.go index b8527ae233..9ad584a62f 100644 --- a/services/convert/issue_comment.go +++ b/services/convert/issue_comment.go @@ -74,7 +74,7 @@ func ToTimelineComment(ctx context.Context, repo *repo_model.Repository, c *issu c.Content[0] == '|' { // TimeTracking Comments from v1.21 on store the seconds instead of an formatted string // so we check for the "|" delimiter and convert new to legacy format on demand - c.Content = util.SecToTime(c.Content[1:]) + c.Content = util.SecToHours(c.Content[1:]) } if c.Type == issues_model.CommentTypeChangeTimeEstimate { diff --git a/services/convert/pull.go b/services/convert/pull.go index a1ab7eeb8e..8f9679f649 100644 --- a/services/convert/pull.go +++ b/services/convert/pull.go @@ -14,10 +14,14 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/gitdiff" ) // ToAPIPullRequest assumes following fields have been assigned with valid values: @@ -25,8 +29,8 @@ import ( // Optional - Merger func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *user_model.User) *api.PullRequest { var ( - baseBranch *git.Branch - headBranch *git.Branch + baseBranch string + headBranch string baseCommit *git.Commit err error ) @@ -57,14 +61,14 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u doerID = doer.ID } - const repoDoerPermCacheKey = "repo_doer_perm_cache" - p, err := cache.GetWithContextCache(ctx, repoDoerPermCacheKey, fmt.Sprintf("%d_%d", pr.BaseRepoID, doerID), - func() (access_model.Permission, error) { + repoUserPerm, err := cache.GetWithContextCache(ctx, cachegroup.RepoUserPermission, fmt.Sprintf("%d-%d", pr.BaseRepoID, doerID), + func(ctx context.Context, _ string) (access_model.Permission, error) { return access_model.GetUserRepoPermission(ctx, pr.BaseRepo, doer) - }) + }, + ) if err != nil { log.Error("GetUserRepoPermission[%d]: %v", pr.BaseRepoID, err) - p.AccessMode = perm.AccessModeNone + repoUserPerm.AccessMode = perm.AccessModeNone } apiPullRequest := &api.PullRequest{ @@ -77,7 +81,7 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u Labels: apiIssue.Labels, Milestone: apiIssue.Milestone, Assignee: apiIssue.Assignee, - Assignees: apiIssue.Assignees, + Assignees: util.SliceNilAsEmpty(apiIssue.Assignees), State: apiIssue.State, Draft: pr.IsWorkInProgress(ctx), IsLocked: apiIssue.IsLocked, @@ -92,7 +96,11 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u Deadline: apiIssue.Deadline, Created: pr.Issue.CreatedUnix.AsTimePtr(), Updated: pr.Issue.UpdatedUnix.AsTimePtr(), - PinOrder: apiIssue.PinOrder, + PinOrder: util.Iif(apiIssue.PinOrder == -1, 0, apiIssue.PinOrder), + + // output "[]" rather than null to align to github outputs + RequestedReviewers: []*api.User{}, + RequestedReviewersTeams: []*api.Team{}, AllowMaintainerEdit: pr.AllowMaintainerEdit, @@ -100,7 +108,7 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u Name: pr.BaseBranch, Ref: pr.BaseBranch, RepoID: pr.BaseRepoID, - Repository: ToRepo(ctx, pr.BaseRepo, p), + Repository: ToRepo(ctx, pr.BaseRepo, repoUserPerm), }, Head: &api.PRBranchInfo{ Name: pr.HeadBranch, @@ -143,16 +151,16 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u } defer gitRepo.Close() - baseBranch, err = gitRepo.GetBranch(pr.BaseBranch) - if err != nil && !git.IsErrBranchNotExist(err) { + exist, err := git_model.IsBranchExist(ctx, pr.BaseRepoID, pr.BaseBranch) + if err != nil { log.Error("GetBranch[%s]: %v", pr.BaseBranch, err) return nil } - if err == nil { - baseCommit, err = baseBranch.GetCommit() + if exist { + baseCommit, err = gitRepo.GetBranchCommit(pr.BaseBranch) if err != nil && !git.IsErrNotExist(err) { - log.Error("GetCommit[%s]: %v", baseBranch.Name, err) + log.Error("GetCommit[%s]: %v", baseBranch, err) return nil } @@ -162,13 +170,6 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u } if pr.Flow == issues_model.PullRequestFlowAGit { - gitRepo, err := gitrepo.OpenRepository(ctx, pr.BaseRepo) - if err != nil { - log.Error("OpenRepository[%s]: %v", pr.GetGitRefName(), err) - return nil - } - defer gitRepo.Close() - apiPullRequest.Head.Sha, err = gitRepo.GetRefCommitID(pr.GetGitRefName()) if err != nil { log.Error("GetRefCommitID[%s]: %v", pr.GetGitRefName(), err) @@ -196,8 +197,8 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u } defer headGitRepo.Close() - headBranch, err = headGitRepo.GetBranch(pr.HeadBranch) - if err != nil && !git.IsErrBranchNotExist(err) { + exist, err = git_model.IsBranchExist(ctx, pr.HeadRepoID, pr.HeadBranch) + if err != nil { log.Error("GetBranch[%s]: %v", pr.HeadBranch, err) return nil } @@ -208,7 +209,7 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u endCommitID string ) - if git.IsErrBranchNotExist(err) { + if !exist { headCommitID, err := headGitRepo.GetRefCommitID(apiPullRequest.Head.Ref) if err != nil && !git.IsErrNotExist(err) { log.Error("GetCommit[%s]: %v", pr.HeadBranch, err) @@ -219,9 +220,9 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u endCommitID = headCommitID } } else { - commit, err := headBranch.GetCommit() + commit, err := headGitRepo.GetBranchCommit(pr.HeadBranch) if err != nil && !git.IsErrNotExist(err) { - log.Error("GetCommit[%s]: %v", headBranch.Name, err) + log.Error("GetCommit[%s]: %v", headBranch, err) return nil } if err == nil { @@ -234,9 +235,13 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u // Calculate diff startCommitID = pr.MergeBase - apiPullRequest.ChangedFiles, apiPullRequest.Additions, apiPullRequest.Deletions, err = gitRepo.GetDiffShortStat(startCommitID, endCommitID) + diffShortStats, err := gitdiff.GetDiffShortStat(gitRepo, startCommitID, endCommitID) if err != nil { log.Error("GetDiffShortStat: %v", err) + } else { + apiPullRequest.ChangedFiles = &diffShortStats.NumFiles + apiPullRequest.Additions = &diffShortStats.TotalAddition + apiPullRequest.Deletions = &diffShortStats.TotalDeletion } } @@ -299,6 +304,9 @@ func ToAPIPullRequests(ctx context.Context, baseRepo *repo_model.Repository, prs if err := issueList.LoadAssignees(ctx); err != nil { return nil, err } + if err = issueList.LoadPinOrder(ctx); err != nil { + return nil, err + } reviews, err := prs.LoadReviews(ctx) if err != nil { @@ -363,7 +371,7 @@ func ToAPIPullRequests(ctx context.Context, baseRepo *repo_model.Repository, prs Deadline: apiIssue.Deadline, Created: pr.Issue.CreatedUnix.AsTimePtr(), Updated: pr.Issue.UpdatedUnix.AsTimePtr(), - PinOrder: apiIssue.PinOrder, + PinOrder: util.Iif(apiIssue.PinOrder == -1, 0, apiIssue.PinOrder), AllowMaintainerEdit: pr.AllowMaintainerEdit, @@ -375,7 +383,7 @@ func ToAPIPullRequests(ctx context.Context, baseRepo *repo_model.Repository, prs }, Head: &api.PRBranchInfo{ Name: pr.HeadBranch, - Ref: fmt.Sprintf("%s%d/head", git.PullPrefix, pr.Index), + Ref: pr.GetGitRefName(), RepoID: -1, }, } @@ -408,88 +416,43 @@ func ToAPIPullRequests(ctx context.Context, baseRepo *repo_model.Repository, prs return nil, err } } - if baseBranch != nil { apiPullRequest.Base.Sha = baseBranch.CommitID } + if pr.HeadRepoID == pr.BaseRepoID { + apiPullRequest.Head.Repository = apiPullRequest.Base.Repository + } - if pr.Flow == issues_model.PullRequestFlowAGit { - apiPullRequest.Head.Sha, err = gitRepo.GetRefCommitID(pr.GetGitRefName()) + // pull request head branch, both repository and branch could not exist + if pr.HeadRepo != nil { + apiPullRequest.Head.RepoID = pr.HeadRepo.ID + exist, err := git_model.IsBranchExist(ctx, pr.HeadRepo.ID, pr.HeadBranch) if err != nil { - log.Error("GetRefCommitID[%s]: %v", pr.GetGitRefName(), err) + log.Error("IsBranchExist[%d]: %v", pr.HeadRepo.ID, err) return nil, err } - apiPullRequest.Head.RepoID = pr.BaseRepoID - apiPullRequest.Head.Repository = apiPullRequest.Base.Repository - apiPullRequest.Head.Name = "" - } - - var headGitRepo *git.Repository - if pr.HeadRepo != nil && pr.Flow == issues_model.PullRequestFlowGithub { - if pr.HeadRepoID == pr.BaseRepoID { - apiPullRequest.Head.RepoID = pr.HeadRepo.ID - apiPullRequest.Head.Repository = apiRepo - headGitRepo = gitRepo - } else { + if exist { + apiPullRequest.Head.Ref = pr.HeadBranch + } + if pr.HeadRepoID != pr.BaseRepoID { p, err := access_model.GetUserRepoPermission(ctx, pr.HeadRepo, doer) if err != nil { log.Error("GetUserRepoPermission[%d]: %v", pr.HeadRepoID, err) p.AccessMode = perm.AccessModeNone } - - apiPullRequest.Head.RepoID = pr.HeadRepo.ID apiPullRequest.Head.Repository = ToRepo(ctx, pr.HeadRepo, p) - - headGitRepo, err = gitrepo.OpenRepository(ctx, pr.HeadRepo) - if err != nil { - log.Error("OpenRepository[%s]: %v", pr.HeadRepo.RepoPath(), err) - return nil, err - } - defer headGitRepo.Close() } + } + if apiPullRequest.Head.Ref == "" { + apiPullRequest.Head.Ref = pr.GetGitRefName() + } - headBranch, err := headGitRepo.GetBranch(pr.HeadBranch) - if err != nil && !git.IsErrBranchNotExist(err) { - log.Error("GetBranch[%s]: %v", pr.HeadBranch, err) - return nil, err - } - - // Outer scope variables to be used in diff calculation - var ( - startCommitID string - endCommitID string - ) - - if git.IsErrBranchNotExist(err) { - headCommitID, err := headGitRepo.GetRefCommitID(apiPullRequest.Head.Ref) - if err != nil && !git.IsErrNotExist(err) { - log.Error("GetCommit[%s]: %v", pr.HeadBranch, err) - return nil, err - } - if err == nil { - apiPullRequest.Head.Sha = headCommitID - endCommitID = headCommitID - } - } else { - commit, err := headBranch.GetCommit() - if err != nil && !git.IsErrNotExist(err) { - log.Error("GetCommit[%s]: %v", headBranch.Name, err) - return nil, err - } - if err == nil { - apiPullRequest.Head.Ref = pr.HeadBranch - apiPullRequest.Head.Sha = commit.ID.String() - endCommitID = commit.ID.String() - } - } - - // Calculate diff - startCommitID = pr.MergeBase - - apiPullRequest.ChangedFiles, apiPullRequest.Additions, apiPullRequest.Deletions, err = gitRepo.GetDiffShortStat(startCommitID, endCommitID) - if err != nil { - log.Error("GetDiffShortStat: %v", err) - } + if pr.Flow == issues_model.PullRequestFlowAGit { + apiPullRequest.Head.Name = "" + } + apiPullRequest.Head.Sha, err = gitRepo.GetRefCommitID(pr.GetGitRefName()) + if err != nil { + log.Error("GetRefCommitID[%s]: %v", pr.GetGitRefName(), err) } if len(apiPullRequest.Head.Sha) == 0 && len(apiPullRequest.Head.Ref) != 0 { @@ -510,6 +473,12 @@ func ToAPIPullRequests(ctx context.Context, baseRepo *repo_model.Repository, prs apiPullRequest.MergedBy = ToUser(ctx, pr.Merger, nil) } + // Do not provide "ChangeFiles/Additions/Deletions" for the PR list, because the "diff" is quite slow + // If callers are interested in these values, they should do a separate request to get the PR details + if apiPullRequest.ChangedFiles != nil || apiPullRequest.Additions != nil || apiPullRequest.Deletions != nil { + setting.PanicInDevOrTesting("ChangedFiles/Additions/Deletions should not be set in PR list") + } + apiPullRequests = append(apiPullRequests, apiPullRequest) } diff --git a/services/convert/pull_review_test.go b/services/convert/pull_review_test.go index a1296fafd4..d0a077ab24 100644 --- a/services/convert/pull_review_test.go +++ b/services/convert/pull_review_test.go @@ -19,8 +19,8 @@ func Test_ToPullReview(t *testing.T) { reviewer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) review := unittest.AssertExistsAndLoadBean(t, &issues_model.Review{ID: 6}) - assert.EqualValues(t, reviewer.ID, review.ReviewerID) - assert.EqualValues(t, issues_model.ReviewTypePending, review.Type) + assert.Equal(t, reviewer.ID, review.ReviewerID) + assert.Equal(t, issues_model.ReviewTypePending, review.Type) reviewList := []*issues_model.Review{review} diff --git a/services/convert/pull_test.go b/services/convert/pull_test.go index e069fa4a68..dfbe24d184 100644 --- a/services/convert/pull_test.go +++ b/services/convert/pull_test.go @@ -27,7 +27,7 @@ func TestPullRequest_APIFormat(t *testing.T) { assert.NoError(t, pr.LoadIssue(db.DefaultContext)) apiPullRequest := ToAPIPullRequest(git.DefaultContext, pr, nil) assert.NotNil(t, apiPullRequest) - assert.EqualValues(t, &structs.PRBranchInfo{ + assert.Equal(t, &structs.PRBranchInfo{ Name: "branch1", Ref: "refs/pull/2/head", Sha: "4a357436d925b5c974181ff12a994538ddc5a269", @@ -46,4 +46,11 @@ func TestPullRequest_APIFormat(t *testing.T) { assert.NotNil(t, apiPullRequest) assert.Nil(t, apiPullRequest.Head.Repository) assert.EqualValues(t, -1, apiPullRequest.Head.RepoID) + + apiPullRequests, err := ToAPIPullRequests(git.DefaultContext, pr.BaseRepo, []*issues_model.PullRequest{pr}, nil) + assert.NoError(t, err) + assert.Len(t, apiPullRequests, 1) + assert.NotNil(t, apiPullRequests[0]) + assert.Nil(t, apiPullRequests[0].Head.Repository) + assert.EqualValues(t, -1, apiPullRequests[0].Head.RepoID) } diff --git a/services/convert/release_test.go b/services/convert/release_test.go index 201b27e16d..bb618c9ca3 100644 --- a/services/convert/release_test.go +++ b/services/convert/release_test.go @@ -23,6 +23,6 @@ func TestRelease_ToRelease(t *testing.T) { apiRelease := ToAPIRelease(db.DefaultContext, repo1, release1) assert.NotNil(t, apiRelease) assert.EqualValues(t, 1, apiRelease.ID) - assert.EqualValues(t, "https://try.gitea.io/api/v1/repos/user2/repo1/releases/1", apiRelease.URL) - assert.EqualValues(t, "https://try.gitea.io/api/v1/repos/user2/repo1/releases/1/assets", apiRelease.UploadURL) + assert.Equal(t, "https://try.gitea.io/api/v1/repos/user2/repo1/releases/1", apiRelease.URL) + assert.Equal(t, "https://try.gitea.io/api/v1/repos/user2/repo1/releases/1/assets", apiRelease.UploadURL) } diff --git a/services/convert/repository.go b/services/convert/repository.go index 632b6392d5..614eb58a88 100644 --- a/services/convert/repository.go +++ b/services/convert/repository.go @@ -14,6 +14,7 @@ import ( unit_model "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/log" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" ) // ToRepo converts a Repository to api.Repository @@ -97,6 +98,8 @@ func innerToRepo(ctx context.Context, repo *repo_model.Repository, permissionInR allowSquash := false allowFastForwardOnly := false allowRebaseUpdate := false + allowManualMerge := true + autodetectManualMerge := false defaultDeleteBranchAfterMerge := false defaultMergeStyle := repo_model.MergeStyleMerge defaultAllowMaintainerEdit := false @@ -110,6 +113,8 @@ func innerToRepo(ctx context.Context, repo *repo_model.Repository, permissionInR allowSquash = config.AllowSquash allowFastForwardOnly = config.AllowFastForwardOnly allowRebaseUpdate = config.AllowRebaseUpdate + allowManualMerge = config.AllowManualMerge + autodetectManualMerge = config.AutodetectManualMerge defaultDeleteBranchAfterMerge = config.DefaultDeleteBranchAfterMerge defaultMergeStyle = config.GetDefaultMergeStyle() defaultAllowMaintainerEdit = config.DefaultAllowMaintainerEdit @@ -234,6 +239,8 @@ func innerToRepo(ctx context.Context, repo *repo_model.Repository, permissionInR AllowSquash: allowSquash, AllowFastForwardOnly: allowFastForwardOnly, AllowRebaseUpdate: allowRebaseUpdate, + AllowManualMerge: allowManualMerge, + AutodetectManualMerge: autodetectManualMerge, DefaultDeleteBranchAfterMerge: defaultDeleteBranchAfterMerge, DefaultMergeStyle: string(defaultMergeStyle), DefaultAllowMaintainerEdit: defaultAllowMaintainerEdit, @@ -242,7 +249,7 @@ func innerToRepo(ctx context.Context, repo *repo_model.Repository, permissionInR MirrorInterval: mirrorInterval, MirrorUpdated: mirrorUpdated, RepoTransfer: transfer, - Topics: repo.Topics, + Topics: util.SliceNilAsEmpty(repo.Topics), ObjectFormatName: repo.ObjectFormatName, Licenses: repoLicenses.StringList(), } diff --git a/services/convert/status.go b/services/convert/status.go index 6cef63c1cd..b4864a0307 100644 --- a/services/convert/status.go +++ b/services/convert/status.go @@ -5,6 +5,7 @@ package convert import ( "context" + "net/url" git_model "code.gitea.io/gitea/models/git" user_model "code.gitea.io/gitea/models/user" @@ -32,34 +33,29 @@ func ToCommitStatus(ctx context.Context, status *git_model.CommitStatus) *api.Co return apiStatus } +func ToCommitStatuses(ctx context.Context, statuses []*git_model.CommitStatus) []*api.CommitStatus { + apiStatuses := make([]*api.CommitStatus, len(statuses)) + for i, status := range statuses { + apiStatuses[i] = ToCommitStatus(ctx, status) + } + return apiStatuses +} + // ToCombinedStatus converts List of CommitStatus to a CombinedStatus func ToCombinedStatus(ctx context.Context, statuses []*git_model.CommitStatus, repo *api.Repository) *api.CombinedStatus { if len(statuses) == 0 { return nil } - retStatus := &api.CombinedStatus{ - SHA: statuses[0].SHA, + combinedStatus := git_model.CalcCommitStatus(statuses) + + return &api.CombinedStatus{ + State: combinedStatus.State, + Statuses: ToCommitStatuses(ctx, statuses), + SHA: combinedStatus.SHA, TotalCount: len(statuses), Repository: repo, - URL: "", + CommitURL: repo.URL + "/commits/" + url.PathEscape(combinedStatus.SHA), + URL: repo.URL + "/commits/" + url.PathEscape(combinedStatus.SHA) + "/status", } - - retStatus.Statuses = make([]*api.CommitStatus, 0, len(statuses)) - for _, status := range statuses { - retStatus.Statuses = append(retStatus.Statuses, ToCommitStatus(ctx, status)) - if retStatus.State == "" || status.State.NoBetterThan(retStatus.State) { - retStatus.State = status.State - } - } - // According to https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#get-the-combined-status-for-a-specific-reference - // > Additionally, a combined state is returned. The state is one of: - // > failure if any of the contexts report as error or failure - // > pending if there are no statuses or a context is pending - // > success if the latest status for all contexts is success - if retStatus.State.IsError() { - retStatus.State = api.CommitStatusFailure - } - - return retStatus } diff --git a/services/convert/user_test.go b/services/convert/user_test.go index 4b1effc7aa..199d500732 100644 --- a/services/convert/user_test.go +++ b/services/convert/user_test.go @@ -30,11 +30,11 @@ func TestUser_ToUser(t *testing.T) { apiUser = toUser(db.DefaultContext, user1, false, false) assert.False(t, apiUser.IsAdmin) - assert.EqualValues(t, api.VisibleTypePublic.String(), apiUser.Visibility) + assert.Equal(t, api.VisibleTypePublic.String(), apiUser.Visibility) user31 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 31, IsAdmin: false, Visibility: api.VisibleTypePrivate}) apiUser = toUser(db.DefaultContext, user31, true, true) assert.False(t, apiUser.IsAdmin) - assert.EqualValues(t, api.VisibleTypePrivate.String(), apiUser.Visibility) + assert.Equal(t, api.VisibleTypePrivate.String(), apiUser.Visibility) } diff --git a/services/convert/utils.go b/services/convert/utils.go index 5e9d32cc8e..b59884ec50 100644 --- a/services/convert/utils.go +++ b/services/convert/utils.go @@ -36,6 +36,8 @@ func ToGitServiceType(value string) structs.GitServiceType { return structs.OneDevService case "gitbucket": return structs.GitBucketService + case "codebase": + return structs.CodebaseService case "codecommit": return structs.CodeCommitService default: diff --git a/services/convert/utils_test.go b/services/convert/utils_test.go index 1ac03a3097..7965624e2b 100644 --- a/services/convert/utils_test.go +++ b/services/convert/utils_test.go @@ -10,10 +10,10 @@ import ( ) func TestToCorrectPageSize(t *testing.T) { - assert.EqualValues(t, 30, ToCorrectPageSize(0)) - assert.EqualValues(t, 30, ToCorrectPageSize(-10)) - assert.EqualValues(t, 20, ToCorrectPageSize(20)) - assert.EqualValues(t, 50, ToCorrectPageSize(100)) + assert.Equal(t, 30, ToCorrectPageSize(0)) + assert.Equal(t, 30, ToCorrectPageSize(-10)) + assert.Equal(t, 20, ToCorrectPageSize(20)) + assert.Equal(t, 50, ToCorrectPageSize(100)) } func TestToGitServiceType(t *testing.T) { @@ -21,6 +21,8 @@ func TestToGitServiceType(t *testing.T) { typ string enum int }{{ + typ: "trash", enum: 1, + }, { typ: "github", enum: 2, }, { typ: "gitea", enum: 3, @@ -29,7 +31,13 @@ func TestToGitServiceType(t *testing.T) { }, { typ: "gogs", enum: 5, }, { - typ: "trash", enum: 1, + typ: "onedev", enum: 6, + }, { + typ: "gitbucket", enum: 7, + }, { + typ: "codebase", enum: 8, + }, { + typ: "codecommit", enum: 9, }} for _, test := range tc { assert.EqualValues(t, test.enum, ToGitServiceType(test.typ)) diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index fb5938745e..841981787d 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -54,7 +54,7 @@ func registerRepoHealthCheck() { RunAtStart: false, Schedule: "@midnight", }, - Timeout: 60 * time.Second, + Timeout: time.Duration(setting.Git.Timeout.Default) * time.Second, Args: []string{}, }, func(ctx context.Context, _ *user_model.User, config Config) error { rhcConfig := config.(*RepoHealthCheckConfig) diff --git a/services/doctor/actions.go b/services/doctor/actions.go index 7c44fb8392..28e26c88eb 100644 --- a/services/doctor/actions.go +++ b/services/doctor/actions.go @@ -19,7 +19,7 @@ func disableMirrorActionsUnit(ctx context.Context, logger log.Logger, autofix bo var reposToFix []*repo_model.Repository for page := 1; ; page++ { - repos, _, err := repo_model.SearchRepository(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.SearchRepository(ctx, repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ PageSize: repo_model.RepositoryListDefaultPageSize, Page: page, diff --git a/services/doctor/authorizedkeys.go b/services/doctor/authorizedkeys.go index 8d6fc9cb5e..46e7099dce 100644 --- a/services/doctor/authorizedkeys.go +++ b/services/doctor/authorizedkeys.go @@ -7,6 +7,7 @@ import ( "bufio" "bytes" "context" + "errors" "fmt" "os" "path/filepath" @@ -78,7 +79,7 @@ func checkAuthorizedKeys(ctx context.Context, logger log.Logger, autofix bool) e fPath, "gitea admin regenerate keys", "gitea doctor --run authorized-keys --fix") - return fmt.Errorf(`authorized_keys is out of date and should be regenerated with "gitea admin regenerate keys" or "gitea doctor --run authorized-keys --fix"`) + return errors.New(`authorized_keys is out of date and should be regenerated with "gitea admin regenerate keys" or "gitea doctor --run authorized-keys --fix"`) } logger.Warn("authorized_keys is out of date. Attempting rewrite...") err = asymkey_service.RewriteAllPublicKeys(ctx) diff --git a/services/doctor/dbconsistency.go b/services/doctor/dbconsistency.go index 7cb7445148..d5a133d8b2 100644 --- a/services/doctor/dbconsistency.go +++ b/services/doctor/dbconsistency.go @@ -12,8 +12,10 @@ import ( issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/migrations" repo_model "code.gitea.io/gitea/models/repo" + secret_model "code.gitea.io/gitea/models/secret" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + issue_service "code.gitea.io/gitea/services/issue" ) type consistencyCheck struct { @@ -92,7 +94,7 @@ func prepareDBConsistencyChecks() []consistencyCheck { // find issues without existing repository Name: "Orphaned Issues without existing repository", Counter: issues_model.CountOrphanedIssues, - Fixer: asFixer(issues_model.DeleteOrphanedIssues), + Fixer: asFixer(issue_service.DeleteOrphanedIssues), }, // find releases without existing repository genericOrphanCheck("Orphaned Releases without existing repository", @@ -164,6 +166,24 @@ func prepareDBConsistencyChecks() []consistencyCheck { Fixer: repo_model.DeleteOrphanedTopics, FixedMessage: "Removed", }, + { + Name: "Repository level Runners with non-zero owner_id", + Counter: actions_model.CountWrongRepoLevelRunners, + Fixer: actions_model.UpdateWrongRepoLevelRunners, + FixedMessage: "Corrected", + }, + { + Name: "Repository level Variables with non-zero owner_id", + Counter: actions_model.CountWrongRepoLevelVariables, + Fixer: actions_model.UpdateWrongRepoLevelVariables, + FixedMessage: "Corrected", + }, + { + Name: "Repository level Secrets with non-zero owner_id", + Counter: secret_model.CountWrongRepoLevelSecrets, + Fixer: secret_model.UpdateWrongRepoLevelSecrets, + FixedMessage: "Corrected", + }, } // TODO: function to recalc all counters diff --git a/services/doctor/dbconsistency_test.go b/services/doctor/dbconsistency_test.go index 4e4ac535b7..eb427dee73 100644 --- a/services/doctor/dbconsistency_test.go +++ b/services/doctor/dbconsistency_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestConsistencyCheck(t *testing.T) { @@ -21,9 +22,7 @@ func TestConsistencyCheck(t *testing.T) { idx := slices.IndexFunc(checks, func(check consistencyCheck) bool { return check.Name == "Orphaned OAuth2Application without existing User" }) - if !assert.NotEqual(t, -1, idx) { - return - } + require.NotEqual(t, -1, idx) _ = db.TruncateBeans(db.DefaultContext, &auth.OAuth2Application{}, &user.User{}) _ = db.TruncateBeans(db.DefaultContext, &auth.OAuth2Application{}, &auth.OAuth2Application{}) diff --git a/services/doctor/dbversion.go b/services/doctor/dbversion.go index 2a102b2194..34279a45e7 100644 --- a/services/doctor/dbversion.go +++ b/services/doctor/dbversion.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/migrations" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/services/versioned_migration" ) func checkDBVersion(ctx context.Context, logger log.Logger, autofix bool) error { @@ -21,7 +22,7 @@ func checkDBVersion(ctx context.Context, logger log.Logger, autofix bool) error logger.Warn("Got Error: %v during ensure up to date", err) logger.Warn("Attempting to migrate to the latest DB version to fix this.") - err = db.InitEngineWithMigration(ctx, migrations.Migrate) + err = db.InitEngineWithMigration(ctx, versioned_migration.Migrate) if err != nil { logger.Critical("Error: %v during migration", err) } diff --git a/services/doctor/doctor.go b/services/doctor/doctor.go index a4eb5e16b9..c6810a5fa0 100644 --- a/services/doctor/doctor.go +++ b/services/doctor/doctor.go @@ -48,7 +48,7 @@ type doctorCheckLogger struct { var _ log.BaseLogger = (*doctorCheckLogger)(nil) -func (d *doctorCheckLogger) Log(skip int, level log.Level, format string, v ...any) { +func (d *doctorCheckLogger) Log(skip int, event *log.Event, format string, v ...any) { _, _ = fmt.Fprintf(os.Stdout, format+"\n", v...) } @@ -62,11 +62,11 @@ type doctorCheckStepLogger struct { var _ log.BaseLogger = (*doctorCheckStepLogger)(nil) -func (d *doctorCheckStepLogger) Log(skip int, level log.Level, format string, v ...any) { - levelChar := fmt.Sprintf("[%s]", strings.ToUpper(level.String()[0:1])) +func (d *doctorCheckStepLogger) Log(skip int, event *log.Event, format string, v ...any) { + levelChar := fmt.Sprintf("[%s]", strings.ToUpper(event.Level.String()[0:1])) var levelArg any = levelChar if d.colorize { - levelArg = log.NewColoredValue(levelChar, level.ColorAttributes()...) + levelArg = log.NewColoredValue(levelChar, event.Level.ColorAttributes()...) } args := append([]any{levelArg}, v...) _, _ = fmt.Fprintf(os.Stdout, " - %s "+format+"\n", args...) diff --git a/services/doctor/fix16961_test.go b/services/doctor/fix16961_test.go index 498ed9c8d5..11a128620c 100644 --- a/services/doctor/fix16961_test.go +++ b/services/doctor/fix16961_test.go @@ -19,12 +19,6 @@ func Test_fixUnitConfig_16961(t *testing.T) { wantErr bool }{ { - name: "empty", - bs: "", - wantFixed: true, - wantErr: false, - }, - { name: "normal: {}", bs: "{}", wantFixed: false, @@ -221,7 +215,7 @@ func Test_fixPullRequestsConfig_16961(t *testing.T) { if gotFixed != tt.wantFixed { t.Errorf("fixPullRequestsConfig_16961() = %v, want %v", gotFixed, tt.wantFixed) } - assert.EqualValues(t, &tt.expected, cfg) + assert.Equal(t, &tt.expected, cfg) }) } } @@ -265,7 +259,7 @@ func Test_fixIssuesConfig_16961(t *testing.T) { if gotFixed != tt.wantFixed { t.Errorf("fixIssuesConfig_16961() = %v, want %v", gotFixed, tt.wantFixed) } - assert.EqualValues(t, &tt.expected, cfg) + assert.Equal(t, &tt.expected, cfg) }) } } diff --git a/services/doctor/heads.go b/services/doctor/heads.go index 41fca01d57..bbfd40da5e 100644 --- a/services/doctor/heads.go +++ b/services/doctor/heads.go @@ -18,9 +18,9 @@ func synchronizeRepoHeads(ctx context.Context, logger log.Logger, autofix bool) numReposUpdated := 0 err := iterateRepositories(ctx, func(repo *repo_model.Repository) error { numRepos++ - _, _, defaultBranchErr := git.NewCommand(ctx, "rev-parse").AddDashesAndList(repo.DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + _, _, defaultBranchErr := git.NewCommand("rev-parse").AddDashesAndList(repo.DefaultBranch).RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) - head, _, headErr := git.NewCommand(ctx, "symbolic-ref", "--short", "HEAD").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + head, _, headErr := git.NewCommand("symbolic-ref", "--short", "HEAD").RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) // what we expect: default branch is valid, and HEAD points to it if headErr == nil && defaultBranchErr == nil && head == repo.DefaultBranch { @@ -46,7 +46,7 @@ func synchronizeRepoHeads(ctx context.Context, logger log.Logger, autofix bool) } // otherwise, let's try fixing HEAD - err := git.NewCommand(ctx, "symbolic-ref").AddDashesAndList("HEAD", git.BranchPrefix+repo.DefaultBranch).Run(&git.RunOpts{Dir: repo.RepoPath()}) + err := git.NewCommand("symbolic-ref").AddDashesAndList("HEAD", git.BranchPrefix+repo.DefaultBranch).Run(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil { logger.Warn("Failed to fix HEAD for %s/%s: %v", repo.OwnerName, repo.Name, err) return nil diff --git a/services/doctor/lfs.go b/services/doctor/lfs.go index 5f110b8f97..a90f394450 100644 --- a/services/doctor/lfs.go +++ b/services/doctor/lfs.go @@ -5,7 +5,7 @@ package doctor import ( "context" - "fmt" + "errors" "time" "code.gitea.io/gitea/modules/log" @@ -27,7 +27,7 @@ func init() { func garbageCollectLFSCheck(ctx context.Context, logger log.Logger, autofix bool) error { if !setting.LFS.StartServer { - return fmt.Errorf("LFS support is disabled") + return errors.New("LFS support is disabled") } if err := repository.GarbageCollectLFSMetaObjects(ctx, repository.GarbageCollectLFSMetaObjectsOptions{ diff --git a/services/doctor/mergebase.go b/services/doctor/mergebase.go index de460c4190..482bcd0a46 100644 --- a/services/doctor/mergebase.go +++ b/services/doctor/mergebase.go @@ -42,17 +42,17 @@ func checkPRMergeBase(ctx context.Context, logger log.Logger, autofix bool) erro if !pr.HasMerged { var err error - pr.MergeBase, _, err = git.NewCommand(ctx, "merge-base").AddDashesAndList(pr.BaseBranch, pr.GetGitRefName()).RunStdString(&git.RunOpts{Dir: repoPath}) + pr.MergeBase, _, err = git.NewCommand("merge-base").AddDashesAndList(pr.BaseBranch, pr.GetGitRefName()).RunStdString(ctx, &git.RunOpts{Dir: repoPath}) if err != nil { var err2 error - pr.MergeBase, _, err2 = git.NewCommand(ctx, "rev-parse").AddDynamicArguments(git.BranchPrefix + pr.BaseBranch).RunStdString(&git.RunOpts{Dir: repoPath}) + pr.MergeBase, _, err2 = git.NewCommand("rev-parse").AddDynamicArguments(git.BranchPrefix+pr.BaseBranch).RunStdString(ctx, &git.RunOpts{Dir: repoPath}) if err2 != nil { logger.Warn("Unable to get merge base for PR ID %d, #%d onto %s in %s/%s. Error: %v & %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err, err2) return nil } } } else { - parentsString, _, err := git.NewCommand(ctx, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoPath}) + parentsString, _, err := git.NewCommand("rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(ctx, &git.RunOpts{Dir: repoPath}) if err != nil { logger.Warn("Unable to get parents for merged PR ID %d, #%d onto %s in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err) return nil @@ -64,8 +64,8 @@ func checkPRMergeBase(ctx context.Context, logger log.Logger, autofix bool) erro refs := append([]string{}, parents[1:]...) refs = append(refs, pr.GetGitRefName()) - cmd := git.NewCommand(ctx, "merge-base").AddDashesAndList(refs...) - pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath}) + cmd := git.NewCommand("merge-base").AddDashesAndList(refs...) + pr.MergeBase, _, err = cmd.RunStdString(ctx, &git.RunOpts{Dir: repoPath}) if err != nil { logger.Warn("Unable to get merge base for merged PR ID %d, #%d onto %s in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err) return nil diff --git a/services/doctor/misc.go b/services/doctor/misc.go index 9300c3a25c..1269d088c3 100644 --- a/services/doctor/misc.go +++ b/services/doctor/misc.go @@ -8,7 +8,7 @@ import ( "fmt" "os" "os/exec" - "path" + "path/filepath" "strings" "code.gitea.io/gitea/models" @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -50,14 +49,14 @@ func checkScriptType(ctx context.Context, logger log.Logger, autofix bool) error func checkHooks(ctx context.Context, logger log.Logger, autofix bool) error { if err := iterateRepositories(ctx, func(repo *repo_model.Repository) error { - results, err := repository.CheckDelegateHooks(repo.RepoPath()) + results, err := gitrepo.CheckDelegateHooks(ctx, repo) if err != nil { logger.Critical("Unable to check delegate hooks for repo %-v. ERROR: %v", repo, err) return fmt.Errorf("Unable to check delegate hooks for repo %-v. ERROR: %w", repo, err) } if len(results) > 0 && autofix { logger.Warn("Regenerated hooks for %s", repo.FullName()) - if err := repository.CreateDelegateHooks(repo.RepoPath()); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo); err != nil { logger.Critical("Unable to recreate delegate hooks for %-v. ERROR: %v", repo, err) return fmt.Errorf("Unable to recreate delegate hooks for %-v. ERROR: %w", repo, err) } @@ -99,11 +98,11 @@ func checkEnablePushOptions(ctx context.Context, logger log.Logger, autofix bool defer r.Close() if autofix { - _, _, err := git.NewCommand(ctx, "config", "receive.advertisePushOptions", "true").RunStdString(&git.RunOpts{Dir: r.Path}) + _, _, err := git.NewCommand("config", "receive.advertisePushOptions", "true").RunStdString(ctx, &git.RunOpts{Dir: r.Path}) return err } - value, _, err := git.NewCommand(ctx, "config", "receive.advertisePushOptions").RunStdString(&git.RunOpts{Dir: r.Path}) + value, _, err := git.NewCommand("config", "receive.advertisePushOptions").RunStdString(ctx, &git.RunOpts{Dir: r.Path}) if err != nil { return err } @@ -149,7 +148,7 @@ func checkDaemonExport(ctx context.Context, logger log.Logger, autofix bool) err } // Create/Remove git-daemon-export-ok for git-daemon... - daemonExportFile := path.Join(repo.RepoPath(), `git-daemon-export-ok`) + daemonExportFile := filepath.Join(repo.RepoPath(), `git-daemon-export-ok`) isExist, err := util.IsExist(daemonExportFile) if err != nil { log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err) @@ -197,7 +196,7 @@ func checkCommitGraph(ctx context.Context, logger log.Logger, autofix bool) erro commitGraphExists := func() (bool, error) { // Check commit-graph exists - commitGraphFile := path.Join(repo.RepoPath(), `objects/info/commit-graph`) + commitGraphFile := filepath.Join(repo.RepoPath(), `objects/info/commit-graph`) isExist, err := util.IsExist(commitGraphFile) if err != nil { logger.Error("Unable to check if %s exists. Error: %v", commitGraphFile, err) @@ -205,7 +204,7 @@ func checkCommitGraph(ctx context.Context, logger log.Logger, autofix bool) erro } if !isExist { - commitGraphsDir := path.Join(repo.RepoPath(), `objects/info/commit-graphs`) + commitGraphsDir := filepath.Join(repo.RepoPath(), `objects/info/commit-graphs`) isExist, err = util.IsExist(commitGraphsDir) if err != nil { logger.Error("Unable to check if %s exists. Error: %v", commitGraphsDir, err) diff --git a/services/doctor/paths.go b/services/doctor/paths.go index 3f62d587ab..4214c36b1a 100644 --- a/services/doctor/paths.go +++ b/services/doctor/paths.go @@ -99,15 +99,14 @@ func checkConfigurationFiles(ctx context.Context, logger log.Logger, autofix boo func isWritableDir(path string) error { // There's no platform-independent way of checking if a directory is writable // https://stackoverflow.com/questions/20026320/how-to-tell-if-folder-exists-and-is-writable - tmpFile, err := os.CreateTemp(path, "doctors-order") if err != nil { return err } if err := os.Remove(tmpFile.Name()); err != nil { - fmt.Printf("Warning: can't remove temporary file: '%s'\n", tmpFile.Name()) //nolint:forbidigo + log.Warn("can't remove temporary file: %q", tmpFile.Name()) } - tmpFile.Close() + _ = tmpFile.Close() return nil } diff --git a/services/doctor/repository.go b/services/doctor/repository.go index 6c33426636..359c4a17e0 100644 --- a/services/doctor/repository.go +++ b/services/doctor/repository.go @@ -7,7 +7,6 @@ import ( "context" "code.gitea.io/gitea/models/db" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" repo_service "code.gitea.io/gitea/services/repository" @@ -39,7 +38,6 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) { batchSize := db.MaxBatchInsertSize("repository") e := db.GetEngine(ctx) var deleted int64 - adminUser := &user_model.User{IsAdmin: true} for { select { @@ -60,7 +58,7 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) { } for _, id := range ids { - if err := repo_service.DeleteRepositoryDirectly(ctx, adminUser, id, true); err != nil { + if err := repo_service.DeleteRepositoryDirectly(ctx, id, true); err != nil { return deleted, err } deleted++ diff --git a/services/doctor/storage.go b/services/doctor/storage.go index 3f3b562c37..77fc6d65df 100644 --- a/services/doctor/storage.go +++ b/services/doctor/storage.go @@ -121,7 +121,7 @@ func checkStorage(opts *checkStorageOptions) func(ctx context.Context, logger lo storer: storage.LFS, isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) { // The oid of an LFS stored object is the name but with all the path.Separators removed - oid := strings.ReplaceAll(path, "/", "") + oid := strings.ReplaceAll(strings.ReplaceAll(path, "\\", ""), "/", "") exists, err := git.ExistsLFSObject(ctx, oid) return !exists, err }, diff --git a/services/externalaccount/link.go b/services/externalaccount/link.go index d6e2ea7e94..ab853140cb 100644 --- a/services/externalaccount/link.go +++ b/services/externalaccount/link.go @@ -5,7 +5,7 @@ package externalaccount import ( "context" - "fmt" + "errors" user_model "code.gitea.io/gitea/models/user" @@ -23,7 +23,7 @@ type Store interface { func LinkAccountFromStore(ctx context.Context, store Store, user *user_model.User) error { gothUser := store.Get("linkAccountGothUser") if gothUser == nil { - return fmt.Errorf("not in LinkAccount session") + return errors.New("not in LinkAccount session") } return LinkAccountToUser(ctx, user, gothUser.(goth.User)) diff --git a/services/feed/feed.go b/services/feed/feed.go index 93bf875fd0..1dbd2e0e26 100644 --- a/services/feed/feed.go +++ b/services/feed/feed.go @@ -5,11 +5,158 @@ package feed import ( "context" + "fmt" + "strings" activities_model "code.gitea.io/gitea/models/activities" + "code.gitea.io/gitea/models/db" + access_model "code.gitea.io/gitea/models/perm/access" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unit" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) +func GetFeedsForDashboard(ctx context.Context, opts activities_model.GetFeedsOptions) (activities_model.ActionList, int, error) { + opts.DontCount = opts.RequestedTeam == nil && opts.Date == "" + results, cnt, err := activities_model.GetFeeds(ctx, opts) + return results, util.Iif(opts.DontCount, -1, int(cnt)), err +} + // GetFeeds returns actions according to the provided options func GetFeeds(ctx context.Context, opts activities_model.GetFeedsOptions) (activities_model.ActionList, int64, error) { return activities_model.GetFeeds(ctx, opts) } + +// notifyWatchers creates batch of actions for every watcher. +// It could insert duplicate actions for a repository action, like this: +// * Original action: UserID=1 (the real actor), ActUserID=1 +// * Organization action: UserID=100 (the repo's org), ActUserID=1 +// * Watcher action: UserID=20 (a user who is watching a repo), ActUserID=1 +func notifyWatchers(ctx context.Context, act *activities_model.Action, watchers []*repo_model.Watch, permCode, permIssue, permPR []bool) error { + // MySQL has TEXT length limit 65535. + // Sometimes the content is "field1|field2|field3", sometimes the content is JSON (ActionMirrorSyncPush, ActionCommitRepo, ActionPushTag, etc...) + if left, right := util.EllipsisDisplayStringX(act.Content, 65535); right != "" { + if strings.HasPrefix(act.Content, `{"`) && strings.HasSuffix(act.Content, `}`) { + // FIXME: at the moment we can do nothing if the content is JSON and it is too long + act.Content = "{}" + } else { + act.Content = left + } + } + + // Add feed for actor. + act.UserID = act.ActUserID + if err := db.Insert(ctx, act); err != nil { + return fmt.Errorf("insert new actioner: %w", err) + } + + // Add feed for organization + if act.Repo.Owner.IsOrganization() && act.ActUserID != act.Repo.Owner.ID { + act.ID = 0 + act.UserID = act.Repo.Owner.ID + if err := db.Insert(ctx, act); err != nil { + return fmt.Errorf("insert new actioner: %w", err) + } + } + + for i, watcher := range watchers { + if act.ActUserID == watcher.UserID { + continue + } + act.ID = 0 + act.UserID = watcher.UserID + act.Repo.Units = nil + + switch act.OpType { + case activities_model.ActionCommitRepo, activities_model.ActionPushTag, activities_model.ActionDeleteTag, activities_model.ActionPublishRelease, activities_model.ActionDeleteBranch: + if !permCode[i] { + continue + } + case activities_model.ActionCreateIssue, activities_model.ActionCommentIssue, activities_model.ActionCloseIssue, activities_model.ActionReopenIssue: + if !permIssue[i] { + continue + } + case activities_model.ActionCreatePullRequest, activities_model.ActionCommentPull, activities_model.ActionMergePullRequest, activities_model.ActionClosePullRequest, activities_model.ActionReopenPullRequest, activities_model.ActionAutoMergePullRequest: + if !permPR[i] { + continue + } + default: + } + + if err := db.Insert(ctx, act); err != nil { + return fmt.Errorf("insert new action: %w", err) + } + } + + return nil +} + +// NotifyWatchers creates batch of actions for every watcher. +func NotifyWatchers(ctx context.Context, acts ...*activities_model.Action) error { + return db.WithTx(ctx, func(ctx context.Context) error { + if len(acts) == 0 { + return nil + } + + repoID := acts[0].RepoID + if repoID == 0 { + setting.PanicInDevOrTesting("action should belong to a repo") + return nil + } + if err := acts[0].LoadRepo(ctx); err != nil { + return err + } + repo := acts[0].Repo + if err := repo.LoadOwner(ctx); err != nil { + return err + } + + actUserID := acts[0].ActUserID + + // Add feeds for user self and all watchers. + watchers, err := repo_model.GetWatchers(ctx, repoID) + if err != nil { + return fmt.Errorf("get watchers: %w", err) + } + + permCode := make([]bool, len(watchers)) + permIssue := make([]bool, len(watchers)) + permPR := make([]bool, len(watchers)) + for i, watcher := range watchers { + user, err := user_model.GetUserByID(ctx, watcher.UserID) + if err != nil { + permCode[i] = false + permIssue[i] = false + permPR[i] = false + continue + } + perm, err := access_model.GetUserRepoPermission(ctx, repo, user) + if err != nil { + permCode[i] = false + permIssue[i] = false + permPR[i] = false + continue + } + permCode[i] = perm.CanRead(unit.TypeCode) + permIssue[i] = perm.CanRead(unit.TypeIssues) + permPR[i] = perm.CanRead(unit.TypePullRequests) + } + + for _, act := range acts { + if act.RepoID != repoID { + setting.PanicInDevOrTesting("action should belong to the same repo, expected[%d], got[%d] ", repoID, act.RepoID) + } + if act.ActUserID != actUserID { + setting.PanicInDevOrTesting("action should have the same actor, expected[%d], got[%d] ", actUserID, act.ActUserID) + } + + act.Repo = repo + if err := notifyWatchers(ctx, act, watchers, permCode, permIssue, permPR); err != nil { + return err + } + } + return nil + }) +} diff --git a/services/feed/feed_test.go b/services/feed/feed_test.go index 1e4d029e18..705d42a2eb 100644 --- a/services/feed/feed_test.go +++ b/services/feed/feed_test.go @@ -30,7 +30,7 @@ func TestGetFeeds(t *testing.T) { assert.NoError(t, err) if assert.Len(t, actions, 1) { assert.EqualValues(t, 1, actions[0].ID) - assert.EqualValues(t, user.ID, actions[0].UserID) + assert.Equal(t, user.ID, actions[0].UserID) } assert.Equal(t, int64(1), count) @@ -107,7 +107,7 @@ func TestGetFeeds2(t *testing.T) { assert.Len(t, actions, 1) if assert.Len(t, actions, 1) { assert.EqualValues(t, 2, actions[0].ID) - assert.EqualValues(t, org.ID, actions[0].UserID) + assert.Equal(t, org.ID, actions[0].UserID) } assert.Equal(t, int64(1), count) @@ -147,7 +147,7 @@ func TestRepoActions(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) _ = db.TruncateBeans(db.DefaultContext, &activities_model.Action{}) - for i := 0; i < 3; i++ { + for i := range 3 { _ = db.Insert(db.DefaultContext, &activities_model.Action{ UserID: 2 + int64(i), ActUserID: 2, @@ -163,3 +163,40 @@ func TestRepoActions(t *testing.T) { assert.NoError(t, err) assert.Len(t, actions, 1) } + +func TestNotifyWatchers(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + action := &activities_model.Action{ + ActUserID: 8, + RepoID: 1, + OpType: activities_model.ActionStarRepo, + } + assert.NoError(t, NotifyWatchers(db.DefaultContext, action)) + + // One watchers are inactive, thus action is only created for user 8, 1, 4, 11 + unittest.AssertExistsAndLoadBean(t, &activities_model.Action{ + ActUserID: action.ActUserID, + UserID: 8, + RepoID: action.RepoID, + OpType: action.OpType, + }) + unittest.AssertExistsAndLoadBean(t, &activities_model.Action{ + ActUserID: action.ActUserID, + UserID: 1, + RepoID: action.RepoID, + OpType: action.OpType, + }) + unittest.AssertExistsAndLoadBean(t, &activities_model.Action{ + ActUserID: action.ActUserID, + UserID: 4, + RepoID: action.RepoID, + OpType: action.OpType, + }) + unittest.AssertExistsAndLoadBean(t, &activities_model.Action{ + ActUserID: action.ActUserID, + UserID: 11, + RepoID: action.RepoID, + OpType: action.OpType, + }) +} diff --git a/services/feed/notifier.go b/services/feed/notifier.go index 702eb5ad53..64aeccdfd2 100644 --- a/services/feed/notifier.go +++ b/services/feed/notifier.go @@ -49,7 +49,7 @@ func (a *actionNotifier) NewIssue(ctx context.Context, issue *issues_model.Issue } repo := issue.Repo - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: issue.Poster.ID, ActUser: issue.Poster, OpType: activities_model.ActionCreateIssue, @@ -90,7 +90,7 @@ func (a *actionNotifier) IssueChangeStatus(ctx context.Context, doer *user_model } // Notify watchers for whatever action comes in, ignore if no action type. - if err := activities_model.NotifyWatchers(ctx, act); err != nil { + if err := NotifyWatchers(ctx, act); err != nil { log.Error("NotifyWatchers: %v", err) } } @@ -126,7 +126,7 @@ func (a *actionNotifier) CreateIssueComment(ctx context.Context, doer *user_mode } // Notify watchers for whatever action comes in, ignore if no action type. - if err := activities_model.NotifyWatchers(ctx, act); err != nil { + if err := NotifyWatchers(ctx, act); err != nil { log.Error("NotifyWatchers: %v", err) } } @@ -145,7 +145,7 @@ func (a *actionNotifier) NewPullRequest(ctx context.Context, pull *issues_model. return } - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: pull.Issue.Poster.ID, ActUser: pull.Issue.Poster, OpType: activities_model.ActionCreatePullRequest, @@ -159,7 +159,7 @@ func (a *actionNotifier) NewPullRequest(ctx context.Context, pull *issues_model. } func (a *actionNotifier) RenameRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, oldRepoName string) { - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: doer.ID, ActUser: doer, OpType: activities_model.ActionRenameRepo, @@ -173,7 +173,7 @@ func (a *actionNotifier) RenameRepository(ctx context.Context, doer *user_model. } func (a *actionNotifier) TransferRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, oldOwnerName string) { - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: doer.ID, ActUser: doer, OpType: activities_model.ActionTransferRepo, @@ -187,7 +187,7 @@ func (a *actionNotifier) TransferRepository(ctx context.Context, doer *user_mode } func (a *actionNotifier) CreateRepository(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository) { - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: doer.ID, ActUser: doer, OpType: activities_model.ActionCreateRepo, @@ -200,7 +200,7 @@ func (a *actionNotifier) CreateRepository(ctx context.Context, doer, u *user_mod } func (a *actionNotifier) ForkRepository(ctx context.Context, doer *user_model.User, oldRepo, repo *repo_model.Repository) { - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: doer.ID, ActUser: doer, OpType: activities_model.ActionCreateRepo, @@ -265,13 +265,13 @@ func (a *actionNotifier) PullRequestReview(ctx context.Context, pr *issues_model actions = append(actions, action) } - if err := activities_model.NotifyWatchersActions(ctx, actions); err != nil { + if err := NotifyWatchers(ctx, actions...); err != nil { log.Error("notify watchers '%d/%d': %v", review.Reviewer.ID, review.Issue.RepoID, err) } } func (*actionNotifier) MergePullRequest(ctx context.Context, doer *user_model.User, pr *issues_model.PullRequest) { - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: doer.ID, ActUser: doer, OpType: activities_model.ActionMergePullRequest, @@ -285,7 +285,7 @@ func (*actionNotifier) MergePullRequest(ctx context.Context, doer *user_model.Us } func (*actionNotifier) AutoMergePullRequest(ctx context.Context, doer *user_model.User, pr *issues_model.PullRequest) { - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: doer.ID, ActUser: doer, OpType: activities_model.ActionAutoMergePullRequest, @@ -303,7 +303,7 @@ func (*actionNotifier) NotifyPullRevieweDismiss(ctx context.Context, doer *user_ if len(review.OriginalAuthor) > 0 { reviewerName = review.OriginalAuthor } - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: doer.ID, ActUser: doer, OpType: activities_model.ActionPullReviewDismissed, @@ -337,7 +337,7 @@ func (a *actionNotifier) PushCommits(ctx context.Context, pusher *user_model.Use opType = activities_model.ActionDeleteBranch } - if err = activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err = NotifyWatchers(ctx, &activities_model.Action{ ActUserID: pusher.ID, ActUser: pusher, OpType: opType, @@ -357,7 +357,7 @@ func (a *actionNotifier) CreateRef(ctx context.Context, doer *user_model.User, r // has sent same action in `PushCommits`, so skip it. return } - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: doer.ID, ActUser: doer, OpType: opType, @@ -376,7 +376,7 @@ func (a *actionNotifier) DeleteRef(ctx context.Context, doer *user_model.User, r // has sent same action in `PushCommits`, so skip it. return } - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: doer.ID, ActUser: doer, OpType: opType, @@ -402,7 +402,7 @@ func (a *actionNotifier) SyncPushCommits(ctx context.Context, pusher *user_model return } - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: repo.OwnerID, ActUser: repo.MustOwner(ctx), OpType: activities_model.ActionMirrorSyncPush, @@ -423,7 +423,7 @@ func (a *actionNotifier) SyncCreateRef(ctx context.Context, doer *user_model.Use return } - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: repo.OwnerID, ActUser: repo.MustOwner(ctx), OpType: activities_model.ActionMirrorSyncCreate, @@ -443,7 +443,7 @@ func (a *actionNotifier) SyncDeleteRef(ctx context.Context, doer *user_model.Use return } - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: repo.OwnerID, ActUser: repo.MustOwner(ctx), OpType: activities_model.ActionMirrorSyncDelete, @@ -461,7 +461,7 @@ func (a *actionNotifier) NewRelease(ctx context.Context, rel *repo_model.Release log.Error("LoadAttributes: %v", err) return } - if err := activities_model.NotifyWatchers(ctx, &activities_model.Action{ + if err := NotifyWatchers(ctx, &activities_model.Action{ ActUserID: rel.PublisherID, ActUser: rel.Publisher, OpType: activities_model.ActionPublishRelease, @@ -469,7 +469,7 @@ func (a *actionNotifier) NewRelease(ctx context.Context, rel *repo_model.Release Repo: rel.Repo, IsPrivate: rel.Repo.IsPrivate, Content: rel.Title, - RefName: rel.TagName, // FIXME: use a full ref name? + RefName: git.RefNameFromTag(rel.TagName).String(), // Other functions in this file all use "refFullName.String()" }); err != nil { log.Error("NotifyWatchers: %v", err) } diff --git a/services/forms/auth_form.go b/services/forms/auth_form.go index c9f3182b3a..a8f97572b1 100644 --- a/services/forms/auth_form.go +++ b/services/forms/auth_form.go @@ -14,9 +14,11 @@ import ( // AuthenticationForm form for authentication type AuthenticationForm struct { - ID int64 - Type int `binding:"Range(2,7)"` - Name string `binding:"Required;MaxSize(30)"` + ID int64 + Type int `binding:"Range(2,7)"` + Name string `binding:"Required;MaxSize(30)"` + TwoFactorPolicy string + Host string Port int BindDN string @@ -74,7 +76,6 @@ type AuthenticationForm struct { Oauth2RestrictedGroup string Oauth2GroupTeamMap string `binding:"ValidGroupTeamMap"` Oauth2GroupTeamMapRemoval bool - SkipLocalTwoFA bool SSPIAutoCreateUsers bool SSPIAutoActivateUsers bool SSPIStripDomainNames bool diff --git a/services/forms/org.go b/services/forms/org.go index db182f7e96..2ac18ef25c 100644 --- a/services/forms/org.go +++ b/services/forms/org.go @@ -36,7 +36,6 @@ func (f *CreateOrgForm) Validate(req *http.Request, errs binding.Errors) binding // UpdateOrgSettingForm form for updating organization settings type UpdateOrgSettingForm struct { - Name string `binding:"Required;Username;MaxSize(40)" locale:"org.org_name_holder"` FullName string `binding:"MaxSize(100)"` Email string `binding:"MaxSize(255)"` Description string `binding:"MaxSize(255)"` @@ -53,6 +52,11 @@ func (f *UpdateOrgSettingForm) Validate(req *http.Request, errs binding.Errors) return middleware.Validate(errs, ctx.Data, f, ctx.Locale) } +type RenameOrgForm struct { + OrgName string `binding:"Required"` + NewOrgName string `binding:"Required;Username;MaxSize(40)" locale:"org.org_name_holder"` +} + // ___________ // \__ ___/___ _____ _____ // | |_/ __ \\__ \ / \ diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index b14171787e..d116bb9f11 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -10,7 +10,6 @@ import ( issues_model "code.gitea.io/gitea/models/issues" project_model "code.gitea.io/gitea/models/project" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/web/middleware" "code.gitea.io/gitea/services/context" @@ -110,12 +109,13 @@ type RepoSettingForm struct { EnablePrune bool // Advanced settings - EnableCode bool - EnableWiki bool - EnableExternalWiki bool - DefaultWikiBranch string - DefaultWikiEveryoneAccess string - ExternalWikiURL string + EnableCode bool + + EnableWiki bool + EnableExternalWiki bool + DefaultWikiBranch string + ExternalWikiURL string + EnableIssues bool EnableExternalTracker bool ExternalTrackerURL string @@ -123,28 +123,34 @@ type RepoSettingForm struct { TrackerIssueStyle string ExternalTrackerRegexpPattern string EnableCloseIssuesViaCommitInAnyBranch bool - EnableProjects bool - ProjectsMode string - EnableReleases bool - EnablePackages bool - EnablePulls bool - EnableActions bool - PullsIgnoreWhitespace bool - PullsAllowMerge bool - PullsAllowRebase bool - PullsAllowRebaseMerge bool - PullsAllowSquash bool - PullsAllowFastForwardOnly bool - PullsAllowManualMerge bool - PullsDefaultMergeStyle string - EnableAutodetectManualMerge bool - PullsAllowRebaseUpdate bool - DefaultDeleteBranchAfterMerge bool - DefaultAllowMaintainerEdit bool - EnableTimetracker bool - AllowOnlyContributorsToTrackTime bool - EnableIssueDependencies bool - IsArchived bool + + EnableProjects bool + ProjectsMode string + + EnableReleases bool + + EnablePackages bool + + EnablePulls bool + PullsIgnoreWhitespace bool + PullsAllowMerge bool + PullsAllowRebase bool + PullsAllowRebaseMerge bool + PullsAllowSquash bool + PullsAllowFastForwardOnly bool + PullsAllowManualMerge bool + PullsDefaultMergeStyle string + EnableAutodetectManualMerge bool + PullsAllowRebaseUpdate bool + DefaultDeleteBranchAfterMerge bool + DefaultAllowMaintainerEdit bool + EnableTimetracker bool + AllowOnlyContributorsToTrackTime bool + EnableIssueDependencies bool + + EnableActions bool + + IsArchived bool // Signing Settings TrustModel string @@ -160,13 +166,6 @@ func (f *RepoSettingForm) Validate(req *http.Request, errs binding.Errors) bindi return middleware.Validate(errs, ctx.Data, f, ctx.Locale) } -// __________ .__ -// \______ \____________ ____ ____ | |__ -// | | _/\_ __ \__ \ / \_/ ___\| | \ -// | | \ | | \// __ \| | \ \___| Y \ -// |______ / |__| (____ /___| /\___ >___| / -// \/ \/ \/ \/ \/ - // ProtectBranchForm form for changing protected branch settings type ProtectBranchForm struct { RuleName string `binding:"Required"` @@ -209,26 +208,18 @@ type ProtectBranchPriorityForm struct { IDs []int64 } -// __ __ ___. .__ __ -// / \ / \ ____\_ |__ | |__ ____ ____ | | __ -// \ \/\/ // __ \| __ \| | \ / _ \ / _ \| |/ / -// \ /\ ___/| \_\ \ Y ( <_> | <_> ) < -// \__/\ / \___ >___ /___| /\____/ \____/|__|_ \ -// \/ \/ \/ \/ \/ - // WebhookForm form for changing web hook type WebhookForm struct { Events string Create bool Delete bool Fork bool + Push bool Issues bool IssueAssign bool IssueLabel bool IssueMilestone bool IssueComment bool - Release bool - Push bool PullRequest bool PullRequestAssign bool PullRequestLabel bool @@ -239,7 +230,11 @@ type WebhookForm struct { PullRequestReviewRequest bool Wiki bool Repository bool + Release bool Package bool + Status bool + WorkflowRun bool + WorkflowJob bool Active bool BranchFilter string `binding:"GlobPattern"` AuthorizationHeader string @@ -478,22 +473,6 @@ func (i *IssueLockForm) Validate(req *http.Request, errs binding.Errors) binding return middleware.Validate(errs, ctx.Data, i, ctx.Locale) } -// HasValidReason checks to make sure that the reason submitted in -// the form matches any of the values in the config -func (i IssueLockForm) HasValidReason() bool { - if strings.TrimSpace(i.Reason) == "" { - return true - } - - for _, v := range setting.Repository.Issue.LockReasons { - if v == i.Reason { - return true - } - } - - return false -} - // CreateProjectForm form for creating a project type CreateProjectForm struct { Title string `binding:"Required;MaxSize(100)"` @@ -524,12 +503,13 @@ func (f *CreateMilestoneForm) Validate(req *http.Request, errs binding.Errors) b // CreateLabelForm form for creating label type CreateLabelForm struct { - ID int64 - Title string `binding:"Required;MaxSize(50)" locale:"repo.issues.label_title"` - Exclusive bool `form:"exclusive"` - IsArchived bool `form:"is_archived"` - Description string `binding:"MaxSize(200)" locale:"repo.issues.label_description"` - Color string `binding:"Required;MaxSize(7)" locale:"repo.issues.label_color"` + ID int64 + Title string `binding:"Required;MaxSize(50)" locale:"repo.issues.label_title"` + Exclusive bool `form:"exclusive"` + ExclusiveOrder int `form:"exclusive_order"` + IsArchived bool `form:"is_archived"` + Description string `binding:"MaxSize(200)" locale:"repo.issues.label_description"` + Color string `binding:"Required;MaxSize(7)" locale:"repo.issues.label_color"` } // Validate validates the fields @@ -651,8 +631,8 @@ type NewReleaseForm struct { Target string `form:"tag_target" binding:"Required;MaxSize(255)"` Title string `binding:"MaxSize(255)"` Content string - Draft string - TagOnly string + Draft bool + TagOnly bool Prerelease bool AddTagMsg bool Files []string @@ -700,125 +680,6 @@ func (f *NewWikiForm) Validate(req *http.Request, errs binding.Errors) binding.E return middleware.Validate(errs, ctx.Data, f, ctx.Locale) } -// ___________ .___.__ __ -// \_ _____/ __| _/|__|/ |_ -// | __)_ / __ | | \ __\ -// | \/ /_/ | | || | -// /_______ /\____ | |__||__| -// \/ \/ - -// EditRepoFileForm form for changing repository file -type EditRepoFileForm struct { - TreePath string `binding:"Required;MaxSize(500)"` - Content string - CommitSummary string `binding:"MaxSize(100)"` - CommitMessage string - CommitChoice string `binding:"Required;MaxSize(50)"` - NewBranchName string `binding:"GitRefName;MaxSize(100)"` - LastCommit string - Signoff bool -} - -// Validate validates the fields -func (f *EditRepoFileForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { - ctx := context.GetValidateContext(req) - return middleware.Validate(errs, ctx.Data, f, ctx.Locale) -} - -// EditPreviewDiffForm form for changing preview diff -type EditPreviewDiffForm struct { - Content string -} - -// Validate validates the fields -func (f *EditPreviewDiffForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { - ctx := context.GetValidateContext(req) - return middleware.Validate(errs, ctx.Data, f, ctx.Locale) -} - -// _________ .__ __________.__ __ -// \_ ___ \| |__ __________________ ___.__. \______ \__| ____ | | __ -// / \ \/| | \_/ __ \_ __ \_ __ < | | | ___/ |/ ___\| |/ / -// \ \___| Y \ ___/| | \/| | \/\___ | | | | \ \___| < -// \______ /___| /\___ >__| |__| / ____| |____| |__|\___ >__|_ \ -// \/ \/ \/ \/ \/ \/ - -// CherryPickForm form for changing repository file -type CherryPickForm struct { - CommitSummary string `binding:"MaxSize(100)"` - CommitMessage string - CommitChoice string `binding:"Required;MaxSize(50)"` - NewBranchName string `binding:"GitRefName;MaxSize(100)"` - LastCommit string - Revert bool - Signoff bool -} - -// Validate validates the fields -func (f *CherryPickForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { - ctx := context.GetValidateContext(req) - return middleware.Validate(errs, ctx.Data, f, ctx.Locale) -} - -// ____ ___ .__ .___ -// | | \______ | | _________ __| _/ -// | | /\____ \| | / _ \__ \ / __ | -// | | / | |_> > |_( <_> ) __ \_/ /_/ | -// |______/ | __/|____/\____(____ /\____ | -// |__| \/ \/ -// - -// UploadRepoFileForm form for uploading repository file -type UploadRepoFileForm struct { - TreePath string `binding:"MaxSize(500)"` - CommitSummary string `binding:"MaxSize(100)"` - CommitMessage string - CommitChoice string `binding:"Required;MaxSize(50)"` - NewBranchName string `binding:"GitRefName;MaxSize(100)"` - Files []string - Signoff bool -} - -// Validate validates the fields -func (f *UploadRepoFileForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { - ctx := context.GetValidateContext(req) - return middleware.Validate(errs, ctx.Data, f, ctx.Locale) -} - -// RemoveUploadFileForm form for removing uploaded file -type RemoveUploadFileForm struct { - File string `binding:"Required;MaxSize(50)"` -} - -// Validate validates the fields -func (f *RemoveUploadFileForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { - ctx := context.GetValidateContext(req) - return middleware.Validate(errs, ctx.Data, f, ctx.Locale) -} - -// ________ .__ __ -// \______ \ ____ | | _____/ |_ ____ -// | | \_/ __ \| | _/ __ \ __\/ __ \ -// | ` \ ___/| |_\ ___/| | \ ___/ -// /_______ /\___ >____/\___ >__| \___ > -// \/ \/ \/ \/ - -// DeleteRepoFileForm form for deleting repository file -type DeleteRepoFileForm struct { - CommitSummary string `binding:"MaxSize(100)"` - CommitMessage string - CommitChoice string `binding:"Required;MaxSize(50)"` - NewBranchName string `binding:"GitRefName;MaxSize(100)"` - LastCommit string - Signoff bool -} - -// Validate validates the fields -func (f *DeleteRepoFileForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { - ctx := context.GetValidateContext(req) - return middleware.Validate(errs, ctx.Data, f, ctx.Locale) -} - // ___________.__ ___________ __ // \__ ___/|__| _____ ____ \__ ___/___________ ____ | | __ ___________ // | | | |/ \_/ __ \ | | \_ __ \__ \ _/ ___\| |/ // __ \_ __ \ diff --git a/services/forms/repo_form_editor.go b/services/forms/repo_form_editor.go new file mode 100644 index 0000000000..3ad2eae75d --- /dev/null +++ b/services/forms/repo_form_editor.go @@ -0,0 +1,57 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package forms + +import ( + "net/http" + + "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/web/middleware" + "code.gitea.io/gitea/services/context" + + "gitea.com/go-chi/binding" +) + +type CommitCommonForm struct { + TreePath string `binding:"MaxSize(500)"` + CommitSummary string `binding:"MaxSize(100)"` + CommitMessage string + CommitChoice string `binding:"Required;MaxSize(50)"` + NewBranchName string `binding:"GitRefName;MaxSize(100)"` + LastCommit string + Signoff bool + CommitEmail string +} + +func (f *CommitCommonForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetValidateContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} + +type CommitCommonFormInterface interface { + GetCommitCommonForm() *CommitCommonForm +} + +func (f *CommitCommonForm) GetCommitCommonForm() *CommitCommonForm { + return f +} + +type EditRepoFileForm struct { + CommitCommonForm + Content optional.Option[string] +} + +type DeleteRepoFileForm struct { + CommitCommonForm +} + +type UploadRepoFileForm struct { + CommitCommonForm + Files []string +} + +type CherryPickForm struct { + CommitCommonForm + Revert bool +} diff --git a/services/forms/repo_form_test.go b/services/forms/repo_form_test.go index 2c5a8e2c0f..a0c67fe0f8 100644 --- a/services/forms/repo_form_test.go +++ b/services/forms/repo_form_test.go @@ -6,8 +6,6 @@ package forms import ( "testing" - "code.gitea.io/gitea/modules/setting" - "github.com/stretchr/testify/assert" ) @@ -39,26 +37,3 @@ func TestSubmitReviewForm_IsEmpty(t *testing.T) { assert.Equal(t, v.expected, v.form.HasEmptyContent()) } } - -func TestIssueLock_HasValidReason(t *testing.T) { - // Init settings - _ = setting.Repository - - cases := []struct { - form IssueLockForm - expected bool - }{ - {IssueLockForm{""}, true}, // an empty reason is accepted - {IssueLockForm{"Off-topic"}, true}, - {IssueLockForm{"Too heated"}, true}, - {IssueLockForm{"Spam"}, true}, - {IssueLockForm{"Resolved"}, true}, - - {IssueLockForm{"ZZZZ"}, false}, - {IssueLockForm{"I want to lock this issue"}, false}, - } - - for _, v := range cases { - assert.Equal(t, v.expected, v.form.HasValidReason()) - } -} diff --git a/services/forms/user_form.go b/services/forms/user_form.go index ed79936add..ddf2bd09b0 100644 --- a/services/forms/user_form.go +++ b/services/forms/user_form.go @@ -7,9 +7,7 @@ package forms import ( "mime/multipart" "net/http" - "strings" - auth_model "code.gitea.io/gitea/models/auth" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/web/middleware" @@ -325,8 +323,9 @@ func (f *AddKeyForm) Validate(req *http.Request, errs binding.Errors) binding.Er // AddSecretForm for adding secrets type AddSecretForm struct { - Name string `binding:"Required;MaxSize(255)"` - Data string `binding:"Required;MaxSize(65535)"` + Name string `binding:"Required;MaxSize(255)"` + Data string `binding:"Required;MaxSize(65535)"` + Description string `binding:"MaxSize(65535)"` } // Validate validates the fields @@ -336,8 +335,9 @@ func (f *AddSecretForm) Validate(req *http.Request, errs binding.Errors) binding } type EditVariableForm struct { - Name string `binding:"Required;MaxSize(255)"` - Data string `binding:"Required;MaxSize(65535)"` + Name string `binding:"Required;MaxSize(255)"` + Data string `binding:"Required;MaxSize(65535)"` + Description string `binding:"MaxSize(65535)"` } func (f *EditVariableForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { @@ -347,8 +347,7 @@ func (f *EditVariableForm) Validate(req *http.Request, errs binding.Errors) bind // NewAccessTokenForm form for creating access token type NewAccessTokenForm struct { - Name string `binding:"Required;MaxSize(255)" locale:"settings.token_name"` - Scope []string + Name string `binding:"Required;MaxSize(255)" locale:"settings.token_name"` } // Validate validates the fields @@ -357,12 +356,6 @@ func (f *NewAccessTokenForm) Validate(req *http.Request, errs binding.Errors) bi return middleware.Validate(errs, ctx.Data, f, ctx.Locale) } -func (f *NewAccessTokenForm) GetScope() (auth_model.AccessTokenScope, error) { - scope := strings.Join(f.Scope, ",") - s, err := auth_model.AccessTokenScope(scope).Normalize() - return s, err -} - // EditOAuth2ApplicationForm form for editing oauth2 applications type EditOAuth2ApplicationForm struct { Name string `binding:"Required;MaxSize(255)" form:"application_name"` diff --git a/services/forms/user_form_test.go b/services/forms/user_form_test.go index 66050187c9..09e9ec0f65 100644 --- a/services/forms/user_form_test.go +++ b/services/forms/user_form_test.go @@ -4,11 +4,10 @@ package forms import ( - "strconv" "testing" - auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/gobwas/glob" "github.com/stretchr/testify/assert" @@ -28,12 +27,7 @@ func TestRegisterForm_IsDomainAllowed_Empty(t *testing.T) { } func TestRegisterForm_IsDomainAllowed_InvalidEmail(t *testing.T) { - oldService := setting.Service - defer func() { - setting.Service = oldService - }() - - setting.Service.EmailDomainAllowList = []glob.Glob{glob.MustCompile("gitea.io")} + defer test.MockVariableValue(&setting.Service.EmailDomainAllowList, []glob.Glob{glob.MustCompile("gitea.io")})() tt := []struct { email string @@ -50,12 +44,7 @@ func TestRegisterForm_IsDomainAllowed_InvalidEmail(t *testing.T) { } func TestRegisterForm_IsDomainAllowed_AllowedEmail(t *testing.T) { - oldService := setting.Service - defer func() { - setting.Service = oldService - }() - - setting.Service.EmailDomainAllowList = []glob.Glob{glob.MustCompile("gitea.io"), glob.MustCompile("*.allow")} + defer test.MockVariableValue(&setting.Service.EmailDomainAllowList, []glob.Glob{glob.MustCompile("gitea.io"), glob.MustCompile("*.allow")})() tt := []struct { email string @@ -78,13 +67,7 @@ func TestRegisterForm_IsDomainAllowed_AllowedEmail(t *testing.T) { } func TestRegisterForm_IsDomainAllowed_BlockedEmail(t *testing.T) { - oldService := setting.Service - defer func() { - setting.Service = oldService - }() - - setting.Service.EmailDomainAllowList = nil - setting.Service.EmailDomainBlockList = []glob.Glob{glob.MustCompile("gitea.io"), glob.MustCompile("*.block")} + defer test.MockVariableValue(&setting.Service.EmailDomainBlockList, []glob.Glob{glob.MustCompile("gitea.io"), glob.MustCompile("*.block")})() tt := []struct { email string @@ -104,28 +87,3 @@ func TestRegisterForm_IsDomainAllowed_BlockedEmail(t *testing.T) { assert.Equal(t, v.valid, form.IsEmailDomainAllowed()) } } - -func TestNewAccessTokenForm_GetScope(t *testing.T) { - tests := []struct { - form NewAccessTokenForm - scope auth_model.AccessTokenScope - expectedErr error - }{ - { - form: NewAccessTokenForm{Name: "test", Scope: []string{"read:repository"}}, - scope: "read:repository", - }, - { - form: NewAccessTokenForm{Name: "test", Scope: []string{"read:repository", "write:user"}}, - scope: "read:repository,write:user", - }, - } - - for i, test := range tests { - t.Run(strconv.Itoa(i), func(t *testing.T) { - scope, err := test.form.GetScope() - assert.Equal(t, test.expectedErr, err) - assert.Equal(t, test.scope, scope) - }) - } -} diff --git a/services/git/commit.go b/services/git/commit.go new file mode 100644 index 0000000000..2e0e8a5096 --- /dev/null +++ b/services/git/commit.go @@ -0,0 +1,97 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "context" + + asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/git" + asymkey_service "code.gitea.io/gitea/services/asymkey" +) + +// ParseCommitsWithSignature checks if signaute of commits are corresponding to users gpg keys. +func ParseCommitsWithSignature(ctx context.Context, repo *repo_model.Repository, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType) ([]*asymkey_model.SignCommit, error) { + newCommits := make([]*asymkey_model.SignCommit, 0, len(oldCommits)) + keyMap := map[string]bool{} + + emails := make(container.Set[string]) + for _, c := range oldCommits { + if c.Committer != nil { + emails.Add(c.Committer.Email) + } + } + + emailUsers, err := user_model.GetUsersByEmails(ctx, emails.Values()) + if err != nil { + return nil, err + } + + for _, c := range oldCommits { + committerUser := emailUsers.GetByEmail(c.Committer.Email) // FIXME: why ValidateCommitsWithEmails uses "Author", but ParseCommitsWithSignature uses "Committer"? + if committerUser == nil { + committerUser = &user_model.User{ + Name: c.Committer.Name, + Email: c.Committer.Email, + } + } + + signCommit := &asymkey_model.SignCommit{ + UserCommit: c, + Verification: asymkey_service.ParseCommitWithSignatureCommitter(ctx, c.Commit, committerUser), + } + + isOwnerMemberCollaborator := func(user *user_model.User) (bool, error) { + return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID) + } + + _ = asymkey_model.CalculateTrustStatus(signCommit.Verification, repoTrustModel, isOwnerMemberCollaborator, &keyMap) + + newCommits = append(newCommits, signCommit) + } + return newCommits, nil +} + +// ConvertFromGitCommit converts git commits into SignCommitWithStatuses +func ConvertFromGitCommit(ctx context.Context, commits []*git.Commit, repo *repo_model.Repository) ([]*git_model.SignCommitWithStatuses, error) { + validatedCommits, err := user_model.ValidateCommitsWithEmails(ctx, commits) + if err != nil { + return nil, err + } + signedCommits, err := ParseCommitsWithSignature( + ctx, + repo, + validatedCommits, + repo.GetTrustModel(), + ) + if err != nil { + return nil, err + } + return ParseCommitsWithStatus(ctx, signedCommits, repo) +} + +// ParseCommitsWithStatus checks commits latest statuses and calculates its worst status state +func ParseCommitsWithStatus(ctx context.Context, oldCommits []*asymkey_model.SignCommit, repo *repo_model.Repository) ([]*git_model.SignCommitWithStatuses, error) { + newCommits := make([]*git_model.SignCommitWithStatuses, 0, len(oldCommits)) + + for _, c := range oldCommits { + commit := &git_model.SignCommitWithStatuses{ + SignCommit: c, + } + statuses, err := git_model.GetLatestCommitStatus(ctx, repo.ID, commit.ID.String(), db.ListOptionsAll) + if err != nil { + return nil, err + } + + commit.Statuses = statuses + commit.Status = git_model.CalcCommitStatus(statuses) + newCommits = append(newCommits, commit) + } + return newCommits, nil +} diff --git a/services/gitdiff/csv.go b/services/gitdiff/csv.go index 8db73c56a3..c10ee14490 100644 --- a/services/gitdiff/csv.go +++ b/services/gitdiff/csv.go @@ -134,7 +134,7 @@ func createCsvDiffSingle(reader *csv.Reader, celltype TableDiffCellType) ([]*Tab return nil, err } cells := make([]*TableDiffCell, len(row)) - for j := 0; j < len(row); j++ { + for j := range row { if celltype == TableDiffCellDel { cells[j] = &TableDiffCell{LeftCell: row[j], Type: celltype} } else { @@ -365,11 +365,11 @@ func getColumnMapping(baseCSVReader, headCSVReader *csvReader) ([]int, []int) { } // Loops through the baseRow and see if there is a match in the head row - for i := 0; i < len(baseRow); i++ { + for i := range baseRow { base2HeadColMap[i] = unmappedColumn baseCell, err := getCell(baseRow, i) if err == nil { - for j := 0; j < len(headRow); j++ { + for j := range headRow { if head2BaseColMap[j] == -1 { headCell, err := getCell(headRow, j) if err == nil && baseCell == headCell { @@ -390,7 +390,7 @@ func getColumnMapping(baseCSVReader, headCSVReader *csvReader) ([]int, []int) { // tryMapColumnsByContent tries to map missing columns by the content of the first lines. func tryMapColumnsByContent(baseCSVReader *csvReader, base2HeadColMap []int, headCSVReader *csvReader, head2BaseColMap []int) { - for i := 0; i < len(base2HeadColMap); i++ { + for i := range base2HeadColMap { headStart := 0 for base2HeadColMap[i] == unmappedColumn && headStart < len(head2BaseColMap) { if head2BaseColMap[headStart] == unmappedColumn { @@ -424,7 +424,7 @@ func getCell(row []string, column int) (string, error) { // countUnmappedColumns returns the count of unmapped columns. func countUnmappedColumns(mapping []int) int { count := 0 - for i := 0; i < len(mapping); i++ { + for i := range mapping { if mapping[i] == unmappedColumn { count++ } diff --git a/services/gitdiff/csv_test.go b/services/gitdiff/csv_test.go index c006a7c2bd..f91e0e9aa7 100644 --- a/services/gitdiff/csv_test.go +++ b/services/gitdiff/csv_test.go @@ -192,23 +192,18 @@ c,d,e`, for n, c := range cases { diff, err := ParsePatch(db.DefaultContext, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, strings.NewReader(c.diff), "") - if err != nil { - t.Errorf("ParsePatch failed: %s", err) - } + assert.NoError(t, err) var baseReader *csv.Reader if len(c.base) > 0 { baseReader, err = csv_module.CreateReaderAndDetermineDelimiter(nil, strings.NewReader(c.base)) - if err != nil { - t.Errorf("CreateReaderAndDetermineDelimiter failed: %s", err) - } + assert.NoError(t, err) } + var headReader *csv.Reader if len(c.head) > 0 { headReader, err = csv_module.CreateReaderAndDetermineDelimiter(nil, strings.NewReader(c.head)) - if err != nil { - t.Errorf("CreateReaderAndDetermineDelimiter failed: %s", err) - } + assert.NoError(t, err) } result, err := CreateCsvDiff(diff.Files[0], baseReader, headReader) diff --git a/services/gitdiff/git_diff_tree.go b/services/gitdiff/git_diff_tree.go new file mode 100644 index 0000000000..ed94bfbfe4 --- /dev/null +++ b/services/gitdiff/git_diff_tree.go @@ -0,0 +1,250 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitdiff + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" +) + +type DiffTree struct { + Files []*DiffTreeRecord +} + +type DiffTreeRecord struct { + // Status is one of 'added', 'deleted', 'modified', 'renamed', 'copied', 'typechanged', 'unmerged', 'unknown' + Status string + + // For renames and copies, the percentage of similarity between the source and target of the move/rename. + Score uint8 + + HeadPath string + BasePath string + HeadMode git.EntryMode + BaseMode git.EntryMode + HeadBlobID string + BaseBlobID string +} + +// GetDiffTree returns the list of path of the files that have changed between the two commits. +// If useMergeBase is true, the diff will be calculated using the merge base of the two commits. +// This is the same behavior as using a three-dot diff in git diff. +func GetDiffTree(ctx context.Context, gitRepo *git.Repository, useMergeBase bool, baseSha, headSha string) (*DiffTree, error) { + gitDiffTreeRecords, err := runGitDiffTree(ctx, gitRepo, useMergeBase, baseSha, headSha) + if err != nil { + return nil, err + } + + return &DiffTree{ + Files: gitDiffTreeRecords, + }, nil +} + +func runGitDiffTree(ctx context.Context, gitRepo *git.Repository, useMergeBase bool, baseSha, headSha string) ([]*DiffTreeRecord, error) { + useMergeBase, baseCommitID, headCommitID, err := validateGitDiffTreeArguments(gitRepo, useMergeBase, baseSha, headSha) + if err != nil { + return nil, err + } + + cmd := git.NewCommand("diff-tree", "--raw", "-r", "--find-renames", "--root") + if useMergeBase { + cmd.AddArguments("--merge-base") + } + cmd.AddDynamicArguments(baseCommitID, headCommitID) + stdout, _, runErr := cmd.RunStdString(ctx, &git.RunOpts{Dir: gitRepo.Path}) + if runErr != nil { + log.Warn("git diff-tree: %v", runErr) + return nil, runErr + } + + return parseGitDiffTree(strings.NewReader(stdout)) +} + +func validateGitDiffTreeArguments(gitRepo *git.Repository, useMergeBase bool, baseSha, headSha string) (shouldUseMergeBase bool, resolvedBaseSha, resolvedHeadSha string, err error) { + // if the head is empty its an error + if headSha == "" { + return false, "", "", errors.New("headSha is empty") + } + + // if the head commit doesn't exist its and error + headCommit, err := gitRepo.GetCommit(headSha) + if err != nil { + return false, "", "", fmt.Errorf("failed to get commit headSha: %v", err) + } + headCommitID := headCommit.ID.String() + + // if the base is empty we should use the parent of the head commit + if baseSha == "" { + // if the headCommit has no parent we should use an empty commit + // this can happen when we are generating a diff against an orphaned commit + if headCommit.ParentCount() == 0 { + objectFormat, err := gitRepo.GetObjectFormat() + if err != nil { + return false, "", "", err + } + + // We set use merge base to false because we have no base commit + return false, objectFormat.EmptyTree().String(), headCommitID, nil + } + + baseCommit, err := headCommit.Parent(0) + if err != nil { + return false, "", "", fmt.Errorf("baseSha is '', attempted to use parent of commit %s, got error: %v", headCommit.ID.String(), err) + } + return useMergeBase, baseCommit.ID.String(), headCommitID, nil + } + + // try and get the base commit + baseCommit, err := gitRepo.GetCommit(baseSha) + // propagate the error if we couldn't get the base commit + if err != nil { + return useMergeBase, "", "", fmt.Errorf("failed to get base commit %s: %v", baseSha, err) + } + + return useMergeBase, baseCommit.ID.String(), headCommit.ID.String(), nil +} + +func parseGitDiffTree(gitOutput io.Reader) ([]*DiffTreeRecord, error) { + /* + The output of `git diff-tree --raw -r --find-renames` is of the form: + + :<old_mode> <new_mode> <old_sha> <new_sha> <status>\t<path> + + or for renames: + + :<old_mode> <new_mode> <old_sha> <new_sha> <status>\t<old_path>\t<new_path> + + See: <https://git-scm.com/docs/git-diff-tree#_raw_output_format> for more details + */ + results := make([]*DiffTreeRecord, 0) + + lines := bufio.NewScanner(gitOutput) + for lines.Scan() { + line := lines.Text() + + if len(line) == 0 { + continue + } + + record, err := parseGitDiffTreeLine(line) + if err != nil { + return nil, err + } + + results = append(results, record) + } + + if err := lines.Err(); err != nil { + return nil, err + } + + return results, nil +} + +func parseGitDiffTreeLine(line string) (*DiffTreeRecord, error) { + line = strings.TrimPrefix(line, ":") + splitSections := strings.SplitN(line, "\t", 2) + if len(splitSections) < 2 { + return nil, fmt.Errorf("unparsable output for diff-tree --raw: `%s`)", line) + } + + fields := strings.Fields(splitSections[0]) + if len(fields) < 5 { + return nil, fmt.Errorf("unparsable output for diff-tree --raw: `%s`, expected 5 space delimited values got %d)", line, len(fields)) + } + + baseMode, err := git.ParseEntryMode(fields[0]) + if err != nil { + return nil, err + } + + headMode, err := git.ParseEntryMode(fields[1]) + if err != nil { + return nil, err + } + + baseBlobID := fields[2] + headBlobID := fields[3] + + status, score, err := statusFromLetter(fields[4]) + if err != nil { + return nil, fmt.Errorf("unparsable output for diff-tree --raw: %s, error: %s", line, err) + } + + filePaths := strings.Split(splitSections[1], "\t") + + var headPath, basePath string + if status == "renamed" { + if len(filePaths) != 2 { + return nil, fmt.Errorf("unparsable output for diff-tree --raw: `%s`, expected 2 paths found %d", line, len(filePaths)) + } + basePath = filePaths[0] + headPath = filePaths[1] + } else { + basePath = filePaths[0] + headPath = filePaths[0] + } + + return &DiffTreeRecord{ + Status: status, + Score: score, + BaseMode: baseMode, + HeadMode: headMode, + BaseBlobID: baseBlobID, + HeadBlobID: headBlobID, + BasePath: basePath, + HeadPath: headPath, + }, nil +} + +func statusFromLetter(rawStatus string) (status string, score uint8, err error) { + if len(rawStatus) < 1 { + return "", 0, errors.New("empty status letter") + } + switch rawStatus[0] { + case 'A': + return "added", 0, nil + case 'D': + return "deleted", 0, nil + case 'M': + return "modified", 0, nil + case 'R': + score, err = tryParseStatusScore(rawStatus) + return "renamed", score, err + case 'C': + score, err = tryParseStatusScore(rawStatus) + return "copied", score, err + case 'T': + return "typechanged", 0, nil + case 'U': + return "unmerged", 0, nil + case 'X': + return "unknown", 0, nil + default: + return "", 0, fmt.Errorf("unknown status letter: '%s'", rawStatus) + } +} + +func tryParseStatusScore(rawStatus string) (uint8, error) { + if len(rawStatus) < 2 { + return 0, errors.New("status score missing") + } + + score, err := strconv.ParseUint(rawStatus[1:], 10, 8) + if err != nil { + return 0, fmt.Errorf("failed to parse status score: %w", err) + } else if score > 100 { + return 0, fmt.Errorf("status score out of range: %d", score) + } + + return uint8(score), nil +} diff --git a/services/gitdiff/git_diff_tree_test.go b/services/gitdiff/git_diff_tree_test.go new file mode 100644 index 0000000000..313d279e95 --- /dev/null +++ b/services/gitdiff/git_diff_tree_test.go @@ -0,0 +1,427 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitdiff + +import ( + "strings" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGitDiffTree(t *testing.T) { + test := []struct { + Name string + RepoPath string + BaseSha string + HeadSha string + useMergeBase bool + Expected *DiffTree + }{ + { + Name: "happy path", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + BaseSha: "72866af952e98d02a73003501836074b286a78f6", + HeadSha: "d8e0bbb45f200e67d9a784ce55bd90821af45ebd", + Expected: &DiffTree{ + Files: []*DiffTreeRecord{ + { + Status: "modified", + HeadPath: "LICENSE", + BasePath: "LICENSE", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "ee469963e76ae1bb7ee83d7510df2864e6c8c640", + BaseBlobID: "c996f4725be8fc8c1d1c776e58c97ddc5d03b336", + }, + { + Status: "modified", + HeadPath: "README.md", + BasePath: "README.md", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "9dfc0a6257d8eff526f0cfaf6a8ea950f55a9dba", + BaseBlobID: "074e590b8e64898b02beef03ece83f962c94f54c", + }, + }, + }, + }, + { + Name: "first commit (no parent)", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + HeadSha: "72866af952e98d02a73003501836074b286a78f6", + Expected: &DiffTree{ + Files: []*DiffTreeRecord{ + { + Status: "added", + HeadPath: ".gitignore", + BasePath: ".gitignore", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeNoEntry, + HeadBlobID: "f1c181ec9c5c921245027c6b452ecfc1d3626364", + BaseBlobID: "0000000000000000000000000000000000000000", + }, + { + Status: "added", + HeadPath: "LICENSE", + BasePath: "LICENSE", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeNoEntry, + HeadBlobID: "c996f4725be8fc8c1d1c776e58c97ddc5d03b336", + BaseBlobID: "0000000000000000000000000000000000000000", + }, + { + Status: "added", + HeadPath: "README.md", + BasePath: "README.md", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeNoEntry, + HeadBlobID: "074e590b8e64898b02beef03ece83f962c94f54c", + BaseBlobID: "0000000000000000000000000000000000000000", + }, + }, + }, + }, + { + Name: "first commit (no parent), merge base = true", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + HeadSha: "72866af952e98d02a73003501836074b286a78f6", + useMergeBase: true, + Expected: &DiffTree{ + Files: []*DiffTreeRecord{ + { + Status: "added", + HeadPath: ".gitignore", + BasePath: ".gitignore", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeNoEntry, + HeadBlobID: "f1c181ec9c5c921245027c6b452ecfc1d3626364", + BaseBlobID: "0000000000000000000000000000000000000000", + }, + { + Status: "added", + HeadPath: "LICENSE", + BasePath: "LICENSE", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeNoEntry, + HeadBlobID: "c996f4725be8fc8c1d1c776e58c97ddc5d03b336", + BaseBlobID: "0000000000000000000000000000000000000000", + }, + { + Status: "added", + HeadPath: "README.md", + BasePath: "README.md", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeNoEntry, + HeadBlobID: "074e590b8e64898b02beef03ece83f962c94f54c", + BaseBlobID: "0000000000000000000000000000000000000000", + }, + }, + }, + }, + { + Name: "base and head same", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + BaseSha: "ed8f4d2fa5b2420706580d191f5dd50c4e491f3f", + HeadSha: "ed8f4d2fa5b2420706580d191f5dd50c4e491f3f", + Expected: &DiffTree{ + Files: []*DiffTreeRecord{}, + }, + }, + { + Name: "useMergeBase false", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + BaseSha: "ed8f4d2fa5b2420706580d191f5dd50c4e491f3f", + HeadSha: "111cac04bd7d20301964e27a93698aabb5781b80", // this commit can be found on the update-readme branch + useMergeBase: false, + Expected: &DiffTree{ + Files: []*DiffTreeRecord{ + { + Status: "modified", + HeadPath: "LICENSE", + BasePath: "LICENSE", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "c996f4725be8fc8c1d1c776e58c97ddc5d03b336", + BaseBlobID: "ed5119b3c1f45547b6785bc03eac7f87570fa17f", + }, + + { + Status: "modified", + HeadPath: "README.md", + BasePath: "README.md", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "fb39771a8865c9a67f2ab9b616c854805664553c", + BaseBlobID: "9dfc0a6257d8eff526f0cfaf6a8ea950f55a9dba", + }, + }, + }, + }, + { + Name: "useMergeBase true", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + BaseSha: "ed8f4d2fa5b2420706580d191f5dd50c4e491f3f", + HeadSha: "111cac04bd7d20301964e27a93698aabb5781b80", // this commit can be found on the update-readme branch + useMergeBase: true, + Expected: &DiffTree{ + Files: []*DiffTreeRecord{ + { + Status: "modified", + HeadPath: "README.md", + BasePath: "README.md", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "fb39771a8865c9a67f2ab9b616c854805664553c", + BaseBlobID: "9dfc0a6257d8eff526f0cfaf6a8ea950f55a9dba", + }, + }, + }, + }, + { + Name: "no base set", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + HeadSha: "d8e0bbb45f200e67d9a784ce55bd90821af45ebd", // this commit can be found on the update-readme branch + useMergeBase: false, + Expected: &DiffTree{ + Files: []*DiffTreeRecord{ + { + Status: "modified", + HeadPath: "LICENSE", + BasePath: "LICENSE", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "ee469963e76ae1bb7ee83d7510df2864e6c8c640", + BaseBlobID: "ed5119b3c1f45547b6785bc03eac7f87570fa17f", + }, + }, + }, + }, + } + + for _, tt := range test { + t.Run(tt.Name, func(t *testing.T) { + gitRepo, err := git.OpenRepository(git.DefaultContext, tt.RepoPath) + assert.NoError(t, err) + defer gitRepo.Close() + + diffPaths, err := GetDiffTree(db.DefaultContext, gitRepo, tt.useMergeBase, tt.BaseSha, tt.HeadSha) + require.NoError(t, err) + + assert.Equal(t, tt.Expected, diffPaths) + }) + } +} + +func TestParseGitDiffTree(t *testing.T) { + test := []struct { + Name string + GitOutput string + Expected []*DiffTreeRecord + }{ + { + Name: "file change", + GitOutput: ":100644 100644 64e43d23bcd08db12563a0a4d84309cadb437e1a 5dbc7792b5bb228647cfcc8dfe65fc649119dedc M\tResources/views/curriculum/edit.blade.php", + Expected: []*DiffTreeRecord{ + { + Status: "modified", + HeadPath: "Resources/views/curriculum/edit.blade.php", + BasePath: "Resources/views/curriculum/edit.blade.php", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "5dbc7792b5bb228647cfcc8dfe65fc649119dedc", + BaseBlobID: "64e43d23bcd08db12563a0a4d84309cadb437e1a", + }, + }, + }, + { + Name: "file added", + GitOutput: ":000000 100644 0000000000000000000000000000000000000000 0063162fb403db15ceb0517b34ab782e4e58b619 A\tResources/views/class/index.blade.php", + Expected: []*DiffTreeRecord{ + { + Status: "added", + HeadPath: "Resources/views/class/index.blade.php", + BasePath: "Resources/views/class/index.blade.php", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeNoEntry, + HeadBlobID: "0063162fb403db15ceb0517b34ab782e4e58b619", + BaseBlobID: "0000000000000000000000000000000000000000", + }, + }, + }, + { + Name: "file deleted", + GitOutput: ":100644 000000 bac4286303c8c0017ea2f0a48c561ddcc0330a14 0000000000000000000000000000000000000000 D\tResources/views/classes/index.blade.php", + Expected: []*DiffTreeRecord{ + { + Status: "deleted", + HeadPath: "Resources/views/classes/index.blade.php", + BasePath: "Resources/views/classes/index.blade.php", + HeadMode: git.EntryModeNoEntry, + BaseMode: git.EntryModeBlob, + HeadBlobID: "0000000000000000000000000000000000000000", + BaseBlobID: "bac4286303c8c0017ea2f0a48c561ddcc0330a14", + }, + }, + }, + { + Name: "file renamed", + GitOutput: ":100644 100644 c8a055cfb45cd39747292983ad1797ceab40f5b1 97248f79a90aaf81fe7fd74b33c1cb182dd41783 R087\tDatabase/Seeders/AdminDatabaseSeeder.php\tDatabase/Seeders/AcademicDatabaseSeeder.php", + Expected: []*DiffTreeRecord{ + { + Status: "renamed", + Score: 87, + HeadPath: "Database/Seeders/AcademicDatabaseSeeder.php", + BasePath: "Database/Seeders/AdminDatabaseSeeder.php", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "97248f79a90aaf81fe7fd74b33c1cb182dd41783", + BaseBlobID: "c8a055cfb45cd39747292983ad1797ceab40f5b1", + }, + }, + }, + { + Name: "no changes", + GitOutput: ``, + Expected: []*DiffTreeRecord{}, + }, + { + Name: "multiple changes", + GitOutput: ":000000 100644 0000000000000000000000000000000000000000 db736b44533a840981f1f17b7029d0f612b69550 A\tHttp/Controllers/ClassController.php\n" + + ":100644 000000 9a4d2344d4d0145db7c91b3f3e123c74367d4ef4 0000000000000000000000000000000000000000 D\tHttp/Controllers/ClassesController.php\n" + + ":100644 100644 f060d6aede65d423f49e7dc248dfa0d8835ef920 b82c8e39a3602dedadb44669956d6eb5b6a7cc86 M\tHttp/Controllers/ProgramDirectorController.php\n", + Expected: []*DiffTreeRecord{ + { + Status: "added", + HeadPath: "Http/Controllers/ClassController.php", + BasePath: "Http/Controllers/ClassController.php", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeNoEntry, + HeadBlobID: "db736b44533a840981f1f17b7029d0f612b69550", + BaseBlobID: "0000000000000000000000000000000000000000", + }, + { + Status: "deleted", + HeadPath: "Http/Controllers/ClassesController.php", + BasePath: "Http/Controllers/ClassesController.php", + HeadMode: git.EntryModeNoEntry, + BaseMode: git.EntryModeBlob, + HeadBlobID: "0000000000000000000000000000000000000000", + BaseBlobID: "9a4d2344d4d0145db7c91b3f3e123c74367d4ef4", + }, + { + Status: "modified", + HeadPath: "Http/Controllers/ProgramDirectorController.php", + BasePath: "Http/Controllers/ProgramDirectorController.php", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "b82c8e39a3602dedadb44669956d6eb5b6a7cc86", + BaseBlobID: "f060d6aede65d423f49e7dc248dfa0d8835ef920", + }, + }, + }, + { + Name: "spaces in file path", + GitOutput: ":000000 100644 0000000000000000000000000000000000000000 db736b44533a840981f1f17b7029d0f612b69550 A\tHttp /Controllers/Class Controller.php\n" + + ":100644 000000 9a4d2344d4d0145db7c91b3f3e123c74367d4ef4 0000000000000000000000000000000000000000 D\tHttp/Cont rollers/Classes Controller.php\n" + + ":100644 100644 f060d6aede65d423f49e7dc248dfa0d8835ef920 b82c8e39a3602dedadb44669956d6eb5b6a7cc86 R010\tHttp/Controllers/Program Director Controller.php\tHttp/Cont rollers/ProgramDirectorController.php\n", + Expected: []*DiffTreeRecord{ + { + Status: "added", + HeadPath: "Http /Controllers/Class Controller.php", + BasePath: "Http /Controllers/Class Controller.php", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeNoEntry, + HeadBlobID: "db736b44533a840981f1f17b7029d0f612b69550", + BaseBlobID: "0000000000000000000000000000000000000000", + }, + { + Status: "deleted", + HeadPath: "Http/Cont rollers/Classes Controller.php", + BasePath: "Http/Cont rollers/Classes Controller.php", + HeadMode: git.EntryModeNoEntry, + BaseMode: git.EntryModeBlob, + HeadBlobID: "0000000000000000000000000000000000000000", + BaseBlobID: "9a4d2344d4d0145db7c91b3f3e123c74367d4ef4", + }, + { + Status: "renamed", + Score: 10, + HeadPath: "Http/Cont rollers/ProgramDirectorController.php", + BasePath: "Http/Controllers/Program Director Controller.php", + HeadMode: git.EntryModeBlob, + BaseMode: git.EntryModeBlob, + HeadBlobID: "b82c8e39a3602dedadb44669956d6eb5b6a7cc86", + BaseBlobID: "f060d6aede65d423f49e7dc248dfa0d8835ef920", + }, + }, + }, + { + Name: "file type changed", + GitOutput: ":100644 120000 344e0ca8aa791cc4164fb0ea645f334fd40d00f0 a7c2973de00bfdc6ca51d315f401b5199fe01dc3 T\twebpack.mix.js", + Expected: []*DiffTreeRecord{ + { + Status: "typechanged", + HeadPath: "webpack.mix.js", + BasePath: "webpack.mix.js", + HeadMode: git.EntryModeSymlink, + BaseMode: git.EntryModeBlob, + HeadBlobID: "a7c2973de00bfdc6ca51d315f401b5199fe01dc3", + BaseBlobID: "344e0ca8aa791cc4164fb0ea645f334fd40d00f0", + }, + }, + }, + } + + for _, tt := range test { + t.Run(tt.Name, func(t *testing.T) { + entries, err := parseGitDiffTree(strings.NewReader(tt.GitOutput)) + assert.NoError(t, err) + assert.Equal(t, tt.Expected, entries) + }) + } +} + +func TestGitDiffTreeErrors(t *testing.T) { + test := []struct { + Name string + RepoPath string + BaseSha string + HeadSha string + }{ + { + Name: "head doesn't exist", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + BaseSha: "f32b0a9dfd09a60f616f29158f772cedd89942d2", + HeadSha: "asdfasdfasdf", + }, + { + Name: "base doesn't exist", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + BaseSha: "asdfasdfasdf", + HeadSha: "f32b0a9dfd09a60f616f29158f772cedd89942d2", + }, + { + Name: "head not set", + RepoPath: "../../modules/git/tests/repos/repo5_pulls", + BaseSha: "f32b0a9dfd09a60f616f29158f772cedd89942d2", + }, + } + + for _, tt := range test { + t.Run(tt.Name, func(t *testing.T) { + gitRepo, err := git.OpenRepository(git.DefaultContext, tt.RepoPath) + assert.NoError(t, err) + defer gitRepo.Close() + + diffPaths, err := GetDiffTree(db.DefaultContext, gitRepo, true, tt.BaseSha, tt.HeadSha) + assert.Error(t, err) + assert.Nil(t, diffPaths) + }) + } +} diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index f42686bb71..9964329876 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -25,12 +25,14 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" "github.com/sergi/go-diff/diffmatchpatch" stdcharset "golang.org/x/net/html/charset" @@ -75,12 +77,12 @@ const ( // DiffLine represents a line difference in a DiffSection. type DiffLine struct { - LeftIdx int - RightIdx int - Match int + LeftIdx int // line number, 1-based + RightIdx int // line number, 1-based + Match int // the diff matched index. -1: no match. 0: plain and no need to match. >0: for add/del, "Lines" slice index of the other side Type DiffLineType Content string - Comments []*issues_model.Comment + Comments issues_model.CommentList // related PR code comments SectionInfo *DiffLineSectionInfo } @@ -95,9 +97,18 @@ type DiffLineSectionInfo struct { RightHunkSize int } +// DiffHTMLOperation is the HTML version of diffmatchpatch.Diff +type DiffHTMLOperation struct { + Type diffmatchpatch.Operation + HTML template.HTML +} + // BlobExcerptChunkSize represent max lines of excerpt const BlobExcerptChunkSize = 20 +// MaxDiffHighlightEntireFileSize is the maximum file size that will be highlighted with "entire file diff" +const MaxDiffHighlightEntireFileSize = 1 * 1024 * 1024 + // GetType returns the type of DiffLine. func (d *DiffLine) GetType() int { return int(d.Type) @@ -112,8 +123,9 @@ func (d *DiffLine) GetHTMLDiffLineType() string { return "del" case DiffLineSection: return "tag" + default: + return "same" } - return "same" } // CanComment returns whether a line can get commented @@ -192,89 +204,20 @@ func getLineContent(content string, locale translation.Locale) DiffInline { type DiffSection struct { file *DiffFile FileName string - Name string Lines []*DiffLine } -var ( - addedCodePrefix = []byte(`<span class="added-code">`) - removedCodePrefix = []byte(`<span class="removed-code">`) - codeTagSuffix = []byte(`</span>`) -) - -func diffToHTML(lineWrapperTags []string, diffs []diffmatchpatch.Diff, lineType DiffLineType) string { - buf := bytes.NewBuffer(nil) - // restore the line wrapper tags <span class="line"> and <span class="cl">, if necessary - for _, tag := range lineWrapperTags { - buf.WriteString(tag) - } - for _, diff := range diffs { - switch { - case diff.Type == diffmatchpatch.DiffEqual: - buf.WriteString(diff.Text) - case diff.Type == diffmatchpatch.DiffInsert && lineType == DiffLineAdd: - buf.Write(addedCodePrefix) - buf.WriteString(diff.Text) - buf.Write(codeTagSuffix) - case diff.Type == diffmatchpatch.DiffDelete && lineType == DiffLineDel: - buf.Write(removedCodePrefix) - buf.WriteString(diff.Text) - buf.Write(codeTagSuffix) - } - } - for range lineWrapperTags { - buf.WriteString("</span>") - } - return buf.String() -} - -// GetLine gets a specific line by type (add or del) and file line number -func (diffSection *DiffSection) GetLine(lineType DiffLineType, idx int) *DiffLine { - var ( - difference = 0 - addCount = 0 - delCount = 0 - matchDiffLine *DiffLine - ) - -LOOP: - for _, diffLine := range diffSection.Lines { - switch diffLine.Type { - case DiffLineAdd: - addCount++ - case DiffLineDel: - delCount++ - default: - if matchDiffLine != nil { - break LOOP - } - difference = diffLine.RightIdx - diffLine.LeftIdx - addCount = 0 - delCount = 0 - } - - switch lineType { - case DiffLineDel: - if diffLine.RightIdx == 0 && diffLine.LeftIdx == idx-difference { - matchDiffLine = diffLine - } - case DiffLineAdd: - if diffLine.LeftIdx == 0 && diffLine.RightIdx == idx+difference { - matchDiffLine = diffLine - } - } - } - - if addCount == delCount { - return matchDiffLine +func (diffSection *DiffSection) GetLine(idx int) *DiffLine { + if idx <= 0 { + return nil } - return nil + return diffSection.Lines[idx] } -var diffMatchPatch = diffmatchpatch.New() - -func init() { - diffMatchPatch.DiffEditCost = 100 +func defaultDiffMatchPatch() *diffmatchpatch.DiffMatchPatch { + dmp := diffmatchpatch.New() + dmp.DiffEditCost = 100 + return dmp } // DiffInline is a struct that has a content and escape status @@ -283,97 +226,125 @@ type DiffInline struct { Content template.HTML } -// DiffInlineWithUnicodeEscape makes a DiffInline with hidden unicode characters escaped +// DiffInlineWithUnicodeEscape makes a DiffInline with hidden Unicode characters escaped func DiffInlineWithUnicodeEscape(s template.HTML, locale translation.Locale) DiffInline { status, content := charset.EscapeControlHTML(s, locale) return DiffInline{EscapeStatus: status, Content: content} } -// DiffInlineWithHighlightCode makes a DiffInline with code highlight and hidden unicode characters escaped -func DiffInlineWithHighlightCode(fileName, language, code string, locale translation.Locale) DiffInline { - highlighted, _ := highlight.Code(fileName, language, code) - status, content := charset.EscapeControlHTML(highlighted, locale) - return DiffInline{EscapeStatus: status, Content: content} -} - -// GetComputedInlineDiffFor computes inline diff for the given line. -func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine, locale translation.Locale) DiffInline { +func (diffSection *DiffSection) getLineContentForRender(lineIdx int, diffLine *DiffLine, fileLanguage string, highlightLines map[int]template.HTML) template.HTML { + h, ok := highlightLines[lineIdx-1] + if ok { + return h + } + if diffLine.Content == "" { + return "" + } if setting.Git.DisableDiffHighlight { - return getLineContent(diffLine.Content[1:], locale) + return template.HTML(html.EscapeString(diffLine.Content[1:])) } + h, _ = highlight.Code(diffSection.FileName, fileLanguage, diffLine.Content[1:]) + return h +} - var ( - compareDiffLine *DiffLine - diff1 string - diff2 string - ) - - language := "" +func (diffSection *DiffSection) getDiffLineForRender(diffLineType DiffLineType, leftLine, rightLine *DiffLine, locale translation.Locale) DiffInline { + var fileLanguage string + var highlightedLeftLines, highlightedRightLines map[int]template.HTML + // when a "diff section" is manually prepared by ExcerptBlob, it doesn't have "file" information if diffSection.file != nil { - language = diffSection.file.Language + fileLanguage = diffSection.file.Language + highlightedLeftLines, highlightedRightLines = diffSection.file.highlightedLeftLines, diffSection.file.highlightedRightLines } + var lineHTML template.HTML + hcd := newHighlightCodeDiff() + if diffLineType == DiffLinePlain { + // left and right are the same, no need to do line-level diff + if leftLine != nil { + lineHTML = diffSection.getLineContentForRender(leftLine.LeftIdx, leftLine, fileLanguage, highlightedLeftLines) + } else if rightLine != nil { + lineHTML = diffSection.getLineContentForRender(rightLine.RightIdx, rightLine, fileLanguage, highlightedRightLines) + } + } else { + var diff1, diff2 template.HTML + if leftLine != nil { + diff1 = diffSection.getLineContentForRender(leftLine.LeftIdx, leftLine, fileLanguage, highlightedLeftLines) + } + if rightLine != nil { + diff2 = diffSection.getLineContentForRender(rightLine.RightIdx, rightLine, fileLanguage, highlightedRightLines) + } + if diff1 != "" && diff2 != "" { + // if only some parts of a line are changed, highlight these changed parts as "deleted/added". + lineHTML = hcd.diffLineWithHighlight(diffLineType, diff1, diff2) + } else { + // if left is empty or right is empty (a line is fully deleted or added), then we do not need to diff anymore. + // the tmpl code already adds background colors for these cases. + lineHTML = util.Iif(diffLineType == DiffLineDel, diff1, diff2) + } + } + return DiffInlineWithUnicodeEscape(lineHTML, locale) +} + +// GetComputedInlineDiffFor computes inline diff for the given line. +func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine, locale translation.Locale) DiffInline { // try to find equivalent diff line. ignore, otherwise switch diffLine.Type { case DiffLineSection: return getLineContent(diffLine.Content[1:], locale) case DiffLineAdd: - compareDiffLine = diffSection.GetLine(DiffLineDel, diffLine.RightIdx) - if compareDiffLine == nil { - return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:], locale) - } - diff1 = compareDiffLine.Content - diff2 = diffLine.Content + compareDiffLine := diffSection.GetLine(diffLine.Match) + return diffSection.getDiffLineForRender(DiffLineAdd, compareDiffLine, diffLine, locale) case DiffLineDel: - compareDiffLine = diffSection.GetLine(DiffLineAdd, diffLine.LeftIdx) - if compareDiffLine == nil { - return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:], locale) - } - diff1 = diffLine.Content - diff2 = compareDiffLine.Content - default: - if strings.IndexByte(" +-", diffLine.Content[0]) > -1 { - return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content[1:], locale) - } - return DiffInlineWithHighlightCode(diffSection.FileName, language, diffLine.Content, locale) + compareDiffLine := diffSection.GetLine(diffLine.Match) + return diffSection.getDiffLineForRender(DiffLineDel, diffLine, compareDiffLine, locale) + default: // Plain + // TODO: there was an "if" check: `if diffLine.Content >strings.IndexByte(" +-", diffLine.Content[0]) > -1 { ... } else { ... }` + // no idea why it needs that check, it seems that the "if" should be always true, so try to simplify the code + return diffSection.getDiffLineForRender(DiffLinePlain, nil, diffLine, locale) } - - hcd := newHighlightCodeDiff() - diffRecord := hcd.diffWithHighlight(diffSection.FileName, language, diff1[1:], diff2[1:]) - // it seems that Gitea doesn't need the line wrapper of Chroma, so do not add them back - // if the line wrappers are still needed in the future, it can be added back by "diffToHTML(hcd.lineWrapperTags. ...)" - diffHTML := diffToHTML(nil, diffRecord, diffLine.Type) - return DiffInlineWithUnicodeEscape(template.HTML(diffHTML), locale) } // DiffFile represents a file diff. type DiffFile struct { - Name string - NameHash string - OldName string - Index int - Addition, Deletion int - Type DiffFileType - IsCreated bool - IsDeleted bool - IsBin bool - IsLFSFile bool - IsRenamed bool - IsAmbiguous bool - Sections []*DiffSection - IsIncomplete bool - IsIncompleteLineTooLong bool - IsProtected bool - IsGenerated bool - IsVendored bool + // only used internally to parse Ambiguous filenames + isAmbiguous bool + + // basic fields (parsed from diff result) + Name string + NameHash string + OldName string + Addition int + Deletion int + Type DiffFileType + Mode string + OldMode string + IsCreated bool + IsDeleted bool + IsBin bool + IsLFSFile bool + IsRenamed bool + IsSubmodule bool + // basic fields but for render purpose only + Sections []*DiffSection + IsIncomplete bool + IsIncompleteLineTooLong bool + + // will be filled by the extra loop in GitDiffForRender + Language string + IsGenerated bool + IsVendored bool + SubmoduleDiffInfo *SubmoduleDiffInfo // IsSubmodule==true, then there must be a SubmoduleDiffInfo + + // will be filled by route handler + IsProtected bool + + // will be filled by SyncUserSpecificDiff IsViewed bool // User specific HasChangedSinceLastReview bool // User specific - Language string - Mode string - OldMode string - IsSubmodule bool // if IsSubmodule==true, then there must be a SubmoduleDiffInfo - SubmoduleDiffInfo *SubmoduleDiffInfo + // for render purpose only, will be filled by the extra loop in GitDiffForRender + highlightedLeftLines map[int]template.HTML + highlightedRightLines map[int]template.HTML } // GetType returns type of diff file. @@ -381,18 +352,30 @@ func (diffFile *DiffFile) GetType() int { return int(diffFile.Type) } -// GetTailSection creates a fake DiffLineSection if the last section is not the end of the file -func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommit, rightCommit *git.Commit) *DiffSection { - if len(diffFile.Sections) == 0 || diffFile.Type != DiffFileChange || diffFile.IsBin || diffFile.IsLFSFile { - return nil - } +type DiffLimitedContent struct { + LeftContent, RightContent *limitByteWriter +} +// GetTailSectionAndLimitedContent creates a fake DiffLineSection if the last section is not the end of the file +func (diffFile *DiffFile) GetTailSectionAndLimitedContent(leftCommit, rightCommit *git.Commit) (_ *DiffSection, diffLimitedContent DiffLimitedContent) { + var leftLineCount, rightLineCount int + diffLimitedContent = DiffLimitedContent{} + if diffFile.IsBin || diffFile.IsLFSFile { + return nil, diffLimitedContent + } + if (diffFile.Type == DiffFileDel || diffFile.Type == DiffFileChange) && leftCommit != nil { + leftLineCount, diffLimitedContent.LeftContent = getCommitFileLineCountAndLimitedContent(leftCommit, diffFile.OldName) + } + if (diffFile.Type == DiffFileAdd || diffFile.Type == DiffFileChange) && rightCommit != nil { + rightLineCount, diffLimitedContent.RightContent = getCommitFileLineCountAndLimitedContent(rightCommit, diffFile.OldName) + } + if len(diffFile.Sections) == 0 || diffFile.Type != DiffFileChange { + return nil, diffLimitedContent + } lastSection := diffFile.Sections[len(diffFile.Sections)-1] lastLine := lastSection.Lines[len(lastSection.Lines)-1] - leftLineCount := getCommitFileLineCount(leftCommit, diffFile.Name) - rightLineCount := getCommitFileLineCount(rightCommit, diffFile.Name) if leftLineCount <= lastLine.LeftIdx || rightLineCount <= lastLine.RightIdx { - return nil + return nil, diffLimitedContent } tailDiffLine := &DiffLine{ Type: DiffLineSection, @@ -406,7 +389,7 @@ func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommit, ri }, } tailSection := &DiffSection{FileName: diffFile.Name, Lines: []*DiffLine{tailDiffLine}} - return tailSection + return tailSection, diffLimitedContent } // GetDiffFileName returns the name of the diff file, or its old name in case it was deleted @@ -438,26 +421,37 @@ func (diffFile *DiffFile) ModeTranslationKey(mode string) string { } } -func getCommitFileLineCount(commit *git.Commit, filePath string) int { +type limitByteWriter struct { + buf bytes.Buffer + limit int +} + +func (l *limitByteWriter) Write(p []byte) (n int, err error) { + if l.buf.Len()+len(p) > l.limit { + p = p[:l.limit-l.buf.Len()] + } + return l.buf.Write(p) +} + +func getCommitFileLineCountAndLimitedContent(commit *git.Commit, filePath string) (lineCount int, limitWriter *limitByteWriter) { blob, err := commit.GetBlobByPath(filePath) if err != nil { - return 0 + return 0, nil } - lineCount, err := blob.GetBlobLineCount() + w := &limitByteWriter{limit: MaxDiffHighlightEntireFileSize + 1} + lineCount, err = blob.GetBlobLineCount(w) if err != nil { - return 0 + return 0, nil } - return lineCount + return lineCount, w } // Diff represents a difference between two git trees. type Diff struct { - Start, End string - NumFiles int - TotalAddition, TotalDeletion int - Files []*DiffFile - IsIncomplete bool - NumViewedFiles int // user-specific + Start, End string + Files []*DiffFile + IsIncomplete bool + NumViewedFiles int // user-specific } // LoadComments loads comments into each line @@ -501,10 +495,7 @@ func ParsePatch(ctx context.Context, maxLines, maxLineCharacters, maxFiles int, // OK let's set a reasonable buffer size. // This should be at least the size of maxLineCharacters or 4096 whichever is larger. - readerSize := maxLineCharacters - if readerSize < 4096 { - readerSize = 4096 - } + readerSize := max(maxLineCharacters, 4096) input := bufio.NewReaderSize(reader, readerSize) line, err := input.ReadString('\n') @@ -528,13 +519,13 @@ parsingLoop: } if maxFiles > -1 && len(diff.Files) >= maxFiles { - lastFile := createDiffFile(diff, line) + lastFile := createDiffFile(line) diff.End = lastFile.Name diff.IsIncomplete = true break parsingLoop } - curFile = createDiffFile(diff, line) + curFile = createDiffFile(line) if skipping { if curFile.Name != skipToFile { line, err = skipToNextDiffHead(input) @@ -617,28 +608,28 @@ parsingLoop: case strings.HasPrefix(line, "rename from "): curFile.IsRenamed = true curFile.Type = DiffFileRename - if curFile.IsAmbiguous { + if curFile.isAmbiguous { curFile.OldName = prepareValue(line, "rename from ") } case strings.HasPrefix(line, "rename to "): curFile.IsRenamed = true curFile.Type = DiffFileRename - if curFile.IsAmbiguous { + if curFile.isAmbiguous { curFile.Name = prepareValue(line, "rename to ") - curFile.IsAmbiguous = false + curFile.isAmbiguous = false } case strings.HasPrefix(line, "copy from "): curFile.IsRenamed = true curFile.Type = DiffFileCopy - if curFile.IsAmbiguous { + if curFile.isAmbiguous { curFile.OldName = prepareValue(line, "copy from ") } case strings.HasPrefix(line, "copy to "): curFile.IsRenamed = true curFile.Type = DiffFileCopy - if curFile.IsAmbiguous { + if curFile.isAmbiguous { curFile.Name = prepareValue(line, "copy to ") - curFile.IsAmbiguous = false + curFile.isAmbiguous = false } case strings.HasPrefix(line, "new file"): curFile.Type = DiffFileAdd @@ -665,7 +656,7 @@ parsingLoop: curFile.IsBin = true case strings.HasPrefix(line, "--- "): // Handle ambiguous filenames - if curFile.IsAmbiguous { + if curFile.isAmbiguous { // The shortest string that can end up here is: // "--- a\t\n" without the quotes. // This line has a len() of 7 but doesn't contain a oldName. @@ -683,7 +674,7 @@ parsingLoop: // Otherwise do nothing with this line case strings.HasPrefix(line, "+++ "): // Handle ambiguous filenames - if curFile.IsAmbiguous { + if curFile.isAmbiguous { if len(line) > 6 && line[4] == 'b' { curFile.Name = line[6 : len(line)-1] if line[len(line)-2] == '\t' { @@ -695,12 +686,10 @@ parsingLoop: } else { curFile.Name = curFile.OldName } - curFile.IsAmbiguous = false + curFile.isAmbiguous = false } // Otherwise do nothing with this line, but now switch to parsing hunks lineBytes, isFragment, err := parseHunks(ctx, curFile, maxLines, maxLineCharacters, input) - diff.TotalAddition += curFile.Addition - diff.TotalDeletion += curFile.Deletion if err != nil { if err != io.EOF { return diff, err @@ -773,7 +762,6 @@ parsingLoop: } } - diff.NumFiles = len(diff.Files) return diff, nil } @@ -1011,7 +999,7 @@ func parseHunks(ctx context.Context, curFile *DiffFile, maxLines, maxLineCharact } } -func createDiffFile(diff *Diff, line string) *DiffFile { +func createDiffFile(line string) *DiffFile { // The a/ and b/ filenames are the same unless rename/copy is involved. // Especially, even for a creation or a deletion, /dev/null is not used // in place of the a/ or b/ filenames. @@ -1022,12 +1010,11 @@ func createDiffFile(diff *Diff, line string) *DiffFile { // // Path names are quoted if necessary. // - // This means that you should always be able to determine the file name even when there + // This means that you should always be able to determine the file name even when // there is potential ambiguity... // // but we can be simpler with our heuristics by just forcing git to prefix things nicely curFile := &DiffFile{ - Index: len(diff.Files) + 1, Type: DiffFileChange, Sections: make([]*DiffSection, 0, 10), } @@ -1039,7 +1026,7 @@ func createDiffFile(diff *Diff, line string) *DiffFile { curFile.OldName, oldNameAmbiguity = readFileName(rd) curFile.Name, newNameAmbiguity = readFileName(rd) if oldNameAmbiguity && newNameAmbiguity { - curFile.IsAmbiguous = true + curFile.isAmbiguous = true // OK we should bet that the oldName and the newName are the same if they can be made to be same // So we need to start again ... if (len(line)-len(cmdDiffHead)-1)%2 == 0 { @@ -1104,53 +1091,48 @@ type DiffOptions struct { MaxFiles int WhitespaceBehavior git.TrustedCmdArgs DirectComparison bool - FileOnly bool } -// GetDiff builds a Diff between two commits of a repository. +func guessBeforeCommitForDiff(gitRepo *git.Repository, beforeCommitID string, afterCommit *git.Commit) (actualBeforeCommit *git.Commit, actualBeforeCommitID git.ObjectID, err error) { + commitObjectFormat := afterCommit.ID.Type() + isBeforeCommitIDEmpty := beforeCommitID == "" || beforeCommitID == commitObjectFormat.EmptyObjectID().String() + + if isBeforeCommitIDEmpty && afterCommit.ParentCount() == 0 { + actualBeforeCommitID = commitObjectFormat.EmptyTree() + } else { + if isBeforeCommitIDEmpty { + actualBeforeCommit, err = afterCommit.Parent(0) + } else { + actualBeforeCommit, err = gitRepo.GetCommit(beforeCommitID) + } + if err != nil { + return nil, nil, err + } + actualBeforeCommitID = actualBeforeCommit.ID + } + return actualBeforeCommit, actualBeforeCommitID, nil +} + +// getDiffBasic builds a Diff between two commits of a repository. // Passing the empty string as beforeCommitID returns a diff from the parent commit. // The whitespaceBehavior is either an empty string or a git flag -func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, files ...string) (*Diff, error) { +// Returned beforeCommit could be nil if the afterCommit doesn't have parent commit +func getDiffBasic(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, files ...string) (_ *Diff, beforeCommit, afterCommit *git.Commit, err error) { repoPath := gitRepo.Path - var beforeCommit *git.Commit - commit, err := gitRepo.GetCommit(opts.AfterCommitID) + afterCommit, err = gitRepo.GetCommit(opts.AfterCommitID) if err != nil { - return nil, err + return nil, nil, nil, err } - cmdCtx, cmdCancel := context.WithCancel(ctx) - defer cmdCancel() - - cmdDiff := git.NewCommand(cmdCtx) - objectFormat, err := gitRepo.GetObjectFormat() + beforeCommit, beforeCommitID, err := guessBeforeCommitForDiff(gitRepo, opts.BeforeCommitID, afterCommit) if err != nil { - return nil, err + return nil, nil, nil, err } - if (len(opts.BeforeCommitID) == 0 || opts.BeforeCommitID == objectFormat.EmptyObjectID().String()) && commit.ParentCount() == 0 { - cmdDiff.AddArguments("diff", "--src-prefix=\\a/", "--dst-prefix=\\b/", "-M"). - AddArguments(opts.WhitespaceBehavior...). - AddDynamicArguments(objectFormat.EmptyTree().String()). - AddDynamicArguments(opts.AfterCommitID) - } else { - actualBeforeCommitID := opts.BeforeCommitID - if len(actualBeforeCommitID) == 0 { - parentCommit, _ := commit.Parent(0) - actualBeforeCommitID = parentCommit.ID.String() - } - - cmdDiff.AddArguments("diff", "--src-prefix=\\a/", "--dst-prefix=\\b/", "-M"). - AddArguments(opts.WhitespaceBehavior...). - AddDynamicArguments(actualBeforeCommitID, opts.AfterCommitID) - opts.BeforeCommitID = actualBeforeCommitID - - var err error - beforeCommit, err = gitRepo.GetCommit(opts.BeforeCommitID) - if err != nil { - return nil, err - } - } + cmdDiff := git.NewCommand(). + AddArguments("diff", "--src-prefix=\\a/", "--dst-prefix=\\b/", "-M"). + AddArguments(opts.WhitespaceBehavior...) // In git 2.31, git diff learned --skip-to which we can use to shortcut skip to file // so if we are using at least this version of git we don't have to tell ParsePatch to do @@ -1161,8 +1143,12 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi parsePatchSkipToFile = "" } + cmdDiff.AddDynamicArguments(beforeCommitID.String(), opts.AfterCommitID) cmdDiff.AddDashesAndList(files...) + cmdCtx, cmdCancel := context.WithCancel(ctx) + defer cmdCancel() + reader, writer := io.Pipe() defer func() { _ = reader.Close() @@ -1171,7 +1157,7 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi go func() { stderr := &bytes.Buffer{} - if err := cmdDiff.Run(&git.RunOpts{ + if err := cmdDiff.Run(cmdCtx, &git.RunOpts{ Timeout: time.Duration(setting.Git.Timeout.Default) * time.Second, Dir: repoPath, Stdout: writer, @@ -1187,32 +1173,44 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi // Ensure the git process is killed if it didn't exit already cmdCancel() if err != nil { - return nil, fmt.Errorf("unable to ParsePatch: %w", err) + return nil, nil, nil, fmt.Errorf("unable to ParsePatch: %w", err) } diff.Start = opts.SkipTo + return diff, beforeCommit, afterCommit, nil +} + +func GetDiffForAPI(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, files ...string) (*Diff, error) { + diff, _, _, err := getDiffBasic(ctx, gitRepo, opts, files...) + return diff, err +} + +func GetDiffForRender(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, files ...string) (*Diff, error) { + diff, beforeCommit, afterCommit, err := getDiffBasic(ctx, gitRepo, opts, files...) + if err != nil { + return nil, err + } - checker, deferable := gitRepo.CheckAttributeReader(opts.AfterCommitID) - defer deferable() + checker, err := attribute.NewBatchChecker(gitRepo, opts.AfterCommitID, []string{attribute.LinguistVendored, attribute.LinguistGenerated, attribute.LinguistLanguage, attribute.GitlabLanguage}) + if err != nil { + return nil, err + } + defer checker.Close() for _, diffFile := range diff.Files { isVendored := optional.None[bool]() isGenerated := optional.None[bool]() - if checker != nil { - attrs, err := checker.CheckPath(diffFile.Name) - if err == nil { - isVendored = git.AttributeToBool(attrs, git.AttributeLinguistVendored) - isGenerated = git.AttributeToBool(attrs, git.AttributeLinguistGenerated) - - language := git.TryReadLanguageAttribute(attrs) - if language.Has() { - diffFile.Language = language.Value() - } + attrs, err := checker.CheckPath(diffFile.Name) + if err == nil { + isVendored, isGenerated = attrs.GetVendored(), attrs.GetGenerated() + language := attrs.GetLanguage() + if language.Has() { + diffFile.Language = language.Value() } } // Populate Submodule URLs if diffFile.SubmoduleDiffInfo != nil { - diffFile.SubmoduleDiffInfo.PopulateURL(diffFile, beforeCommit, commit) + diffFile.SubmoduleDiffInfo.PopulateURL(diffFile, beforeCommit, afterCommit) } if !isVendored.Has() { @@ -1224,76 +1222,80 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi isGenerated = optional.Some(analyze.IsGenerated(diffFile.Name)) } diffFile.IsGenerated = isGenerated.Value() - - tailSection := diffFile.GetTailSection(gitRepo, beforeCommit, commit) + tailSection, limitedContent := diffFile.GetTailSectionAndLimitedContent(beforeCommit, afterCommit) if tailSection != nil { diffFile.Sections = append(diffFile.Sections, tailSection) } - } - - if opts.FileOnly { - return diff, nil - } - stats, err := GetPullDiffStats(gitRepo, opts) - if err != nil { - return nil, err + if !setting.Git.DisableDiffHighlight { + if limitedContent.LeftContent != nil && limitedContent.LeftContent.buf.Len() < MaxDiffHighlightEntireFileSize { + diffFile.highlightedLeftLines = highlightCodeLines(diffFile, true /* left */, limitedContent.LeftContent.buf.String()) + } + if limitedContent.RightContent != nil && limitedContent.RightContent.buf.Len() < MaxDiffHighlightEntireFileSize { + diffFile.highlightedRightLines = highlightCodeLines(diffFile, false /* right */, limitedContent.RightContent.buf.String()) + } + } } - diff.NumFiles, diff.TotalAddition, diff.TotalDeletion = stats.NumFiles, stats.TotalAddition, stats.TotalDeletion - return diff, nil } -type PullDiffStats struct { +func highlightCodeLines(diffFile *DiffFile, isLeft bool, content string) map[int]template.HTML { + highlightedNewContent, _ := highlight.Code(diffFile.Name, diffFile.Language, content) + splitLines := strings.Split(string(highlightedNewContent), "\n") + lines := make(map[int]template.HTML, len(splitLines)) + // only save the highlighted lines we need, but not the whole file, to save memory + for _, sec := range diffFile.Sections { + for _, ln := range sec.Lines { + lineIdx := ln.LeftIdx + if !isLeft { + lineIdx = ln.RightIdx + } + if lineIdx >= 1 { + idx := lineIdx - 1 + if idx < len(splitLines) { + lines[idx] = template.HTML(splitLines[idx]) + } + } + } + } + return lines +} + +type DiffShortStat struct { NumFiles, TotalAddition, TotalDeletion int } -// GetPullDiffStats -func GetPullDiffStats(gitRepo *git.Repository, opts *DiffOptions) (*PullDiffStats, error) { +func GetDiffShortStat(gitRepo *git.Repository, beforeCommitID, afterCommitID string) (*DiffShortStat, error) { repoPath := gitRepo.Path - diff := &PullDiffStats{} - - separator := "..." - if opts.DirectComparison { - separator = ".." - } - - objectFormat, err := gitRepo.GetObjectFormat() + afterCommit, err := gitRepo.GetCommit(afterCommitID) if err != nil { return nil, err } - diffPaths := []string{opts.BeforeCommitID + separator + opts.AfterCommitID} - if len(opts.BeforeCommitID) == 0 || opts.BeforeCommitID == objectFormat.EmptyObjectID().String() { - diffPaths = []string{objectFormat.EmptyTree().String(), opts.AfterCommitID} + _, actualBeforeCommitID, err := guessBeforeCommitForDiff(gitRepo, beforeCommitID, afterCommit) + if err != nil { + return nil, err } - diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) - if err != nil && strings.Contains(err.Error(), "no merge base") { - // git >= 2.28 now returns an error if base and head have become unrelated. - // previously it would return the results of git diff --shortstat base head so let's try that... - diffPaths = []string{opts.BeforeCommitID, opts.AfterCommitID} - diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) - } + diff := &DiffShortStat{} + diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStatByCmdArgs(gitRepo.Ctx, repoPath, nil, actualBeforeCommitID.String(), afterCommitID) if err != nil { return nil, err } - return diff, nil } -// SyncAndGetUserSpecificDiff is like GetDiff, except that user specific data such as which files the given user has already viewed on the given PR will also be set -// Additionally, the database asynchronously is updated if files have changed since the last review -func SyncAndGetUserSpecificDiff(ctx context.Context, userID int64, pull *issues_model.PullRequest, gitRepo *git.Repository, opts *DiffOptions, files ...string) (*Diff, error) { - diff, err := GetDiff(ctx, gitRepo, opts, files...) +// SyncUserSpecificDiff inserts user-specific data such as which files the user has already viewed on the given diff +// Additionally, the database is updated asynchronously if files have changed since the last review +func SyncUserSpecificDiff(ctx context.Context, userID int64, pull *issues_model.PullRequest, gitRepo *git.Repository, diff *Diff, opts *DiffOptions) (*pull_model.ReviewState, error) { + review, err := pull_model.GetNewestReviewState(ctx, userID, pull.ID) if err != nil { return nil, err } - review, err := pull_model.GetNewestReviewState(ctx, userID, pull.ID) - if err != nil || review == nil || review.UpdatedFiles == nil { - return diff, err + if review == nil || len(review.UpdatedFiles) == 0 { + return review, nil } latestCommit := opts.AfterCommitID @@ -1301,13 +1303,13 @@ func SyncAndGetUserSpecificDiff(ctx context.Context, userID int64, pull *issues_ latestCommit = pull.HeadBranch // opts.AfterCommitID is preferred because it handles PRs from forks correctly and the branch name doesn't } - changedFiles, err := gitRepo.GetFilesChangedBetween(review.CommitSHA, latestCommit) + changedFiles, errIgnored := gitRepo.GetFilesChangedBetween(review.CommitSHA, latestCommit) // There are way too many possible errors. // Examples are various git errors such as the commit the review was based on was gc'ed and hence doesn't exist anymore as well as unrecoverable errors where we should serve a 500 response // Due to the current architecture and physical limitation of needing to compare explicit error messages, we can only choose one approach without the code getting ugly // For SOME of the errors such as the gc'ed commit, it would be best to mark all files as changed // But as that does not work for all potential errors, we simply mark all files as unchanged and drop the error which always works, even if not as good as possible - if err != nil { + if errIgnored != nil { log.Error("Could not get changed files between %s and %s for pull request %d in repo with path %s. Assuming no changes. Error: %w", review.CommitSHA, latestCommit, pull.Index, gitRepo.Path, err) } @@ -1350,7 +1352,7 @@ outer: } } - return diff, nil + return review, nil } // CommentAsDiff returns c.Patch as *Diff @@ -1396,10 +1398,8 @@ func GetWhitespaceFlag(whitespaceBehavior string) git.TrustedCmdArgs { "ignore-eol": {"--ignore-space-at-eol"}, "show-all": nil, } - if flag, ok := whitespaceFlags[whitespaceBehavior]; ok { return flag } - log.Warn("unknown whitespace behavior: %q, default to 'show-all'", whitespaceBehavior) return nil } diff --git a/services/gitdiff/gitdiff_test.go b/services/gitdiff/gitdiff_test.go index 2351c5da87..b84530043a 100644 --- a/services/gitdiff/gitdiff_test.go +++ b/services/gitdiff/gitdiff_test.go @@ -17,26 +17,10 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/setting" - dmp "github.com/sergi/go-diff/diffmatchpatch" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestDiffToHTML(t *testing.T) { - assert.Equal(t, "foo <span class=\"added-code\">bar</span> biz", diffToHTML(nil, []dmp.Diff{ - {Type: dmp.DiffEqual, Text: "foo "}, - {Type: dmp.DiffInsert, Text: "bar"}, - {Type: dmp.DiffDelete, Text: " baz"}, - {Type: dmp.DiffEqual, Text: " biz"}, - }, DiffLineAdd)) - - assert.Equal(t, "foo <span class=\"removed-code\">bar</span> biz", diffToHTML(nil, []dmp.Diff{ - {Type: dmp.DiffEqual, Text: "foo "}, - {Type: dmp.DiffDelete, Text: "bar"}, - {Type: dmp.DiffInsert, Text: " baz"}, - {Type: dmp.DiffEqual, Text: " biz"}, - }, DiffLineDel)) -} - func TestParsePatch_skipTo(t *testing.T) { type testcase struct { name string @@ -181,16 +165,10 @@ diff --git "\\a/README.md" "\\b/README.md" } gotMarshaled, _ := json.MarshalIndent(got, "", " ") - if got.NumFiles != 1 { + if len(got.Files) != 1 { t.Errorf("ParsePath(%q) did not receive 1 file:\n%s", testcase.name, string(gotMarshaled)) return } - if got.TotalAddition != testcase.addition { - t.Errorf("ParsePath(%q) does not have correct totalAddition %d, wanted %d", testcase.name, got.TotalAddition, testcase.addition) - } - if got.TotalDeletion != testcase.deletion { - t.Errorf("ParsePath(%q) did not have correct totalDeletion %d, wanted %d", testcase.name, got.TotalDeletion, testcase.deletion) - } file := got.Files[0] if file.Addition != testcase.addition { t.Errorf("ParsePath(%q) does not have correct file addition %d, wanted %d", testcase.name, file.Addition, testcase.addition) @@ -406,16 +384,10 @@ index 6961180..9ba1a00 100644 } gotMarshaled, _ := json.MarshalIndent(got, "", " ") - if got.NumFiles != 1 { + if len(got.Files) != 1 { t.Errorf("ParsePath(%q) did not receive 1 file:\n%s", testcase.name, string(gotMarshaled)) return } - if got.TotalAddition != testcase.addition { - t.Errorf("ParsePath(%q) does not have correct totalAddition %d, wanted %d", testcase.name, got.TotalAddition, testcase.addition) - } - if got.TotalDeletion != testcase.deletion { - t.Errorf("ParsePath(%q) did not have correct totalDeletion %d, wanted %d", testcase.name, got.TotalDeletion, testcase.deletion) - } file := got.Files[0] if file.Addition != testcase.addition { t.Errorf("ParsePath(%q) does not have correct file addition %d, wanted %d", testcase.name, file.Addition, testcase.addition) @@ -444,7 +416,7 @@ index 0000000..6bb8f39 ` diffBuilder.WriteString(diff) - for i := 0; i < 35; i++ { + for i := range 35 { diffBuilder.WriteString("+line" + strconv.Itoa(i) + "\n") } diff = diffBuilder.String() @@ -481,11 +453,11 @@ index 0000000..6bb8f39 diffBuilder.Reset() diffBuilder.WriteString(diff) - for i := 0; i < 33; i++ { + for i := range 33 { diffBuilder.WriteString("+line" + strconv.Itoa(i) + "\n") } diffBuilder.WriteString("+line33") - for i := 0; i < 512; i++ { + for range 512 { diffBuilder.WriteString("0123456789ABCDEF") } diffBuilder.WriteByte('\n') @@ -627,24 +599,25 @@ func TestDiffLine_GetCommentSide(t *testing.T) { } func TestGetDiffRangeWithWhitespaceBehavior(t *testing.T) { - gitRepo, err := git.OpenRepository(git.DefaultContext, "./testdata/academic-module") - if !assert.NoError(t, err) { - return - } + gitRepo, err := git.OpenRepository(t.Context(), "../../modules/git/tests/repos/repo5_pulls") + require.NoError(t, err) + defer gitRepo.Close() for _, behavior := range []git.TrustedCmdArgs{{"-w"}, {"--ignore-space-at-eol"}, {"-b"}, nil} { - diffs, err := GetDiff(db.DefaultContext, gitRepo, + diffs, err := GetDiffForAPI(t.Context(), gitRepo, &DiffOptions{ - AfterCommitID: "bd7063cc7c04689c4d082183d32a604ed27a24f9", - BeforeCommitID: "559c156f8e0178b71cb44355428f24001b08fc68", + AfterCommitID: "d8e0bbb45f200e67d9a784ce55bd90821af45ebd", + BeforeCommitID: "72866af952e98d02a73003501836074b286a78f6", MaxLines: setting.Git.MaxGitDiffLines, MaxLineCharacters: setting.Git.MaxGitDiffLineCharacters, - MaxFiles: setting.Git.MaxGitDiffFiles, + MaxFiles: 1, WhitespaceBehavior: behavior, }) - assert.NoError(t, err, "Error when diff with %s", behavior) + require.NoError(t, err, "Error when diff with WhitespaceBehavior=%s", behavior) + assert.True(t, diffs.IsIncomplete) + assert.Len(t, diffs.Files, 1) for _, f := range diffs.Files { - assert.NotEmpty(t, f.Sections, "%s should have sections", f.Name) + assert.NotEmpty(t, f.Sections, "Diff file %q should have sections", f.Name) } } } diff --git a/services/gitdiff/highlightdiff.go b/services/gitdiff/highlightdiff.go index 35d4844550..e8be063e69 100644 --- a/services/gitdiff/highlightdiff.go +++ b/services/gitdiff/highlightdiff.go @@ -4,23 +4,24 @@ package gitdiff import ( + "bytes" + "html/template" "strings" - "code.gitea.io/gitea/modules/highlight" - "github.com/sergi/go-diff/diffmatchpatch" ) // token is a html tag or entity, eg: "<span ...>", "</span>", "<" func extractHTMLToken(s string) (before, token, after string, valid bool) { for pos1 := 0; pos1 < len(s); pos1++ { - if s[pos1] == '<' { + switch s[pos1] { + case '<': pos2 := strings.IndexByte(s[pos1:], '>') if pos2 == -1 { return "", "", s, false } return s[:pos1], s[pos1 : pos1+pos2+1], s[pos1+pos2+1:], true - } else if s[pos1] == '&' { + case '&': pos2 := strings.IndexByte(s[pos1:], ';') if pos2 == -1 { return "", "", s, false @@ -77,7 +78,7 @@ func (hcd *highlightCodeDiff) isInPlaceholderRange(r rune) bool { return hcd.placeholderBegin <= r && r < hcd.placeholderBegin+rune(hcd.placeholderMaxCount) } -func (hcd *highlightCodeDiff) collectUsedRunes(code string) { +func (hcd *highlightCodeDiff) collectUsedRunes(code template.HTML) { for _, r := range code { if hcd.isInPlaceholderRange(r) { // put the existing rune (used by code) in map, then this rune won't be used a placeholder anymore. @@ -86,27 +87,76 @@ func (hcd *highlightCodeDiff) collectUsedRunes(code string) { } } -func (hcd *highlightCodeDiff) diffWithHighlight(filename, language, codeA, codeB string) []diffmatchpatch.Diff { +func (hcd *highlightCodeDiff) diffLineWithHighlight(lineType DiffLineType, codeA, codeB template.HTML) template.HTML { + return hcd.diffLineWithHighlightWrapper(nil, lineType, codeA, codeB) +} + +func (hcd *highlightCodeDiff) diffLineWithHighlightWrapper(lineWrapperTags []string, lineType DiffLineType, codeA, codeB template.HTML) template.HTML { hcd.collectUsedRunes(codeA) hcd.collectUsedRunes(codeB) - highlightCodeA, _ := highlight.Code(filename, language, codeA) - highlightCodeB, _ := highlight.Code(filename, language, codeB) + convertedCodeA := hcd.convertToPlaceholders(codeA) + convertedCodeB := hcd.convertToPlaceholders(codeB) + + dmp := defaultDiffMatchPatch() + diffs := dmp.DiffMain(convertedCodeA, convertedCodeB, true) + diffs = dmp.DiffCleanupSemantic(diffs) + + buf := bytes.NewBuffer(nil) - convertedCodeA := hcd.convertToPlaceholders(string(highlightCodeA)) - convertedCodeB := hcd.convertToPlaceholders(string(highlightCodeB)) + // restore the line wrapper tags <span class="line"> and <span class="cl">, if necessary + for _, tag := range lineWrapperTags { + buf.WriteString(tag) + } - diffs := diffMatchPatch.DiffMain(convertedCodeA, convertedCodeB, true) - diffs = diffMatchPatch.DiffCleanupEfficiency(diffs) + addedCodePrefix := hcd.registerTokenAsPlaceholder(`<span class="added-code">`) + removedCodePrefix := hcd.registerTokenAsPlaceholder(`<span class="removed-code">`) + codeTagSuffix := hcd.registerTokenAsPlaceholder(`</span>`) - for i := range diffs { - hcd.recoverOneDiff(&diffs[i]) + if codeTagSuffix != 0 { + for _, diff := range diffs { + switch { + case diff.Type == diffmatchpatch.DiffEqual: + buf.WriteString(diff.Text) + case diff.Type == diffmatchpatch.DiffInsert && lineType == DiffLineAdd: + buf.WriteRune(addedCodePrefix) + buf.WriteString(diff.Text) + buf.WriteRune(codeTagSuffix) + case diff.Type == diffmatchpatch.DiffDelete && lineType == DiffLineDel: + buf.WriteRune(removedCodePrefix) + buf.WriteString(diff.Text) + buf.WriteRune(codeTagSuffix) + } + } + } else { + // placeholder map space is exhausted + for _, diff := range diffs { + take := diff.Type == diffmatchpatch.DiffEqual || (diff.Type == diffmatchpatch.DiffInsert && lineType == DiffLineAdd) || (diff.Type == diffmatchpatch.DiffDelete && lineType == DiffLineDel) + if take { + buf.WriteString(diff.Text) + } + } } - return diffs + for range lineWrapperTags { + buf.WriteString("</span>") + } + return hcd.recoverOneDiff(buf.String()) +} + +func (hcd *highlightCodeDiff) registerTokenAsPlaceholder(token string) rune { + placeholder, ok := hcd.tokenPlaceholderMap[token] + if !ok { + placeholder = hcd.nextPlaceholder() + if placeholder != 0 { + hcd.tokenPlaceholderMap[token] = placeholder + hcd.placeholderTokenMap[placeholder] = token + } + } + return placeholder } // convertToPlaceholders totally depends on Chroma's valid HTML output and its structure, do not use these functions for other purposes. -func (hcd *highlightCodeDiff) convertToPlaceholders(htmlCode string) string { +func (hcd *highlightCodeDiff) convertToPlaceholders(htmlContent template.HTML) string { var tagStack []string res := strings.Builder{} @@ -115,6 +165,7 @@ func (hcd *highlightCodeDiff) convertToPlaceholders(htmlCode string) string { var beforeToken, token string var valid bool + htmlCode := string(htmlContent) // the standard chroma highlight HTML is "<span class="line [hl]"><span class="cl"> ... </span></span>" for { beforeToken, token, htmlCode, valid = extractHTMLToken(htmlCode) @@ -151,14 +202,7 @@ func (hcd *highlightCodeDiff) convertToPlaceholders(htmlCode string) string { } // else: impossible // remember the placeholder and token in the map - placeholder, ok := hcd.tokenPlaceholderMap[tokenInMap] - if !ok { - placeholder = hcd.nextPlaceholder() - if placeholder != 0 { - hcd.tokenPlaceholderMap[tokenInMap] = placeholder - hcd.placeholderTokenMap[placeholder] = tokenInMap - } - } + placeholder := hcd.registerTokenAsPlaceholder(tokenInMap) if placeholder != 0 { res.WriteRune(placeholder) // use the placeholder to replace the token @@ -179,11 +223,11 @@ func (hcd *highlightCodeDiff) convertToPlaceholders(htmlCode string) string { return res.String() } -func (hcd *highlightCodeDiff) recoverOneDiff(diff *diffmatchpatch.Diff) { +func (hcd *highlightCodeDiff) recoverOneDiff(str string) template.HTML { sb := strings.Builder{} var tagStack []string - for _, r := range diff.Text { + for _, r := range str { token, ok := hcd.placeholderTokenMap[r] if !ok || token == "" { sb.WriteRune(r) // if the rune is not a placeholder, write it as it is @@ -217,6 +261,5 @@ func (hcd *highlightCodeDiff) recoverOneDiff(diff *diffmatchpatch.Diff) { } // else: impossible. every tag was pushed into the stack by the code above and is valid HTML opening tag } } - - diff.Text = sb.String() + return template.HTML(sb.String()) } diff --git a/services/gitdiff/highlightdiff_test.go b/services/gitdiff/highlightdiff_test.go index 545a060e20..aebe38ae7c 100644 --- a/services/gitdiff/highlightdiff_test.go +++ b/services/gitdiff/highlightdiff_test.go @@ -5,121 +5,82 @@ package gitdiff import ( "fmt" + "html/template" "strings" "testing" - "github.com/sergi/go-diff/diffmatchpatch" "github.com/stretchr/testify/assert" ) func TestDiffWithHighlight(t *testing.T) { - hcd := newHighlightCodeDiff() - diffs := hcd.diffWithHighlight( - "main.v", "", - " run('<>')\n", - " run(db)\n", - ) - - expected := ` <span class="n">run</span><span class="o">(</span><span class="removed-code"><span class="k">'</span><span class="o"><</span><span class="o">></span><span class="k">'</span></span><span class="o">)</span>` - output := diffToHTML(nil, diffs, DiffLineDel) - assert.Equal(t, expected, output) - - expected = ` <span class="n">run</span><span class="o">(</span><span class="added-code"><span class="n">db</span></span><span class="o">)</span>` - output = diffToHTML(nil, diffs, DiffLineAdd) - assert.Equal(t, expected, output) - - hcd = newHighlightCodeDiff() - hcd.placeholderTokenMap['O'] = "<span>" - hcd.placeholderTokenMap['C'] = "</span>" - diff := diffmatchpatch.Diff{} - - diff.Text = "OC" - hcd.recoverOneDiff(&diff) - assert.Equal(t, "<span></span>", diff.Text) - - diff.Text = "O" - hcd.recoverOneDiff(&diff) - assert.Equal(t, "<span></span>", diff.Text) - - diff.Text = "C" - hcd.recoverOneDiff(&diff) - assert.Equal(t, "", diff.Text) + t.Run("DiffLineAddDel", func(t *testing.T) { + hcd := newHighlightCodeDiff() + codeA := template.HTML(`x <span class="k">foo</span> y`) + codeB := template.HTML(`x <span class="k">bar</span> y`) + outDel := hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB) + assert.Equal(t, `x <span class="k"><span class="removed-code">foo</span></span> y`, string(outDel)) + outAdd := hcd.diffLineWithHighlight(DiffLineAdd, codeA, codeB) + assert.Equal(t, `x <span class="k"><span class="added-code">bar</span></span> y`, string(outAdd)) + }) + + t.Run("CleanUp", func(t *testing.T) { + hcd := newHighlightCodeDiff() + codeA := template.HTML(`<span class="cm>this is a comment</span>`) + codeB := template.HTML(`<span class="cm>this is updated comment</span>`) + outDel := hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB) + assert.Equal(t, `<span class="cm>this is <span class="removed-code">a</span> comment</span>`, string(outDel)) + outAdd := hcd.diffLineWithHighlight(DiffLineAdd, codeA, codeB) + assert.Equal(t, `<span class="cm>this is <span class="added-code">updated</span> comment</span>`, string(outAdd)) + }) + + t.Run("OpenCloseTags", func(t *testing.T) { + hcd := newHighlightCodeDiff() + hcd.placeholderTokenMap['O'], hcd.placeholderTokenMap['C'] = "<span>", "</span>" + assert.Equal(t, "<span></span>", string(hcd.recoverOneDiff("OC"))) + assert.Equal(t, "<span></span>", string(hcd.recoverOneDiff("O"))) + assert.Empty(t, string(hcd.recoverOneDiff("C"))) + }) } func TestDiffWithHighlightPlaceholder(t *testing.T) { hcd := newHighlightCodeDiff() - diffs := hcd.diffWithHighlight( - "main.js", "", - "a='\U00100000'", - "a='\U0010FFFD''", - ) - assert.Equal(t, "", hcd.placeholderTokenMap[0x00100000]) - assert.Equal(t, "", hcd.placeholderTokenMap[0x0010FFFD]) - - expected := fmt.Sprintf(`<span class="nx">a</span><span class="o">=</span><span class="s1">'</span><span class="removed-code">%s</span>'`, "\U00100000") - output := diffToHTML(hcd.lineWrapperTags, diffs, DiffLineDel) - assert.Equal(t, expected, output) + output := hcd.diffLineWithHighlight(DiffLineDel, "a='\U00100000'", "a='\U0010FFFD''") + assert.Empty(t, hcd.placeholderTokenMap[0x00100000]) + assert.Empty(t, hcd.placeholderTokenMap[0x0010FFFD]) + expected := fmt.Sprintf(`a='<span class="removed-code">%s</span>'`, "\U00100000") + assert.Equal(t, expected, string(output)) hcd = newHighlightCodeDiff() - diffs = hcd.diffWithHighlight( - "main.js", "", - "a='\U00100000'", - "a='\U0010FFFD'", - ) - expected = fmt.Sprintf(`<span class="nx">a</span><span class="o">=</span><span class="s1">'</span><span class="added-code">%s</span>'`, "\U0010FFFD") - output = diffToHTML(nil, diffs, DiffLineAdd) - assert.Equal(t, expected, output) + output = hcd.diffLineWithHighlight(DiffLineAdd, "a='\U00100000'", "a='\U0010FFFD'") + expected = fmt.Sprintf(`a='<span class="added-code">%s</span>'`, "\U0010FFFD") + assert.Equal(t, expected, string(output)) } func TestDiffWithHighlightPlaceholderExhausted(t *testing.T) { hcd := newHighlightCodeDiff() hcd.placeholderMaxCount = 0 - diffs := hcd.diffWithHighlight( - "main.js", "", - "'", - ``, - ) - output := diffToHTML(nil, diffs, DiffLineDel) - expected := fmt.Sprintf(`<span class="removed-code">%s#39;</span>`, "\uFFFD") - assert.Equal(t, expected, output) - - hcd = newHighlightCodeDiff() - hcd.placeholderMaxCount = 0 - diffs = hcd.diffWithHighlight( - "main.js", "", - "a < b", - "a > b", - ) - output = diffToHTML(nil, diffs, DiffLineDel) - expected = fmt.Sprintf(`a %s<span class="removed-code">l</span>t; b`, "\uFFFD") - assert.Equal(t, expected, output) - - output = diffToHTML(nil, diffs, DiffLineAdd) - expected = fmt.Sprintf(`a %s<span class="added-code">g</span>t; b`, "\uFFFD") - assert.Equal(t, expected, output) + placeHolderAmp := string(rune(0xFFFD)) + output := hcd.diffLineWithHighlight(DiffLineDel, `<span class="k"><</span>`, `<span class="k">></span>`) + assert.Equal(t, placeHolderAmp+"lt;", string(output)) + output = hcd.diffLineWithHighlight(DiffLineAdd, `<span class="k"><</span>`, `<span class="k">></span>`) + assert.Equal(t, placeHolderAmp+"gt;", string(output)) } func TestDiffWithHighlightTagMatch(t *testing.T) { - totalOverflow := 0 - for i := 0; i < 100; i++ { - hcd := newHighlightCodeDiff() - hcd.placeholderMaxCount = i - diffs := hcd.diffWithHighlight( - "main.js", "", - "a='1'", - "b='2'", - ) - totalOverflow += hcd.placeholderOverflowCount - - output := diffToHTML(nil, diffs, DiffLineDel) - c1 := strings.Count(output, "<span") - c2 := strings.Count(output, "</span") - assert.Equal(t, c1, c2) - - output = diffToHTML(nil, diffs, DiffLineAdd) - c1 = strings.Count(output, "<span") - c2 = strings.Count(output, "</span") - assert.Equal(t, c1, c2) + f := func(t *testing.T, lineType DiffLineType) { + totalOverflow := 0 + for i := 0; ; i++ { + hcd := newHighlightCodeDiff() + hcd.placeholderMaxCount = i + output := string(hcd.diffLineWithHighlight(lineType, `<span class="k"><</span>`, `<span class="k">></span>`)) + totalOverflow += hcd.placeholderOverflowCount + assert.Equal(t, strings.Count(output, "<span"), strings.Count(output, "</span")) + if hcd.placeholderOverflowCount == 0 { + break + } + } + assert.NotZero(t, totalOverflow) } - assert.NotZero(t, totalOverflow) + t.Run("DiffLineAdd", func(t *testing.T) { f(t, DiffLineAdd) }) + t.Run("DiffLineDel", func(t *testing.T) { f(t, DiffLineDel) }) } diff --git a/services/gitdiff/submodule_test.go b/services/gitdiff/submodule_test.go index 89f32c0e0c..152c5b7066 100644 --- a/services/gitdiff/submodule_test.go +++ b/services/gitdiff/submodule_test.go @@ -4,7 +4,6 @@ package gitdiff import ( - "context" "strings" "testing" @@ -204,7 +203,6 @@ index 0000000..68972a9 } for _, testcase := range tests { - testcase := testcase t.Run(testcase.name, func(t *testing.T) { diff, err := ParsePatch(db.DefaultContext, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, strings.NewReader(testcase.gitdiff), "") assert.NoError(t, err) @@ -224,13 +222,13 @@ func TestSubmoduleInfo(t *testing.T) { PreviousRefID: "aaaa", NewRefID: "bbbb", } - ctx := context.Background() + ctx := t.Context() assert.EqualValues(t, "1111", sdi.CommitRefIDLinkHTML(ctx, "1111")) assert.EqualValues(t, "aaaa...bbbb", sdi.CompareRefIDLinkHTML(ctx)) assert.EqualValues(t, "name", sdi.SubmoduleRepoLinkHTML(ctx)) sdi.SubmoduleFile = git.NewCommitSubmoduleFile("https://github.com/owner/repo", "1234") - assert.EqualValues(t, `<a href="https://github.com/owner/repo/commit/1111">1111</a>`, sdi.CommitRefIDLinkHTML(ctx, "1111")) + assert.EqualValues(t, `<a href="https://github.com/owner/repo/tree/1111">1111</a>`, sdi.CommitRefIDLinkHTML(ctx, "1111")) assert.EqualValues(t, `<a href="https://github.com/owner/repo/compare/aaaa...bbbb">aaaa...bbbb</a>`, sdi.CompareRefIDLinkHTML(ctx)) assert.EqualValues(t, `<a href="https://github.com/owner/repo">name</a>`, sdi.SubmoduleRepoLinkHTML(ctx)) } diff --git a/services/gitdiff/testdata/academic-module/HEAD b/services/gitdiff/testdata/academic-module/HEAD deleted file mode 100644 index cb089cd89a..0000000000 --- a/services/gitdiff/testdata/academic-module/HEAD +++ /dev/null @@ -1 +0,0 @@ -ref: refs/heads/master diff --git a/services/gitdiff/testdata/academic-module/config b/services/gitdiff/testdata/academic-module/config deleted file mode 100644 index 1bc26be514..0000000000 --- a/services/gitdiff/testdata/academic-module/config +++ /dev/null @@ -1,10 +0,0 @@ -[core] - repositoryformatversion = 0 - filemode = true - bare = false - logallrefupdates = true - ignorecase = true - precomposeunicode = true -[branch "master"] - remote = origin - merge = refs/heads/master diff --git a/services/gitdiff/testdata/academic-module/index b/services/gitdiff/testdata/academic-module/index Binary files differdeleted file mode 100644 index e712c906e3..0000000000 --- a/services/gitdiff/testdata/academic-module/index +++ /dev/null diff --git a/services/gitdiff/testdata/academic-module/logs/HEAD b/services/gitdiff/testdata/academic-module/logs/HEAD deleted file mode 100644 index 16b2e1c0f6..0000000000 --- a/services/gitdiff/testdata/academic-module/logs/HEAD +++ /dev/null @@ -1 +0,0 @@ -0000000000000000000000000000000000000000 bd7063cc7c04689c4d082183d32a604ed27a24f9 Lunny Xiao <xiaolunwen@gmail.com> 1574829684 +0800 clone: from https://try.gitea.io/shemgp-aiias/academic-module diff --git a/services/gitdiff/testdata/academic-module/logs/refs/heads/master b/services/gitdiff/testdata/academic-module/logs/refs/heads/master deleted file mode 100644 index 16b2e1c0f6..0000000000 --- a/services/gitdiff/testdata/academic-module/logs/refs/heads/master +++ /dev/null @@ -1 +0,0 @@ -0000000000000000000000000000000000000000 bd7063cc7c04689c4d082183d32a604ed27a24f9 Lunny Xiao <xiaolunwen@gmail.com> 1574829684 +0800 clone: from https://try.gitea.io/shemgp-aiias/academic-module diff --git a/services/gitdiff/testdata/academic-module/logs/refs/remotes/origin/HEAD b/services/gitdiff/testdata/academic-module/logs/refs/remotes/origin/HEAD deleted file mode 100644 index 16b2e1c0f6..0000000000 --- a/services/gitdiff/testdata/academic-module/logs/refs/remotes/origin/HEAD +++ /dev/null @@ -1 +0,0 @@ -0000000000000000000000000000000000000000 bd7063cc7c04689c4d082183d32a604ed27a24f9 Lunny Xiao <xiaolunwen@gmail.com> 1574829684 +0800 clone: from https://try.gitea.io/shemgp-aiias/academic-module diff --git a/services/gitdiff/testdata/academic-module/objects/pack/pack-597efbc3613c7ba790e33b178fd9fc1fe17b4245.idx b/services/gitdiff/testdata/academic-module/objects/pack/pack-597efbc3613c7ba790e33b178fd9fc1fe17b4245.idx Binary files differdeleted file mode 100644 index 4d759aa504..0000000000 --- a/services/gitdiff/testdata/academic-module/objects/pack/pack-597efbc3613c7ba790e33b178fd9fc1fe17b4245.idx +++ /dev/null diff --git a/services/gitdiff/testdata/academic-module/objects/pack/pack-597efbc3613c7ba790e33b178fd9fc1fe17b4245.pack b/services/gitdiff/testdata/academic-module/objects/pack/pack-597efbc3613c7ba790e33b178fd9fc1fe17b4245.pack Binary files differdeleted file mode 100644 index 2dc49cfded..0000000000 --- a/services/gitdiff/testdata/academic-module/objects/pack/pack-597efbc3613c7ba790e33b178fd9fc1fe17b4245.pack +++ /dev/null diff --git a/services/gitdiff/testdata/academic-module/packed-refs b/services/gitdiff/testdata/academic-module/packed-refs deleted file mode 100644 index 13b5611650..0000000000 --- a/services/gitdiff/testdata/academic-module/packed-refs +++ /dev/null @@ -1,2 +0,0 @@ -# pack-refs with: peeled fully-peeled sorted -bd7063cc7c04689c4d082183d32a604ed27a24f9 refs/remotes/origin/master diff --git a/services/gitdiff/testdata/academic-module/refs/heads/master b/services/gitdiff/testdata/academic-module/refs/heads/master deleted file mode 100644 index bd2b56eaf4..0000000000 --- a/services/gitdiff/testdata/academic-module/refs/heads/master +++ /dev/null @@ -1 +0,0 @@ -bd7063cc7c04689c4d082183d32a604ed27a24f9 diff --git a/services/gitdiff/testdata/academic-module/refs/remotes/origin/HEAD b/services/gitdiff/testdata/academic-module/refs/remotes/origin/HEAD deleted file mode 100644 index 6efe28fff8..0000000000 --- a/services/gitdiff/testdata/academic-module/refs/remotes/origin/HEAD +++ /dev/null @@ -1 +0,0 @@ -ref: refs/remotes/origin/master diff --git a/services/issue/assignee.go b/services/issue/assignee.go index c7e2495568..ba9c91e0ed 100644 --- a/services/issue/assignee.go +++ b/services/issue/assignee.go @@ -54,6 +54,8 @@ func ToggleAssigneeWithNotify(ctx context.Context, issue *issues_model.Issue, do if err != nil { return false, nil, err } + issue.AssigneeID = assigneeID + issue.Assignee = assignee notify_service.IssueChangeAssignee(ctx, doer, issue, assignee, removed, comment) @@ -302,7 +304,7 @@ func CanDoerChangeReviewRequests(ctx context.Context, doer *user_model.User, rep // If the repo's owner is an organization, members of teams with read permission on pull requests can change reviewers if repo.Owner.IsOrganization() { - teams, err := organization.GetTeamsWithAccessToRepo(ctx, repo.OwnerID, repo.ID, perm.AccessModeRead) + teams, err := organization.GetTeamsWithAccessToAnyRepoUnit(ctx, repo.OwnerID, repo.ID, perm.AccessModeRead, unit.TypePullRequests) if err != nil { log.Error("GetTeamsWithAccessToRepo: %v", err) return false diff --git a/services/issue/comments.go b/services/issue/comments.go index 33b5702a00..10c81198d5 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -5,6 +5,7 @@ package issue import ( "context" + "errors" "fmt" "code.gitea.io/gitea/models/db" @@ -12,14 +13,17 @@ import ( access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/timeutil" + git_service "code.gitea.io/gitea/services/git" notify_service "code.gitea.io/gitea/services/notify" ) // CreateRefComment creates a commit reference comment to issue. func CreateRefComment(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, issue *issues_model.Issue, content, commitSHA string) error { if len(commitSHA) == 0 { - return fmt.Errorf("cannot create reference with empty commit SHA") + return errors.New("cannot create reference with empty commit SHA") } if user_model.IsUserBlockedBy(ctx, doer, issue.PosterID, repo.OwnerID) { @@ -139,3 +143,40 @@ func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_m return nil } + +// LoadCommentPushCommits Load push commits +func LoadCommentPushCommits(ctx context.Context, c *issues_model.Comment) (err error) { + if c.Content == "" || c.Commits != nil || c.Type != issues_model.CommentTypePullRequestPush { + return nil + } + + var data issues_model.PushActionContent + err = json.Unmarshal([]byte(c.Content), &data) + if err != nil { + return err + } + + c.IsForcePush = data.IsForcePush + + if c.IsForcePush { + if len(data.CommitIDs) != 2 { + return nil + } + c.OldCommit = data.CommitIDs[0] + c.NewCommit = data.CommitIDs[1] + } else { + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, c.Issue.Repo) + if err != nil { + return err + } + defer closer.Close() + + c.Commits, err = git_service.ConvertFromGitCommit(ctx, gitRepo.GetCommitsFromIDs(data.CommitIDs), c.Issue.Repo) + if err != nil { + return err + } + c.CommitsNum = int64(len(c.Commits)) + } + + return err +} diff --git a/services/issue/issue.go b/services/issue/issue.go index c6a52cc0fe..2cb5f2801d 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -92,8 +92,12 @@ func ChangeTitle(ctx context.Context, issue *issues_model.Issue, doer *user_mode var reviewNotifiers []*ReviewRequestNotifier if issue.IsPull && issues_model.HasWorkInProgressPrefix(oldTitle) && !issues_model.HasWorkInProgressPrefix(title) { + if err := issue.LoadPullRequest(ctx); err != nil { + return err + } + var err error - reviewNotifiers, err = PullRequestCodeOwnersReview(ctx, issue, issue.PullRequest) + reviewNotifiers, err = PullRequestCodeOwnersReview(ctx, issue.PullRequest) if err != nil { log.Error("PullRequestCodeOwnersReview: %v", err) } @@ -186,9 +190,13 @@ func DeleteIssue(ctx context.Context, doer *user_model.User, gitRepo *git.Reposi } // delete entries in database - if err := deleteIssue(ctx, issue); err != nil { + attachmentPaths, err := deleteIssue(ctx, issue) + if err != nil { return err } + for _, attachmentPath := range attachmentPaths { + system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", attachmentPath) + } // delete pull request related git data if issue.IsPull && gitRepo != nil { @@ -197,13 +205,6 @@ func DeleteIssue(ctx context.Context, doer *user_model.User, gitRepo *git.Reposi } } - // If the Issue is pinned, we should unpin it before deletion to avoid problems with other pinned Issues - if issue.IsPinned() { - if err := issue.Unpin(ctx, doer); err != nil { - return err - } - } - notify_service.DeleteIssue(ctx, doer, issue) return nil @@ -250,53 +251,54 @@ func GetRefEndNamesAndURLs(issues []*issues_model.Issue, repoLink string) (map[i issueRefURLs := make(map[int64]string, len(issues)) for _, issue := range issues { if issue.Ref != "" { - issueRefEndNames[issue.ID] = git.RefName(issue.Ref).ShortName() - issueRefURLs[issue.ID] = git.RefURL(repoLink, issue.Ref) + ref := git.RefName(issue.Ref) + issueRefEndNames[issue.ID] = ref.ShortName() + issueRefURLs[issue.ID] = repoLink + "/src/" + ref.RefWebLinkPath() } } return issueRefEndNames, issueRefURLs } // deleteIssue deletes the issue -func deleteIssue(ctx context.Context, issue *issues_model.Issue) error { +func deleteIssue(ctx context.Context, issue *issues_model.Issue) ([]string, error) { ctx, committer, err := db.TxContext(ctx) if err != nil { - return err + return nil, err } defer committer.Close() - e := db.GetEngine(ctx) - if _, err := e.ID(issue.ID).NoAutoCondition().Delete(issue); err != nil { - return err + if _, err := db.GetEngine(ctx).ID(issue.ID).NoAutoCondition().Delete(issue); err != nil { + return nil, err } // update the total issue numbers if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, false); err != nil { - return err + return nil, err } // if the issue is closed, update the closed issue numbers if issue.IsClosed { if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, true); err != nil { - return err + return nil, err } } if err := issues_model.UpdateMilestoneCounters(ctx, issue.MilestoneID); err != nil { - return fmt.Errorf("error updating counters for milestone id %d: %w", + return nil, fmt.Errorf("error updating counters for milestone id %d: %w", issue.MilestoneID, err) } if err := activities_model.DeleteIssueActions(ctx, issue.RepoID, issue.ID, issue.Index); err != nil { - return err + return nil, err } // find attachments related to this issue and remove them - if err := issue.LoadAttributes(ctx); err != nil { - return err + if err := issue.LoadAttachments(ctx); err != nil { + return nil, err } + var attachmentPaths []string for i := range issue.Attachments { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", issue.Attachments[i].RelativePath()) + attachmentPaths = append(attachmentPaths, issue.Attachments[i].RelativePath()) } // delete all database data still assigned to this issue @@ -318,9 +320,70 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue) error { &issues_model.Comment{RefIssueID: issue.ID}, &issues_model.IssueDependency{DependencyID: issue.ID}, &issues_model.Comment{DependentIssueID: issue.ID}, + &issues_model.IssuePin{IssueID: issue.ID}, ); err != nil { + return nil, err + } + + if err := committer.Commit(); err != nil { + return nil, err + } + return attachmentPaths, nil +} + +// DeleteOrphanedIssues delete issues without a repo +func DeleteOrphanedIssues(ctx context.Context) error { + var attachmentPaths []string + err := db.WithTx(ctx, func(ctx context.Context) error { + repoIDs, err := issues_model.GetOrphanedIssueRepoIDs(ctx) + if err != nil { + return err + } + for i := range repoIDs { + paths, err := DeleteIssuesByRepoID(ctx, repoIDs[i]) + if err != nil { + return err + } + attachmentPaths = append(attachmentPaths, paths...) + } + return nil + }) + if err != nil { return err } - return committer.Commit() + // Remove issue attachment files. + for i := range attachmentPaths { + system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", attachmentPaths[i]) + } + return nil +} + +// DeleteIssuesByRepoID deletes issues by repositories id +func DeleteIssuesByRepoID(ctx context.Context, repoID int64) (attachmentPaths []string, err error) { + for { + issues := make([]*issues_model.Issue, 0, db.DefaultMaxInSize) + if err := db.GetEngine(ctx). + Where("repo_id = ?", repoID). + OrderBy("id"). + Limit(db.DefaultMaxInSize). + Find(&issues); err != nil { + return nil, err + } + + if len(issues) == 0 { + break + } + + for _, issue := range issues { + issueAttachPaths, err := deleteIssue(ctx, issue) + if err != nil { + return nil, fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err) + } + + attachmentPaths = append(attachmentPaths, issueAttachPaths...) + } + } + + return attachmentPaths, err } diff --git a/services/issue/issue_test.go b/services/issue/issue_test.go index 8806cec0e7..bad0d65d1e 100644 --- a/services/issue/issue_test.go +++ b/services/issue/issue_test.go @@ -24,8 +24,8 @@ func TestGetRefEndNamesAndURLs(t *testing.T) { repoLink := "/foo/bar" endNames, urls := GetRefEndNamesAndURLs(issues, repoLink) - assert.EqualValues(t, map[int64]string{1: "branch1", 2: "tag1", 3: "c0ffee"}, endNames) - assert.EqualValues(t, map[int64]string{ + assert.Equal(t, map[int64]string{1: "branch1", 2: "tag1", 3: "c0ffee"}, endNames) + assert.Equal(t, map[int64]string{ 1: repoLink + "/src/branch/branch1", 2: repoLink + "/src/tag/tag1", 3: repoLink + "/src/commit/c0ffee", @@ -44,7 +44,7 @@ func TestIssue_DeleteIssue(t *testing.T) { ID: issueIDs[2], } - err = deleteIssue(db.DefaultContext, issue) + _, err = deleteIssue(db.DefaultContext, issue) assert.NoError(t, err) issueIDs, err = issues_model.GetIssueIDsByRepoID(db.DefaultContext, 1) assert.NoError(t, err) @@ -55,7 +55,7 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) issue, err = issues_model.GetIssueByID(db.DefaultContext, 4) assert.NoError(t, err) - err = deleteIssue(db.DefaultContext, issue) + _, err = deleteIssue(db.DefaultContext, issue) assert.NoError(t, err) assert.Len(t, attachments, 2) for i := range attachments { @@ -78,7 +78,7 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) assert.False(t, left) - err = deleteIssue(db.DefaultContext, issue2) + _, err = deleteIssue(db.DefaultContext, issue2) assert.NoError(t, err) left, err = issues_model.IssueNoDependenciesLeft(db.DefaultContext, issue1) assert.NoError(t, err) diff --git a/services/issue/milestone.go b/services/issue/milestone.go index beb6f131a9..afca70794d 100644 --- a/services/issue/milestone.go +++ b/services/issue/milestone.go @@ -5,6 +5,7 @@ package issue import ( "context" + "errors" "fmt" "code.gitea.io/gitea/models/db" @@ -21,7 +22,7 @@ func changeMilestoneAssign(ctx context.Context, doer *user_model.User, issue *is return fmt.Errorf("HasMilestoneByRepoID: %w", err) } if !has { - return fmt.Errorf("HasMilestoneByRepoID: issue doesn't exist") + return errors.New("HasMilestoneByRepoID: issue doesn't exist") } } diff --git a/services/issue/pull.go b/services/issue/pull.go index 896802108d..3543b05b18 100644 --- a/services/issue/pull.go +++ b/services/issue/pull.go @@ -6,6 +6,7 @@ package issue import ( "context" "fmt" + "slices" "time" issues_model "code.gitea.io/gitea/models/issues" @@ -40,20 +41,27 @@ type ReviewRequestNotifier struct { ReviewTeam *org_model.Team } -func PullRequestCodeOwnersReview(ctx context.Context, issue *issues_model.Issue, pr *issues_model.PullRequest) ([]*ReviewRequestNotifier, error) { - files := []string{"CODEOWNERS", "docs/CODEOWNERS", ".gitea/CODEOWNERS"} +var codeOwnerFiles = []string{"CODEOWNERS", "docs/CODEOWNERS", ".gitea/CODEOWNERS"} +func IsCodeOwnerFile(f string) bool { + return slices.Contains(codeOwnerFiles, f) +} + +func PullRequestCodeOwnersReview(ctx context.Context, pr *issues_model.PullRequest) ([]*ReviewRequestNotifier, error) { + if err := pr.LoadIssue(ctx); err != nil { + return nil, err + } + issue := pr.Issue if pr.IsWorkInProgress(ctx) { return nil, nil } - if err := pr.LoadHeadRepo(ctx); err != nil { return nil, err } - if err := pr.LoadBaseRepo(ctx); err != nil { return nil, err } + pr.Issue.Repo = pr.BaseRepo if pr.BaseRepo.IsFork { return nil, nil @@ -71,7 +79,7 @@ func PullRequestCodeOwnersReview(ctx context.Context, issue *issues_model.Issue, } var data string - for _, file := range files { + for _, file := range codeOwnerFiles { if blob, err := commit.GetBlobByPath(file); err == nil { data, err = blob.GetBlobContent(setting.UI.MaxDisplayFileSize) if err == nil { @@ -79,8 +87,14 @@ func PullRequestCodeOwnersReview(ctx context.Context, issue *issues_model.Issue, } } } + if data == "" { + return nil, nil + } rules, _ := issues_model.GetCodeOwnersFromContent(ctx, data) + if len(rules) == 0 { + return nil, nil + } // get the mergebase mergeBase, err := getMergeBase(repo, pr, git.BranchPrefix+pr.BaseBranch, pr.GetGitRefName()) @@ -116,13 +130,31 @@ func PullRequestCodeOwnersReview(ctx context.Context, issue *issues_model.Issue, return nil, err } + // load all reviews from database + latestReivews, _, err := issues_model.GetReviewsByIssueID(ctx, pr.IssueID) + if err != nil { + return nil, err + } + + contain := func(list issues_model.ReviewList, u *user_model.User) bool { + for _, review := range list { + if review.ReviewerTeamID == 0 && review.ReviewerID == u.ID { + return true + } + } + return false + } + for _, u := range uniqUsers { - if u.ID != issue.Poster.ID { + if u.ID != issue.Poster.ID && !contain(latestReivews, u) { comment, err := issues_model.AddReviewRequest(ctx, issue, u, issue.Poster) if err != nil { log.Warn("Failed add assignee user: %s to PR review: %s#%d, error: %s", u.Name, pr.BaseRepo.Name, pr.ID, err) return nil, err } + if comment == nil { // comment maybe nil if review type is ReviewTypeRequest + continue + } notifiers = append(notifiers, &ReviewRequestNotifier{ Comment: comment, IsAdd: true, @@ -130,12 +162,16 @@ func PullRequestCodeOwnersReview(ctx context.Context, issue *issues_model.Issue, }) } } + for _, t := range uniqTeams { comment, err := issues_model.AddTeamReviewRequest(ctx, issue, t, issue.Poster) if err != nil { log.Warn("Failed add assignee team: %s to PR review: %s#%d, error: %s", t.Name, pr.BaseRepo.Name, pr.ID, err) return nil, err } + if comment == nil { // comment maybe nil if review type is ReviewTypeRequest + continue + } notifiers = append(notifiers, &ReviewRequestNotifier{ Comment: comment, IsAdd: true, diff --git a/services/issue/status.go b/services/issue/status.go index e18b891175..f9d7dca841 100644 --- a/services/issue/status.go +++ b/services/issue/status.go @@ -24,14 +24,14 @@ func CloseIssue(ctx context.Context, issue *issues_model.Issue, doer *user_model comment, err := issues_model.CloseIssue(dbCtx, issue, doer) if err != nil { if issues_model.IsErrDependenciesLeft(err) { - if err := issues_model.FinishIssueStopwatchIfPossible(dbCtx, doer, issue); err != nil { + if _, err := issues_model.FinishIssueStopwatch(dbCtx, doer, issue); err != nil { log.Error("Unable to stop stopwatch for issue[%d]#%d: %v", issue.ID, issue.Index, err) } } return err } - if err := issues_model.FinishIssueStopwatchIfPossible(dbCtx, doer, issue); err != nil { + if _, err := issues_model.FinishIssueStopwatch(dbCtx, doer, issue); err != nil { return err } diff --git a/services/issue/suggestion.go b/services/issue/suggestion.go new file mode 100644 index 0000000000..22eddb1904 --- /dev/null +++ b/services/issue/suggestion.go @@ -0,0 +1,73 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package issue + +import ( + "context" + "strconv" + + issues_model "code.gitea.io/gitea/models/issues" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/structs" +) + +func GetSuggestion(ctx context.Context, repo *repo_model.Repository, isPull optional.Option[bool], keyword string) ([]*structs.Issue, error) { + var issues issues_model.IssueList + var err error + pageSize := 5 + if keyword == "" { + issues, err = issues_model.FindLatestUpdatedIssues(ctx, repo.ID, isPull, pageSize) + if err != nil { + return nil, err + } + } else { + indexKeyword, _ := strconv.ParseInt(keyword, 10, 64) + var issueByIndex *issues_model.Issue + var excludedID int64 + if indexKeyword > 0 { + issueByIndex, err = issues_model.GetIssueByIndex(ctx, repo.ID, indexKeyword) + if err != nil && !issues_model.IsErrIssueNotExist(err) { + return nil, err + } + if issueByIndex != nil { + excludedID = issueByIndex.ID + pageSize-- + } + } + + issues, err = issues_model.FindIssuesSuggestionByKeyword(ctx, repo.ID, keyword, isPull, excludedID, pageSize) + if err != nil { + return nil, err + } + + if issueByIndex != nil { + issues = append([]*issues_model.Issue{issueByIndex}, issues...) + } + } + + if err := issues.LoadPullRequests(ctx); err != nil { + return nil, err + } + + suggestions := make([]*structs.Issue, 0, len(issues)) + for _, issue := range issues { + suggestion := &structs.Issue{ + ID: issue.ID, + Index: issue.Index, + Title: issue.Title, + State: issue.State(), + } + + if issue.IsPull && issue.PullRequest != nil { + suggestion.PullRequest = &structs.PullRequestMeta{ + HasMerged: issue.PullRequest.HasMerged, + IsWorkInProgress: issue.PullRequest.IsWorkInProgress(ctx), + } + } + suggestions = append(suggestions, suggestion) + } + + return suggestions, nil +} diff --git a/services/issue/suggestion_test.go b/services/issue/suggestion_test.go new file mode 100644 index 0000000000..a5b39d27bb --- /dev/null +++ b/services/issue/suggestion_test.go @@ -0,0 +1,57 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package issue + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/optional" + + "github.com/stretchr/testify/assert" +) + +func Test_Suggestion(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + testCases := []struct { + keyword string + isPull optional.Option[bool] + expectedIndexes []int64 + }{ + { + keyword: "", + expectedIndexes: []int64{5, 1, 4, 2, 3}, + }, + { + keyword: "1", + expectedIndexes: []int64{1}, + }, + { + keyword: "issue", + expectedIndexes: []int64{4, 1, 2, 3}, + }, + { + keyword: "pull", + expectedIndexes: []int64{5}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.keyword, func(t *testing.T) { + issues, err := GetSuggestion(db.DefaultContext, repo1, testCase.isPull, testCase.keyword) + assert.NoError(t, err) + + issueIndexes := make([]int64, 0, len(issues)) + for _, issue := range issues { + issueIndexes = append(issueIndexes, issue.Index) + } + assert.Equal(t, testCase.expectedIndexes, issueIndexes) + }) + } +} diff --git a/services/lfs/locks.go b/services/lfs/locks.go index 1d464f4a66..264001f0f9 100644 --- a/services/lfs/locks.go +++ b/services/lfs/locks.go @@ -74,10 +74,7 @@ func GetListLockHandler(ctx *context.Context) { } ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) - cursor := ctx.FormInt("cursor") - if cursor < 0 { - cursor = 0 - } + cursor := max(ctx.FormInt("cursor"), 0) limit := ctx.FormInt("limit") if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 { limit = setting.LFS.LocksPagingNum @@ -239,10 +236,7 @@ func VerifyLockHandler(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) - cursor := ctx.FormInt("cursor") - if cursor < 0 { - cursor = 0 - } + cursor := max(ctx.FormInt("cursor"), 0) limit := ctx.FormInt("limit") if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 { limit = setting.LFS.LocksPagingNum diff --git a/services/lfs/server.go b/services/lfs/server.go index a77623fdc1..c44cc35e53 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "net/url" "path" @@ -26,6 +27,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/auth/httpauth" "code.gitea.io/gitea/modules/json" lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" @@ -134,7 +136,9 @@ func DownloadHandler(ctx *context.Context) { } contentLength := toByte + 1 - fromByte - ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10)) + contentLengthStr := strconv.FormatInt(contentLength, 10) + ctx.Resp.Header().Set("Content-Length", contentLengthStr) + ctx.Resp.Header().Set("X-Gitea-LFS-Content-Length", contentLengthStr) // we need this header to make sure it won't be affected by reverse proxy or compression ctx.Resp.Header().Set("Content-Type", "application/octet-stream") filename := ctx.PathParam("filename") @@ -162,11 +166,12 @@ func BatchHandler(ctx *context.Context) { } var isUpload bool - if br.Operation == "upload" { + switch br.Operation { + case "upload": isUpload = true - } else if br.Operation == "download" { + case "download": isUpload = false - } else { + default: log.Trace("Attempt to BATCH with invalid operation: %s", br.Operation) writeStatus(ctx, http.StatusBadRequest) return @@ -199,7 +204,7 @@ func BatchHandler(ctx *context.Context) { exists, err := contentStore.Exists(p) if err != nil { - log.Error("Unable to check if LFS OID[%s] exist. Error: %v", p.Oid, rc.User, rc.Repo, err) + log.Error("Unable to check if LFS object with ID '%s' exists for %s/%s. Error: %v", p.Oid, rc.User, rc.Repo, err) writeStatus(ctx, http.StatusInternalServerError) return } @@ -477,9 +482,7 @@ func buildObjectResponse(rc *requestContext, pointer lfs_module.Pointer, downloa rep.Actions["upload"] = &lfs_module.Link{Href: rc.UploadLink(pointer), Header: header} verifyHeader := make(map[string]string) - for key, value := range header { - verifyHeader[key] = value - } + maps.Copy(verifyHeader, header) // This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662 verifyHeader["Accept"] = lfs_module.AcceptHeader @@ -569,15 +572,15 @@ func handleLFSToken(ctx stdCtx.Context, tokenSHA string, target *repo_model.Repo claims, claimsOk := token.Claims.(*Claims) if !token.Valid || !claimsOk { - return nil, fmt.Errorf("invalid token claim") + return nil, errors.New("invalid token claim") } if claims.RepoID != target.ID { - return nil, fmt.Errorf("invalid token claim") + return nil, errors.New("invalid token claim") } if mode == perm_model.AccessModeWrite && claims.Op != "upload" { - return nil, fmt.Errorf("invalid token claim") + return nil, errors.New("invalid token claim") } u, err := user_model.GetUserByID(ctx, claims.UserID) @@ -590,21 +593,13 @@ func handleLFSToken(ctx stdCtx.Context, tokenSHA string, target *repo_model.Repo func parseToken(ctx stdCtx.Context, authorization string, target *repo_model.Repository, mode perm_model.AccessMode) (*user_model.User, error) { if authorization == "" { - return nil, fmt.Errorf("no token") - } - - parts := strings.SplitN(authorization, " ", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("no token") + return nil, errors.New("no token") } - tokenSHA := parts[1] - switch strings.ToLower(parts[0]) { - case "bearer": - fallthrough - case "token": - return handleLFSToken(ctx, tokenSHA, target, mode) + parsed, ok := httpauth.ParseAuthorizationHeader(authorization) + if !ok || parsed.BearerToken == nil { + return nil, errors.New("token not found") } - return nil, fmt.Errorf("token not found") + return handleLFSToken(ctx, parsed.BearerToken.Token, target, mode) } func requireAuth(ctx *context.Context) { diff --git a/services/mailer/mail.go b/services/mailer/mail.go index 52e19bde6f..aa51cbdbcf 100644 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -7,46 +7,30 @@ package mailer import ( "bytes" "context" + "encoding/base64" + "errors" "fmt" "html/template" + "io" "mime" "regexp" - "strconv" "strings" texttmpl "text/template" - "time" - activities_model "code.gitea.io/gitea/models/activities" - issues_model "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/models/renderhelper" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/emoji" + "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/templates" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/translation" - incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/typesniffer" sender_service "code.gitea.io/gitea/services/mailer/sender" - "code.gitea.io/gitea/services/mailer/token" -) - -const ( - mailAuthActivate templates.TplName = "auth/activate" - mailAuthActivateEmail templates.TplName = "auth/activate_email" - mailAuthResetPassword templates.TplName = "auth/reset_passwd" - mailAuthRegisterNotify templates.TplName = "auth/register_notify" - - mailNotifyCollaborator templates.TplName = "notify/collaborator" - mailRepoTransferNotify templates.TplName = "notify/repo_transfer" - - // There's no actual limit for subject in RFC 5322 - mailMaxSubjectRunes = 256 + "golang.org/x/net/html" ) +const mailMaxSubjectRunes = 256 // There's no actual limit for subject in RFC 5322 + var ( bodyTemplates *template.Template subjectTemplates *texttmpl.Template @@ -62,475 +46,114 @@ func SendTestMail(email string) error { return sender_service.Send(sender, sender_service.NewMessage(email, "Gitea Test Email!", "Gitea Test Email!")) } -// sendUserMail sends a mail to the user -func sendUserMail(language string, u *user_model.User, tpl templates.TplName, code, subject, info string) { - locale := translation.NewLocale(language) - data := map[string]any{ - "locale": locale, - "DisplayName": u.DisplayName(), - "ActiveCodeLives": timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, locale), - "ResetPwdCodeLives": timeutil.MinutesToFriendly(setting.Service.ResetPwdCodeLives, locale), - "Code": code, - "Language": locale.Language(), - } - - var content bytes.Buffer - - if err := bodyTemplates.ExecuteTemplate(&content, string(tpl), data); err != nil { - log.Error("Template: %v", err) - return - } - - msg := sender_service.NewMessage(u.EmailTo(), subject, content.String()) - msg.Info = fmt.Sprintf("UID: %d, %s", u.ID, info) - - SendAsync(msg) -} - -// SendActivateAccountMail sends an activation mail to the user (new user registration) -func SendActivateAccountMail(locale translation.Locale, u *user_model.User) { - if setting.MailService == nil { - // No mail service configured - return - } - opts := &user_model.TimeLimitCodeOptions{Purpose: user_model.TimeLimitCodeActivateAccount} - sendUserMail(locale.Language(), u, mailAuthActivate, user_model.GenerateUserTimeLimitCode(opts, u), locale.TrString("mail.activate_account"), "activate account") -} - -// SendResetPasswordMail sends a password reset mail to the user -func SendResetPasswordMail(u *user_model.User) { - if setting.MailService == nil { - // No mail service configured - return - } - locale := translation.NewLocale(u.Language) - opts := &user_model.TimeLimitCodeOptions{Purpose: user_model.TimeLimitCodeResetPassword} - sendUserMail(u.Language, u, mailAuthResetPassword, user_model.GenerateUserTimeLimitCode(opts, u), locale.TrString("mail.reset_password"), "recover account") -} - -// SendActivateEmailMail sends confirmation email to confirm new email address -func SendActivateEmailMail(u *user_model.User, email string) { - if setting.MailService == nil { - // No mail service configured - return - } - locale := translation.NewLocale(u.Language) - opts := &user_model.TimeLimitCodeOptions{Purpose: user_model.TimeLimitCodeActivateEmail, NewEmail: email} - data := map[string]any{ - "locale": locale, - "DisplayName": u.DisplayName(), - "ActiveCodeLives": timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, locale), - "Code": user_model.GenerateUserTimeLimitCode(opts, u), - "Email": email, - "Language": locale.Language(), - } - - var content bytes.Buffer - - if err := bodyTemplates.ExecuteTemplate(&content, string(mailAuthActivateEmail), data); err != nil { - log.Error("Template: %v", err) - return +func sanitizeSubject(subject string) string { + runes := []rune(strings.TrimSpace(subjectRemoveSpaces.ReplaceAllLiteralString(subject, " "))) + if len(runes) > mailMaxSubjectRunes { + runes = runes[:mailMaxSubjectRunes] } - - msg := sender_service.NewMessage(email, locale.TrString("mail.activate_email"), content.String()) - msg.Info = fmt.Sprintf("UID: %d, activate email", u.ID) - - SendAsync(msg) + // Encode non-ASCII characters + return mime.QEncoding.Encode("utf-8", string(runes)) } -// SendRegisterNotifyMail triggers a notify e-mail by admin created a account. -func SendRegisterNotifyMail(u *user_model.User) { - if setting.MailService == nil || !u.IsActive { - // No mail service configured OR user is inactive - return - } - locale := translation.NewLocale(u.Language) - - data := map[string]any{ - "locale": locale, - "DisplayName": u.DisplayName(), - "Username": u.Name, - "Language": locale.Language(), - } - - var content bytes.Buffer - - if err := bodyTemplates.ExecuteTemplate(&content, string(mailAuthRegisterNotify), data); err != nil { - log.Error("Template: %v", err) - return - } - - msg := sender_service.NewMessage(u.EmailTo(), locale.TrString("mail.register_notify", setting.AppName), content.String()) - msg.Info = fmt.Sprintf("UID: %d, registration notify", u.ID) - - SendAsync(msg) +type mailAttachmentBase64Embedder struct { + doer *user_model.User + repo *repo_model.Repository + maxSize int64 + estimateSize int64 } -// SendCollaboratorMail sends mail notification to new collaborator. -func SendCollaboratorMail(u, doer *user_model.User, repo *repo_model.Repository) { - if setting.MailService == nil || !u.IsActive { - // No mail service configured OR the user is inactive - return - } - locale := translation.NewLocale(u.Language) - repoName := repo.FullName() - - subject := locale.TrString("mail.repo.collaborator.added.subject", doer.DisplayName(), repoName) - data := map[string]any{ - "locale": locale, - "Subject": subject, - "RepoName": repoName, - "Link": repo.HTMLURL(), - "Language": locale.Language(), - } - - var content bytes.Buffer - - if err := bodyTemplates.ExecuteTemplate(&content, string(mailNotifyCollaborator), data); err != nil { - log.Error("Template: %v", err) - return - } - - msg := sender_service.NewMessage(u.EmailTo(), subject, content.String()) - msg.Info = fmt.Sprintf("UID: %d, add collaborator", u.ID) - - SendAsync(msg) +func newMailAttachmentBase64Embedder(doer *user_model.User, repo *repo_model.Repository, maxSize int64) *mailAttachmentBase64Embedder { + return &mailAttachmentBase64Embedder{doer: doer, repo: repo, maxSize: maxSize} } -func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipients []*user_model.User, fromMention bool, info string) ([]*sender_service.Message, error) { - var ( - subject string - link string - prefix string - // Fall back subject for bad templates, make sure subject is never empty - fallback string - reviewComments []*issues_model.Comment - ) - - commentType := issues_model.CommentTypeComment - if ctx.Comment != nil { - commentType = ctx.Comment.Type - link = ctx.Issue.HTMLURL() + "#" + ctx.Comment.HashTag() - } else { - link = ctx.Issue.HTMLURL() - } - - reviewType := issues_model.ReviewTypeComment - if ctx.Comment != nil && ctx.Comment.Review != nil { - reviewType = ctx.Comment.Review.Type - } - - // This is the body of the new issue or comment, not the mail body - rctx := renderhelper.NewRenderContextRepoComment(ctx.Context, ctx.Issue.Repo).WithUseAbsoluteLink(true) - body, err := markdown.RenderString(rctx, - ctx.Content) +func (b64embedder *mailAttachmentBase64Embedder) Base64InlineImages(ctx context.Context, body template.HTML) (template.HTML, error) { + doc, err := html.Parse(strings.NewReader(string(body))) if err != nil { - return nil, err - } - - actType, actName, tplName := actionToTemplate(ctx.Issue, ctx.ActionType, commentType, reviewType) - - if actName != "new" { - prefix = "Re: " - } - fallback = prefix + fallbackMailSubject(ctx.Issue) - - if ctx.Comment != nil && ctx.Comment.Review != nil { - reviewComments = make([]*issues_model.Comment, 0, 10) - for _, lines := range ctx.Comment.Review.CodeComments { - for _, comments := range lines { - reviewComments = append(reviewComments, comments...) + return "", fmt.Errorf("html.Parse failed: %w", err) + } + + b64embedder.estimateSize = int64(len(string(body))) + + var processNode func(*html.Node) + processNode = func(n *html.Node) { + if n.Type == html.ElementNode { + if n.Data == "img" { + for i, attr := range n.Attr { + if attr.Key == "src" { + attachmentSrc := attr.Val + dataURI, err := b64embedder.AttachmentSrcToBase64DataURI(ctx, attachmentSrc) + if err != nil { + // Not an error, just skip. This is probably an image from outside the gitea instance. + log.Trace("Unable to embed attachment %q to mail body: %v", attachmentSrc, err) + } else { + n.Attr[i].Val = dataURI + } + break + } + } } } - } - locale := translation.NewLocale(lang) - - mailMeta := map[string]any{ - "locale": locale, - "FallbackSubject": fallback, - "Body": body, - "Link": link, - "Issue": ctx.Issue, - "Comment": ctx.Comment, - "IsPull": ctx.Issue.IsPull, - "User": ctx.Issue.Repo.MustOwner(ctx), - "Repo": ctx.Issue.Repo.FullName(), - "Doer": ctx.Doer, - "IsMention": fromMention, - "SubjectPrefix": prefix, - "ActionType": actType, - "ActionName": actName, - "ReviewComments": reviewComments, - "Language": locale.Language(), - "CanReply": setting.IncomingEmail.Enabled && commentType != issues_model.CommentTypePullRequestPush, - } - - var mailSubject bytes.Buffer - if err := subjectTemplates.ExecuteTemplate(&mailSubject, tplName, mailMeta); err == nil { - subject = sanitizeSubject(mailSubject.String()) - if subject == "" { - subject = fallback + for c := n.FirstChild; c != nil; c = c.NextSibling { + processNode(c) } - } else { - log.Error("ExecuteTemplate [%s]: %v", tplName+"/subject", err) } - subject = emoji.ReplaceAliases(subject) - - mailMeta["Subject"] = subject - - var mailBody bytes.Buffer - - if err := bodyTemplates.ExecuteTemplate(&mailBody, tplName, mailMeta); err != nil { - log.Error("ExecuteTemplate [%s]: %v", tplName+"/body", err) - } + processNode(doc) - // Make sure to compose independent messages to avoid leaking user emails - msgID := generateMessageIDForIssue(ctx.Issue, ctx.Comment, ctx.ActionType) - reference := generateMessageIDForIssue(ctx.Issue, nil, activities_model.ActionType(0)) - - var replyPayload []byte - if ctx.Comment != nil { - if ctx.Comment.Type.HasMailReplySupport() { - replyPayload, err = incoming_payload.CreateReferencePayload(ctx.Comment) - } - } else { - replyPayload, err = incoming_payload.CreateReferencePayload(ctx.Issue) - } + var buf bytes.Buffer + err = html.Render(&buf, doc) if err != nil { - return nil, err + return "", fmt.Errorf("html.Render failed: %w", err) } - - unsubscribePayload, err := incoming_payload.CreateReferencePayload(ctx.Issue) - if err != nil { - return nil, err - } - - msgs := make([]*sender_service.Message, 0, len(recipients)) - for _, recipient := range recipients { - msg := sender_service.NewMessageFrom( - recipient.Email, - fromDisplayName(ctx.Doer), - setting.MailService.FromEmail, - subject, - mailBody.String(), - ) - msg.Info = fmt.Sprintf("Subject: %s, %s", subject, info) - - msg.SetHeader("Message-ID", msgID) - msg.SetHeader("In-Reply-To", reference) - - references := []string{reference} - listUnsubscribe := []string{"<" + ctx.Issue.HTMLURL() + ">"} - - if setting.IncomingEmail.Enabled { - if replyPayload != nil { - token, err := token.CreateToken(token.ReplyHandlerType, recipient, replyPayload) - if err != nil { - log.Error("CreateToken failed: %v", err) - } else { - replyAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1) - msg.ReplyTo = replyAddress - msg.SetHeader("List-Post", fmt.Sprintf("<mailto:%s>", replyAddress)) - - references = append(references, fmt.Sprintf("<reply-%s@%s>", token, setting.Domain)) - } - } - - token, err := token.CreateToken(token.UnsubscribeHandlerType, recipient, unsubscribePayload) - if err != nil { - log.Error("CreateToken failed: %v", err) - } else { - unsubAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1) - listUnsubscribe = append(listUnsubscribe, "<mailto:"+unsubAddress+">") - } - } - - msg.SetHeader("References", references...) - msg.SetHeader("List-Unsubscribe", listUnsubscribe...) - - for key, value := range generateAdditionalHeaders(ctx, actType, recipient) { - msg.SetHeader(key, value) - } - - msgs = append(msgs, msg) - } - - return msgs, nil + return template.HTML(buf.String()), nil } -func generateMessageIDForIssue(issue *issues_model.Issue, comment *issues_model.Comment, actionType activities_model.ActionType) string { - var path string - if issue.IsPull { - path = "pulls" - } else { - path = "issues" - } - - var extra string - if comment != nil { - extra = fmt.Sprintf("/comment/%d", comment.ID) - } else { - switch actionType { - case activities_model.ActionCloseIssue, activities_model.ActionClosePullRequest: - extra = fmt.Sprintf("/close/%d", time.Now().UnixNano()/1e6) - case activities_model.ActionReopenIssue, activities_model.ActionReopenPullRequest: - extra = fmt.Sprintf("/reopen/%d", time.Now().UnixNano()/1e6) - case activities_model.ActionMergePullRequest, activities_model.ActionAutoMergePullRequest: - extra = fmt.Sprintf("/merge/%d", time.Now().UnixNano()/1e6) - case activities_model.ActionPullRequestReadyForReview: - extra = fmt.Sprintf("/ready/%d", time.Now().UnixNano()/1e6) +func (b64embedder *mailAttachmentBase64Embedder) AttachmentSrcToBase64DataURI(ctx context.Context, attachmentSrc string) (string, error) { + parsedSrc := httplib.ParseGiteaSiteURL(ctx, attachmentSrc) + var attachmentUUID string + if parsedSrc != nil { + var ok bool + attachmentUUID, ok = strings.CutPrefix(parsedSrc.RoutePath, "/attachments/") + if !ok { + attachmentUUID, ok = strings.CutPrefix(parsedSrc.RepoSubPath, "/attachments/") + } + if !ok { + return "", errors.New("not an attachment") } } - - return fmt.Sprintf("<%s/%s/%d%s@%s>", issue.Repo.FullName(), path, issue.Index, extra, setting.Domain) -} - -func generateMessageIDForRelease(release *repo_model.Release) string { - return fmt.Sprintf("<%s/releases/%d@%s>", release.Repo.FullName(), release.ID, setting.Domain) -} - -func generateAdditionalHeaders(ctx *mailCommentContext, reason string, recipient *user_model.User) map[string]string { - repo := ctx.Issue.Repo - - return map[string]string{ - // https://datatracker.ietf.org/doc/html/rfc2919 - "List-ID": fmt.Sprintf("%s <%s.%s.%s>", repo.FullName(), repo.Name, repo.OwnerName, setting.Domain), - - // https://datatracker.ietf.org/doc/html/rfc2369 - "List-Archive": fmt.Sprintf("<%s>", repo.HTMLURL()), - - "X-Mailer": "Gitea", - "X-Gitea-Reason": reason, - "X-Gitea-Sender": ctx.Doer.Name, - "X-Gitea-Recipient": recipient.Name, - "X-Gitea-Recipient-Address": recipient.Email, - "X-Gitea-Repository": repo.Name, - "X-Gitea-Repository-Path": repo.FullName(), - "X-Gitea-Repository-Link": repo.HTMLURL(), - "X-Gitea-Issue-ID": strconv.FormatInt(ctx.Issue.Index, 10), - "X-Gitea-Issue-Link": ctx.Issue.HTMLURL(), - - "X-GitHub-Reason": reason, - "X-GitHub-Sender": ctx.Doer.Name, - "X-GitHub-Recipient": recipient.Name, - "X-GitHub-Recipient-Address": recipient.Email, - - "X-GitLab-NotificationReason": reason, - "X-GitLab-Project": repo.Name, - "X-GitLab-Project-Path": repo.FullName(), - "X-GitLab-Issue-IID": strconv.FormatInt(ctx.Issue.Index, 10), - } -} - -func sanitizeSubject(subject string) string { - runes := []rune(strings.TrimSpace(subjectRemoveSpaces.ReplaceAllLiteralString(subject, " "))) - if len(runes) > mailMaxSubjectRunes { - runes = runes[:mailMaxSubjectRunes] + attachment, err := repo_model.GetAttachmentByUUID(ctx, attachmentUUID) + if err != nil { + return "", err } - // Encode non-ASCII characters - return mime.QEncoding.Encode("utf-8", string(runes)) -} -// SendIssueAssignedMail composes and sends issue assigned email -func SendIssueAssignedMail(ctx context.Context, issue *issues_model.Issue, doer *user_model.User, content string, comment *issues_model.Comment, recipients []*user_model.User) error { - if setting.MailService == nil { - // No mail service configured - return nil + if attachment.RepoID != b64embedder.repo.ID { + return "", errors.New("attachment does not belong to the repository") } - - if err := issue.LoadRepo(ctx); err != nil { - log.Error("Unable to load repo [%d] for issue #%d [%d]. Error: %v", issue.RepoID, issue.Index, issue.ID, err) - return err + if attachment.Size+b64embedder.estimateSize > b64embedder.maxSize { + return "", errors.New("total embedded images exceed max limit") } - langMap := make(map[string][]*user_model.User) - for _, user := range recipients { - if !user.IsActive { - // don't send emails to inactive users - continue - } - langMap[user.Language] = append(langMap[user.Language], user) + fr, err := storage.Attachments.Open(attachment.RelativePath()) + if err != nil { + return "", err } + defer fr.Close() - for lang, tos := range langMap { - msgs, err := composeIssueCommentMessages(&mailCommentContext{ - Context: ctx, - Issue: issue, - Doer: doer, - ActionType: activities_model.ActionType(0), - Content: content, - Comment: comment, - }, lang, tos, false, "issue assigned") - if err != nil { - return err - } - SendAsync(msgs...) + lr := &io.LimitedReader{R: fr, N: b64embedder.maxSize + 1} + content, err := io.ReadAll(lr) + if err != nil { + return "", fmt.Errorf("LimitedReader ReadAll: %w", err) } - return nil -} -// actionToTemplate returns the type and name of the action facing the user -// (slightly different from activities_model.ActionType) and the name of the template to use (based on availability) -func actionToTemplate(issue *issues_model.Issue, actionType activities_model.ActionType, - commentType issues_model.CommentType, reviewType issues_model.ReviewType, -) (typeName, name, template string) { - if issue.IsPull { - typeName = "pull" - } else { - typeName = "issue" - } - switch actionType { - case activities_model.ActionCreateIssue, activities_model.ActionCreatePullRequest: - name = "new" - case activities_model.ActionCommentIssue, activities_model.ActionCommentPull: - name = "comment" - case activities_model.ActionCloseIssue, activities_model.ActionClosePullRequest: - name = "close" - case activities_model.ActionReopenIssue, activities_model.ActionReopenPullRequest: - name = "reopen" - case activities_model.ActionMergePullRequest, activities_model.ActionAutoMergePullRequest: - name = "merge" - case activities_model.ActionPullReviewDismissed: - name = "review_dismissed" - case activities_model.ActionPullRequestReadyForReview: - name = "ready_for_review" - default: - switch commentType { - case issues_model.CommentTypeReview: - switch reviewType { - case issues_model.ReviewTypeApprove: - name = "approve" - case issues_model.ReviewTypeReject: - name = "reject" - default: - name = "review" - } - case issues_model.CommentTypeCode: - name = "code" - case issues_model.CommentTypeAssignees: - name = "assigned" - case issues_model.CommentTypePullRequestPush: - name = "push" - default: - name = "default" - } + mimeType := typesniffer.DetectContentType(content) + if !mimeType.IsImage() { + return "", errors.New("not an image") } - template = typeName + "/" + name - ok := bodyTemplates.Lookup(template) != nil - if !ok && typeName != "issue" { - template = "issue/" + name - ok = bodyTemplates.Lookup(template) != nil - } - if !ok { - template = typeName + "/default" - ok = bodyTemplates.Lookup(template) != nil - } - if !ok { - template = "issue/default" - } - return typeName, name, template + encoded := base64.StdEncoding.EncodeToString(content) + dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType.GetMimeType(), encoded) + b64embedder.estimateSize += int64(len(dataURI)) + return dataURI, nil } func fromDisplayName(u *user_model.User) string { diff --git a/services/mailer/mail_comment.go b/services/mailer/mail_comment.go index 1812441d5a..e8d12e429d 100644 --- a/services/mailer/mail_comment.go +++ b/services/mailer/mail_comment.go @@ -25,9 +25,8 @@ func MailParticipantsComment(ctx context.Context, c *issues_model.Comment, opTyp if c.Type == issues_model.CommentTypePullRequestPush { content = "" } - if err := mailIssueCommentToParticipants( - &mailCommentContext{ - Context: ctx, + if err := mailIssueCommentToParticipants(ctx, + &mailComment{ Issue: issue, Doer: c.Poster, ActionType: opType, @@ -48,9 +47,8 @@ func MailMentionsComment(ctx context.Context, pr *issues_model.PullRequest, c *i visited := make(container.Set[int64], len(mentions)+1) visited.Add(c.Poster.ID) - if err = mailIssueCommentBatch( - &mailCommentContext{ - Context: ctx, + if err = mailIssueCommentBatch(ctx, + &mailComment{ Issue: pr.Issue, Doer: c.Poster, ActionType: activities_model.ActionCommentPull, diff --git a/services/mailer/mail_issue.go b/services/mailer/mail_issue.go index fab3315be2..b854d61a1a 100644 --- a/services/mailer/mail_issue.go +++ b/services/mailer/mail_issue.go @@ -18,38 +18,21 @@ import ( "code.gitea.io/gitea/modules/setting" ) -func fallbackMailSubject(issue *issues_model.Issue) string { - return fmt.Sprintf("[%s] %s (#%d)", issue.Repo.FullName(), issue.Title, issue.Index) -} - -type mailCommentContext struct { - context.Context - Issue *issues_model.Issue - Doer *user_model.User - ActionType activities_model.ActionType - Content string - Comment *issues_model.Comment - ForceDoerNotification bool -} - -const ( - // MailBatchSize set the batch size used in mailIssueCommentBatch - MailBatchSize = 100 -) +const MailBatchSize = 100 // batch size used in mailIssueCommentBatch // mailIssueCommentToParticipants can be used for both new issue creation and comment. // This function sends two list of emails: // 1. Repository watchers (except for WIP pull requests) and users who are participated in comments. // 2. Users who are not in 1. but get mentioned in current issue/comment. -func mailIssueCommentToParticipants(ctx *mailCommentContext, mentions []*user_model.User) error { +func mailIssueCommentToParticipants(ctx context.Context, comment *mailComment, mentions []*user_model.User) error { // Required by the mail composer; make sure to load these before calling the async function - if err := ctx.Issue.LoadRepo(ctx); err != nil { + if err := comment.Issue.LoadRepo(ctx); err != nil { return fmt.Errorf("LoadRepo: %w", err) } - if err := ctx.Issue.LoadPoster(ctx); err != nil { + if err := comment.Issue.LoadPoster(ctx); err != nil { return fmt.Errorf("LoadPoster: %w", err) } - if err := ctx.Issue.LoadPullRequest(ctx); err != nil { + if err := comment.Issue.LoadPullRequest(ctx); err != nil { return fmt.Errorf("LoadPullRequest: %w", err) } @@ -57,35 +40,35 @@ func mailIssueCommentToParticipants(ctx *mailCommentContext, mentions []*user_mo unfiltered := make([]int64, 1, 64) // =========== Original poster =========== - unfiltered[0] = ctx.Issue.PosterID + unfiltered[0] = comment.Issue.PosterID // =========== Assignees =========== - ids, err := issues_model.GetAssigneeIDsByIssue(ctx, ctx.Issue.ID) + ids, err := issues_model.GetAssigneeIDsByIssue(ctx, comment.Issue.ID) if err != nil { - return fmt.Errorf("GetAssigneeIDsByIssue(%d): %w", ctx.Issue.ID, err) + return fmt.Errorf("GetAssigneeIDsByIssue(%d): %w", comment.Issue.ID, err) } unfiltered = append(unfiltered, ids...) // =========== Participants (i.e. commenters, reviewers) =========== - ids, err = issues_model.GetParticipantsIDsByIssueID(ctx, ctx.Issue.ID) + ids, err = issues_model.GetParticipantsIDsByIssueID(ctx, comment.Issue.ID) if err != nil { - return fmt.Errorf("GetParticipantsIDsByIssueID(%d): %w", ctx.Issue.ID, err) + return fmt.Errorf("GetParticipantsIDsByIssueID(%d): %w", comment.Issue.ID, err) } unfiltered = append(unfiltered, ids...) // =========== Issue watchers =========== - ids, err = issues_model.GetIssueWatchersIDs(ctx, ctx.Issue.ID, true) + ids, err = issues_model.GetIssueWatchersIDs(ctx, comment.Issue.ID, true) if err != nil { - return fmt.Errorf("GetIssueWatchersIDs(%d): %w", ctx.Issue.ID, err) + return fmt.Errorf("GetIssueWatchersIDs(%d): %w", comment.Issue.ID, err) } unfiltered = append(unfiltered, ids...) // =========== Repo watchers =========== // Make repo watchers last, since it's likely the list with the most users - if !(ctx.Issue.IsPull && ctx.Issue.PullRequest.IsWorkInProgress(ctx) && ctx.ActionType != activities_model.ActionCreatePullRequest) { - ids, err = repo_model.GetRepoWatchersIDs(ctx, ctx.Issue.RepoID) + if !(comment.Issue.IsPull && comment.Issue.PullRequest.IsWorkInProgress(ctx) && comment.ActionType != activities_model.ActionCreatePullRequest) { + ids, err = repo_model.GetRepoWatchersIDs(ctx, comment.Issue.RepoID) if err != nil { - return fmt.Errorf("GetRepoWatchersIDs(%d): %w", ctx.Issue.RepoID, err) + return fmt.Errorf("GetRepoWatchersIDs(%d): %w", comment.Issue.RepoID, err) } unfiltered = append(ids, unfiltered...) } @@ -93,36 +76,36 @@ func mailIssueCommentToParticipants(ctx *mailCommentContext, mentions []*user_mo visited := make(container.Set[int64], len(unfiltered)+len(mentions)+1) // Avoid mailing the doer - if ctx.Doer.EmailNotificationsPreference != user_model.EmailNotificationsAndYourOwn && !ctx.ForceDoerNotification { - visited.Add(ctx.Doer.ID) + if comment.Doer.EmailNotificationsPreference != user_model.EmailNotificationsAndYourOwn && !comment.ForceDoerNotification { + visited.Add(comment.Doer.ID) } // =========== Mentions =========== - if err = mailIssueCommentBatch(ctx, mentions, visited, true); err != nil { + if err = mailIssueCommentBatch(ctx, comment, mentions, visited, true); err != nil { return fmt.Errorf("mailIssueCommentBatch() mentions: %w", err) } // Avoid mailing explicit unwatched - ids, err = issues_model.GetIssueWatchersIDs(ctx, ctx.Issue.ID, false) + ids, err = issues_model.GetIssueWatchersIDs(ctx, comment.Issue.ID, false) if err != nil { - return fmt.Errorf("GetIssueWatchersIDs(%d): %w", ctx.Issue.ID, err) + return fmt.Errorf("GetIssueWatchersIDs(%d): %w", comment.Issue.ID, err) } visited.AddMultiple(ids...) - unfilteredUsers, err := user_model.GetMaileableUsersByIDs(ctx, unfiltered, false) + unfilteredUsers, err := user_model.GetMailableUsersByIDs(ctx, unfiltered, false) if err != nil { return err } - if err = mailIssueCommentBatch(ctx, unfilteredUsers, visited, false); err != nil { + if err = mailIssueCommentBatch(ctx, comment, unfilteredUsers, visited, false); err != nil { return fmt.Errorf("mailIssueCommentBatch(): %w", err) } return nil } -func mailIssueCommentBatch(ctx *mailCommentContext, users []*user_model.User, visited container.Set[int64], fromMention bool) error { +func mailIssueCommentBatch(ctx context.Context, comment *mailComment, users []*user_model.User, visited container.Set[int64], fromMention bool) error { checkUnit := unit.TypeIssues - if ctx.Issue.IsPull { + if comment.Issue.IsPull { checkUnit = unit.TypePullRequests } @@ -146,7 +129,7 @@ func mailIssueCommentBatch(ctx *mailCommentContext, users []*user_model.User, vi } // test if this user is allowed to see the issue/pull - if !access_model.CheckRepoUnitUser(ctx, ctx.Issue.Repo, user, checkUnit) { + if !access_model.CheckRepoUnitUser(ctx, comment.Issue.Repo, user, checkUnit) { continue } @@ -158,7 +141,7 @@ func mailIssueCommentBatch(ctx *mailCommentContext, users []*user_model.User, vi // working backwards from the last (possibly) incomplete batch. If len(receivers) can be 0 this // starting condition will need to be changed slightly for i := ((len(receivers) - 1) / MailBatchSize) * MailBatchSize; i >= 0; i -= MailBatchSize { - msgs, err := composeIssueCommentMessages(ctx, lang, receivers[i:], fromMention, "issue comments") + msgs, err := composeIssueCommentMessages(ctx, comment, lang, receivers[i:], fromMention, "issue comments") if err != nil { return err } @@ -185,9 +168,8 @@ func MailParticipants(ctx context.Context, issue *issues_model.Issue, doer *user content = "" } forceDoerNotification := opType == activities_model.ActionAutoMergePullRequest - if err := mailIssueCommentToParticipants( - &mailCommentContext{ - Context: ctx, + if err := mailIssueCommentToParticipants(ctx, + &mailComment{ Issue: issue, Doer: doer, ActionType: opType, @@ -199,3 +181,40 @@ func MailParticipants(ctx context.Context, issue *issues_model.Issue, doer *user } return nil } + +// SendIssueAssignedMail composes and sends issue assigned email +func SendIssueAssignedMail(ctx context.Context, issue *issues_model.Issue, doer *user_model.User, content string, comment *issues_model.Comment, recipients []*user_model.User) error { + if setting.MailService == nil { + // No mail service configured + return nil + } + + if err := issue.LoadRepo(ctx); err != nil { + log.Error("Unable to load repo [%d] for issue #%d [%d]. Error: %v", issue.RepoID, issue.Index, issue.ID, err) + return err + } + + langMap := make(map[string][]*user_model.User) + for _, user := range recipients { + if !user.IsActive { + // don't send emails to inactive users + continue + } + langMap[user.Language] = append(langMap[user.Language], user) + } + + for lang, tos := range langMap { + msgs, err := composeIssueCommentMessages(ctx, &mailComment{ + Issue: issue, + Doer: doer, + ActionType: activities_model.ActionType(0), + Content: content, + Comment: comment, + }, lang, tos, false, "issue assigned") + if err != nil { + return err + } + SendAsync(msgs...) + } + return nil +} diff --git a/services/mailer/mail_issue_common.go b/services/mailer/mail_issue_common.go new file mode 100644 index 0000000000..ebfd52162c --- /dev/null +++ b/services/mailer/mail_issue_common.go @@ -0,0 +1,336 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package mailer + +import ( + "bytes" + "context" + "fmt" + "strconv" + "strings" + "time" + + activities_model "code.gitea.io/gitea/models/activities" + issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/renderhelper" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/emoji" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup/markdown" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation" + incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" + sender_service "code.gitea.io/gitea/services/mailer/sender" + "code.gitea.io/gitea/services/mailer/token" +) + +// maxEmailBodySize is the approximate maximum size of an email body in bytes +// Many e-mail service providers have limitations on the size of the email body, it's usually from 10MB to 25MB +const maxEmailBodySize = 9_000_000 + +func fallbackMailSubject(issue *issues_model.Issue) string { + return fmt.Sprintf("[%s] %s (#%d)", issue.Repo.FullName(), issue.Title, issue.Index) +} + +type mailComment struct { + Issue *issues_model.Issue + Doer *user_model.User + ActionType activities_model.ActionType + Content string + Comment *issues_model.Comment + ForceDoerNotification bool +} + +func composeIssueCommentMessages(ctx context.Context, comment *mailComment, lang string, recipients []*user_model.User, fromMention bool, info string) ([]*sender_service.Message, error) { + var ( + subject string + link string + prefix string + // Fall back subject for bad templates, make sure subject is never empty + fallback string + reviewComments []*issues_model.Comment + ) + + commentType := issues_model.CommentTypeComment + if comment.Comment != nil { + commentType = comment.Comment.Type + link = comment.Issue.HTMLURL() + "#" + comment.Comment.HashTag() + } else { + link = comment.Issue.HTMLURL() + } + + reviewType := issues_model.ReviewTypeComment + if comment.Comment != nil && comment.Comment.Review != nil { + reviewType = comment.Comment.Review.Type + } + + // This is the body of the new issue or comment, not the mail body + rctx := renderhelper.NewRenderContextRepoComment(ctx, comment.Issue.Repo).WithUseAbsoluteLink(true) + body, err := markdown.RenderString(rctx, comment.Content) + if err != nil { + return nil, err + } + + if setting.MailService.EmbedAttachmentImages { + attEmbedder := newMailAttachmentBase64Embedder(comment.Doer, comment.Issue.Repo, maxEmailBodySize) + bodyAfterEmbedding, err := attEmbedder.Base64InlineImages(ctx, body) + if err != nil { + log.Error("Failed to embed images in mail body: %v", err) + } else { + body = bodyAfterEmbedding + } + } + actType, actName, tplName := actionToTemplate(comment.Issue, comment.ActionType, commentType, reviewType) + + if actName != "new" { + prefix = "Re: " + } + fallback = prefix + fallbackMailSubject(comment.Issue) + + if comment.Comment != nil && comment.Comment.Review != nil { + reviewComments = make([]*issues_model.Comment, 0, 10) + for _, lines := range comment.Comment.Review.CodeComments { + for _, comments := range lines { + reviewComments = append(reviewComments, comments...) + } + } + } + locale := translation.NewLocale(lang) + + mailMeta := map[string]any{ + "locale": locale, + "FallbackSubject": fallback, + "Body": body, + "Link": link, + "Issue": comment.Issue, + "Comment": comment.Comment, + "IsPull": comment.Issue.IsPull, + "User": comment.Issue.Repo.MustOwner(ctx), + "Repo": comment.Issue.Repo.FullName(), + "Doer": comment.Doer, + "IsMention": fromMention, + "SubjectPrefix": prefix, + "ActionType": actType, + "ActionName": actName, + "ReviewComments": reviewComments, + "Language": locale.Language(), + "CanReply": setting.IncomingEmail.Enabled && commentType != issues_model.CommentTypePullRequestPush, + } + + var mailSubject bytes.Buffer + if err := subjectTemplates.ExecuteTemplate(&mailSubject, tplName, mailMeta); err == nil { + subject = sanitizeSubject(mailSubject.String()) + if subject == "" { + subject = fallback + } + } else { + log.Error("ExecuteTemplate [%s]: %v", tplName+"/subject", err) + } + + subject = emoji.ReplaceAliases(subject) + + mailMeta["Subject"] = subject + + var mailBody bytes.Buffer + + if err := bodyTemplates.ExecuteTemplate(&mailBody, tplName, mailMeta); err != nil { + log.Error("ExecuteTemplate [%s]: %v", tplName+"/body", err) + } + + // Make sure to compose independent messages to avoid leaking user emails + msgID := generateMessageIDForIssue(comment.Issue, comment.Comment, comment.ActionType) + reference := generateMessageIDForIssue(comment.Issue, nil, activities_model.ActionType(0)) + + var replyPayload []byte + if comment.Comment != nil { + if comment.Comment.Type.HasMailReplySupport() { + replyPayload, err = incoming_payload.CreateReferencePayload(comment.Comment) + } + } else { + replyPayload, err = incoming_payload.CreateReferencePayload(comment.Issue) + } + if err != nil { + return nil, err + } + + unsubscribePayload, err := incoming_payload.CreateReferencePayload(comment.Issue) + if err != nil { + return nil, err + } + + msgs := make([]*sender_service.Message, 0, len(recipients)) + for _, recipient := range recipients { + msg := sender_service.NewMessageFrom( + recipient.Email, + fromDisplayName(comment.Doer), + setting.MailService.FromEmail, + subject, + mailBody.String(), + ) + msg.Info = fmt.Sprintf("Subject: %s, %s", subject, info) + + msg.SetHeader("Message-ID", msgID) + msg.SetHeader("In-Reply-To", reference) + + references := []string{reference} + listUnsubscribe := []string{"<" + comment.Issue.HTMLURL() + ">"} + + if setting.IncomingEmail.Enabled { + if replyPayload != nil { + token, err := token.CreateToken(token.ReplyHandlerType, recipient, replyPayload) + if err != nil { + log.Error("CreateToken failed: %v", err) + } else { + replyAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1) + msg.ReplyTo = replyAddress + msg.SetHeader("List-Post", fmt.Sprintf("<mailto:%s>", replyAddress)) + + references = append(references, fmt.Sprintf("<reply-%s@%s>", token, setting.Domain)) + } + } + + token, err := token.CreateToken(token.UnsubscribeHandlerType, recipient, unsubscribePayload) + if err != nil { + log.Error("CreateToken failed: %v", err) + } else { + unsubAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1) + listUnsubscribe = append(listUnsubscribe, "<mailto:"+unsubAddress+">") + } + } + + msg.SetHeader("References", references...) + msg.SetHeader("List-Unsubscribe", listUnsubscribe...) + + for key, value := range generateAdditionalHeaders(comment, actType, recipient) { + msg.SetHeader(key, value) + } + + msgs = append(msgs, msg) + } + + return msgs, nil +} + +// actionToTemplate returns the type and name of the action facing the user +// (slightly different from activities_model.ActionType) and the name of the template to use (based on availability) +func actionToTemplate(issue *issues_model.Issue, actionType activities_model.ActionType, + commentType issues_model.CommentType, reviewType issues_model.ReviewType, +) (typeName, name, template string) { + if issue.IsPull { + typeName = "pull" + } else { + typeName = "issue" + } + switch actionType { + case activities_model.ActionCreateIssue, activities_model.ActionCreatePullRequest: + name = "new" + case activities_model.ActionCommentIssue, activities_model.ActionCommentPull: + name = "comment" + case activities_model.ActionCloseIssue, activities_model.ActionClosePullRequest: + name = "close" + case activities_model.ActionReopenIssue, activities_model.ActionReopenPullRequest: + name = "reopen" + case activities_model.ActionMergePullRequest, activities_model.ActionAutoMergePullRequest: + name = "merge" + case activities_model.ActionPullReviewDismissed: + name = "review_dismissed" + case activities_model.ActionPullRequestReadyForReview: + name = "ready_for_review" + default: + switch commentType { + case issues_model.CommentTypeReview: + switch reviewType { + case issues_model.ReviewTypeApprove: + name = "approve" + case issues_model.ReviewTypeReject: + name = "reject" + default: + name = "review" + } + case issues_model.CommentTypeCode: + name = "code" + case issues_model.CommentTypeAssignees: + name = "assigned" + case issues_model.CommentTypePullRequestPush: + name = "push" + default: + name = "default" + } + } + + template = typeName + "/" + name + ok := bodyTemplates.Lookup(template) != nil + if !ok && typeName != "issue" { + template = "issue/" + name + ok = bodyTemplates.Lookup(template) != nil + } + if !ok { + template = typeName + "/default" + ok = bodyTemplates.Lookup(template) != nil + } + if !ok { + template = "issue/default" + } + return typeName, name, template +} + +func generateMessageIDForIssue(issue *issues_model.Issue, comment *issues_model.Comment, actionType activities_model.ActionType) string { + var path string + if issue.IsPull { + path = "pulls" + } else { + path = "issues" + } + + var extra string + if comment != nil { + extra = fmt.Sprintf("/comment/%d", comment.ID) + } else { + switch actionType { + case activities_model.ActionCloseIssue, activities_model.ActionClosePullRequest: + extra = fmt.Sprintf("/close/%d", time.Now().UnixNano()/1e6) + case activities_model.ActionReopenIssue, activities_model.ActionReopenPullRequest: + extra = fmt.Sprintf("/reopen/%d", time.Now().UnixNano()/1e6) + case activities_model.ActionMergePullRequest, activities_model.ActionAutoMergePullRequest: + extra = fmt.Sprintf("/merge/%d", time.Now().UnixNano()/1e6) + case activities_model.ActionPullRequestReadyForReview: + extra = fmt.Sprintf("/ready/%d", time.Now().UnixNano()/1e6) + } + } + + return fmt.Sprintf("<%s/%s/%d%s@%s>", issue.Repo.FullName(), path, issue.Index, extra, setting.Domain) +} + +func generateAdditionalHeaders(ctx *mailComment, reason string, recipient *user_model.User) map[string]string { + repo := ctx.Issue.Repo + + return map[string]string{ + // https://datatracker.ietf.org/doc/html/rfc2919 + "List-ID": fmt.Sprintf("%s <%s.%s.%s>", repo.FullName(), repo.Name, repo.OwnerName, setting.Domain), + + // https://datatracker.ietf.org/doc/html/rfc2369 + "List-Archive": fmt.Sprintf("<%s>", repo.HTMLURL()), + + "X-Mailer": "Gitea", + "X-Gitea-Reason": reason, + "X-Gitea-Sender": ctx.Doer.Name, + "X-Gitea-Recipient": recipient.Name, + "X-Gitea-Recipient-Address": recipient.Email, + "X-Gitea-Repository": repo.Name, + "X-Gitea-Repository-Path": repo.FullName(), + "X-Gitea-Repository-Link": repo.HTMLURL(), + "X-Gitea-Issue-ID": strconv.FormatInt(ctx.Issue.Index, 10), + "X-Gitea-Issue-Link": ctx.Issue.HTMLURL(), + + "X-GitHub-Reason": reason, + "X-GitHub-Sender": ctx.Doer.Name, + "X-GitHub-Recipient": recipient.Name, + "X-GitHub-Recipient-Address": recipient.Email, + + "X-GitLab-NotificationReason": reason, + "X-GitLab-Project": repo.Name, + "X-GitLab-Project-Path": repo.FullName(), + "X-GitLab-Issue-IID": strconv.FormatInt(ctx.Issue.Index, 10), + } +} diff --git a/services/mailer/mail_release.go b/services/mailer/mail_release.go index 796d63d27a..bfff73c39c 100644 --- a/services/mailer/mail_release.go +++ b/services/mailer/mail_release.go @@ -6,6 +6,7 @@ package mailer import ( "bytes" "context" + "fmt" "code.gitea.io/gitea/models/renderhelper" repo_model "code.gitea.io/gitea/models/repo" @@ -18,9 +19,11 @@ import ( sender_service "code.gitea.io/gitea/services/mailer/sender" ) -const ( - tplNewReleaseMail templates.TplName = "release" -) +const tplNewReleaseMail templates.TplName = "release" + +func generateMessageIDForRelease(release *repo_model.Release) string { + return fmt.Sprintf("<%s/releases/%d@%s>", release.Repo.FullName(), release.ID, setting.Domain) +} // MailNewRelease send new release notify to all repo watchers. func MailNewRelease(ctx context.Context, rel *repo_model.Release) { @@ -35,9 +38,9 @@ func MailNewRelease(ctx context.Context, rel *repo_model.Release) { return } - recipients, err := user_model.GetMaileableUsersByIDs(ctx, watcherIDList, false) + recipients, err := user_model.GetMailableUsersByIDs(ctx, watcherIDList, false) if err != nil { - log.Error("user_model.GetMaileableUsersByIDs: %v", err) + log.Error("user_model.GetMailableUsersByIDs: %v", err) return } diff --git a/services/mailer/mail_repo.go b/services/mailer/mail_repo.go index 5f80654bcd..b6b2d5ca07 100644 --- a/services/mailer/mail_repo.go +++ b/services/mailer/mail_repo.go @@ -12,10 +12,13 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/translation" sender_service "code.gitea.io/gitea/services/mailer/sender" ) +const mailRepoTransferNotify templates.TplName = "notify/repo_transfer" + // SendRepoTransferNotifyMail triggers a notification e-mail when a pending repository transfer was created func SendRepoTransferNotifyMail(ctx context.Context, doer, newOwner *user_model.User, repo *repo_model.Repository) error { if setting.MailService == nil { diff --git a/services/mailer/mail_team_invite.go b/services/mailer/mail_team_invite.go index 5ca44442f3..f4aa788dec 100644 --- a/services/mailer/mail_team_invite.go +++ b/services/mailer/mail_team_invite.go @@ -6,6 +6,7 @@ package mailer import ( "bytes" "context" + "errors" "fmt" "net/url" @@ -18,9 +19,7 @@ import ( sender_service "code.gitea.io/gitea/services/mailer/sender" ) -const ( - tplTeamInviteMail templates.TplName = "team_invite" -) +const tplTeamInviteMail templates.TplName = "team_invite" // MailTeamInvite sends team invites func MailTeamInvite(ctx context.Context, inviter *user_model.User, team *org_model.Team, invite *org_model.TeamInvite) error { @@ -40,10 +39,10 @@ func MailTeamInvite(ctx context.Context, inviter *user_model.User, team *org_mod if err != nil && !user_model.IsErrUserNotExist(err) { return err } else if user != nil && user.ProhibitLogin { - return fmt.Errorf("login is prohibited for the invited user") + return errors.New("login is prohibited for the invited user") } - inviteRedirect := url.QueryEscape(fmt.Sprintf("/org/invite/%s", invite.Token)) + inviteRedirect := url.QueryEscape("/org/invite/" + invite.Token) inviteURL := fmt.Sprintf("%suser/sign_up?redirect_to=%s", setting.AppURL, inviteRedirect) if (err == nil && user != nil) || setting.Service.DisableRegistration || setting.Service.AllowOnlyExternalRegistration { diff --git a/services/mailer/mail_test.go b/services/mailer/mail_test.go index 185b72f069..b15949f352 100644 --- a/services/mailer/mail_test.go +++ b/services/mailer/mail_test.go @@ -6,6 +6,7 @@ package mailer import ( "bytes" "context" + "encoding/base64" "fmt" "html/template" "io" @@ -23,9 +24,13 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/services/attachment" sender_service "code.gitea.io/gitea/services/mailer/sender" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const subjectTpl = ` @@ -53,22 +58,44 @@ const bodyTpl = ` func prepareMailerTest(t *testing.T) (doer *user_model.User, repo *repo_model.Repository, issue *issues_model.Issue, comment *issues_model.Comment) { assert.NoError(t, unittest.PrepareTestDatabase()) - mailService := setting.Mailer{ - From: "test@gitea.com", - } - - setting.MailService = &mailService + setting.MailService = &setting.Mailer{From: "test@gitea.com"} setting.Domain = "localhost" + setting.AppURL = "https://try.gitea.io/" doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1, Owner: doer}) issue = unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 1, Repo: repo, Poster: doer}) - assert.NoError(t, issue.LoadRepo(db.DefaultContext)) comment = unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 2, Issue: issue}) + require.NoError(t, issue.LoadRepo(db.DefaultContext)) return doer, repo, issue, comment } -func TestComposeIssueCommentMessage(t *testing.T) { +func prepareMailerBase64Test(t *testing.T) (doer *user_model.User, repo *repo_model.Repository, issue *issues_model.Issue, att1, att2 *repo_model.Attachment) { + user, repo, issue, comment := prepareMailerTest(t) + setting.MailService.EmbedAttachmentImages = true + + att1, err := attachment.NewAttachment(t.Context(), &repo_model.Attachment{ + RepoID: repo.ID, + IssueID: issue.ID, + UploaderID: user.ID, + CommentID: comment.ID, + Name: "test.png", + }, bytes.NewReader([]byte("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a")), 8) + require.NoError(t, err) + + att2, err = attachment.NewAttachment(t.Context(), &repo_model.Attachment{ + RepoID: repo.ID, + IssueID: issue.ID, + UploaderID: user.ID, + CommentID: comment.ID, + Name: "test.png", + }, bytes.NewReader([]byte("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a"+strings.Repeat("\x00", 1024))), 8+1024) + require.NoError(t, err) + + return user, repo, issue, att1, att2 +} + +func TestComposeIssueComment(t *testing.T) { doer, _, issue, comment := prepareMailerTest(t) markup.Init(&markup.RenderHelperFuncs{ @@ -84,9 +111,8 @@ func TestComposeIssueCommentMessage(t *testing.T) { bodyTemplates = template.Must(template.New("issue/comment").Parse(bodyTpl)) recipients := []*user_model.User{{Name: "Test", Email: "test@gitea.com"}, {Name: "Test2", Email: "test2@gitea.com"}} - msgs, err := composeIssueCommentMessages(&mailCommentContext{ - Context: context.TODO(), // TODO: use a correct context - Issue: issue, Doer: doer, ActionType: activities_model.ActionCommentIssue, + msgs, err := composeIssueCommentMessages(t.Context(), &mailComment{ + Issue: issue, Doer: doer, ActionType: activities_model.ActionCommentIssue, Content: fmt.Sprintf("test @%s %s#%d body", doer.Name, issue.Repo.FullName(), issue.Index), Comment: comment, }, "en-US", recipients, false, "issue comment") @@ -109,7 +135,8 @@ func TestComposeIssueCommentMessage(t *testing.T) { assert.Len(t, gomailMsg.GetGenHeader("List-Unsubscribe"), 2) // url + mailto var buf bytes.Buffer - gomailMsg.WriteTo(&buf) + _, err = gomailMsg.WriteTo(&buf) + require.NoError(t, err) b, err := io.ReadAll(quotedprintable.NewReader(&buf)) assert.NoError(t, err) @@ -123,6 +150,22 @@ func TestComposeIssueCommentMessage(t *testing.T) { assert.Contains(t, string(b), fmt.Sprintf(`href="%s"`, issue.HTMLURL())) } +func TestMailMentionsComment(t *testing.T) { + doer, _, issue, comment := prepareMailerTest(t) + comment.Poster = doer + subjectTemplates = texttmpl.Must(texttmpl.New("issue/comment").Parse(subjectTpl)) + bodyTemplates = template.Must(template.New("issue/comment").Parse(bodyTpl)) + mails := 0 + + defer test.MockVariableValue(&SendAsync, func(msgs ...*sender_service.Message) { + mails = len(msgs) + })() + + err := MailParticipantsComment(t.Context(), comment, activities_model.ActionCommentIssue, issue, []*user_model.User{}) + require.NoError(t, err) + assert.Equal(t, 3, mails) +} + func TestComposeIssueMessage(t *testing.T) { doer, _, issue, _ := prepareMailerTest(t) @@ -130,9 +173,8 @@ func TestComposeIssueMessage(t *testing.T) { bodyTemplates = template.Must(template.New("issue/new").Parse(bodyTpl)) recipients := []*user_model.User{{Name: "Test", Email: "test@gitea.com"}, {Name: "Test2", Email: "test2@gitea.com"}} - msgs, err := composeIssueCommentMessages(&mailCommentContext{ - Context: context.TODO(), // TODO: use a correct context - Issue: issue, Doer: doer, ActionType: activities_model.ActionCreateIssue, + msgs, err := composeIssueCommentMessages(t.Context(), &mailComment{ + Issue: issue, Doer: doer, ActionType: activities_model.ActionCreateIssue, Content: "test body", }, "en-US", recipients, false, "issue create") assert.NoError(t, err) @@ -177,32 +219,28 @@ func TestTemplateSelection(t *testing.T) { assert.Contains(t, wholemsg, expBody) } - msg := testComposeIssueCommentMessage(t, &mailCommentContext{ - Context: context.TODO(), // TODO: use a correct context - Issue: issue, Doer: doer, ActionType: activities_model.ActionCreateIssue, + msg := testComposeIssueCommentMessage(t, &mailComment{ + Issue: issue, Doer: doer, ActionType: activities_model.ActionCreateIssue, Content: "test body", }, recipients, false, "TestTemplateSelection") expect(t, msg, "issue/new/subject", "issue/new/body") - msg = testComposeIssueCommentMessage(t, &mailCommentContext{ - Context: context.TODO(), // TODO: use a correct context - Issue: issue, Doer: doer, ActionType: activities_model.ActionCommentIssue, + msg = testComposeIssueCommentMessage(t, &mailComment{ + Issue: issue, Doer: doer, ActionType: activities_model.ActionCommentIssue, Content: "test body", Comment: comment, }, recipients, false, "TestTemplateSelection") expect(t, msg, "issue/default/subject", "issue/default/body") pull := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 2, Repo: repo, Poster: doer}) comment = unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 4, Issue: pull}) - msg = testComposeIssueCommentMessage(t, &mailCommentContext{ - Context: context.TODO(), // TODO: use a correct context - Issue: pull, Doer: doer, ActionType: activities_model.ActionCommentPull, + msg = testComposeIssueCommentMessage(t, &mailComment{ + Issue: pull, Doer: doer, ActionType: activities_model.ActionCommentPull, Content: "test body", Comment: comment, }, recipients, false, "TestTemplateSelection") expect(t, msg, "pull/comment/subject", "pull/comment/body") - msg = testComposeIssueCommentMessage(t, &mailCommentContext{ - Context: context.TODO(), // TODO: use a correct context - Issue: issue, Doer: doer, ActionType: activities_model.ActionCloseIssue, + msg = testComposeIssueCommentMessage(t, &mailComment{ + Issue: issue, Doer: doer, ActionType: activities_model.ActionCloseIssue, Content: "test body", Comment: comment, }, recipients, false, "TestTemplateSelection") expect(t, msg, "Re: [user2/repo1] issue1 (#1)", "issue/close/body") @@ -219,9 +257,8 @@ func TestTemplateServices(t *testing.T) { bodyTemplates = template.Must(template.New("issue/default").Parse(tplBody)) recipients := []*user_model.User{{Name: "Test", Email: "test@gitea.com"}} - msg := testComposeIssueCommentMessage(t, &mailCommentContext{ - Context: context.TODO(), // TODO: use a correct context - Issue: issue, Doer: doer, ActionType: actionType, + msg := testComposeIssueCommentMessage(t, &mailComment{ + Issue: issue, Doer: doer, ActionType: actionType, Content: "test body", Comment: comment, }, recipients, fromMention, "TestTemplateServices") @@ -253,8 +290,8 @@ func TestTemplateServices(t *testing.T) { "//Re: //") } -func testComposeIssueCommentMessage(t *testing.T, ctx *mailCommentContext, recipients []*user_model.User, fromMention bool, info string) *sender_service.Message { - msgs, err := composeIssueCommentMessages(ctx, "en-US", recipients, fromMention, info) +func testComposeIssueCommentMessage(t *testing.T, ctx *mailComment, recipients []*user_model.User, fromMention bool, info string) *sender_service.Message { + msgs, err := composeIssueCommentMessages(t.Context(), ctx, "en-US", recipients, fromMention, info) assert.NoError(t, err) assert.Len(t, msgs, 1) return msgs[0] @@ -263,10 +300,10 @@ func testComposeIssueCommentMessage(t *testing.T, ctx *mailCommentContext, recip func TestGenerateAdditionalHeaders(t *testing.T) { doer, _, issue, _ := prepareMailerTest(t) - ctx := &mailCommentContext{Context: context.TODO() /* TODO: use a correct context */, Issue: issue, Doer: doer} + comment := &mailComment{Issue: issue, Doer: doer} recipient := &user_model.User{Name: "test", Email: "test@gitea.com"} - headers := generateAdditionalHeaders(ctx, "dummy-reason", recipient) + headers := generateAdditionalHeaders(comment, "dummy-reason", recipient) expected := map[string]string{ "List-ID": "user2/repo1 <repo1.user2.localhost>", @@ -390,9 +427,7 @@ func TestGenerateMessageIDForIssue(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := generateMessageIDForIssue(tt.args.issue, tt.args.comment, tt.args.actionType) - if !strings.HasPrefix(got, tt.prefix) { - t.Errorf("generateMessageIDForIssue() = %v, want %v", got, tt.prefix) - } + assert.True(t, strings.HasPrefix(got, tt.prefix), "%v, want %v", got, tt.prefix) }) } } @@ -406,9 +441,9 @@ func TestGenerateMessageIDForRelease(t *testing.T) { } func TestFromDisplayName(t *testing.T) { - template, err := texttmpl.New("mailFrom").Parse("{{ .DisplayName }}") + tmpl, err := texttmpl.New("mailFrom").Parse("{{ .DisplayName }}") assert.NoError(t, err) - setting.MailService = &setting.Mailer{FromDisplayNameFormatTemplate: template} + setting.MailService = &setting.Mailer{FromDisplayNameFormatTemplate: tmpl} defer func() { setting.MailService = nil }() tests := []struct { @@ -432,14 +467,14 @@ func TestFromDisplayName(t *testing.T) { t.Run(tc.userDisplayName, func(t *testing.T) { user := &user_model.User{FullName: tc.userDisplayName, Name: "tmp"} got := fromDisplayName(user) - assert.EqualValues(t, tc.fromDisplayName, got) + assert.Equal(t, tc.fromDisplayName, got) }) } t.Run("template with all available vars", func(t *testing.T) { - template, err = texttmpl.New("mailFrom").Parse("{{ .DisplayName }} (by {{ .AppName }} on [{{ .Domain }}])") + tmpl, err = texttmpl.New("mailFrom").Parse("{{ .DisplayName }} (by {{ .AppName }} on [{{ .Domain }}])") assert.NoError(t, err) - setting.MailService = &setting.Mailer{FromDisplayNameFormatTemplate: template} + setting.MailService = &setting.Mailer{FromDisplayNameFormatTemplate: tmpl} oldAppName := setting.AppName setting.AppName = "Code IT" oldDomain := setting.Domain @@ -449,6 +484,74 @@ func TestFromDisplayName(t *testing.T) { setting.Domain = oldDomain }() - assert.EqualValues(t, "Mister X (by Code IT on [code.it])", fromDisplayName(&user_model.User{FullName: "Mister X", Name: "tmp"})) + assert.Equal(t, "Mister X (by Code IT on [code.it])", fromDisplayName(&user_model.User{FullName: "Mister X", Name: "tmp"})) + }) +} + +func TestEmbedBase64Images(t *testing.T) { + user, repo, issue, att1, att2 := prepareMailerBase64Test(t) + // comment := &mailComment{Issue: issue, Doer: user} + + imgExternalURL := "https://via.placeholder.com/10" + imgExternalImg := fmt.Sprintf(`<img src="%s"/>`, imgExternalURL) + + att1URL := setting.AppURL + repo.Owner.Name + "/" + repo.Name + "/attachments/" + att1.UUID + att1Img := fmt.Sprintf(`<img src="%s"/>`, att1URL) + att1Base64 := "data:image/png;base64,iVBORw0KGgo=" + att1ImgBase64 := fmt.Sprintf(`<img src="%s"/>`, att1Base64) + + att2URL := setting.AppURL + repo.Owner.Name + "/" + repo.Name + "/attachments/" + att2.UUID + att2Img := fmt.Sprintf(`<img src="%s"/>`, att2URL) + att2File, err := storage.Attachments.Open(att2.RelativePath()) + require.NoError(t, err) + defer att2File.Close() + att2Bytes, err := io.ReadAll(att2File) + require.NoError(t, err) + require.Greater(t, len(att2Bytes), 1024) + att2Base64 := "data:image/png;base64," + base64.StdEncoding.EncodeToString(att2Bytes) + att2ImgBase64 := fmt.Sprintf(`<img src="%s"/>`, att2Base64) + + t.Run("ComposeMessage", func(t *testing.T) { + subjectTemplates = texttmpl.Must(texttmpl.New("issue/new").Parse(subjectTpl)) + bodyTemplates = template.Must(template.New("issue/new").Parse(bodyTpl)) + + issue.Content = fmt.Sprintf(`MSG-BEFORE <image src="attachments/%s"> MSG-AFTER`, att1.UUID) + require.NoError(t, issues_model.UpdateIssueCols(t.Context(), issue, "content")) + + recipients := []*user_model.User{{Name: "Test", Email: "test@gitea.com"}} + msgs, err := composeIssueCommentMessages(t.Context(), &mailComment{ + Issue: issue, + Doer: user, + ActionType: activities_model.ActionCreateIssue, + Content: issue.Content, + }, "en-US", recipients, false, "issue create") + require.NoError(t, err) + + mailBody := msgs[0].Body + assert.Regexp(t, `MSG-BEFORE <a[^>]+><img src="data:image/png;base64,iVBORw0KGgo=".*/></a> MSG-AFTER`, mailBody) + }) + + t.Run("EmbedInstanceImageSkipExternalImage", func(t *testing.T) { + mailBody := "<html><head></head><body><p>Test1</p>" + imgExternalImg + "<p>Test2</p>" + att1Img + "<p>Test3</p></body></html>" + expectedMailBody := "<html><head></head><body><p>Test1</p>" + imgExternalImg + "<p>Test2</p>" + att1ImgBase64 + "<p>Test3</p></body></html>" + b64embedder := newMailAttachmentBase64Embedder(user, repo, 1024) + resultMailBody, err := b64embedder.Base64InlineImages(t.Context(), template.HTML(mailBody)) + require.NoError(t, err) + assert.Equal(t, expectedMailBody, string(resultMailBody)) + }) + + t.Run("LimitedEmailBodySize", func(t *testing.T) { + mailBody := fmt.Sprintf("<html><head></head><body>%s%s</body></html>", att1Img, att2Img) + b64embedder := newMailAttachmentBase64Embedder(user, repo, 1024) + resultMailBody, err := b64embedder.Base64InlineImages(t.Context(), template.HTML(mailBody)) + require.NoError(t, err) + expected := fmt.Sprintf("<html><head></head><body>%s%s</body></html>", att1ImgBase64, att2Img) + assert.Equal(t, expected, string(resultMailBody)) + + b64embedder = newMailAttachmentBase64Embedder(user, repo, 4096) + resultMailBody, err = b64embedder.Base64InlineImages(t.Context(), template.HTML(mailBody)) + require.NoError(t, err) + expected = fmt.Sprintf("<html><head></head><body>%s%s</body></html>", att1ImgBase64, att2ImgBase64) + assert.Equal(t, expected, string(resultMailBody)) }) } diff --git a/services/mailer/mail_user.go b/services/mailer/mail_user.go new file mode 100644 index 0000000000..5a200a5fa7 --- /dev/null +++ b/services/mailer/mail_user.go @@ -0,0 +1,161 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package mailer + +import ( + "bytes" + "fmt" + + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/translation" + sender_service "code.gitea.io/gitea/services/mailer/sender" +) + +const ( + mailAuthActivate templates.TplName = "auth/activate" + mailAuthActivateEmail templates.TplName = "auth/activate_email" + mailAuthResetPassword templates.TplName = "auth/reset_passwd" + mailAuthRegisterNotify templates.TplName = "auth/register_notify" + mailNotifyCollaborator templates.TplName = "notify/collaborator" +) + +// sendUserMail sends a mail to the user +func sendUserMail(language string, u *user_model.User, tpl templates.TplName, code, subject, info string) { + locale := translation.NewLocale(language) + data := map[string]any{ + "locale": locale, + "DisplayName": u.DisplayName(), + "ActiveCodeLives": timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, locale), + "ResetPwdCodeLives": timeutil.MinutesToFriendly(setting.Service.ResetPwdCodeLives, locale), + "Code": code, + "Language": locale.Language(), + } + + var content bytes.Buffer + + if err := bodyTemplates.ExecuteTemplate(&content, string(tpl), data); err != nil { + log.Error("Template: %v", err) + return + } + + msg := sender_service.NewMessage(u.EmailTo(), subject, content.String()) + msg.Info = fmt.Sprintf("UID: %d, %s", u.ID, info) + + SendAsync(msg) +} + +// SendActivateAccountMail sends an activation mail to the user (new user registration) +func SendActivateAccountMail(locale translation.Locale, u *user_model.User) { + if setting.MailService == nil { + // No mail service configured + return + } + opts := &user_model.TimeLimitCodeOptions{Purpose: user_model.TimeLimitCodeActivateAccount} + sendUserMail(locale.Language(), u, mailAuthActivate, user_model.GenerateUserTimeLimitCode(opts, u), locale.TrString("mail.activate_account"), "activate account") +} + +// SendResetPasswordMail sends a password reset mail to the user +func SendResetPasswordMail(u *user_model.User) { + if setting.MailService == nil { + // No mail service configured + return + } + locale := translation.NewLocale(u.Language) + opts := &user_model.TimeLimitCodeOptions{Purpose: user_model.TimeLimitCodeResetPassword} + sendUserMail(u.Language, u, mailAuthResetPassword, user_model.GenerateUserTimeLimitCode(opts, u), locale.TrString("mail.reset_password"), "recover account") +} + +// SendActivateEmailMail sends confirmation email to confirm new email address +func SendActivateEmailMail(u *user_model.User, email string) { + if setting.MailService == nil { + // No mail service configured + return + } + locale := translation.NewLocale(u.Language) + opts := &user_model.TimeLimitCodeOptions{Purpose: user_model.TimeLimitCodeActivateEmail, NewEmail: email} + data := map[string]any{ + "locale": locale, + "DisplayName": u.DisplayName(), + "ActiveCodeLives": timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, locale), + "Code": user_model.GenerateUserTimeLimitCode(opts, u), + "Email": email, + "Language": locale.Language(), + } + + var content bytes.Buffer + + if err := bodyTemplates.ExecuteTemplate(&content, string(mailAuthActivateEmail), data); err != nil { + log.Error("Template: %v", err) + return + } + + msg := sender_service.NewMessage(email, locale.TrString("mail.activate_email"), content.String()) + msg.Info = fmt.Sprintf("UID: %d, activate email", u.ID) + + SendAsync(msg) +} + +// SendRegisterNotifyMail triggers a notify e-mail by admin created a account. +func SendRegisterNotifyMail(u *user_model.User) { + if setting.MailService == nil || !u.IsActive { + // No mail service configured OR user is inactive + return + } + locale := translation.NewLocale(u.Language) + + data := map[string]any{ + "locale": locale, + "DisplayName": u.DisplayName(), + "Username": u.Name, + "Language": locale.Language(), + } + + var content bytes.Buffer + + if err := bodyTemplates.ExecuteTemplate(&content, string(mailAuthRegisterNotify), data); err != nil { + log.Error("Template: %v", err) + return + } + + msg := sender_service.NewMessage(u.EmailTo(), locale.TrString("mail.register_notify", setting.AppName), content.String()) + msg.Info = fmt.Sprintf("UID: %d, registration notify", u.ID) + + SendAsync(msg) +} + +// SendCollaboratorMail sends mail notification to new collaborator. +func SendCollaboratorMail(u, doer *user_model.User, repo *repo_model.Repository) { + if setting.MailService == nil || !u.IsActive { + // No mail service configured OR the user is inactive + return + } + locale := translation.NewLocale(u.Language) + repoName := repo.FullName() + + subject := locale.TrString("mail.repo.collaborator.added.subject", doer.DisplayName(), repoName) + data := map[string]any{ + "locale": locale, + "Subject": subject, + "RepoName": repoName, + "Link": repo.HTMLURL(), + "Language": locale.Language(), + } + + var content bytes.Buffer + + if err := bodyTemplates.ExecuteTemplate(&content, string(mailNotifyCollaborator), data); err != nil { + log.Error("Template: %v", err) + return + } + + msg := sender_service.NewMessage(u.EmailTo(), subject, content.String()) + msg.Info = fmt.Sprintf("UID: %d, add collaborator", u.ID) + + SendAsync(msg) +} diff --git a/services/mailer/notify.go b/services/mailer/notify.go index e48b5d399d..77c366fe31 100644 --- a/services/mailer/notify.go +++ b/services/mailer/notify.go @@ -12,6 +12,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" + issue_service "code.gitea.io/gitea/services/issue" notify_service "code.gitea.io/gitea/services/notify" ) @@ -30,15 +31,16 @@ func (m *mailNotifier) CreateIssueComment(ctx context.Context, doer *user_model. issue *issues_model.Issue, comment *issues_model.Comment, mentions []*user_model.User, ) { var act activities_model.ActionType - if comment.Type == issues_model.CommentTypeClose { + switch comment.Type { + case issues_model.CommentTypeClose: act = activities_model.ActionCloseIssue - } else if comment.Type == issues_model.CommentTypeReopen { + case issues_model.CommentTypeReopen: act = activities_model.ActionReopenIssue - } else if comment.Type == issues_model.CommentTypeComment { + case issues_model.CommentTypeComment: act = activities_model.ActionCommentIssue - } else if comment.Type == issues_model.CommentTypeCode { + case issues_model.CommentTypeCode: act = activities_model.ActionCommentIssue - } else if comment.Type == issues_model.CommentTypePullRequestPush { + case issues_model.CommentTypePullRequestPush: act = 0 } @@ -94,11 +96,12 @@ func (m *mailNotifier) NewPullRequest(ctx context.Context, pr *issues_model.Pull func (m *mailNotifier) PullRequestReview(ctx context.Context, pr *issues_model.PullRequest, r *issues_model.Review, comment *issues_model.Comment, mentions []*user_model.User) { var act activities_model.ActionType - if comment.Type == issues_model.CommentTypeClose { + switch comment.Type { + case issues_model.CommentTypeClose: act = activities_model.ActionCloseIssue - } else if comment.Type == issues_model.CommentTypeReopen { + case issues_model.CommentTypeReopen: act = activities_model.ActionReopenIssue - } else if comment.Type == issues_model.CommentTypeComment { + case issues_model.CommentTypeComment: act = activities_model.ActionCommentPull } if err := MailParticipantsComment(ctx, comment, act, pr.Issue, mentions); err != nil { @@ -169,7 +172,7 @@ func (m *mailNotifier) PullRequestPushCommits(ctx context.Context, doer *user_mo log.Error("comment.Issue.PullRequest.LoadBaseRepo: %v", err) return } - if err := comment.LoadPushCommits(ctx); err != nil { + if err := issue_service.LoadCommentPushCommits(ctx, comment); err != nil { log.Error("comment.LoadPushCommits: %v", err) } m.CreateIssueComment(ctx, doer, comment.Issue.Repo, comment.Issue, comment, nil) diff --git a/services/mailer/sender/message_test.go b/services/mailer/sender/message_test.go index 63d0bc349a..ae153ebf05 100644 --- a/services/mailer/sender/message_test.go +++ b/services/mailer/sender/message_test.go @@ -108,9 +108,9 @@ func extractMailHeaderAndContent(t *testing.T, mail string) (map[string]string, } content := strings.TrimSpace("boundary=" + parts[1]) - hParts := strings.Split(parts[0], "\n") + hParts := strings.SplitSeq(parts[0], "\n") - for _, hPart := range hParts { + for hPart := range hParts { parts := strings.SplitN(hPart, ":", 2) hk := strings.TrimSpace(parts[0]) if hk != "" { diff --git a/services/mailer/sender/smtp.go b/services/mailer/sender/smtp.go index c53c3da997..8dc1b40b74 100644 --- a/services/mailer/sender/smtp.go +++ b/services/mailer/sender/smtp.go @@ -5,6 +5,7 @@ package sender import ( "crypto/tls" + "errors" "fmt" "io" "net" @@ -99,7 +100,7 @@ func (s *SMTPSender) Send(from string, to []string, msg io.WriterTo) error { canAuth, options := client.Extension("AUTH") if len(opts.User) > 0 { if !canAuth { - return fmt.Errorf("SMTP server does not support AUTH, but credentials provided") + return errors.New("SMTP server does not support AUTH, but credentials provided") } var auth smtp.Auth diff --git a/services/mailer/sender/smtp_auth.go b/services/mailer/sender/smtp_auth.go index 260b12437b..c60e0dbfbb 100644 --- a/services/mailer/sender/smtp_auth.go +++ b/services/mailer/sender/smtp_auth.go @@ -4,6 +4,7 @@ package sender import ( + "errors" "fmt" "github.com/Azure/go-ntlmssp" @@ -60,7 +61,7 @@ func (a *ntlmAuth) Start(server *smtp.ServerInfo) (string, []byte, error) { func (a *ntlmAuth) Next(fromServer []byte, more bool) ([]byte, error) { if more { if len(fromServer) == 0 { - return nil, fmt.Errorf("ntlm ChallengeMessage is empty") + return nil, errors.New("ntlm ChallengeMessage is empty") } authenticateMessage, err := ntlmssp.ProcessChallenge(fromServer, a.username, a.password, a.domainNeeded) return authenticateMessage, err diff --git a/services/markup/renderhelper.go b/services/markup/renderhelper.go index 4b9852b48b..ea494146a7 100644 --- a/services/markup/renderhelper.go +++ b/services/markup/renderhelper.go @@ -21,8 +21,8 @@ func FormalRenderHelperFuncs() *markup.RenderHelperFuncs { return false } - giteaCtx, ok := ctx.(*gitea_context.Context) - if !ok { + giteaCtx := gitea_context.GetWebContext(ctx) + if giteaCtx == nil { // when using general context, use user's visibility to check return mentionedUser.Visibility.IsPublic() } diff --git a/services/markup/renderhelper_codepreview.go b/services/markup/renderhelper_codepreview.go index 170c70c409..fa1eb824a2 100644 --- a/services/markup/renderhelper_codepreview.go +++ b/services/markup/renderhelper_codepreview.go @@ -6,7 +6,7 @@ package markup import ( "bufio" "context" - "fmt" + "errors" "html/template" "strings" @@ -14,13 +14,13 @@ import ( "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/charset" + "code.gitea.io/gitea/modules/git/languagestats" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/indexer/code" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" gitea_context "code.gitea.io/gitea/services/context" - "code.gitea.io/gitea/services/repository/files" ) func renderRepoFileCodePreview(ctx context.Context, opts markup.RenderCodePreviewOptions) (template.HTML, error) { @@ -36,9 +36,9 @@ func renderRepoFileCodePreview(ctx context.Context, opts markup.RenderCodePrevie return "", err } - webCtx, ok := ctx.Value(gitea_context.WebContextKey).(*gitea_context.Context) - if !ok { - return "", fmt.Errorf("context is not a web context") + webCtx := gitea_context.GetWebContext(ctx) + if webCtx == nil { + return "", errors.New("context is not a web context") } doer := webCtx.Doer @@ -61,14 +61,14 @@ func renderRepoFileCodePreview(ctx context.Context, opts markup.RenderCodePrevie return "", err } - language, _ := files.TryGetContentLanguage(gitRepo, opts.CommitID, opts.FilePath) + language, _ := languagestats.GetFileLanguage(ctx, gitRepo, opts.CommitID, opts.FilePath) blob, err := commit.GetBlobByPath(opts.FilePath) if err != nil { return "", err } if blob.Size() > setting.UI.MaxDisplayFileSize { - return "", fmt.Errorf("file is too large") + return "", errors.New("file is too large") } dataRc, err := blob.DataAsync() diff --git a/services/markup/renderhelper_issueicontitle.go b/services/markup/renderhelper_issueicontitle.go index 53a508e908..27b5595fa9 100644 --- a/services/markup/renderhelper_issueicontitle.go +++ b/services/markup/renderhelper_issueicontitle.go @@ -5,6 +5,7 @@ package markup import ( "context" + "errors" "fmt" "html/template" @@ -18,9 +19,9 @@ import ( ) func renderRepoIssueIconTitle(ctx context.Context, opts markup.RenderIssueIconTitleOptions) (_ template.HTML, err error) { - webCtx, ok := ctx.Value(gitea_context.WebContextKey).(*gitea_context.Context) - if !ok { - return "", fmt.Errorf("context is not a web context") + webCtx := gitea_context.GetWebContext(ctx) + if webCtx == nil { + return "", errors.New("context is not a web context") } textIssueIndex := fmt.Sprintf("(#%d)", opts.IssueIndex) diff --git a/services/markup/renderhelper_mention_test.go b/services/markup/renderhelper_mention_test.go index c244fa3d21..d54ab13a48 100644 --- a/services/markup/renderhelper_mention_test.go +++ b/services/markup/renderhelper_mention_test.go @@ -4,7 +4,6 @@ package markup import ( - "context" "net/http" "net/http/httptest" "testing" @@ -32,13 +31,13 @@ func TestRenderHelperMention(t *testing.T) { unittest.AssertCount(t, &user.User{Name: userNoSuch}, 0) // when using general context, use user's visibility to check - assert.True(t, FormalRenderHelperFuncs().IsUsernameMentionable(context.Background(), userPublic)) - assert.False(t, FormalRenderHelperFuncs().IsUsernameMentionable(context.Background(), userLimited)) - assert.False(t, FormalRenderHelperFuncs().IsUsernameMentionable(context.Background(), userPrivate)) - assert.False(t, FormalRenderHelperFuncs().IsUsernameMentionable(context.Background(), userNoSuch)) + assert.True(t, FormalRenderHelperFuncs().IsUsernameMentionable(t.Context(), userPublic)) + assert.False(t, FormalRenderHelperFuncs().IsUsernameMentionable(t.Context(), userLimited)) + assert.False(t, FormalRenderHelperFuncs().IsUsernameMentionable(t.Context(), userPrivate)) + assert.False(t, FormalRenderHelperFuncs().IsUsernameMentionable(t.Context(), userNoSuch)) // when using web context, use user.IsUserVisibleToViewer to check - req, err := http.NewRequest("GET", "/", nil) + req, err := http.NewRequest(http.MethodGet, "/", nil) assert.NoError(t, err) base := gitea_context.NewBaseContextForTest(httptest.NewRecorder(), req) giteaCtx := gitea_context.NewWebContext(base, &contexttest.MockRender{}, nil) diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index 492fc908e9..240c7bcdc9 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -66,7 +66,6 @@ type codebaseUser struct { // from Codebase type CodebaseDownloader struct { base.NullDownloader - ctx context.Context client *http.Client baseURL *url.URL projectURL *url.URL @@ -77,17 +76,11 @@ type CodebaseDownloader struct { commitMap map[string]string } -// SetContext set context -func (d *CodebaseDownloader) SetContext(ctx context.Context) { - d.ctx = ctx -} - // NewCodebaseDownloader creates a new downloader -func NewCodebaseDownloader(ctx context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader { +func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader { baseURL, _ := url.Parse("https://api3.codebasehq.com") downloader := &CodebaseDownloader{ - ctx: ctx, baseURL: baseURL, projectURL: projectURL, project: project, @@ -127,7 +120,7 @@ func (d *CodebaseDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr return opts.CloneAddr, nil } -func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]string, result any) error { +func (d *CodebaseDownloader) callAPI(ctx context.Context, endpoint string, parameter map[string]string, result any) error { u, err := d.baseURL.Parse(endpoint) if err != nil { return err @@ -141,7 +134,7 @@ func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]strin u.RawQuery = query.Encode() } - req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return err } @@ -158,7 +151,7 @@ func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]strin // GetRepoInfo returns repository information // https://support.codebasehq.com/kb/projects -func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) { +func (d *CodebaseDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { var rawRepository struct { XMLName xml.Name `xml:"repository"` Name string `xml:"name"` @@ -169,6 +162,7 @@ func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) { } err := d.callAPI( + ctx, fmt.Sprintf("/%s/%s", d.project, d.repoName), nil, &rawRepository, @@ -187,7 +181,7 @@ func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) { // GetMilestones returns milestones // https://support.codebasehq.com/kb/tickets-and-milestones/milestones -func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) { +func (d *CodebaseDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { var rawMilestones struct { XMLName xml.Name `xml:"ticketing-milestone"` Type string `xml:"type,attr"` @@ -209,6 +203,7 @@ func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) { } err := d.callAPI( + ctx, fmt.Sprintf("/%s/milestones", d.project), nil, &rawMilestones, @@ -245,7 +240,7 @@ func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) { // GetLabels returns labels // https://support.codebasehq.com/kb/tickets-and-milestones/statuses-priorities-and-categories -func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) { +func (d *CodebaseDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) { var rawTypes struct { XMLName xml.Name `xml:"ticketing-types"` Type string `xml:"type,attr"` @@ -259,6 +254,7 @@ func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) { } err := d.callAPI( + ctx, fmt.Sprintf("/%s/tickets/types", d.project), nil, &rawTypes, @@ -284,7 +280,7 @@ type codebaseIssueContext struct { // GetIssues returns issues, limits are not supported // https://support.codebasehq.com/kb/tickets-and-milestones // https://support.codebasehq.com/kb/tickets-and-milestones/updating-tickets -func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (d *CodebaseDownloader) GetIssues(ctx context.Context, _, _ int) ([]*base.Issue, bool, error) { var rawIssues struct { XMLName xml.Name `xml:"tickets"` Type string `xml:"type,attr"` @@ -324,6 +320,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, } err := d.callAPI( + ctx, fmt.Sprintf("/%s/tickets", d.project), nil, &rawIssues, @@ -358,6 +355,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, } `xml:"ticket-note"` } err := d.callAPI( + ctx, fmt.Sprintf("/%s/tickets/%d/notes", d.project, issue.TicketID.Value), nil, ¬es, @@ -370,7 +368,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, if len(note.Content) == 0 { continue } - poster := d.tryGetUser(note.UserID.Value) + poster := d.tryGetUser(ctx, note.UserID.Value) comments = append(comments, &base.Comment{ IssueIndex: issue.TicketID.Value, Index: note.ID.Value, @@ -390,7 +388,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, if issue.Status.TreatAsClosed.Value { state = "closed" } - poster := d.tryGetUser(issue.ReporterID.Value) + poster := d.tryGetUser(ctx, issue.ReporterID.Value) issues = append(issues, &base.Issue{ Title: issue.Summary, Number: issue.TicketID.Value, @@ -419,7 +417,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments -func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (d *CodebaseDownloader) GetComments(_ context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { context, ok := commentable.GetContext().(codebaseIssueContext) if !ok { return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) @@ -430,7 +428,7 @@ func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base. // GetPullRequests returns pull requests // https://support.codebasehq.com/kb/repositories/merge-requests -func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (d *CodebaseDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { var rawMergeRequests struct { XMLName xml.Name `xml:"merge-requests"` Type string `xml:"type,attr"` @@ -443,6 +441,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq } err := d.callAPI( + ctx, fmt.Sprintf("/%s/%s/merge_requests", d.project, d.repoName), map[string]string{ "query": `"Target Project" is "` + d.repoName + `"`, @@ -503,6 +502,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq } `xml:"comments"` } err := d.callAPI( + ctx, fmt.Sprintf("/%s/%s/merge_requests/%d", d.project, d.repoName, mr.ID.Value), nil, &rawMergeRequest, @@ -531,7 +531,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq } continue } - poster := d.tryGetUser(comment.UserID.Value) + poster := d.tryGetUser(ctx, comment.UserID.Value) comments = append(comments, &base.Comment{ IssueIndex: number, Index: comment.ID.Value, @@ -547,7 +547,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq comments = append(comments, &base.Comment{}) } - poster := d.tryGetUser(rawMergeRequest.UserID.Value) + poster := d.tryGetUser(ctx, rawMergeRequest.UserID.Value) pullRequests = append(pullRequests, &base.PullRequest{ Title: rawMergeRequest.Subject, @@ -563,12 +563,12 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq MergedTime: mergedTime, Head: base.PullRequestBranch{ Ref: rawMergeRequest.SourceRef, - SHA: d.getHeadCommit(rawMergeRequest.SourceRef), + SHA: d.getHeadCommit(ctx, rawMergeRequest.SourceRef), RepoName: d.repoName, }, Base: base.PullRequestBranch{ Ref: rawMergeRequest.TargetRef, - SHA: d.getHeadCommit(rawMergeRequest.TargetRef), + SHA: d.getHeadCommit(ctx, rawMergeRequest.TargetRef), RepoName: d.repoName, }, ForeignIndex: rawMergeRequest.ID.Value, @@ -584,7 +584,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq return pullRequests, true, nil } -func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { +func (d *CodebaseDownloader) tryGetUser(ctx context.Context, userID int64) *codebaseUser { if len(d.userMap) == 0 { var rawUsers struct { XMLName xml.Name `xml:"users"` @@ -602,6 +602,7 @@ func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { } err := d.callAPI( + ctx, "/users", nil, &rawUsers, @@ -627,7 +628,7 @@ func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { return user } -func (d *CodebaseDownloader) getHeadCommit(ref string) string { +func (d *CodebaseDownloader) getHeadCommit(ctx context.Context, ref string) string { commitRef, ok := d.commitMap[ref] if !ok { var rawCommits struct { @@ -638,6 +639,7 @@ func (d *CodebaseDownloader) getHeadCommit(ref string) string { } `xml:"commit"` } err := d.callAPI( + ctx, fmt.Sprintf("/%s/%s/commits/%s", d.project, d.repoName, ref), nil, &rawCommits, diff --git a/services/migrations/codebase_test.go b/services/migrations/codebase_test.go index 68721e0641..6cd52e5e59 100644 --- a/services/migrations/codebase_test.go +++ b/services/migrations/codebase_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "net/url" "os" "testing" @@ -30,9 +29,9 @@ func TestCodebaseDownloadRepo(t *testing.T) { if cloneUser != "" { u.User = url.UserPassword(cloneUser, clonePassword) } - + ctx := t.Context() factory := &CodebaseDownloaderFactory{} - downloader, err := factory.New(context.Background(), base.MigrateOptions{ + downloader, err := factory.New(ctx, base.MigrateOptions{ CloneAddr: u.String(), AuthUsername: apiUser, AuthPassword: apiPassword, @@ -40,7 +39,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { if err != nil { t.Fatalf("Error creating Codebase downloader: %v", err) } - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ Name: "test", @@ -50,7 +49,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { OriginalURL: cloneAddr, }, repo) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -65,11 +64,11 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assert.Len(t, labels, 4) - issues, isEnd, err := downloader.GetIssues(1, 2) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 2) assert.NoError(t, err) assert.True(t, isEnd) assertIssuesEqual(t, []*base.Issue{ @@ -106,7 +105,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(issues[0]) + comments, _, err := downloader.GetComments(ctx, issues[0]) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -119,7 +118,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, comments) - prs, _, err := downloader.GetPullRequests(1, 1) + prs, _, err := downloader.GetPullRequests(ctx, 1, 1) assert.NoError(t, err) assertPullRequestsEqual(t, []*base.PullRequest{ { @@ -144,7 +143,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(prs[0]) + rvs, err := downloader.GetReviews(ctx, prs[0]) assert.NoError(t, err) assert.Empty(t, rvs) } diff --git a/services/migrations/codecommit.go b/services/migrations/codecommit.go index fead527f5b..d08b2e6d4a 100644 --- a/services/migrations/codecommit.go +++ b/services/migrations/codecommit.go @@ -5,7 +5,7 @@ package migrations import ( "context" - "fmt" + "errors" "net/url" "strconv" "strings" @@ -42,13 +42,13 @@ func (c *CodeCommitDownloaderFactory) New(ctx context.Context, opts base.Migrate hostElems := strings.Split(u.Host, ".") if len(hostElems) != 4 { - return nil, fmt.Errorf("cannot get the region from clone URL") + return nil, errors.New("cannot get the region from clone URL") } region := hostElems[1] pathElems := strings.Split(u.Path, "/") if len(pathElems) == 0 { - return nil, fmt.Errorf("cannot get the repo name from clone URL") + return nil, errors.New("cannot get the repo name from clone URL") } repoName := pathElems[len(pathElems)-1] @@ -62,9 +62,8 @@ func (c *CodeCommitDownloaderFactory) GitServiceType() structs.GitServiceType { return structs.CodeCommitService } -func NewCodeCommitDownloader(ctx context.Context, repoName, baseURL, accessKeyID, secretAccessKey, region string) *CodeCommitDownloader { +func NewCodeCommitDownloader(_ context.Context, repoName, baseURL, accessKeyID, secretAccessKey, region string) *CodeCommitDownloader { downloader := CodeCommitDownloader{ - ctx: ctx, repoName: repoName, baseURL: baseURL, codeCommitClient: codecommit.New(codecommit.Options{ @@ -79,21 +78,15 @@ func NewCodeCommitDownloader(ctx context.Context, repoName, baseURL, accessKeyID // CodeCommitDownloader implements a downloader for AWS CodeCommit type CodeCommitDownloader struct { base.NullDownloader - ctx context.Context codeCommitClient *codecommit.Client repoName string baseURL string allPullRequestIDs []string } -// SetContext set context -func (c *CodeCommitDownloader) SetContext(ctx context.Context) { - c.ctx = ctx -} - // GetRepoInfo returns a repository information -func (c *CodeCommitDownloader) GetRepoInfo() (*base.Repository, error) { - output, err := c.codeCommitClient.GetRepository(c.ctx, &codecommit.GetRepositoryInput{ +func (c *CodeCommitDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { + output, err := c.codeCommitClient.GetRepository(ctx, &codecommit.GetRepositoryInput{ RepositoryName: util.ToPointer(c.repoName), }) if err != nil { @@ -117,14 +110,14 @@ func (c *CodeCommitDownloader) GetRepoInfo() (*base.Repository, error) { } // GetComments returns comments of an issue or PR -func (c *CodeCommitDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (c *CodeCommitDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { var ( nextToken *string comments []*base.Comment ) for { - resp, err := c.codeCommitClient.GetCommentsForPullRequest(c.ctx, &codecommit.GetCommentsForPullRequestInput{ + resp, err := c.codeCommitClient.GetCommentsForPullRequest(ctx, &codecommit.GetCommentsForPullRequestInput{ NextToken: nextToken, PullRequestId: util.ToPointer(strconv.FormatInt(commentable.GetForeignIndex(), 10)), }) @@ -155,22 +148,19 @@ func (c *CodeCommitDownloader) GetComments(commentable base.Commentable) ([]*bas } // GetPullRequests returns pull requests according page and perPage -func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { - allPullRequestIDs, err := c.getAllPullRequestIDs() +func (c *CodeCommitDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { + allPullRequestIDs, err := c.getAllPullRequestIDs(ctx) if err != nil { return nil, false, err } startIndex := (page - 1) * perPage - endIndex := page * perPage - if endIndex > len(allPullRequestIDs) { - endIndex = len(allPullRequestIDs) - } + endIndex := min(page*perPage, len(allPullRequestIDs)) batch := allPullRequestIDs[startIndex:endIndex] prs := make([]*base.PullRequest, 0, len(batch)) for _, id := range batch { - output, err := c.codeCommitClient.GetPullRequest(c.ctx, &codecommit.GetPullRequestInput{ + output, err := c.codeCommitClient.GetPullRequest(ctx, &codecommit.GetPullRequestInput{ PullRequestId: util.ToPointer(id), }) if err != nil { @@ -187,11 +177,15 @@ func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullR continue } target := orig.PullRequestTargets[0] + description := "" + if orig.Description != nil { + description = *orig.Description + } pr := &base.PullRequest{ Number: number, Title: *orig.Title, PosterName: c.getUsernameFromARN(*orig.AuthorArn), - Content: *orig.Description, + Content: description, State: "open", Created: *orig.CreationDate, Updated: *orig.LastActivityDate, @@ -213,6 +207,10 @@ func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullR pr.State = "closed" pr.Closed = orig.LastActivityDate } + if pr.Merged { + pr.MergeCommitSHA = *target.MergeMetadata.MergeCommitId + pr.MergedTime = orig.LastActivityDate + } _ = CheckAndEnsureSafePR(pr, c.baseURL, c) prs = append(prs, pr) @@ -231,7 +229,7 @@ func (c *CodeCommitDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr st return u.String(), nil } -func (c *CodeCommitDownloader) getAllPullRequestIDs() ([]string, error) { +func (c *CodeCommitDownloader) getAllPullRequestIDs(ctx context.Context) ([]string, error) { if len(c.allPullRequestIDs) > 0 { return c.allPullRequestIDs, nil } @@ -242,7 +240,7 @@ func (c *CodeCommitDownloader) getAllPullRequestIDs() ([]string, error) { ) for { - output, err := c.codeCommitClient.ListPullRequests(c.ctx, &codecommit.ListPullRequestsInput{ + output, err := c.codeCommitClient.ListPullRequests(ctx, &codecommit.ListPullRequestsInput{ RepositoryName: util.ToPointer(c.repoName), NextToken: nextToken, }) diff --git a/services/migrations/dump.go b/services/migrations/dump.go index 07812002af..b4ca1e41e0 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -32,7 +32,6 @@ var _ base.Uploader = &RepositoryDumper{} // RepositoryDumper implements an Uploader to the local directory type RepositoryDumper struct { - ctx context.Context baseDir string repoOwner string repoName string @@ -56,7 +55,6 @@ func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName strin return nil, err } return &RepositoryDumper{ - ctx: ctx, opts: opts, baseDir: baseDir, repoOwner: repoOwner, @@ -105,7 +103,7 @@ func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) { } // CreateRepo creates a repository -func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { +func (g *RepositoryDumper) CreateRepo(ctx context.Context, repo *base.Repository, opts base.MigrateOptions) error { f, err := os.Create(filepath.Join(g.baseDir, "repo.yml")) if err != nil { return err @@ -149,7 +147,7 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp return err } - err = git.Clone(g.ctx, remoteAddr, repoPath, git.CloneRepoOptions{ + err = git.Clone(ctx, remoteAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -158,19 +156,19 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp if err != nil { return fmt.Errorf("Clone: %w", err) } - if err := git.WriteCommitGraph(g.ctx, repoPath); err != nil { + if err := git.WriteCommitGraph(ctx, repoPath); err != nil { return err } if opts.Wiki { wikiPath := g.wikiPath() - wikiRemotePath := repository.WikiRemoteURL(g.ctx, remoteAddr) + wikiRemotePath := repository.WikiRemoteURL(ctx, remoteAddr) if len(wikiRemotePath) > 0 { if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil { return fmt.Errorf("Failed to remove %s: %w", wikiPath, err) } - if err := git.Clone(g.ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ + if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -181,13 +179,13 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp if err := os.RemoveAll(wikiPath); err != nil { return fmt.Errorf("Failed to remove %s: %w", wikiPath, err) } - } else if err := git.WriteCommitGraph(g.ctx, wikiPath); err != nil { + } else if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { return err } } } - g.gitRepo, err = git.OpenRepository(g.ctx, g.gitPath()) + g.gitRepo, err = git.OpenRepository(ctx, g.gitPath()) return err } @@ -220,7 +218,7 @@ func (g *RepositoryDumper) Close() { } // CreateTopics creates topics -func (g *RepositoryDumper) CreateTopics(topics ...string) error { +func (g *RepositoryDumper) CreateTopics(_ context.Context, topics ...string) error { f, err := os.Create(filepath.Join(g.baseDir, "topic.yml")) if err != nil { return err @@ -242,7 +240,7 @@ func (g *RepositoryDumper) CreateTopics(topics ...string) error { } // CreateMilestones creates milestones -func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error { +func (g *RepositoryDumper) CreateMilestones(_ context.Context, milestones ...*base.Milestone) error { var err error if g.milestoneFile == nil { g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml")) @@ -264,7 +262,7 @@ func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error } // CreateLabels creates labels -func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { +func (g *RepositoryDumper) CreateLabels(_ context.Context, labels ...*base.Label) error { var err error if g.labelFile == nil { g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml")) @@ -286,7 +284,7 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { } // CreateReleases creates releases -func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { +func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.Release) error { if g.opts.ReleaseAssets { for _, release := range releases { attachDir := filepath.Join("release_assets", release.TagName) @@ -354,12 +352,12 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { } // SyncTags syncs releases with tags in the database -func (g *RepositoryDumper) SyncTags() error { +func (g *RepositoryDumper) SyncTags(ctx context.Context) error { return nil } // CreateIssues creates issues -func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { +func (g *RepositoryDumper) CreateIssues(_ context.Context, issues ...*base.Issue) error { var err error if g.issueFile == nil { g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml")) @@ -412,7 +410,7 @@ func (g *RepositoryDumper) encodeItems(number int64, items []any, dir string, it } // CreateComments creates comments of issues -func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { +func (g *RepositoryDumper) CreateComments(_ context.Context, comments ...*base.Comment) error { commentsMap := make(map[int64][]any, len(comments)) for _, comment := range comments { commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment) @@ -421,7 +419,7 @@ func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { return g.createItems(g.commentDir(), g.commentFiles, commentsMap) } -func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { +func (g *RepositoryDumper) handlePullRequest(ctx context.Context, pr *base.PullRequest) error { // SECURITY: this pr must have been ensured safe if !pr.EnsuredSafe { log.Error("PR #%d in %s/%s has not been checked for safety ... We will ignore this.", pr.Number, g.repoOwner, g.repoName) @@ -490,7 +488,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { if pr.Head.CloneURL == "" || pr.Head.Ref == "" { // Set head information if pr.Head.SHA is available if pr.Head.SHA != "" { - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.gitPath()}) if err != nil { log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err) } @@ -520,7 +518,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { if !ok { // Set head information if pr.Head.SHA is available if pr.Head.SHA != "" { - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.gitPath()}) if err != nil { log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err) } @@ -555,7 +553,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { fetchArg = git.BranchPrefix + fetchArg } - _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + _, _, err = git.NewCommand("fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(ctx, &git.RunOpts{Dir: g.gitPath()}) if err != nil { log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) // We need to continue here so that the Head.Ref is reset and we attempt to set the gitref for the PR @@ -579,7 +577,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { pr.Head.SHA = headSha } if pr.Head.SHA != "" { - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.gitPath()}) if err != nil { log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) } @@ -589,7 +587,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { } // CreatePullRequests creates pull requests -func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { +func (g *RepositoryDumper) CreatePullRequests(ctx context.Context, prs ...*base.PullRequest) error { var err error if g.pullrequestFile == nil { if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { @@ -607,7 +605,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { count := 0 for i := 0; i < len(prs); i++ { pr := prs[i] - if err := g.handlePullRequest(pr); err != nil { + if err := g.handlePullRequest(ctx, pr); err != nil { log.Error("PR #%d in %s/%s failed - skipping", pr.Number, g.repoOwner, g.repoName, err) continue } @@ -620,7 +618,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { } // CreateReviews create pull request reviews -func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { +func (g *RepositoryDumper) CreateReviews(_ context.Context, reviews ...*base.Review) error { reviewsMap := make(map[int64][]any, len(reviews)) for _, review := range reviews { reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review) @@ -636,7 +634,7 @@ func (g *RepositoryDumper) Rollback() error { } // Finish when migrating succeed, this will update something. -func (g *RepositoryDumper) Finish() error { +func (g *RepositoryDumper) Finish(_ context.Context) error { return nil } diff --git a/services/migrations/error.go b/services/migrations/error.go index c7d912f50b..9b470149bf 100644 --- a/services/migrations/error.go +++ b/services/migrations/error.go @@ -7,7 +7,7 @@ package migrations import ( "errors" - "github.com/google/go-github/v61/github" + "github.com/google/go-github/v71/github" ) // ErrRepoNotCreated returns the error that repository not created diff --git a/services/migrations/git.go b/services/migrations/git.go index 22ffd5e765..1ed99499a1 100644 --- a/services/migrations/git.go +++ b/services/migrations/git.go @@ -28,12 +28,8 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl } } -// SetContext set context -func (g *PlainGitDownloader) SetContext(ctx context.Context) { -} - // GetRepoInfo returns a repository information -func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { +func (g *PlainGitDownloader) GetRepoInfo(_ context.Context) (*base.Repository, error) { // convert github repo to stand Repo return &base.Repository{ Owner: g.ownerName, @@ -43,6 +39,6 @@ func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { } // GetTopics return empty string slice -func (g PlainGitDownloader) GetTopics() ([]string, error) { +func (g PlainGitDownloader) GetTopics(_ context.Context) ([]string, error) { return []string{}, nil } diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index 272bf02e11..5d48d2f003 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -67,7 +67,6 @@ func (f *GiteaDownloaderFactory) GitServiceType() structs.GitServiceType { // GiteaDownloader implements a Downloader interface to get repository information's type GiteaDownloader struct { base.NullDownloader - ctx context.Context client *gitea_sdk.Client baseURL string repoOwner string @@ -114,7 +113,6 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo } return &GiteaDownloader{ - ctx: ctx, client: giteaClient, baseURL: baseURL, repoOwner: path[0], @@ -124,11 +122,6 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo }, nil } -// SetContext set context -func (g *GiteaDownloader) SetContext(ctx context.Context) { - g.ctx = ctx -} - // String implements Stringer func (g *GiteaDownloader) String() string { return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) @@ -142,7 +135,7 @@ func (g *GiteaDownloader) LogString() string { } // GetRepoInfo returns a repository information -func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) { +func (g *GiteaDownloader) GetRepoInfo(_ context.Context) (*base.Repository, error) { if g == nil { return nil, errors.New("error: GiteaDownloader is nil") } @@ -164,19 +157,19 @@ func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) { } // GetTopics return gitea topics -func (g *GiteaDownloader) GetTopics() ([]string, error) { +func (g *GiteaDownloader) GetTopics(_ context.Context) ([]string, error) { topics, _, err := g.client.ListRepoTopics(g.repoOwner, g.repoName, gitea_sdk.ListRepoTopicsOptions{}) return topics, err } // GetMilestones returns milestones -func (g *GiteaDownloader) GetMilestones() ([]*base.Milestone, error) { +func (g *GiteaDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { milestones := make([]*base.Milestone, 0, g.maxPerPage) for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, nil default: } @@ -235,13 +228,13 @@ func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label } // GetLabels returns labels -func (g *GiteaDownloader) GetLabels() ([]*base.Label, error) { +func (g *GiteaDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) { labels := make([]*base.Label, 0, g.maxPerPage) for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, nil default: } @@ -305,7 +298,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele } // FIXME: for a private download? - req, err := http.NewRequest("GET", assetDownloadURL, nil) + req, err := http.NewRequest(http.MethodGet, assetDownloadURL, nil) if err != nil { return nil, err } @@ -323,13 +316,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele } // GetReleases returns releases -func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) { +func (g *GiteaDownloader) GetReleases(ctx context.Context) ([]*base.Release, error) { releases := make([]*base.Release, 0, g.maxPerPage) for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, nil default: } @@ -395,7 +388,7 @@ func (g *GiteaDownloader) getCommentReactions(commentID int64) ([]*base.Reaction } // GetIssues returns issues according start and limit -func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (g *GiteaDownloader) GetIssues(_ context.Context, page, perPage int) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -458,13 +451,13 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err } // GetComments returns comments according issueNumber -func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GiteaDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, g.maxPerPage) for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, false, nil default: } @@ -504,7 +497,7 @@ func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Com } // GetPullRequests returns pull requests according page and perPage -func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (g *GiteaDownloader) GetPullRequests(_ context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -624,7 +617,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } // GetReviews returns pull requests review -func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GiteaDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { log.Info("GiteaDownloader: instance to old, skip GetReviews") return nil, nil @@ -635,7 +628,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, nil default: } diff --git a/services/migrations/gitea_downloader_test.go b/services/migrations/gitea_downloader_test.go index c37c70947e..bb1760e889 100644 --- a/services/migrations/gitea_downloader_test.go +++ b/services/migrations/gitea_downloader_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "net/http" "os" "sort" @@ -14,6 +13,7 @@ import ( base "code.gitea.io/gitea/modules/migration" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGiteaDownloadRepo(t *testing.T) { @@ -27,16 +27,12 @@ func TestGiteaDownloadRepo(t *testing.T) { if err != nil || resp.StatusCode != http.StatusOK { t.Skipf("Can't reach https://gitea.com, skipping %s", t.Name()) } + ctx := t.Context() + downloader, err := NewGiteaDownloader(ctx, "https://gitea.com", "gitea/test_repo", "", "", giteaToken) + require.NoError(t, err, "NewGiteaDownloader error occur") + require.NotNil(t, downloader, "NewGiteaDownloader is nil") - downloader, err := NewGiteaDownloader(context.Background(), "https://gitea.com", "gitea/test_repo", "", "", giteaToken) - if downloader == nil { - t.Fatal("NewGitlabDownloader is nil") - } - if !assert.NoError(t, err) { - t.Fatal("NewGitlabDownloader error occur") - } - - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ Name: "test_repo", @@ -48,12 +44,12 @@ func TestGiteaDownloadRepo(t *testing.T) { DefaultBranch: "master", }, repo) - topics, err := downloader.GetTopics() + topics, err := downloader.GetTopics(ctx) assert.NoError(t, err) sort.Strings(topics) - assert.EqualValues(t, []string{"ci", "gitea", "migration", "test"}, topics) + assert.Equal(t, []string{"ci", "gitea", "migration", "test"}, topics) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assertLabelsEqual(t, []*base.Label{ { @@ -83,7 +79,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, labels) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -103,7 +99,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, milestones) - releases, err := downloader.GetReleases() + releases, err := downloader.GetReleases(ctx) assert.NoError(t, err) assertReleasesEqual(t, []*base.Release{ { @@ -134,13 +130,13 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, releases) - issues, isEnd, err := downloader.GetIssues(1, 50) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 50) assert.NoError(t, err) assert.True(t, isEnd) assert.Len(t, issues, 7) - assert.EqualValues(t, "open", issues[0].State) + assert.Equal(t, "open", issues[0].State) - issues, isEnd, err = downloader.GetIssues(3, 2) + issues, isEnd, err = downloader.GetIssues(ctx, 3, 2) assert.NoError(t, err) assert.False(t, isEnd) @@ -197,7 +193,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{Number: 4, ForeignIndex: 4}) + comments, _, err := downloader.GetComments(ctx, &base.Issue{Number: 4, ForeignIndex: 4}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -220,11 +216,11 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, comments) - prs, isEnd, err := downloader.GetPullRequests(1, 50) + prs, isEnd, err := downloader.GetPullRequests(ctx, 1, 50) assert.NoError(t, err) assert.True(t, isEnd) assert.Len(t, prs, 6) - prs, isEnd, err = downloader.GetPullRequests(1, 3) + prs, isEnd, err = downloader.GetPullRequests(ctx, 1, 3) assert.NoError(t, err) assert.False(t, isEnd) assert.Len(t, prs, 3) @@ -262,7 +258,7 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignIndex: 7}) + reviews, err := downloader.GetReviews(ctx, &base.Issue{Number: 7, ForeignIndex: 7}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 9e06b77b66..737bff24d0 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -41,7 +41,6 @@ var _ base.Uploader = &GiteaLocalUploader{} // GiteaLocalUploader implements an Uploader to gitea sites type GiteaLocalUploader struct { - ctx context.Context doer *user_model.User repoOwner string repoName string @@ -58,9 +57,8 @@ type GiteaLocalUploader struct { } // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 -func NewGiteaLocalUploader(ctx context.Context, doer *user_model.User, repoOwner, repoName string) *GiteaLocalUploader { +func NewGiteaLocalUploader(_ context.Context, doer *user_model.User, repoOwner, repoName string) *GiteaLocalUploader { return &GiteaLocalUploader{ - ctx: ctx, doer: doer, repoOwner: repoOwner, repoName: repoName, @@ -93,15 +91,15 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int { } // CreateRepo creates a repository -func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { - owner, err := user_model.GetUserByName(g.ctx, g.repoOwner) +func (g *GiteaLocalUploader) CreateRepo(ctx context.Context, repo *base.Repository, opts base.MigrateOptions) error { + owner, err := user_model.GetUserByName(ctx, g.repoOwner) if err != nil { return err } var r *repo_model.Repository if opts.MigrateToRepoID <= 0 { - r, err = repo_service.CreateRepositoryDirectly(g.ctx, g.doer, owner, repo_service.CreateRepoOptions{ + r, err = repo_service.CreateRepositoryDirectly(ctx, g.doer, owner, repo_service.CreateRepoOptions{ Name: g.repoName, Description: repo.Description, OriginalURL: repo.OriginalURL, @@ -109,9 +107,9 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate IsPrivate: opts.Private || setting.Repository.ForcePrivate, IsMirror: opts.Mirror, Status: repo_model.RepositoryBeingMigrated, - }) + }, false) } else { - r, err = repo_model.GetRepositoryByID(g.ctx, opts.MigrateToRepoID) + r, err = repo_model.GetRepositoryByID(ctx, opts.MigrateToRepoID) } if err != nil { return err @@ -119,7 +117,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate r.DefaultBranch = repo.DefaultBranch r.Description = repo.Description - r, err = repo_service.MigrateRepositoryGitData(g.ctx, owner, r, base.MigrateOptions{ + r, err = repo_service.MigrateRepositoryGitData(ctx, owner, r, base.MigrateOptions{ RepoName: g.repoName, Description: repo.Description, OriginalURL: repo.OriginalURL, @@ -139,7 +137,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate if err != nil { return err } - g.gitRepo, err = gitrepo.OpenRepository(g.ctx, g.repo) + g.gitRepo, err = gitrepo.OpenRepository(ctx, g.repo) if err != nil { return err } @@ -150,7 +148,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate return err } g.repo.ObjectFormatName = objectFormat.Name() - return repo_model.UpdateRepositoryCols(g.ctx, g.repo, "object_format_name") + return repo_model.UpdateRepositoryColsNoAutoTime(ctx, g.repo, "object_format_name") } // Close closes this uploader @@ -161,7 +159,7 @@ func (g *GiteaLocalUploader) Close() { } // CreateTopics creates topics -func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { +func (g *GiteaLocalUploader) CreateTopics(ctx context.Context, topics ...string) error { // Ignore topics too long for the db c := 0 for _, topic := range topics { @@ -173,11 +171,11 @@ func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { c++ } topics = topics[:c] - return repo_model.SaveTopics(g.ctx, g.repo.ID, topics...) + return repo_model.SaveTopics(ctx, g.repo.ID, topics...) } // CreateMilestones creates milestones -func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error { +func (g *GiteaLocalUploader) CreateMilestones(ctx context.Context, milestones ...*base.Milestone) error { mss := make([]*issues_model.Milestone, 0, len(milestones)) for _, milestone := range milestones { var deadline timeutil.TimeStamp @@ -216,7 +214,7 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err mss = append(mss, &ms) } - err := issues_model.InsertMilestones(g.ctx, mss...) + err := issues_model.InsertMilestones(ctx, mss...) if err != nil { return err } @@ -228,7 +226,7 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err } // CreateLabels creates labels -func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { +func (g *GiteaLocalUploader) CreateLabels(ctx context.Context, labels ...*base.Label) error { lbs := make([]*issues_model.Label, 0, len(labels)) for _, l := range labels { if color, err := label.NormalizeColor(l.Color); err != nil { @@ -247,7 +245,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { }) } - err := issues_model.NewLabels(g.ctx, lbs...) + err := issues_model.NewLabels(ctx, lbs...) if err != nil { return err } @@ -258,7 +256,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { } // CreateReleases creates releases -func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { +func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*base.Release) error { rels := make([]*repo_model.Release, 0, len(releases)) for _, release := range releases { if release.Created.IsZero() { @@ -292,7 +290,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { CreatedUnix: timeutil.TimeStamp(release.Created.Unix()), } - if err := g.remapUser(release, &rel); err != nil { + if err := g.remapUser(ctx, release, &rel); err != nil { return err } @@ -361,16 +359,16 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { rels = append(rels, &rel) } - return repo_model.InsertReleases(g.ctx, rels...) + return repo_model.InsertReleases(ctx, rels...) } // SyncTags syncs releases with tags in the database -func (g *GiteaLocalUploader) SyncTags() error { - return repo_module.SyncReleasesWithTags(g.ctx, g.repo, g.gitRepo) +func (g *GiteaLocalUploader) SyncTags(ctx context.Context) error { + return repo_module.SyncReleasesWithTags(ctx, g.repo, g.gitRepo) } // CreateIssues creates issues -func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { +func (g *GiteaLocalUploader) CreateIssues(ctx context.Context, issues ...*base.Issue) error { iss := make([]*issues_model.Issue, 0, len(issues)) for _, issue := range issues { var labels []*issues_model.Label @@ -419,7 +417,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()), } - if err := g.remapUser(issue, &is); err != nil { + if err := g.remapUser(ctx, issue, &is); err != nil { return err } @@ -432,7 +430,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { Type: reaction.Content, CreatedUnix: timeutil.TimeStampNow(), } - if err := g.remapUser(reaction, &res); err != nil { + if err := g.remapUser(ctx, reaction, &res); err != nil { return err } is.Reactions = append(is.Reactions, &res) @@ -441,7 +439,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } if len(iss) > 0 { - if err := issues_model.InsertIssues(g.ctx, iss...); err != nil { + if err := issues_model.InsertIssues(ctx, iss...); err != nil { return err } @@ -454,7 +452,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } // CreateComments creates comments of issues -func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { +func (g *GiteaLocalUploader) CreateComments(ctx context.Context, comments ...*base.Comment) error { cms := make([]*issues_model.Comment, 0, len(comments)) for _, comment := range comments { var issue *issues_model.Issue @@ -513,7 +511,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { default: } - if err := g.remapUser(comment, &cm); err != nil { + if err := g.remapUser(ctx, comment, &cm); err != nil { return err } @@ -523,7 +521,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { Type: reaction.Content, CreatedUnix: timeutil.TimeStampNow(), } - if err := g.remapUser(reaction, &res); err != nil { + if err := g.remapUser(ctx, reaction, &res); err != nil { return err } cm.Reactions = append(cm.Reactions, &res) @@ -535,35 +533,35 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { if len(cms) == 0 { return nil } - return issues_model.InsertIssueComments(g.ctx, cms) + return issues_model.InsertIssueComments(ctx, cms) } // CreatePullRequests creates pull requests -func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error { +func (g *GiteaLocalUploader) CreatePullRequests(ctx context.Context, prs ...*base.PullRequest) error { gprs := make([]*issues_model.PullRequest, 0, len(prs)) for _, pr := range prs { - gpr, err := g.newPullRequest(pr) + gpr, err := g.newPullRequest(ctx, pr) if err != nil { return err } - if err := g.remapUser(pr, gpr.Issue); err != nil { + if err := g.remapUser(ctx, pr, gpr.Issue); err != nil { return err } gprs = append(gprs, gpr) } - if err := issues_model.InsertPullRequests(g.ctx, gprs...); err != nil { + if err := issues_model.InsertPullRequests(ctx, gprs...); err != nil { return err } for _, pr := range gprs { g.issues[pr.Issue.Index] = pr.Issue - pull.AddToTaskQueue(g.ctx, pr) + pull.StartPullRequestCheckImmediately(ctx, pr) } return nil } -func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { +func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *base.PullRequest) (head string, err error) { // SECURITY: this pr must have been must have been ensured safe if !pr.EnsuredSafe { log.Error("PR #%d in %s/%s has not been checked for safety.", pr.Number, g.repoOwner, g.repoName) @@ -664,7 +662,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head fetchArg = git.BranchPrefix + fetchArg } - _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + _, _, err = git.NewCommand("fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(ctx, &git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) return head, nil @@ -683,7 +681,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head pr.Head.SHA = headSha } - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { return "", err } @@ -700,13 +698,13 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head // The SHA is empty log.Warn("Empty reference, no pull head for PR #%d in %s/%s", pr.Number, g.repoOwner, g.repoName) } else { - _, _, err = git.NewCommand(g.ctx, "rev-list", "--quiet", "-1").AddDynamicArguments(pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + _, _, err = git.NewCommand("rev-list", "--quiet", "-1").AddDynamicArguments(pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { // Git update-ref remove bad references with a relative path log.Warn("Deprecated local head %s for PR #%d in %s/%s, removing %s", pr.Head.SHA, pr.Number, g.repoOwner, g.repoName, pr.GetGitRefName()) } else { // set head information - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) } @@ -716,7 +714,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head return head, nil } -func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model.PullRequest, error) { +func (g *GiteaLocalUploader) newPullRequest(ctx context.Context, pr *base.PullRequest) (*issues_model.PullRequest, error) { var labels []*issues_model.Label for _, label := range pr.Labels { lb, ok := g.labels[label.Name] @@ -727,7 +725,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model milestoneID := g.milestones[pr.Milestone] - head, err := g.updateGitForPullRequest(pr) + head, err := g.updateGitForPullRequest(ctx, pr) if err != nil { return nil, fmt.Errorf("updateGitForPullRequest: %w", err) } @@ -767,7 +765,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model issue := issues_model.Issue{ RepoID: g.repo.ID, Repo: g.repo, - Title: prTitle, + Title: util.TruncateRunes(prTitle, 255), Index: pr.Number, Content: pr.Content, MilestoneID: milestoneID, @@ -779,7 +777,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()), } - if err := g.remapUser(pr, &issue); err != nil { + if err := g.remapUser(ctx, pr, &issue); err != nil { return nil, err } @@ -789,7 +787,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model Type: reaction.Content, CreatedUnix: timeutil.TimeStampNow(), } - if err := g.remapUser(reaction, &res); err != nil { + if err := g.remapUser(ctx, reaction, &res); err != nil { return nil, err } issue.Reactions = append(issue.Reactions, &res) @@ -839,7 +837,7 @@ func convertReviewState(state string) issues_model.ReviewType { } // CreateReviews create pull request reviews of currently migrated issues -func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { +func (g *GiteaLocalUploader) CreateReviews(ctx context.Context, reviews ...*base.Review) error { cms := make([]*issues_model.Review, 0, len(reviews)) for _, review := range reviews { var issue *issues_model.Issue @@ -860,7 +858,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()), } - if err := g.remapUser(review, &cm); err != nil { + if err := g.remapUser(ctx, review, &cm); err != nil { return err } @@ -870,7 +868,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { pr, ok := g.prCache[issue.ID] if !ok { var err error - pr, err = issues_model.GetPullRequestByIssueIDWithNoAttributes(g.ctx, issue.ID) + pr, err = issues_model.GetPullRequestByIssueIDWithNoAttributes(ctx, issue.ID) if err != nil { return err } @@ -940,7 +938,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()), } - if err := g.remapUser(review, &c); err != nil { + if err := g.remapUser(ctx, review, &c); err != nil { return err } @@ -948,7 +946,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { } } - return issues_model.InsertReviews(g.ctx, cms) + return issues_model.InsertReviews(ctx, cms) } // Rollback when migrating failed, this will rollback all the changes. @@ -962,31 +960,31 @@ func (g *GiteaLocalUploader) Rollback() error { } // Finish when migrating success, this will do some status update things. -func (g *GiteaLocalUploader) Finish() error { +func (g *GiteaLocalUploader) Finish(ctx context.Context) error { if g.repo == nil || g.repo.ID <= 0 { return ErrRepoNotCreated } // update issue_index - if err := issues_model.RecalculateIssueIndexForRepo(g.ctx, g.repo.ID); err != nil { + if err := issues_model.RecalculateIssueIndexForRepo(ctx, g.repo.ID); err != nil { return err } - if err := models.UpdateRepoStats(g.ctx, g.repo.ID); err != nil { + if err := models.UpdateRepoStats(ctx, g.repo.ID); err != nil { return err } g.repo.Status = repo_model.RepositoryReady - return repo_model.UpdateRepositoryCols(g.ctx, g.repo, "status") + return repo_model.UpdateRepositoryColsWithAutoTime(ctx, g.repo, "status") } -func (g *GiteaLocalUploader) remapUser(source user_model.ExternalUserMigrated, target user_model.ExternalUserRemappable) error { +func (g *GiteaLocalUploader) remapUser(ctx context.Context, source user_model.ExternalUserMigrated, target user_model.ExternalUserRemappable) error { var userID int64 var err error if g.sameApp { - userID, err = g.remapLocalUser(source) + userID, err = g.remapLocalUser(ctx, source) } else { - userID, err = g.remapExternalUser(source) + userID, err = g.remapExternalUser(ctx, source) } if err != nil { return err @@ -998,10 +996,10 @@ func (g *GiteaLocalUploader) remapUser(source user_model.ExternalUserMigrated, t return target.RemapExternalUser(source.GetExternalName(), source.GetExternalID(), g.doer.ID) } -func (g *GiteaLocalUploader) remapLocalUser(source user_model.ExternalUserMigrated) (int64, error) { +func (g *GiteaLocalUploader) remapLocalUser(ctx context.Context, source user_model.ExternalUserMigrated) (int64, error) { userid, ok := g.userMap[source.GetExternalID()] if !ok { - name, err := user_model.GetUserNameByID(g.ctx, source.GetExternalID()) + name, err := user_model.GetUserNameByID(ctx, source.GetExternalID()) if err != nil { return 0, err } @@ -1016,10 +1014,10 @@ func (g *GiteaLocalUploader) remapLocalUser(source user_model.ExternalUserMigrat return userid, nil } -func (g *GiteaLocalUploader) remapExternalUser(source user_model.ExternalUserMigrated) (userid int64, err error) { +func (g *GiteaLocalUploader) remapExternalUser(ctx context.Context, source user_model.ExternalUserMigrated) (userid int64, err error) { userid, ok := g.userMap[source.GetExternalID()] if !ok { - userid, err = user_model.GetUserIDByExternalUserID(g.ctx, g.gitServiceType.Name(), fmt.Sprintf("%d", source.GetExternalID())) + userid, err = user_model.GetUserIDByExternalUserID(ctx, g.gitServiceType.Name(), strconv.FormatInt(source.GetExternalID(), 10)) if err != nil { log.Error("GetUserIDByExternalUserID: %v", err) return 0, err diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index f2379dadf8..1970c0550c 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -5,7 +5,6 @@ package migrations import ( - "context" "fmt" "os" "path/filepath" @@ -40,7 +39,7 @@ func TestGiteaUploadRepo(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) var ( - ctx = context.Background() + ctx = t.Context() downloader = NewGithubDownloaderV3(ctx, "https://github.com", "", "", "", "go-xorm", "builder") repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05") uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) @@ -65,7 +64,7 @@ func TestGiteaUploadRepo(t *testing.T) { repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerID: user.ID, Name: repoName}) assert.True(t, repo.HasWiki()) - assert.EqualValues(t, repo_model.RepositoryReady, repo.Status) + assert.Equal(t, repo_model.RepositoryReady, repo.Status) milestones, err := db.Find[issues_model.Milestone](db.DefaultContext, issues_model.FindMilestoneOptions{ RepoID: repo.ID, @@ -132,8 +131,9 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + ctx := t.Context() repoName := "migrated" - uploader := NewGiteaLocalUploader(context.Background(), doer, doer.Name, repoName) + uploader := NewGiteaLocalUploader(ctx, doer, doer.Name, repoName) // call remapLocalUser uploader.sameApp = true @@ -150,9 +150,9 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { // target := repo_model.Release{} uploader.userMap = make(map[int64]int64) - err := uploader.remapUser(&source, &target) + err := uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, doer.ID, target.GetUserID()) + assert.Equal(t, doer.ID, target.GetUserID()) // // The externalID matches a known user but the name does not match, @@ -161,9 +161,9 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { source.PublisherID = user.ID target = repo_model.Release{} uploader.userMap = make(map[int64]int64) - err = uploader.remapUser(&source, &target) + err = uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, doer.ID, target.GetUserID()) + assert.Equal(t, doer.ID, target.GetUserID()) // // The externalID and externalName match an existing user, everything @@ -172,17 +172,17 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { source.PublisherName = user.Name target = repo_model.Release{} uploader.userMap = make(map[int64]int64) - err = uploader.remapUser(&source, &target) + err = uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, user.ID, target.GetUserID()) + assert.Equal(t, user.ID, target.GetUserID()) } func TestGiteaUploadRemapExternalUser(t *testing.T) { unittest.PrepareTestEnv(t) doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - + ctx := t.Context() repoName := "migrated" - uploader := NewGiteaLocalUploader(context.Background(), doer, doer.Name, repoName) + uploader := NewGiteaLocalUploader(ctx, doer, doer.Name, repoName) uploader.gitServiceType = structs.GiteaService // call remapExternalUser uploader.sameApp = false @@ -200,9 +200,9 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) { // uploader.userMap = make(map[int64]int64) target := repo_model.Release{} - err := uploader.remapUser(&source, &target) + err := uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, doer.ID, target.GetUserID()) + assert.Equal(t, doer.ID, target.GetUserID()) // // Link the external ID to an existing user @@ -223,9 +223,9 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) { // uploader.userMap = make(map[int64]int64) target = repo_model.Release{} - err = uploader.remapUser(&source, &target) + err = uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, linkedUser.ID, target.GetUserID()) + assert.Equal(t, linkedUser.ID, target.GetUserID()) } func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { @@ -237,9 +237,9 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { fromRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) baseRef := "master" assert.NoError(t, git.InitRepository(git.DefaultContext, fromRepo.RepoPath(), false, fromRepo.ObjectFormatName)) - err := git.NewCommand(git.DefaultContext, "symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef).Run(&git.RunOpts{Dir: fromRepo.RepoPath()}) + err := git.NewCommand("symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef).Run(git.DefaultContext, &git.RunOpts{Dir: fromRepo.RepoPath()}) assert.NoError(t, err) - assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# Testing Repository\n\nOriginally created in: %s", fromRepo.RepoPath())), 0o644)) + assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte("# Testing Repository\n\nOriginally created in: "+fromRepo.RepoPath()), 0o644)) assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true)) signature := git.Signature{ Email: "test@example.com", @@ -261,7 +261,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { // fromRepo branch1 // headRef := "branch1" - _, _, err = git.NewCommand(git.DefaultContext, "checkout", "-b").AddDynamicArguments(headRef).RunStdString(&git.RunOpts{Dir: fromRepo.RepoPath()}) + _, _, err = git.NewCommand("checkout", "-b").AddDynamicArguments(headRef).RunStdString(git.DefaultContext, &git.RunOpts{Dir: fromRepo.RepoPath()}) assert.NoError(t, err) assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte("SOMETHING"), 0o644)) assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true)) @@ -285,9 +285,9 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { assert.NoError(t, git.CloneWithArgs(git.DefaultContext, nil, fromRepo.RepoPath(), forkRepo.RepoPath(), git.CloneRepoOptions{ Branch: headRef, })) - _, _, err = git.NewCommand(git.DefaultContext, "checkout", "-b").AddDynamicArguments(forkHeadRef).RunStdString(&git.RunOpts{Dir: forkRepo.RepoPath()}) + _, _, err = git.NewCommand("checkout", "-b").AddDynamicArguments(forkHeadRef).RunStdString(git.DefaultContext, &git.RunOpts{Dir: forkRepo.RepoPath()}) assert.NoError(t, err) - assert.NoError(t, os.WriteFile(filepath.Join(forkRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# branch2 %s", forkRepo.RepoPath())), 0o644)) + assert.NoError(t, os.WriteFile(filepath.Join(forkRepo.RepoPath(), "README.md"), []byte("# branch2 "+forkRepo.RepoPath()), 0o644)) assert.NoError(t, git.AddChanges(forkRepo.RepoPath(), true)) assert.NoError(t, git.CommitChanges(forkRepo.RepoPath(), git.CommitChangesOptions{ Committer: &signature, @@ -301,11 +301,12 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { assert.NoError(t, err) toRepoName := "migrated" - uploader := NewGiteaLocalUploader(context.Background(), fromRepoOwner, fromRepoOwner.Name, toRepoName) + ctx := t.Context() + uploader := NewGiteaLocalUploader(ctx, fromRepoOwner, fromRepoOwner.Name, toRepoName) uploader.gitServiceType = structs.GiteaService - assert.NoError(t, repo_service.Init(context.Background())) - assert.NoError(t, uploader.CreateRepo(&base.Repository{ + assert.NoError(t, repo_service.Init(t.Context())) + assert.NoError(t, uploader.CreateRepo(ctx, &base.Repository{ Description: "description", OriginalURL: fromRepo.RepoPath(), CloneURL: fromRepo.RepoPath(), @@ -505,16 +506,16 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { testCase.pr.EnsuredSafe = true - head, err := uploader.updateGitForPullRequest(&testCase.pr) + head, err := uploader.updateGitForPullRequest(ctx, &testCase.pr) assert.NoError(t, err) - assert.EqualValues(t, testCase.head, head) + assert.Equal(t, testCase.head, head) log.Info(stopMark) logFiltered, logStopped := logChecker.Check(5 * time.Second) assert.True(t, logStopped) if len(testCase.logFilter) > 0 { - assert.EqualValues(t, testCase.logFiltered, logFiltered, "for log message filters: %v", testCase.logFilter) + assert.Equal(t, testCase.logFiltered, logFiltered, "for log message filters: %v", testCase.logFilter) } }) } diff --git a/services/migrations/github.go b/services/migrations/github.go index 604ab84b39..2ce11615c6 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -20,7 +20,7 @@ import ( "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/structs" - "github.com/google/go-github/v61/github" + "github.com/google/go-github/v71/github" "golang.org/x/oauth2" ) @@ -64,7 +64,6 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { // from github via APIv3 type GithubDownloaderV3 struct { base.NullDownloader - ctx context.Context clients []*github.Client baseURL string repoOwner string @@ -79,20 +78,19 @@ type GithubDownloaderV3 struct { } // NewGithubDownloaderV3 creates a github Downloader via github v3 API -func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 { +func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 { downloader := GithubDownloaderV3{ userName: userName, baseURL: baseURL, password: password, - ctx: ctx, repoOwner: repoOwner, repoName: repoName, maxPerPage: 100, } if token != "" { - tokens := strings.Split(token, ",") - for _, token := range tokens { + tokens := strings.SplitSeq(token, ",") + for token := range tokens { token = strings.TrimSpace(token) ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, @@ -135,18 +133,13 @@ func (g *GithubDownloaderV3) LogString() string { func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) { githubClient := github.NewClient(client) if baseURL != "https://github.com" { - githubClient, _ = github.NewClient(client).WithEnterpriseURLs(baseURL, baseURL) + githubClient, _ = githubClient.WithEnterpriseURLs(baseURL, baseURL) } g.clients = append(g.clients, githubClient) g.rates = append(g.rates, nil) } -// SetContext set context -func (g *GithubDownloaderV3) SetContext(ctx context.Context) { - g.ctx = ctx -} - -func (g *GithubDownloaderV3) waitAndPickClient() { +func (g *GithubDownloaderV3) waitAndPickClient(ctx context.Context) { var recentIdx int var maxRemaining int for i := 0; i < len(g.clients); i++ { @@ -160,13 +153,13 @@ func (g *GithubDownloaderV3) waitAndPickClient() { for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining { timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time)) select { - case <-g.ctx.Done(): + case <-ctx.Done(): timer.Stop() return case <-timer.C: } - err := g.RefreshRate() + err := g.RefreshRate(ctx) if err != nil { log.Error("g.getClient().RateLimit.Get: %s", err) } @@ -174,8 +167,8 @@ func (g *GithubDownloaderV3) waitAndPickClient() { } // RefreshRate update the current rate (doesn't count in rate limit) -func (g *GithubDownloaderV3) RefreshRate() error { - rates, _, err := g.getClient().RateLimit.Get(g.ctx) +func (g *GithubDownloaderV3) RefreshRate(ctx context.Context) error { + rates, _, err := g.getClient().RateLimit.Get(ctx) if err != nil { // if rate limit is not enabled, ignore it if strings.Contains(err.Error(), "404") { @@ -198,9 +191,9 @@ func (g *GithubDownloaderV3) setRate(rate *github.Rate) { } // GetRepoInfo returns a repository information -func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { - g.waitAndPickClient() - gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) +func (g *GithubDownloaderV3) GetRepoInfo(ctx context.Context) (*base.Repository, error) { + g.waitAndPickClient(ctx) + gr, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -219,9 +212,9 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { } // GetTopics return github topics -func (g *GithubDownloaderV3) GetTopics() ([]string, error) { - g.waitAndPickClient() - r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) +func (g *GithubDownloaderV3) GetTopics(ctx context.Context) ([]string, error) { + g.waitAndPickClient(ctx) + r, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -230,12 +223,12 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) { } // GetMilestones returns milestones -func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { +func (g *GithubDownloaderV3) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { perPage := g.maxPerPage milestones := make([]*base.Milestone, 0, perPage) for i := 1; ; i++ { - g.waitAndPickClient() - ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, + g.waitAndPickClient(ctx) + ms, resp, err := g.getClient().Issues.ListMilestones(ctx, g.repoOwner, g.repoName, &github.MilestoneListOptions{ State: "all", ListOptions: github.ListOptions{ @@ -279,12 +272,12 @@ func convertGithubLabel(label *github.Label) *base.Label { } // GetLabels returns labels -func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { +func (g *GithubDownloaderV3) GetLabels(ctx context.Context) ([]*base.Label, error) { perPage := g.maxPerPage labels := make([]*base.Label, 0, perPage) for i := 1; ; i++ { - g.waitAndPickClient() - ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, + g.waitAndPickClient(ctx) + ls, resp, err := g.getClient().Issues.ListLabels(ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, @@ -304,7 +297,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { return labels, nil } -func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release { +func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *github.RepositoryRelease) *base.Release { // GitHub allows commitish to be a reference. // In this case, we need to remove the prefix, i.e. convert "refs/heads/main" to "main". targetCommitish := strings.TrimPrefix(rel.GetTargetCommitish(), git.BranchPrefix) @@ -339,12 +332,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) Created: asset.CreatedAt.Time, Updated: asset.UpdatedAt.Time, DownloadFunc: func() (io.ReadCloser, error) { - g.waitAndPickClient() - readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil) + g.waitAndPickClient(ctx) + readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(ctx, g.repoOwner, g.repoName, assetID, nil) if err != nil { return nil, err } - if err := g.RefreshRate(); err != nil { + if err := g.RefreshRate(ctx); err != nil { log.Error("g.getClient().RateLimits: %s", err) } @@ -364,13 +357,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) return io.NopCloser(strings.NewReader(redirectURL)), nil } - g.waitAndPickClient() - req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil) + g.waitAndPickClient(ctx) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, redirectURL, nil) if err != nil { return nil, err } resp, err := httpClient.Do(req) - err1 := g.RefreshRate() + err1 := g.RefreshRate(ctx) if err1 != nil { log.Error("g.RefreshRate(): %s", err1) } @@ -385,12 +378,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) } // GetReleases returns releases -func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { +func (g *GithubDownloaderV3) GetReleases(ctx context.Context) ([]*base.Release, error) { perPage := g.maxPerPage releases := make([]*base.Release, 0, perPage) for i := 1; ; i++ { - g.waitAndPickClient() - ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, + g.waitAndPickClient(ctx) + ls, resp, err := g.getClient().Repositories.ListReleases(ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, @@ -401,7 +394,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { g.setRate(&resp.Rate) for _, release := range ls { - releases = append(releases, g.convertGithubRelease(release)) + releases = append(releases, g.convertGithubRelease(ctx, release)) } if len(ls) < perPage { break @@ -411,7 +404,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { } // GetIssues returns issues according start and limit -func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -426,8 +419,8 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, } allIssues := make([]*base.Issue, 0, perPage) - g.waitAndPickClient() - issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) + g.waitAndPickClient(ctx) + issues, resp, err := g.getClient().Issues.ListByRepo(ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } @@ -447,10 +440,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{ - Page: i, - PerPage: perPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: perPage, + }, }) if err != nil { return nil, false, err @@ -503,12 +498,12 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { } // GetComments returns comments according issueNumber -func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - comments, err := g.getComments(commentable) +func (g *GithubDownloaderV3) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { + comments, err := g.getComments(ctx, commentable) return comments, false, err } -func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -522,8 +517,8 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. }, } for { - g.waitAndPickClient() - comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt) + g.waitAndPickClient(ctx) + comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } @@ -533,10 +528,12 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ - Page: i, - PerPage: g.maxPerPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: g.maxPerPage, + }, }) if err != nil { return nil, err @@ -576,7 +573,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. } // GetAllComments returns repository comments according page and perPageSize -func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) { +func (g *GithubDownloaderV3) GetAllComments(ctx context.Context, page, perPage int) ([]*base.Comment, bool, error) { var ( allComments = make([]*base.Comment, 0, perPage) created = "created" @@ -594,8 +591,8 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, }, } - g.waitAndPickClient() - comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt) + g.waitAndPickClient(ctx) + comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, 0, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } @@ -608,10 +605,12 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ - Page: i, - PerPage: g.maxPerPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: g.maxPerPage, + }, }) if err != nil { return nil, false, err @@ -648,7 +647,7 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, } // GetPullRequests returns pull requests according page and perPage -func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -662,8 +661,8 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, } allPRs := make([]*base.PullRequest, 0, perPage) - g.waitAndPickClient() - prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) + g.waitAndPickClient(ctx) + prs, resp, err := g.getClient().PullRequests.List(ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } @@ -679,10 +678,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{ - Page: i, - PerPage: perPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: perPage, + }, }) if err != nil { return nil, false, err @@ -702,7 +703,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq } // download patch and saved as tmp file - g.waitAndPickClient() + g.waitAndPickClient(ctx) allPRs = append(allPRs, &base.PullRequest{ Title: pr.GetTitle(), @@ -759,17 +760,19 @@ func convertGithubReview(r *github.PullRequestReview) *base.Review { } } -func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) { +func (g *GithubDownloaderV3) convertGithubReviewComments(ctx context.Context, cs []*github.PullRequestComment) ([]*base.ReviewComment, error) { rcs := make([]*base.ReviewComment, 0, len(cs)) for _, c := range cs { // get reactions var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{ - Page: i, - PerPage: g.maxPerPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: g.maxPerPage, + }, }) if err != nil { return nil, err @@ -806,7 +809,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } // GetReviews returns pull requests review -func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { allReviews := make([]*base.Review, 0, g.maxPerPage) if g.SkipReviews { return allReviews, nil @@ -816,8 +819,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } // Get approve/request change reviews for { - g.waitAndPickClient() - reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) + g.waitAndPickClient(ctx) + reviews, resp, err := g.getClient().PullRequests.ListReviews(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } @@ -830,14 +833,14 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev PerPage: g.maxPerPage, } for { - g.waitAndPickClient() - reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) + g.waitAndPickClient(ctx) + reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } g.setRate(&resp.Rate) - cs, err := g.convertGithubReviewComments(reviewComments) + cs, err := g.convertGithubReviewComments(ctx, reviewComments) if err != nil { return nil, err } @@ -856,8 +859,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } // Get requested reviews for { - g.waitAndPickClient() - reviewers, resp, err := g.getClient().PullRequests.ListReviewers(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) + g.waitAndPickClient(ctx) + reviewers, resp, err := g.getClient().PullRequests.ListReviewers(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } @@ -879,3 +882,18 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } return allReviews, nil } + +// FormatCloneURL add authentication into remote URLs +func (g *GithubDownloaderV3) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) { + u, err := url.Parse(remoteAddr) + if err != nil { + return "", err + } + if len(opts.AuthToken) > 0 { + // "multiple tokens" are used to benefit more "API rate limit quota" + // git clone doesn't count for rate limits, so only use the first token. + // source: https://github.com/orgs/community/discussions/44515 + u.User = url.UserPassword("oauth2", strings.Split(opts.AuthToken, ",")[0]) + } + return u.String(), nil +} diff --git a/services/migrations/github_test.go b/services/migrations/github_test.go index 2b89e6dc0f..6d1a5378b9 100644 --- a/services/migrations/github_test.go +++ b/services/migrations/github_test.go @@ -5,7 +5,6 @@ package migrations import ( - "context" "os" "testing" "time" @@ -13,6 +12,7 @@ import ( base "code.gitea.io/gitea/modules/migration" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGitHubDownloadRepo(t *testing.T) { @@ -21,11 +21,12 @@ func TestGitHubDownloadRepo(t *testing.T) { if token == "" { t.Skip("Skipping GitHub migration test because GITHUB_READ_TOKEN is empty") } - downloader := NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", token, "go-gitea", "test_repo") - err := downloader.RefreshRate() + ctx := t.Context() + downloader := NewGithubDownloaderV3(ctx, "https://github.com", "", "", token, "go-gitea", "test_repo") + err := downloader.RefreshRate(ctx) assert.NoError(t, err) - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ Name: "test_repo", @@ -36,11 +37,11 @@ func TestGitHubDownloadRepo(t *testing.T) { DefaultBranch: "master", }, repo) - topics, err := downloader.GetTopics() + topics, err := downloader.GetTopics(ctx) assert.NoError(t, err) assert.Contains(t, topics, "gitea") - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -63,7 +64,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assertLabelsEqual(t, []*base.Label{ { @@ -113,7 +114,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, labels) - releases, err := downloader.GetReleases() + releases, err := downloader.GetReleases(ctx) assert.NoError(t, err) assertReleasesEqual(t, []*base.Release{ { @@ -129,7 +130,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, releases) // downloader.GetIssues() - issues, isEnd, err := downloader.GetIssues(1, 2) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 2) assert.NoError(t, err) assert.False(t, isEnd) assertIssuesEqual(t, []*base.Issue{ @@ -218,7 +219,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 2, ForeignIndex: 2}) + comments, _, err := downloader.GetComments(ctx, &base.Issue{Number: 2, ForeignIndex: 2}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -248,7 +249,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, comments) // downloader.GetPullRequests() - prs, _, err := downloader.GetPullRequests(1, 2) + prs, _, err := downloader.GetPullRequests(ctx, 1, 2) assert.NoError(t, err) assertPullRequestsEqual(t, []*base.PullRequest{ { @@ -338,7 +339,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, prs) - reviews, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignIndex: 3}) + reviews, err := downloader.GetReviews(ctx, &base.PullRequest{Number: 3, ForeignIndex: 3}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -370,7 +371,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignIndex: 4}) + reviews, err = downloader.GetReviews(ctx, &base.PullRequest{Number: 4, ForeignIndex: 4}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -429,3 +430,36 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) } + +func TestGithubMultiToken(t *testing.T) { + testCases := []struct { + desc string + token string + expectedCloneURL string + }{ + { + desc: "Single Token", + token: "single_token", + expectedCloneURL: "https://oauth2:single_token@github.com", + }, + { + desc: "Multi Token", + token: "token1,token2", + expectedCloneURL: "https://oauth2:token1@github.com", + }, + } + factory := GithubDownloaderV3Factory{} + + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + opts := base.MigrateOptions{CloneAddr: "https://github.com/go-gitea/gitea", AuthToken: tC.token} + client, err := factory.New(t.Context(), opts) + require.NoError(t, err) + + cloneURL, err := client.FormatCloneURL(opts, "https://github.com") + require.NoError(t, err) + + assert.Equal(t, tC.expectedCloneURL, cloneURL) + }) + } +} diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index 07d5040b5b..a19a04bc44 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -16,6 +16,7 @@ import ( "time" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" @@ -80,7 +81,6 @@ func (r *gitlabIIDResolver) generatePullRequestNumber(mrIID int) int64 { // because Gitlab has individual Issue and Pull Request numbers. type GitlabDownloader struct { base.NullDownloader - ctx context.Context client *gitlab.Client baseURL string repoID int @@ -143,7 +143,6 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw } return &GitlabDownloader{ - ctx: ctx, client: gitlabClient, baseURL: baseURL, repoID: gr.ID, @@ -164,14 +163,9 @@ func (g *GitlabDownloader) LogString() string { return fmt.Sprintf("<GitlabDownloader %s [%d]/%s>", g.baseURL, g.repoID, g.repoName) } -// SetContext set context -func (g *GitlabDownloader) SetContext(ctx context.Context) { - g.ctx = ctx -} - // GetRepoInfo returns a repository information -func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) { - gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { + gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -207,8 +201,8 @@ func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) { } // GetTopics return gitlab topics -func (g *GitlabDownloader) GetTopics() ([]string, error) { - gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetTopics(ctx context.Context) ([]string, error) { + gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -216,7 +210,7 @@ func (g *GitlabDownloader) GetTopics() ([]string, error) { } // GetMilestones returns milestones -func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) { +func (g *GitlabDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { perPage := g.maxPerPage state := "all" milestones := make([]*base.Milestone, 0, perPage) @@ -227,7 +221,7 @@ func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) { Page: i, PerPage: perPage, }, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -288,14 +282,14 @@ func (g *GitlabDownloader) normalizeColor(val string) string { } // GetLabels returns labels -func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) { +func (g *GitlabDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) { perPage := g.maxPerPage labels := make([]*base.Label, 0, perPage) for i := 1; ; i++ { ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{ Page: i, PerPage: perPage, - }}, nil, gitlab.WithContext(g.ctx)) + }}, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -314,7 +308,7 @@ func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) { return labels, nil } -func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release { +func (g *GitlabDownloader) convertGitlabRelease(ctx context.Context, rel *gitlab.Release) *base.Release { var zero int r := &base.Release{ TagName: rel.TagName, @@ -337,7 +331,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea Size: &zero, DownloadCount: &zero, DownloadFunc: func() (io.ReadCloser, error) { - link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, assetID, gitlab.WithContext(g.ctx)) + link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, assetID, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -347,11 +341,11 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea return io.NopCloser(strings.NewReader(link.URL)), nil } - req, err := http.NewRequest("GET", link.URL, nil) + req, err := http.NewRequest(http.MethodGet, link.URL, nil) if err != nil { return nil, err } - req = req.WithContext(g.ctx) + req = req.WithContext(ctx) resp, err := httpClient.Do(req) if err != nil { return nil, err @@ -366,7 +360,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea } // GetReleases returns releases -func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { +func (g *GitlabDownloader) GetReleases(ctx context.Context) ([]*base.Release, error) { perPage := g.maxPerPage releases := make([]*base.Release, 0, perPage) for i := 1; ; i++ { @@ -375,13 +369,13 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { Page: i, PerPage: perPage, }, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } for _, release := range ls { - releases = append(releases, g.convertGitlabRelease(release)) + releases = append(releases, g.convertGitlabRelease(ctx, release)) } if len(ls) < perPage { break @@ -397,7 +391,7 @@ type gitlabIssueContext struct { // GetIssues returns issues according start and limit // // Note: issue label description and colors are not supported by the go-gitlab library at this time -func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (g *GitlabDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { state := "all" sort := "asc" @@ -416,7 +410,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er allIssues := make([]*base.Issue, 0, perPage) - issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(g.ctx)) + issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(ctx)) if err != nil { return nil, false, fmt.Errorf("error while listing issues: %w", err) } @@ -436,7 +430,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er var reactions []*gitlab.AwardEmoji awardPage := 1 for { - awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx)) + awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(ctx)) if err != nil { return nil, false, fmt.Errorf("error while listing issue awards: %w", err) } @@ -477,7 +471,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er // GetComments returns comments according issueNumber // TODO: figure out how to transfer comment reactions -func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GitlabDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { context, ok := commentable.GetContext().(gitlabIssueContext) if !ok { return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) @@ -495,12 +489,12 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) } else { comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) } if err != nil { @@ -528,25 +522,29 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co Page: page, PerPage: g.maxPerPage, }, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) } else { stateEvents, resp, err = g.client.ResourceStateEvents.ListIssueStateEvents(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListStateEventsOptions{ ListOptions: gitlab.ListOptions{ Page: page, PerPage: g.maxPerPage, }, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) } if err != nil { return nil, false, fmt.Errorf("error while listing state events: %v %w", g.repoID, err) } for _, stateEvent := range stateEvents { + posterUserID, posterUsername := user.GhostUserID, user.GhostUserName + if stateEvent.User != nil { + posterUserID, posterUsername = int64(stateEvent.User.ID), stateEvent.User.Username + } comment := &base.Comment{ IssueIndex: commentable.GetLocalIndex(), Index: int64(stateEvent.ID), - PosterID: int64(stateEvent.User.ID), - PosterName: stateEvent.User.Username, + PosterID: posterUserID, + PosterName: posterUsername, Content: "", Created: *stateEvent.CreatedAt, } @@ -604,7 +602,7 @@ func (g *GitlabDownloader) convertNoteToComment(localIndex int64, note *gitlab.N } // GetPullRequests returns pull requests according page and perPage -func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (g *GitlabDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -620,7 +618,7 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque allPRs := make([]*base.PullRequest, 0, perPage) - prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(g.ctx)) + prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(ctx)) if err != nil { return nil, false, fmt.Errorf("error while listing merge requests: %w", err) } @@ -673,7 +671,7 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque var reactions []*gitlab.AwardEmoji awardPage := 1 for { - awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx)) + awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(ctx)) if err != nil { return nil, false, fmt.Errorf("error while listing merge requests awards: %w", err) } @@ -733,8 +731,8 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests review -func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { - approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { + approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index 556fe771c5..73a1b6a276 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "fmt" "net/http" "net/http/httptest" @@ -31,12 +30,12 @@ func TestGitlabDownloadRepo(t *testing.T) { if err != nil || resp.StatusCode != http.StatusOK { t.Skipf("Can't access test repo, skipping %s", t.Name()) } - - downloader, err := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken) + ctx := t.Context() + downloader, err := NewGitlabDownloader(ctx, "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken) if err != nil { t.Fatalf("NewGitlabDownloader is nil: %v", err) } - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) // Repo Owner is blank in Gitlab Group repos assertRepositoryEqual(t, &base.Repository{ @@ -48,12 +47,12 @@ func TestGitlabDownloadRepo(t *testing.T) { DefaultBranch: "master", }, repo) - topics, err := downloader.GetTopics() + topics, err := downloader.GetTopics(ctx) assert.NoError(t, err) assert.Len(t, topics, 2) - assert.EqualValues(t, []string{"migration", "test"}, topics) + assert.Equal(t, []string{"migration", "test"}, topics) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -71,7 +70,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assertLabelsEqual(t, []*base.Label{ { @@ -112,7 +111,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, labels) - releases, err := downloader.GetReleases() + releases, err := downloader.GetReleases(ctx) assert.NoError(t, err) assertReleasesEqual(t, []*base.Release{ { @@ -126,7 +125,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, releases) - issues, isEnd, err := downloader.GetIssues(1, 2) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 2) assert.NoError(t, err) assert.False(t, isEnd) @@ -214,7 +213,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{ + comments, _, err := downloader.GetComments(ctx, &base.Issue{ Number: 2, ForeignIndex: 2, Context: gitlabIssueContext{IsMergeRequest: false}, @@ -255,7 +254,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, comments) - prs, _, err := downloader.GetPullRequests(1, 1) + prs, _, err := downloader.GetPullRequests(ctx, 1, 1) assert.NoError(t, err) assertPullRequestsEqual(t, []*base.PullRequest{ { @@ -304,7 +303,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignIndex: 1}) + rvs, err := downloader.GetReviews(ctx, &base.PullRequest{Number: 1, ForeignIndex: 1}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -323,7 +322,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, rvs) - rvs, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignIndex: 2}) + rvs, err = downloader.GetReviews(ctx, &base.PullRequest{Number: 2, ForeignIndex: 2}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -423,9 +422,8 @@ func TestGitlabGetReviews(t *testing.T) { defer gitlabClientMockTeardown(server) repoID := 1324 - + ctx := t.Context() downloader := &GitlabDownloader{ - ctx: context.Background(), client: client, repoID: repoID, } @@ -465,7 +463,7 @@ func TestGitlabGetReviews(t *testing.T) { mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/approvals", testCase.repoID, testCase.prID), mock) id := int64(testCase.prID) - rvs, err := downloader.GetReviews(&base.Issue{Number: id, ForeignIndex: id}) + rvs, err := downloader.GetReviews(ctx, &base.Issue{Number: id, ForeignIndex: id}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{&review}, rvs) } @@ -503,7 +501,7 @@ func TestAwardsToReactions(t *testing.T) { assert.NoError(t, json.Unmarshal([]byte(testResponse), &awards)) reactions := downloader.awardsToReactions(awards) - assert.EqualValues(t, []*base.Reaction{ + assert.Equal(t, []*base.Reaction{ { UserName: "lafriks", UserID: 1241334, @@ -595,7 +593,7 @@ func TestNoteToComment(t *testing.T) { for i, note := range notes { actualComment := *downloader.convertNoteToComment(17, ¬e) - assert.EqualValues(t, actualComment, comments[i]) + assert.Equal(t, actualComment, comments[i]) } } diff --git a/services/migrations/gogs.go b/services/migrations/gogs.go index 72c52d180b..a4f84dbf72 100644 --- a/services/migrations/gogs.go +++ b/services/migrations/gogs.go @@ -13,7 +13,6 @@ import ( "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" - "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/structs" "github.com/gogs/go-gogs-client" @@ -60,16 +59,14 @@ func (f *GogsDownloaderFactory) GitServiceType() structs.GitServiceType { // from gogs via API type GogsDownloader struct { base.NullDownloader - ctx context.Context - client *gogs.Client baseURL string repoOwner string repoName string userName string password string + token string openIssuesFinished bool openIssuesPages int - transport http.RoundTripper } // String implements Stringer @@ -84,53 +81,45 @@ func (g *GogsDownloader) LogString() string { return fmt.Sprintf("<GogsDownloader %s %s/%s>", g.baseURL, g.repoOwner, g.repoName) } -// SetContext set context -func (g *GogsDownloader) SetContext(ctx context.Context) { - g.ctx = ctx -} - // NewGogsDownloader creates a gogs Downloader via gogs API -func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader { +func NewGogsDownloader(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader { downloader := GogsDownloader{ - ctx: ctx, baseURL: baseURL, userName: userName, password: password, + token: token, repoOwner: repoOwner, repoName: repoName, } + return &downloader +} - var client *gogs.Client - if len(token) != 0 { - client = gogs.NewClient(baseURL, token) - downloader.userName = token - } else { - transport := NewMigrationHTTPTransport() - transport.Proxy = func(req *http.Request) (*url.URL, error) { - req.SetBasicAuth(userName, password) - return proxy.Proxy()(req) - } - downloader.transport = transport - - client = gogs.NewClient(baseURL, "") - client.SetHTTPClient(&http.Client{ - Transport: &downloader, - }) - } +type roundTripperFunc func(req *http.Request) (*http.Response, error) - downloader.client = client - return &downloader +func (rt roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return rt(r) } -// RoundTrip wraps the provided request within this downloader's context and passes it to our internal http.Transport. -// This implements http.RoundTripper and makes the gogs client requests cancellable even though it is not cancellable itself -func (g *GogsDownloader) RoundTrip(req *http.Request) (*http.Response, error) { - return g.transport.RoundTrip(req.WithContext(g.ctx)) +func (g *GogsDownloader) client(ctx context.Context) *gogs.Client { + // Gogs client lacks the context support, so we use a custom transport + // Then each request uses a dedicated client with its own context + httpTransport := NewMigrationHTTPTransport() + gogsClient := gogs.NewClient(g.baseURL, g.token) + gogsClient.SetHTTPClient(&http.Client{ + Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + if g.password != "" { + // Gogs client lacks the support for basic auth, this is the only way to set it + req.SetBasicAuth(g.userName, g.password) + } + return httpTransport.RoundTrip(req.WithContext(ctx)) + }), + }) + return gogsClient } // GetRepoInfo returns a repository information -func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) { - gr, err := g.client.GetRepo(g.repoOwner, g.repoName) +func (g *GogsDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { + gr, err := g.client(ctx).GetRepo(g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -148,11 +137,11 @@ func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) { } // GetMilestones returns milestones -func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) { +func (g *GogsDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { perPage := 100 milestones := make([]*base.Milestone, 0, perPage) - ms, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName) + ms, err := g.client(ctx).ListRepoMilestones(g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -171,10 +160,10 @@ func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) { } // GetLabels returns labels -func (g *GogsDownloader) GetLabels() ([]*base.Label, error) { +func (g *GogsDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) { perPage := 100 labels := make([]*base.Label, 0, perPage) - ls, err := g.client.ListRepoLabels(g.repoOwner, g.repoName) + ls, err := g.client(ctx).ListRepoLabels(g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -187,7 +176,7 @@ func (g *GogsDownloader) GetLabels() ([]*base.Label, error) { } // GetIssues returns issues according start and limit, perPage is not supported -func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) { +func (g *GogsDownloader) GetIssues(ctx context.Context, page, _ int) ([]*base.Issue, bool, error) { var state string if g.openIssuesFinished { state = string(gogs.STATE_CLOSED) @@ -197,7 +186,7 @@ func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) { g.openIssuesPages = page } - issues, isEnd, err := g.getIssues(page, state) + issues, isEnd, err := g.getIssues(ctx, page, state) if err != nil { return nil, false, err } @@ -212,10 +201,10 @@ func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) { return issues, false, nil } -func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, error) { +func (g *GogsDownloader) getIssues(ctx context.Context, page int, state string) ([]*base.Issue, bool, error) { allIssues := make([]*base.Issue, 0, 10) - issues, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{ + issues, err := g.client(ctx).ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{ Page: page, State: state, }) @@ -234,10 +223,10 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GogsDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, 100) - comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex()) + comments, err := g.client(ctx).ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex()) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } @@ -261,7 +250,7 @@ func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comm } // GetTopics return repository topics -func (g *GogsDownloader) GetTopics() ([]string, error) { +func (g *GogsDownloader) GetTopics(_ context.Context) ([]string, error) { return []string{}, nil } diff --git a/services/migrations/gogs_test.go b/services/migrations/gogs_test.go index 610af183de..503b669f8e 100644 --- a/services/migrations/gogs_test.go +++ b/services/migrations/gogs_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "net/http" "os" "testing" @@ -28,9 +27,9 @@ func TestGogsDownloadRepo(t *testing.T) { t.Skipf("visit test repo failed, ignored") return } - - downloader := NewGogsDownloader(context.Background(), "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO") - repo, err := downloader.GetRepoInfo() + ctx := t.Context() + downloader := NewGogsDownloader(ctx, "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO") + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ @@ -42,7 +41,7 @@ func TestGogsDownloadRepo(t *testing.T) { DefaultBranch: "master", }, repo) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -51,7 +50,7 @@ func TestGogsDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assertLabelsEqual(t, []*base.Label{ { @@ -85,7 +84,7 @@ func TestGogsDownloadRepo(t *testing.T) { }, labels) // downloader.GetIssues() - issues, isEnd, err := downloader.GetIssues(1, 8) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 8) assert.NoError(t, err) assert.False(t, isEnd) assertIssuesEqual(t, []*base.Issue{ @@ -110,7 +109,7 @@ func TestGogsDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 1, ForeignIndex: 1}) + comments, _, err := downloader.GetComments(ctx, &base.Issue{Number: 1, ForeignIndex: 1}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -134,6 +133,6 @@ func TestGogsDownloadRepo(t *testing.T) { }, comments) // downloader.GetPullRequests() - _, _, err = downloader.GetPullRequests(1, 3) + _, _, err = downloader.GetPullRequests(ctx, 1, 3) assert.Error(t, err) } diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 51b22d6111..eba9c79df5 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -6,6 +6,7 @@ package migrations import ( "context" + "errors" "fmt" "net" "net/url" @@ -74,11 +75,9 @@ func IsMigrateURLAllowed(remoteURL string, doer *user_model.User) error { return &git.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true} } - hostName, _, err := net.SplitHostPort(u.Host) - if err != nil { - // u.Host can be "host" or "host:port" - err = nil //nolint - hostName = u.Host + hostName, _, errIgnored := net.SplitHostPort(u.Host) + if errIgnored != nil { + hostName = u.Host // u.Host can be "host" or "host:port" } // some users only use proxy, there is no DNS resolver. it's safe to ignore the LookupIP error @@ -168,7 +167,7 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio } if setting.Migrations.MaxAttempts > 1 { - downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) + downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) } return downloader, nil } @@ -176,12 +175,12 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio // migrateRepository will download information and then upload it to Uploader, this is a simple // process for small repository. For a big repository, save all the data to disk // before upload is better -func migrateRepository(_ context.Context, doer *user_model.User, downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error { +func migrateRepository(ctx context.Context, doer *user_model.User, downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error { if messenger == nil { messenger = base.NilMessenger } - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -211,7 +210,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if cloneURL.Scheme == "file" || cloneURL.Scheme == "" { if cloneAddrURL.Scheme != "file" && cloneAddrURL.Scheme != "" { - return fmt.Errorf("repo info has changed from external to local filesystem") + return errors.New("repo info has changed from external to local filesystem") } } @@ -220,14 +219,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base log.Trace("migrating git data from %s", repo.CloneURL) messenger("repo.migrate.migrating_git") - if err = uploader.CreateRepo(repo, opts); err != nil { + if err = uploader.CreateRepo(ctx, repo, opts); err != nil { return err } defer uploader.Close() log.Trace("migrating topics") messenger("repo.migrate.migrating_topics") - topics, err := downloader.GetTopics() + topics, err := downloader.GetTopics(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -235,7 +234,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base log.Warn("migrating topics is not supported, ignored") } if len(topics) != 0 { - if err = uploader.CreateTopics(topics...); err != nil { + if err = uploader.CreateTopics(ctx, topics...); err != nil { return err } } @@ -243,7 +242,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if opts.Milestones { log.Trace("migrating milestones") messenger("repo.migrate.migrating_milestones") - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -256,7 +255,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base msBatchSize = len(milestones) } - if err := uploader.CreateMilestones(milestones[:msBatchSize]...); err != nil { + if err := uploader.CreateMilestones(ctx, milestones[:msBatchSize]...); err != nil { return err } milestones = milestones[msBatchSize:] @@ -266,7 +265,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if opts.Labels { log.Trace("migrating labels") messenger("repo.migrate.migrating_labels") - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -280,7 +279,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base lbBatchSize = len(labels) } - if err := uploader.CreateLabels(labels[:lbBatchSize]...); err != nil { + if err := uploader.CreateLabels(ctx, labels[:lbBatchSize]...); err != nil { return err } labels = labels[lbBatchSize:] @@ -290,7 +289,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if opts.Releases { log.Trace("migrating releases") messenger("repo.migrate.migrating_releases") - releases, err := downloader.GetReleases() + releases, err := downloader.GetReleases(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -304,14 +303,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base relBatchSize = len(releases) } - if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil { + if err = uploader.CreateReleases(ctx, releases[:relBatchSize]...); err != nil { return err } releases = releases[relBatchSize:] } // Once all releases (if any) are inserted, sync any remaining non-release tags - if err = uploader.SyncTags(); err != nil { + if err = uploader.SyncTags(ctx); err != nil { return err } } @@ -329,7 +328,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base issueBatchSize := uploader.MaxBatchInsertSize("issue") for i := 1; ; i++ { - issues, isEnd, err := downloader.GetIssues(i, issueBatchSize) + issues, isEnd, err := downloader.GetIssues(ctx, i, issueBatchSize) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -338,7 +337,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base break } - if err := uploader.CreateIssues(issues...); err != nil { + if err := uploader.CreateIssues(ctx, issues...); err != nil { return err } @@ -346,7 +345,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allComments := make([]*base.Comment, 0, commentBatchSize) for _, issue := range issues { log.Trace("migrating issue %d's comments", issue.Number) - comments, _, err := downloader.GetComments(issue) + comments, _, err := downloader.GetComments(ctx, issue) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -357,7 +356,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allComments = append(allComments, comments...) if len(allComments) >= commentBatchSize { - if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + if err = uploader.CreateComments(ctx, allComments[:commentBatchSize]...); err != nil { return err } @@ -366,7 +365,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base } if len(allComments) > 0 { - if err = uploader.CreateComments(allComments...); err != nil { + if err = uploader.CreateComments(ctx, allComments...); err != nil { return err } } @@ -383,7 +382,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base messenger("repo.migrate.migrating_pulls") prBatchSize := uploader.MaxBatchInsertSize("pullrequest") for i := 1; ; i++ { - prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) + prs, isEnd, err := downloader.GetPullRequests(ctx, i, prBatchSize) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -392,7 +391,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base break } - if err := uploader.CreatePullRequests(prs...); err != nil { + if err := uploader.CreatePullRequests(ctx, prs...); err != nil { return err } @@ -402,7 +401,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allComments := make([]*base.Comment, 0, commentBatchSize) for _, pr := range prs { log.Trace("migrating pull request %d's comments", pr.Number) - comments, _, err := downloader.GetComments(pr) + comments, _, err := downloader.GetComments(ctx, pr) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -413,14 +412,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allComments = append(allComments, comments...) if len(allComments) >= commentBatchSize { - if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + if err = uploader.CreateComments(ctx, allComments[:commentBatchSize]...); err != nil { return err } allComments = allComments[commentBatchSize:] } } if len(allComments) > 0 { - if err = uploader.CreateComments(allComments...); err != nil { + if err = uploader.CreateComments(ctx, allComments...); err != nil { return err } } @@ -429,7 +428,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base // migrate reviews allReviews := make([]*base.Review, 0, reviewBatchSize) for _, pr := range prs { - reviews, err := downloader.GetReviews(pr) + reviews, err := downloader.GetReviews(ctx, pr) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -441,14 +440,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allReviews = append(allReviews, reviews...) if len(allReviews) >= reviewBatchSize { - if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { + if err = uploader.CreateReviews(ctx, allReviews[:reviewBatchSize]...); err != nil { return err } allReviews = allReviews[reviewBatchSize:] } } if len(allReviews) > 0 { - if err = uploader.CreateReviews(allReviews...); err != nil { + if err = uploader.CreateReviews(ctx, allReviews...); err != nil { return err } } @@ -463,12 +462,12 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if opts.Comments && supportAllComments { log.Trace("migrating comments") for i := 1; ; i++ { - comments, isEnd, err := downloader.GetAllComments(i, commentBatchSize) + comments, isEnd, err := downloader.GetAllComments(ctx, i, commentBatchSize) if err != nil { return err } - if err := uploader.CreateComments(comments...); err != nil { + if err := uploader.CreateComments(ctx, comments...); err != nil { return err } @@ -478,7 +477,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base } } - return uploader.Finish() + return uploader.Finish(ctx) } // Init migrations service diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index e2f7b771f3..e052cba0cc 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -71,7 +71,6 @@ type onedevUser struct { // from OneDev type OneDevDownloader struct { base.NullDownloader - ctx context.Context client *http.Client baseURL *url.URL repoName string @@ -81,15 +80,9 @@ type OneDevDownloader struct { milestoneMap map[int64]string } -// SetContext set context -func (d *OneDevDownloader) SetContext(ctx context.Context) { - d.ctx = ctx -} - // NewOneDevDownloader creates a new downloader -func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader { +func NewOneDevDownloader(_ context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader { downloader := &OneDevDownloader{ - ctx: ctx, baseURL: baseURL, repoName: repoName, client: &http.Client{ @@ -121,7 +114,7 @@ func (d *OneDevDownloader) LogString() string { return fmt.Sprintf("<OneDevDownloader %s [%d]/%s>", d.baseURL, d.repoID, d.repoName) } -func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result any) error { +func (d *OneDevDownloader) callAPI(ctx context.Context, endpoint string, parameter map[string]string, result any) error { u, err := d.baseURL.Parse(endpoint) if err != nil { return err @@ -135,7 +128,7 @@ func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, u.RawQuery = query.Encode() } - req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return err } @@ -151,7 +144,7 @@ func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, } // GetRepoInfo returns repository information -func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { +func (d *OneDevDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { info := make([]struct { ID int64 `json:"id"` Name string `json:"name"` @@ -159,6 +152,7 @@ func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { }, 0, 1) err := d.callAPI( + ctx, "/api/projects", map[string]string{ "query": `"Name" is "` + d.repoName + `"`, @@ -194,7 +188,7 @@ func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { } // GetMilestones returns milestones -func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { +func (d *OneDevDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { rawMilestones := make([]struct { ID int64 `json:"id"` Name string `json:"name"` @@ -209,6 +203,7 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { offset := 0 for { err := d.callAPI( + ctx, endpoint, map[string]string{ "offset": strconv.Itoa(offset), @@ -243,7 +238,7 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { } // GetLabels returns labels -func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) { +func (d *OneDevDownloader) GetLabels(_ context.Context) ([]*base.Label, error) { return []*base.Label{ { Name: "Bug", @@ -277,7 +272,7 @@ type onedevIssueContext struct { } // GetIssues returns issues -func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (d *OneDevDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { rawIssues := make([]struct { ID int64 `json:"id"` Number int64 `json:"number"` @@ -289,6 +284,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er }, 0, perPage) err := d.callAPI( + ctx, "/api/issues", map[string]string{ "query": `"Project" is "` + d.repoName + `"`, @@ -308,6 +304,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Value string `json:"value"` }, 0, 10) err := d.callAPI( + ctx, fmt.Sprintf("/api/issues/%d/fields", issue.ID), nil, &fields, @@ -329,6 +326,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Name string `json:"name"` }, 0, 10) err = d.callAPI( + ctx, fmt.Sprintf("/api/issues/%d/milestones", issue.ID), nil, &milestones, @@ -345,7 +343,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er if state == "released" { state = "closed" } - poster := d.tryGetUser(issue.SubmitterID) + poster := d.tryGetUser(ctx, issue.SubmitterID) issues = append(issues, &base.Issue{ Title: issue.Title, Number: issue.Number, @@ -370,7 +368,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er } // GetComments returns comments -func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (d *OneDevDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { context, ok := commentable.GetContext().(onedevIssueContext) if !ok { return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) @@ -391,6 +389,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co } err := d.callAPI( + ctx, endpoint, nil, &rawComments, @@ -412,6 +411,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co } err = d.callAPI( + ctx, endpoint, nil, &rawChanges, @@ -425,7 +425,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co if len(comment.Content) == 0 { continue } - poster := d.tryGetUser(comment.UserID) + poster := d.tryGetUser(ctx, comment.UserID) comments = append(comments, &base.Comment{ IssueIndex: commentable.GetLocalIndex(), Index: comment.ID, @@ -450,7 +450,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co continue } - poster := d.tryGetUser(change.UserID) + poster := d.tryGetUser(ctx, change.UserID) comments = append(comments, &base.Comment{ IssueIndex: commentable.GetLocalIndex(), PosterID: poster.ID, @@ -466,7 +466,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co } // GetPullRequests returns pull requests -func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (d *OneDevDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { rawPullRequests := make([]struct { ID int64 `json:"id"` Number int64 `json:"number"` @@ -484,6 +484,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque }, 0, perPage) err := d.callAPI( + ctx, "/api/pull-requests", map[string]string{ "query": `"Target Project" is "` + d.repoName + `"`, @@ -505,6 +506,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque MergeCommitHash string `json:"mergeCommitHash"` } err := d.callAPI( + ctx, fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID), nil, &mergePreview, @@ -525,7 +527,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque mergedTime = pr.CloseInfo.Date } } - poster := d.tryGetUser(pr.SubmitterID) + poster := d.tryGetUser(ctx, pr.SubmitterID) number := pr.Number + d.maxIssueIndex pullRequests = append(pullRequests, &base.PullRequest{ @@ -562,7 +564,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests reviews -func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (d *OneDevDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { rawReviews := make([]struct { ID int64 `json:"id"` UserID int64 `json:"userId"` @@ -574,6 +576,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie }, 0, 100) err := d.callAPI( + ctx, fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignIndex()), nil, &rawReviews, @@ -596,7 +599,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie } } - poster := d.tryGetUser(review.UserID) + poster := d.tryGetUser(ctx, review.UserID) reviews = append(reviews, &base.Review{ IssueIndex: reviewable.GetLocalIndex(), ReviewerID: poster.ID, @@ -610,14 +613,15 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie } // GetTopics return repository topics -func (d *OneDevDownloader) GetTopics() ([]string, error) { +func (d *OneDevDownloader) GetTopics(_ context.Context) ([]string, error) { return []string{}, nil } -func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser { +func (d *OneDevDownloader) tryGetUser(ctx context.Context, userID int64) *onedevUser { user, ok := d.userMap[userID] if !ok { err := d.callAPI( + ctx, fmt.Sprintf("/api/users/%d", userID), nil, &user, diff --git a/services/migrations/onedev_test.go b/services/migrations/onedev_test.go index 48412fec64..a05d6cac6e 100644 --- a/services/migrations/onedev_test.go +++ b/services/migrations/onedev_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "net/http" "net/url" "testing" @@ -22,11 +21,12 @@ func TestOneDevDownloadRepo(t *testing.T) { } u, _ := url.Parse("https://code.onedev.io") - downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo") + ctx := t.Context() + downloader := NewOneDevDownloader(ctx, u, "", "", "go-gitea-test_repo") if err != nil { t.Fatalf("NewOneDevDownloader is nil: %v", err) } - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ Name: "go-gitea-test_repo", @@ -36,7 +36,7 @@ func TestOneDevDownloadRepo(t *testing.T) { OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo", }, repo) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) deadline := time.Unix(1620086400, 0) assertMilestonesEqual(t, []*base.Milestone{ @@ -51,11 +51,11 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assert.Len(t, labels, 6) - issues, isEnd, err := downloader.GetIssues(1, 2) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 2) assert.NoError(t, err) assert.False(t, isEnd) assertIssuesEqual(t, []*base.Issue{ @@ -94,7 +94,7 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{ + comments, _, err := downloader.GetComments(ctx, &base.Issue{ Number: 4, ForeignIndex: 398, Context: onedevIssueContext{IsPullRequest: false}, @@ -110,7 +110,7 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, comments) - prs, _, err := downloader.GetPullRequests(1, 1) + prs, _, err := downloader.GetPullRequests(ctx, 1, 1) assert.NoError(t, err) assertPullRequestsEqual(t, []*base.PullRequest{ { @@ -136,7 +136,7 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignIndex: 186}) + rvs, err := downloader.GetReviews(ctx, &base.PullRequest{Number: 5, ForeignIndex: 186}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/restore.go b/services/migrations/restore.go index fd337b22c7..5686285935 100644 --- a/services/migrations/restore.go +++ b/services/migrations/restore.go @@ -18,7 +18,6 @@ import ( // RepositoryRestorer implements an Downloader from the local directory type RepositoryRestorer struct { base.NullDownloader - ctx context.Context baseDir string repoOwner string repoName string @@ -26,13 +25,12 @@ type RepositoryRestorer struct { } // NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder -func NewRepositoryRestorer(ctx context.Context, baseDir, owner, repoName string, validation bool) (*RepositoryRestorer, error) { +func NewRepositoryRestorer(_ context.Context, baseDir, owner, repoName string, validation bool) (*RepositoryRestorer, error) { baseDir, err := filepath.Abs(baseDir) if err != nil { return nil, err } return &RepositoryRestorer{ - ctx: ctx, baseDir: baseDir, repoOwner: owner, repoName: repoName, @@ -48,11 +46,6 @@ func (r *RepositoryRestorer) reviewDir() string { return filepath.Join(r.baseDir, "reviews") } -// SetContext set context -func (r *RepositoryRestorer) SetContext(ctx context.Context) { - r.ctx = ctx -} - func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) { p := filepath.Join(r.baseDir, "repo.yml") bs, err := os.ReadFile(p) @@ -69,7 +62,7 @@ func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) { } // GetRepoInfo returns a repository information -func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { +func (r *RepositoryRestorer) GetRepoInfo(_ context.Context) (*base.Repository, error) { opts, err := r.getRepoOptions() if err != nil { return nil, err @@ -89,7 +82,7 @@ func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { } // GetTopics return github topics -func (r *RepositoryRestorer) GetTopics() ([]string, error) { +func (r *RepositoryRestorer) GetTopics(_ context.Context) ([]string, error) { p := filepath.Join(r.baseDir, "topic.yml") topics := struct { @@ -112,7 +105,7 @@ func (r *RepositoryRestorer) GetTopics() ([]string, error) { } // GetMilestones returns milestones -func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { +func (r *RepositoryRestorer) GetMilestones(_ context.Context) ([]*base.Milestone, error) { milestones := make([]*base.Milestone, 0, 10) p := filepath.Join(r.baseDir, "milestone.yml") err := base.Load(p, &milestones, r.validation) @@ -127,7 +120,7 @@ func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { } // GetReleases returns releases -func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { +func (r *RepositoryRestorer) GetReleases(_ context.Context) ([]*base.Release, error) { releases := make([]*base.Release, 0, 10) p := filepath.Join(r.baseDir, "release.yml") _, err := os.Stat(p) @@ -158,7 +151,7 @@ func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { } // GetLabels returns labels -func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { +func (r *RepositoryRestorer) GetLabels(_ context.Context) ([]*base.Label, error) { labels := make([]*base.Label, 0, 10) p := filepath.Join(r.baseDir, "label.yml") _, err := os.Stat(p) @@ -182,7 +175,7 @@ func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { } // GetIssues returns issues according start and limit -func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (r *RepositoryRestorer) GetIssues(_ context.Context, _, _ int) ([]*base.Issue, bool, error) { issues := make([]*base.Issue, 0, 10) p := filepath.Join(r.baseDir, "issue.yml") err := base.Load(p, &issues, r.validation) @@ -196,7 +189,7 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (r *RepositoryRestorer) GetComments(_ context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { comments := make([]*base.Comment, 0, 10) p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", commentable.GetForeignIndex())) _, err := os.Stat(p) @@ -220,7 +213,7 @@ func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base. } // GetPullRequests returns pull requests according page and perPage -func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (r *RepositoryRestorer) GetPullRequests(_ context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { pulls := make([]*base.PullRequest, 0, 10) p := filepath.Join(r.baseDir, "pull_request.yml") _, err := os.Stat(p) @@ -248,7 +241,7 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } // GetReviews returns pull requests review -func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (r *RepositoryRestorer) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { reviews := make([]*base.Review, 0, 10) p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignIndex())) _, err := os.Stat(p) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index e029bbb1d6..7fb7fabb75 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -5,7 +5,7 @@ package mirror import ( "context" - "fmt" + "errors" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/log" @@ -29,7 +29,7 @@ func doMirrorSync(ctx context.Context, req *SyncRequest) { } } -var errLimit = fmt.Errorf("reached limit") +var errLimit = errors.New("reached limit") // Update checks and updates mirror repositories. func Update(ctx context.Context, pullLimit, pushLimit int) error { @@ -68,7 +68,7 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error { // Check we've not been cancelled select { case <-ctx.Done(): - return fmt.Errorf("aborted") + return errors.New("aborted") default: } diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 948222a436..cb90af5894 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/git" giturl "code.gitea.io/gitea/modules/git/url" "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" @@ -40,13 +41,13 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error remoteName := m.GetRemoteName() repoPath := m.GetRepository(ctx).RepoPath() // Remove old remote - _, _, err = git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(remoteName).RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommand("remote", "rm").AddDynamicArguments(remoteName).RunStdString(ctx, &git.RunOpts{Dir: repoPath}) if err != nil && !git.IsRemoteNotExistError(err) { return err } - cmd := git.NewCommand(ctx, "remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(addr) - _, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath}) + cmd := git.NewCommand("remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(addr) + _, _, err = cmd.RunStdString(ctx, &git.RunOpts{Dir: repoPath}) if err != nil && !git.IsRemoteNotExistError(err) { return err } @@ -55,13 +56,13 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error wikiPath := m.Repo.WikiPath() wikiRemotePath := repo_module.WikiRemoteURL(ctx, addr) // Remove old remote of wiki - _, _, err = git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(remoteName).RunStdString(&git.RunOpts{Dir: wikiPath}) + _, _, err = git.NewCommand("remote", "rm").AddDynamicArguments(remoteName).RunStdString(ctx, &git.RunOpts{Dir: wikiPath}) if err != nil && !git.IsRemoteNotExistError(err) { return err } - cmd = git.NewCommand(ctx, "remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(wikiRemotePath) - _, _, err = cmd.RunStdString(&git.RunOpts{Dir: wikiPath}) + cmd = git.NewCommand("remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(wikiRemotePath) + _, _, err = cmd.RunStdString(ctx, &git.RunOpts{Dir: wikiPath}) if err != nil && !git.IsRemoteNotExistError(err) { return err } @@ -70,7 +71,7 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error // erase authentication before storing in database u.User = nil m.Repo.OriginalURL = u.String() - return repo_model.UpdateRepositoryCols(ctx, m.Repo, "original_url") + return repo_model.UpdateRepositoryColsNoAutoTime(ctx, m.Repo, "original_url") } // mirrorSyncResult contains information of a updated reference. @@ -87,6 +88,7 @@ type mirrorSyncResult struct { /* // * [new tag] v0.1.8 -> v0.1.8 // * [new branch] master -> origin/master +// * [new ref] refs/pull/2/head -> refs/pull/2/head" // - [deleted] (none) -> origin/test // delete a branch // - [deleted] (none) -> 1 // delete a tag // 957a993..a87ba5f test -> origin/test @@ -117,10 +119,17 @@ func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult { refName: git.RefNameFromBranch(refName), oldCommitID: gitShortEmptySha, }) + case strings.HasPrefix(lines[i], " * [new ref]"): // new reference + results = append(results, &mirrorSyncResult{ + refName: git.RefName(refName), + oldCommitID: gitShortEmptySha, + }) case strings.HasPrefix(lines[i], " - "): // Delete reference isTag := !strings.HasPrefix(refName, remoteName+"/") var refFullName git.RefName - if isTag { + if strings.HasPrefix(refName, "refs/") { + refFullName = git.RefName(refName) + } else if isTag { refFullName = git.RefNameFromTag(refName) } else { refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/")) @@ -143,8 +152,15 @@ func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult { log.Error("Expect two SHAs but not what found: %q", lines[i]) continue } + var refFullName git.RefName + if strings.HasPrefix(refName, "refs/") { + refFullName = git.RefName(refName) + } else { + refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/")) + } + results = append(results, &mirrorSyncResult{ - refName: git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/")), + refName: refFullName, oldCommitID: shas[0], newCommitID: shas[1], }) @@ -159,8 +175,15 @@ func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult { log.Error("Expect two SHAs but not what found: %q", lines[i]) continue } + var refFullName git.RefName + if strings.HasPrefix(refName, "refs/") { + refFullName = git.RefName(refName) + } else { + refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/")) + } + results = append(results, &mirrorSyncResult{ - refName: git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/")), + refName: refFullName, oldCommitID: shas[0], newCommitID: shas[1], }) @@ -186,8 +209,8 @@ func pruneBrokenReferences(ctx context.Context, stderrBuilder.Reset() stdoutBuilder.Reset() - pruneErr := git.NewCommand(ctx, "remote", "prune").AddDynamicArguments(m.GetRemoteName()). - Run(&git.RunOpts{ + pruneErr := git.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()). + Run(ctx, &git.RunOpts{ Timeout: timeout, Dir: repoPath, Stdout: stdoutBuilder, @@ -212,6 +235,24 @@ func pruneBrokenReferences(ctx context.Context, return pruneErr } +// checkRecoverableSyncError takes an error message from a git fetch command and returns false if it should be a fatal/blocking error +func checkRecoverableSyncError(stderrMessage string) bool { + switch { + case strings.Contains(stderrMessage, "unable to resolve reference") && strings.Contains(stderrMessage, "reference broken"): + return true + case strings.Contains(stderrMessage, "remote error") && strings.Contains(stderrMessage, "not our ref"): + return true + case strings.Contains(stderrMessage, "cannot lock ref") && strings.Contains(stderrMessage, "but expected"): + return true + case strings.Contains(stderrMessage, "cannot lock ref") && strings.Contains(stderrMessage, "unable to resolve reference"): + return true + case strings.Contains(stderrMessage, "Unable to create") && strings.Contains(stderrMessage, ".lock"): + return true + default: + return false + } +} + // runSync returns true if sync finished without error. func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() @@ -221,7 +262,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) // use fetch but not remote update because git fetch support --tags but remote update doesn't - cmd := git.NewCommand(ctx, "fetch") + cmd := git.NewCommand("fetch") if m.EnablePrune { cmd.AddArguments("--prune") } @@ -237,7 +278,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo stdoutBuilder := strings.Builder{} stderrBuilder := strings.Builder{} - if err := cmd.Run(&git.RunOpts{ + if err := cmd.Run(ctx, &git.RunOpts{ Timeout: timeout, Dir: repoPath, Env: envs, @@ -252,7 +293,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo stdoutMessage := util.SanitizeCredentialURLs(stdout) // Now check if the error is a resolve reference due to broken reference - if strings.Contains(stderr, "unable to resolve reference") && strings.Contains(stderr, "reference broken") { + if checkRecoverableSyncError(stderr) { log.Warn("SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err) err = nil @@ -262,7 +303,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo // Successful prune - reattempt mirror stderrBuilder.Reset() stdoutBuilder.Reset() - if err = cmd.Run(&git.RunOpts{ + if err = cmd.Run(ctx, &git.RunOpts{ Timeout: timeout, Dir: repoPath, Stdout: &stdoutBuilder, @@ -301,6 +342,15 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo return nil, false } + if m.LFS && setting.LFS.StartServer { + log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) + endpoint := lfs.DetermineEndpoint(remoteURL.String(), m.LFSEndpoint) + lfsClient := lfs.NewClient(endpoint, nil) + if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil { + log.Error("SyncMirrors [repo: %-v]: failed to synchronize LFS objects for repository: %v", m.Repo, err) + } + } + log.Trace("SyncMirrors [repo: %-v]: syncing branches...", m.Repo) if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, m.Repo, gitRepo, 0); err != nil { log.Error("SyncMirrors [repo: %-v]: failed to synchronize branches: %v", m.Repo, err) @@ -310,15 +360,6 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo if err = repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo); err != nil { log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err) } - - if m.LFS && setting.LFS.StartServer { - log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) - endpoint := lfs.DetermineEndpoint(remoteURL.String(), m.LFSEndpoint) - lfsClient := lfs.NewClient(endpoint, nil) - if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil { - log.Error("SyncMirrors [repo: %-v]: failed to synchronize LFS objects for repository: %v", m.Repo, err) - } - } gitRepo.Close() log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) @@ -330,8 +371,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo) stderrBuilder.Reset() stdoutBuilder.Reset() - if err := git.NewCommand(ctx, "remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()). - Run(&git.RunOpts{ + if err := git.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()). + Run(ctx, &git.RunOpts{ Timeout: timeout, Dir: wikiPath, Stdout: &stdoutBuilder, @@ -345,7 +386,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo stdoutMessage := util.SanitizeCredentialURLs(stdout) // Now check if the error is a resolve reference due to broken reference - if strings.Contains(stderrMessage, "unable to resolve reference") && strings.Contains(stderrMessage, "reference broken") { + if checkRecoverableSyncError(stderrMessage) { log.Warn("SyncMirrors [repo: %-v Wiki]: failed to update mirror wiki repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err) err = nil @@ -356,8 +397,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo stderrBuilder.Reset() stdoutBuilder.Reset() - if err = git.NewCommand(ctx, "remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()). - Run(&git.RunOpts{ + if err = git.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()). + Run(ctx, &git.RunOpts{ Timeout: timeout, Dir: wikiPath, Stdout: &stdoutBuilder, @@ -396,13 +437,17 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo } for _, branch := range branches { - cache.Remove(m.Repo.GetCommitsCountCacheKey(branch.Name, true)) + cache.Remove(m.Repo.GetCommitsCountCacheKey(branch, true)) } m.UpdatedUnix = timeutil.TimeStampNow() return parseRemoteUpdateOutput(output, m.GetRemoteName()), true } +func getRepoPullMirrorLockKey(repoID int64) string { + return fmt.Sprintf("repo_pull_mirror_%d", repoID) +} + // SyncPullMirror starts the sync of the pull mirror and schedules the next run. func SyncPullMirror(ctx context.Context, repoID int64) bool { log.Trace("SyncMirrors [repo_id: %v]", repoID) @@ -415,6 +460,13 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { log.Error("PANIC whilst SyncMirrors[repo_id: %d] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) }() + releaser, err := globallock.Lock(ctx, getRepoPullMirrorLockKey(repoID)) + if err != nil { + log.Error("globallock.Lock(): %v", err) + return false + } + defer releaser() + m, err := repo_model.GetMirrorByRepoID(ctx, repoID) if err != nil { log.Error("SyncMirrors [repo_id: %v]: unable to GetMirrorByRepoID: %v", repoID, err) @@ -601,7 +653,7 @@ func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, re } m.Repo.IsEmpty = false // Update the is empty and default_branch columns - if err := repo_model.UpdateRepositoryCols(ctx, m.Repo, "default_branch", "is_empty"); err != nil { + if err := repo_model.UpdateRepositoryColsWithAutoTime(ctx, m.Repo, "default_branch", "is_empty"); err != nil { log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err) desc := fmt.Sprintf("Failed to update default branch of repository '%s': %v", m.Repo.RepoPath(), err) if err = system_model.CreateRepositoryNotice(desc); err != nil { diff --git a/services/mirror/mirror_pull_test.go b/services/mirror/mirror_pull_test.go new file mode 100644 index 0000000000..97859be5b0 --- /dev/null +++ b/services/mirror/mirror_pull_test.go @@ -0,0 +1,94 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package mirror + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_parseRemoteUpdateOutput(t *testing.T) { + output := ` + * [new tag] v0.1.8 -> v0.1.8 + * [new branch] master -> origin/master + - [deleted] (none) -> origin/test1 + - [deleted] (none) -> tag1 + + f895a1e...957a993 test2 -> origin/test2 (forced update) + 957a993..a87ba5f test3 -> origin/test3 + * [new ref] refs/pull/26595/head -> refs/pull/26595/head + * [new ref] refs/pull/26595/merge -> refs/pull/26595/merge + e0639e38fb..6db2410489 refs/pull/25873/head -> refs/pull/25873/head + + 1c97ebc746...976d27d52f refs/pull/25873/merge -> refs/pull/25873/merge (forced update) +` + results := parseRemoteUpdateOutput(output, "origin") + assert.Len(t, results, 10) + assert.Equal(t, "refs/tags/v0.1.8", results[0].refName.String()) + assert.Equal(t, gitShortEmptySha, results[0].oldCommitID) + assert.Empty(t, results[0].newCommitID) + + assert.Equal(t, "refs/heads/master", results[1].refName.String()) + assert.Equal(t, gitShortEmptySha, results[1].oldCommitID) + assert.Empty(t, results[1].newCommitID) + + assert.Equal(t, "refs/heads/test1", results[2].refName.String()) + assert.Empty(t, results[2].oldCommitID) + assert.Equal(t, gitShortEmptySha, results[2].newCommitID) + + assert.Equal(t, "refs/tags/tag1", results[3].refName.String()) + assert.Empty(t, results[3].oldCommitID) + assert.Equal(t, gitShortEmptySha, results[3].newCommitID) + + assert.Equal(t, "refs/heads/test2", results[4].refName.String()) + assert.Equal(t, "f895a1e", results[4].oldCommitID) + assert.Equal(t, "957a993", results[4].newCommitID) + + assert.Equal(t, "refs/heads/test3", results[5].refName.String()) + assert.Equal(t, "957a993", results[5].oldCommitID) + assert.Equal(t, "a87ba5f", results[5].newCommitID) + + assert.Equal(t, "refs/pull/26595/head", results[6].refName.String()) + assert.Equal(t, gitShortEmptySha, results[6].oldCommitID) + assert.Empty(t, results[6].newCommitID) + + assert.Equal(t, "refs/pull/26595/merge", results[7].refName.String()) + assert.Equal(t, gitShortEmptySha, results[7].oldCommitID) + assert.Empty(t, results[7].newCommitID) + + assert.Equal(t, "refs/pull/25873/head", results[8].refName.String()) + assert.Equal(t, "e0639e38fb", results[8].oldCommitID) + assert.Equal(t, "6db2410489", results[8].newCommitID) + + assert.Equal(t, "refs/pull/25873/merge", results[9].refName.String()) + assert.Equal(t, "1c97ebc746", results[9].oldCommitID) + assert.Equal(t, "976d27d52f", results[9].newCommitID) +} + +func Test_checkRecoverableSyncError(t *testing.T) { + cases := []struct { + recoverable bool + message string + }{ + // A race condition in http git-fetch where certain refs were listed on the remote and are no longer there, would exit status 128 + {true, "fatal: remote error: upload-pack: not our ref 988881adc9fc3655077dc2d4d757d480b5ea0e11"}, + // A race condition where a local gc/prune removes a named ref during a git-fetch would exit status 1 + {true, "cannot lock ref 'refs/pull/123456/merge': unable to resolve reference 'refs/pull/134153/merge'"}, + // A race condition in http git-fetch where named refs were listed on the remote and are no longer there + {true, "error: cannot lock ref 'refs/remotes/origin/foo': unable to resolve reference 'refs/remotes/origin/foo': reference broken"}, + // A race condition in http git-fetch where named refs were force-pushed during the update, would exit status 128 + {true, "error: cannot lock ref 'refs/pull/123456/merge': is at 988881adc9fc3655077dc2d4d757d480b5ea0e11 but expected 7f894307ffc9553edbd0b671cab829786866f7b2"}, + // A race condition with other local git operations, such as git-maintenance, would exit status 128 (well, "Unable" the "U" is uppercase) + {true, "fatal: Unable to create '/data/gitea-repositories/foo-org/bar-repo.git/./objects/info/commit-graphs/commit-graph-chain.lock': File exists."}, + // Missing or unauthorized credentials, would exit status 128 + {false, "fatal: Authentication failed for 'https://example.com/foo-does-not-exist/bar.git/'"}, + // A non-existent remote repository, would exit status 128 + {false, "fatal: Could not read from remote repository."}, + // A non-functioning proxy, would exit status 128 + {false, "fatal: unable to access 'https://example.com/foo-does-not-exist/bar.git/': Failed to connect to configured-https-proxy port 1080 after 0 ms: Couldn't connect to server"}, + } + + for _, c := range cases { + assert.Equal(t, c.recoverable, checkRecoverableSyncError(c.message), "test case: %s", c.message) + } +} diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 02ff97b1f0..9b57427d98 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -29,14 +30,14 @@ var stripExitStatus = regexp.MustCompile(`exit status \d+ - `) // AddPushMirrorRemote registers the push mirror remote. func AddPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr string) error { addRemoteAndConfig := func(addr, path string) error { - cmd := git.NewCommand(ctx, "remote", "add", "--mirror=push").AddDynamicArguments(m.RemoteName, addr) - if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: path}); err != nil { + cmd := git.NewCommand("remote", "add", "--mirror=push").AddDynamicArguments(m.RemoteName, addr) + if _, _, err := cmd.RunStdString(ctx, &git.RunOpts{Dir: path}); err != nil { return err } - if _, _, err := git.NewCommand(ctx, "config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { + if _, _, err := git.NewCommand("config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(ctx, &git.RunOpts{Dir: path}); err != nil { return err } - if _, _, err := git.NewCommand(ctx, "config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { + if _, _, err := git.NewCommand("config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(ctx, &git.RunOpts{Dir: path}); err != nil { return err } return nil @@ -60,15 +61,15 @@ func AddPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr str // RemovePushMirrorRemote removes the push mirror remote. func RemovePushMirrorRemote(ctx context.Context, m *repo_model.PushMirror) error { - cmd := git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(m.RemoteName) + cmd := git.NewCommand("remote", "rm").AddDynamicArguments(m.RemoteName) _ = m.GetRepository(ctx) - if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.RepoPath()}); err != nil { + if _, _, err := cmd.RunStdString(ctx, &git.RunOpts{Dir: m.Repo.RepoPath()}); err != nil { return err } if m.Repo.HasWiki() { - if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.WikiPath()}); err != nil { + if _, _, err := cmd.RunStdString(ctx, &git.RunOpts{Dir: m.Repo.WikiPath()}); err != nil { // The wiki remote may not exist log.Warn("Wiki Remote[%d] could not be removed: %v", m.ID, err) } @@ -142,7 +143,7 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { var gitRepo *git.Repository if isWiki { - gitRepo, err = gitrepo.OpenWikiRepository(ctx, repo) + gitRepo, err = gitrepo.OpenRepository(ctx, repo.WikiStorageRepo()) } else { gitRepo, err = gitrepo.OpenRepository(ctx, repo) } @@ -161,11 +162,13 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) + envs := proxy.EnvWithProxy(remoteURL.URL) if err := git.Push(ctx, path, git.PushOptions{ Remote: m.RemoteName, Force: true, Mirror: true, Timeout: timeout, + Env: envs, }); err != nil { log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err) diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go deleted file mode 100644 index 8ad524b608..0000000000 --- a/services/mirror/mirror_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package mirror - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_parseRemoteUpdateOutput(t *testing.T) { - output := ` - * [new tag] v0.1.8 -> v0.1.8 - * [new branch] master -> origin/master - - [deleted] (none) -> origin/test1 - - [deleted] (none) -> tag1 - + f895a1e...957a993 test2 -> origin/test2 (forced update) - 957a993..a87ba5f test3 -> origin/test3 -` - results := parseRemoteUpdateOutput(output, "origin") - assert.Len(t, results, 6) - assert.EqualValues(t, "refs/tags/v0.1.8", results[0].refName.String()) - assert.EqualValues(t, gitShortEmptySha, results[0].oldCommitID) - assert.EqualValues(t, "", results[0].newCommitID) - - assert.EqualValues(t, "refs/heads/master", results[1].refName.String()) - assert.EqualValues(t, gitShortEmptySha, results[1].oldCommitID) - assert.EqualValues(t, "", results[1].newCommitID) - - assert.EqualValues(t, "refs/heads/test1", results[2].refName.String()) - assert.EqualValues(t, "", results[2].oldCommitID) - assert.EqualValues(t, gitShortEmptySha, results[2].newCommitID) - - assert.EqualValues(t, "refs/tags/tag1", results[3].refName.String()) - assert.EqualValues(t, "", results[3].oldCommitID) - assert.EqualValues(t, gitShortEmptySha, results[3].newCommitID) - - assert.EqualValues(t, "refs/heads/test2", results[4].refName.String()) - assert.EqualValues(t, "f895a1e", results[4].oldCommitID) - assert.EqualValues(t, "957a993", results[4].newCommitID) - - assert.EqualValues(t, "refs/heads/test3", results[5].refName.String()) - assert.EqualValues(t, "957a993", results[5].oldCommitID) - assert.EqualValues(t, "a87ba5f", results[5].newCommitID) -} diff --git a/services/notify/notifier.go b/services/notify/notifier.go index 29bbb5702b..875a70e564 100644 --- a/services/notify/notifier.go +++ b/services/notify/notifier.go @@ -6,6 +6,7 @@ package notify import ( "context" + actions_model "code.gitea.io/gitea/models/actions" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" packages_model "code.gitea.io/gitea/models/packages" @@ -77,4 +78,8 @@ type Notifier interface { ChangeDefaultBranch(ctx context.Context, repo *repo_model.Repository) CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, commit *repository.PushCommit, sender *user_model.User, status *git_model.CommitStatus) + + WorkflowRunStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, run *actions_model.ActionRun) + + WorkflowJobStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, job *actions_model.ActionRunJob, task *actions_model.ActionTask) } diff --git a/services/notify/notify.go b/services/notify/notify.go index 3b5f24340b..2416cbd2e0 100644 --- a/services/notify/notify.go +++ b/services/notify/notify.go @@ -6,6 +6,7 @@ package notify import ( "context" + actions_model "code.gitea.io/gitea/models/actions" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" packages_model "code.gitea.io/gitea/models/packages" @@ -45,10 +46,25 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model } } +func shouldSendCommentChangeNotification(ctx context.Context, comment *issues_model.Comment) bool { + if err := comment.LoadReview(ctx); err != nil { + log.Error("LoadReview: %v", err) + return false + } else if comment.Review != nil && comment.Review.Type == issues_model.ReviewTypePending { + // Pending review comments updating should not triggered + return false + } + return true +} + // CreateIssueComment notifies issue comment related message to notifiers func CreateIssueComment(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, issue *issues_model.Issue, comment *issues_model.Comment, mentions []*user_model.User, ) { + if !shouldSendCommentChangeNotification(ctx, comment) { + return + } + for _, notifier := range notifiers { notifier.CreateIssueComment(ctx, doer, repo, issue, comment, mentions) } @@ -155,6 +171,10 @@ func PullReviewDismiss(ctx context.Context, doer *user_model.User, review *issue // UpdateComment notifies update comment to notifiers func UpdateComment(ctx context.Context, doer *user_model.User, c *issues_model.Comment, oldContent string) { + if !shouldSendCommentChangeNotification(ctx, c) { + return + } + for _, notifier := range notifiers { notifier.UpdateComment(ctx, doer, c, oldContent) } @@ -162,6 +182,10 @@ func UpdateComment(ctx context.Context, doer *user_model.User, c *issues_model.C // DeleteComment notifies delete comment to notifiers func DeleteComment(ctx context.Context, doer *user_model.User, c *issues_model.Comment) { + if !shouldSendCommentChangeNotification(ctx, c) { + return + } + for _, notifier := range notifiers { notifier.DeleteComment(ctx, doer, c) } @@ -272,9 +296,9 @@ func MigrateRepository(ctx context.Context, doer, u *user_model.User, repo *repo } // TransferRepository notifies create repository to notifiers -func TransferRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, newOwnerName string) { +func TransferRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, oldOwnerName string) { for _, notifier := range notifiers { - notifier.TransferRepository(ctx, doer, repo, newOwnerName) + notifier.TransferRepository(ctx, doer, repo, oldOwnerName) } } @@ -374,3 +398,15 @@ func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, commit notifier.CreateCommitStatus(ctx, repo, commit, sender, status) } } + +func WorkflowRunStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, run *actions_model.ActionRun) { + for _, notifier := range notifiers { + notifier.WorkflowRunStatusUpdate(ctx, repo, sender, run) + } +} + +func WorkflowJobStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, job *actions_model.ActionRunJob, task *actions_model.ActionTask) { + for _, notifier := range notifiers { + notifier.WorkflowJobStatusUpdate(ctx, repo, sender, job, task) + } +} diff --git a/services/notify/null.go b/services/notify/null.go index 7354efd701..c3085d7c9e 100644 --- a/services/notify/null.go +++ b/services/notify/null.go @@ -6,6 +6,7 @@ package notify import ( "context" + actions_model "code.gitea.io/gitea/models/actions" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" packages_model "code.gitea.io/gitea/models/packages" @@ -212,3 +213,9 @@ func (*NullNotifier) ChangeDefaultBranch(ctx context.Context, repo *repo_model.R func (*NullNotifier) CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, commit *repository.PushCommit, sender *user_model.User, status *git_model.CommitStatus) { } + +func (*NullNotifier) WorkflowRunStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, run *actions_model.ActionRun) { +} + +func (*NullNotifier) WorkflowJobStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, job *actions_model.ActionRunJob, task *actions_model.ActionTask) { +} diff --git a/services/oauth2_provider/access_token.go b/services/oauth2_provider/access_token.go index 5cb6fb64c5..e01777031b 100644 --- a/services/oauth2_provider/access_token.go +++ b/services/oauth2_provider/access_token.go @@ -1,12 +1,13 @@ // Copyright 2024 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package oauth2_provider //nolint +package oauth2_provider import ( "context" "fmt" "slices" + "strconv" "strings" auth "code.gitea.io/gitea/models/auth" @@ -15,7 +16,9 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" "github.com/golang-jwt/jwt/v5" ) @@ -82,7 +85,7 @@ func GrantAdditionalScopes(grantScopes string) auth.AccessTokenScope { } var accessScopes []string // the scopes for access control, but not for general information - for _, scope := range strings.Split(grantScopes, " ") { + for scope := range strings.SplitSeq(grantScopes, " ") { if scope != "" && !slices.Contains(generalScopesSupported, scope) { accessScopes = append(accessScopes, scope) } @@ -177,7 +180,7 @@ func NewAccessTokenResponse(ctx context.Context, grant *auth.OAuth2Grant, server ExpiresAt: jwt.NewNumericDate(expirationDate.AsTime()), Issuer: setting.AppURL, Audience: []string{app.ClientID}, - Subject: fmt.Sprint(grant.UserID), + Subject: strconv.FormatInt(grant.UserID, 10), }, Nonce: grant.Nonce, } @@ -230,12 +233,11 @@ func NewAccessTokenResponse(ctx context.Context, grant *auth.OAuth2Grant, server }, nil } -// returns a list of "org" and "org:team" strings, -// that the given user is a part of. +// GetOAuthGroupsForUser returns a list of "org" and "org:team" strings, that the given user is a part of. func GetOAuthGroupsForUser(ctx context.Context, user *user_model.User, onlyPublicGroups bool) ([]string, error) { orgs, err := db.Find[org_model.Organization](ctx, org_model.FindOrgOptions{ - UserID: user.ID, - IncludePrivate: !onlyPublicGroups, + UserID: user.ID, + IncludeVisibility: util.Iif(onlyPublicGroups, api.VisibleTypePublic, api.VisibleTypePrivate), }) if err != nil { return nil, fmt.Errorf("GetUserOrgList: %w", err) diff --git a/services/oauth2_provider/additional_scopes_test.go b/services/oauth2_provider/additional_scopes_test.go index 2d4df7aea2..5f375346dc 100644 --- a/services/oauth2_provider/additional_scopes_test.go +++ b/services/oauth2_provider/additional_scopes_test.go @@ -1,7 +1,7 @@ // Copyright 2024 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package oauth2_provider //nolint +package oauth2_provider import ( "testing" diff --git a/services/oauth2_provider/init.go b/services/oauth2_provider/init.go index e5958099a6..c412bd6433 100644 --- a/services/oauth2_provider/init.go +++ b/services/oauth2_provider/init.go @@ -1,7 +1,7 @@ // Copyright 2024 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package oauth2_provider //nolint +package oauth2_provider import ( "context" diff --git a/services/oauth2_provider/jwtsigningkey.go b/services/oauth2_provider/jwtsigningkey.go index 6c668db463..03c7403f75 100644 --- a/services/oauth2_provider/jwtsigningkey.go +++ b/services/oauth2_provider/jwtsigningkey.go @@ -1,7 +1,7 @@ // Copyright 2021 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package oauth2_provider //nolint +package oauth2_provider import ( "crypto/ecdsa" @@ -31,7 +31,7 @@ type ErrInvalidAlgorithmType struct { } func (err ErrInvalidAlgorithmType) Error() string { - return fmt.Sprintf("JWT signing algorithm is not supported: %s", err.Algorithm) + return "JWT signing algorithm is not supported: " + err.Algorithm } // JWTSigningKey represents a algorithm/key pair to sign JWTs diff --git a/services/oauth2_provider/token.go b/services/oauth2_provider/token.go index b71b11906e..935c4cc01f 100644 --- a/services/oauth2_provider/token.go +++ b/services/oauth2_provider/token.go @@ -1,9 +1,10 @@ // Copyright 2021 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package oauth2_provider //nolint +package oauth2_provider import ( + "errors" "fmt" "time" @@ -44,12 +45,12 @@ func ParseToken(jwtToken string, signingKey JWTSigningKey) (*Token, error) { return nil, err } if !parsedToken.Valid { - return nil, fmt.Errorf("invalid token") + return nil, errors.New("invalid token") } var token *Token var ok bool if token, ok = parsedToken.Claims.(*Token); !ok || !parsedToken.Valid { - return nil, fmt.Errorf("invalid token") + return nil, errors.New("invalid token") } return token, nil } diff --git a/services/org/team.go b/services/org/team.go index ee3bd898ea..6890dafd90 100644 --- a/services/org/team.go +++ b/services/org/team.go @@ -259,37 +259,6 @@ func AddTeamMember(ctx context.Context, team *organization.Team, user *user_mode } team.NumMembers++ - - // Give access to team repositories. - // update exist access if mode become bigger - subQuery := builder.Select("repo_id").From("team_repo"). - Where(builder.Eq{"team_id": team.ID}) - - if _, err := sess.Where("user_id=?", user.ID). - In("repo_id", subQuery). - And("mode < ?", team.AccessMode). - SetExpr("mode", team.AccessMode). - Update(new(access_model.Access)); err != nil { - return fmt.Errorf("update user accesses: %w", err) - } - - // for not exist access - var repoIDs []int64 - accessSubQuery := builder.Select("repo_id").From("access").Where(builder.Eq{"user_id": user.ID}) - if err := sess.SQL(subQuery.And(builder.NotIn("repo_id", accessSubQuery))).Find(&repoIDs); err != nil { - return fmt.Errorf("select id accesses: %w", err) - } - - accesses := make([]*access_model.Access, 0, 100) - for i, repoID := range repoIDs { - accesses = append(accesses, &access_model.Access{RepoID: repoID, UserID: user.ID, Mode: team.AccessMode}) - if (i%100 == 0 || i == len(repoIDs)-1) && len(accesses) > 0 { - if err = db.Insert(ctx, accesses); err != nil { - return fmt.Errorf("insert new user accesses: %w", err) - } - accesses = accesses[:0] - } - } return nil }) if err != nil { diff --git a/services/org/team_test.go b/services/org/team_test.go index 3791776e46..c1a69d8ee7 100644 --- a/services/org/team_test.go +++ b/services/org/team_test.go @@ -88,7 +88,7 @@ func TestUpdateTeam(t *testing.T) { assert.True(t, strings.HasPrefix(team.Description, "A long description!")) access := unittest.AssertExistsAndLoadBean(t, &access_model.Access{UserID: 4, RepoID: 3}) - assert.EqualValues(t, perm.AccessModeAdmin, access.Mode) + assert.Equal(t, perm.AccessModeAdmin, access.Mode) unittest.CheckConsistencyFor(t, &organization.Team{ID: team.ID}) } @@ -166,24 +166,6 @@ func TestRemoveTeamMember(t *testing.T) { assert.True(t, organization.IsErrLastOrgOwner(err)) } -func TestRepository_RecalculateAccesses3(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - team5 := unittest.AssertExistsAndLoadBean(t, &organization.Team{ID: 5}) - user29 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 29}) - - has, err := db.GetEngine(db.DefaultContext).Get(&access_model.Access{UserID: user29.ID, RepoID: 23}) - assert.NoError(t, err) - assert.False(t, has) - - // adding user29 to team5 should add an explicit access row for repo 23 - // even though repo 23 is public - assert.NoError(t, AddTeamMember(db.DefaultContext, team5, user29)) - - has, err = db.GetEngine(db.DefaultContext).Get(&access_model.Access{UserID: user29.ID, RepoID: 23}) - assert.NoError(t, err) - assert.True(t, has) -} - func TestIncludesAllRepositoriesTeams(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) @@ -222,8 +204,9 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { // Create repos. repoIDs := make([]int64, 0) - for i := 0; i < 3; i++ { - r, err := repo_service.CreateRepositoryDirectly(db.DefaultContext, user, org.AsUser(), repo_service.CreateRepoOptions{Name: fmt.Sprintf("repo-%d", i)}) + for i := range 3 { + r, err := repo_service.CreateRepositoryDirectly(db.DefaultContext, user, org.AsUser(), + repo_service.CreateRepoOptions{Name: fmt.Sprintf("repo-%d", i)}, true) assert.NoError(t, err, "CreateRepository %d", i) if r != nil { repoIDs = append(repoIDs, r.ID) @@ -285,7 +268,7 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { } // Create repo and check teams repositories. - r, err := repo_service.CreateRepositoryDirectly(db.DefaultContext, user, org.AsUser(), repo_service.CreateRepoOptions{Name: "repo-last"}) + r, err := repo_service.CreateRepositoryDirectly(db.DefaultContext, user, org.AsUser(), repo_service.CreateRepoOptions{Name: "repo-last"}, true) assert.NoError(t, err, "CreateRepository last") if r != nil { repoIDs = append(repoIDs, r.ID) @@ -298,7 +281,7 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { } // Remove repo and check teams repositories. - assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, user, repoIDs[0]), "DeleteRepository") + assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, repoIDs[0]), "DeleteRepository") teamRepos[0] = repoIDs[1:] teamRepos[1] = repoIDs[1:] teamRepos[3] = repoIDs[1:3] @@ -310,7 +293,7 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { // Wipe created items. for i, rid := range repoIDs { if i > 0 { // first repo already deleted. - assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, user, rid), "DeleteRepository %d", i) + assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, rid), "DeleteRepository %d", i) } } assert.NoError(t, DeleteOrganization(db.DefaultContext, org, false), "DeleteOrganization") diff --git a/services/org/user.go b/services/org/user.go index 0e74d006bb..26927253d2 100644 --- a/services/org/user.go +++ b/services/org/user.go @@ -64,10 +64,11 @@ func RemoveOrgUser(ctx context.Context, org *organization.Organization, user *us if err != nil { return fmt.Errorf("AccessibleReposEnv: %w", err) } - repoIDs, err := env.RepoIDs(1, org.NumRepos) + repoIDs, err := env.RepoIDs(ctx) if err != nil { return fmt.Errorf("GetUserRepositories [%d]: %w", user.ID, err) } + for _, repoID := range repoIDs { repo, err := repo_model.GetRepositoryByID(ctx, repoID) if err != nil { diff --git a/services/org/user_test.go b/services/org/user_test.go index 96d1a1c8ca..c61d600d90 100644 --- a/services/org/user_test.go +++ b/services/org/user_test.go @@ -53,7 +53,7 @@ func TestRemoveOrgUser(t *testing.T) { assert.NoError(t, RemoveOrgUser(db.DefaultContext, org, user)) unittest.AssertNotExistsBean(t, &organization.OrgUser{OrgID: org.ID, UID: user.ID}) org = unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: org.ID}) - assert.EqualValues(t, expectedNumMembers, org.NumMembers) + assert.Equal(t, expectedNumMembers, org.NumMembers) } org3 := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3}) diff --git a/services/packages/alpine/repository.go b/services/packages/alpine/repository.go index 27e6391980..277c188874 100644 --- a/services/packages/alpine/repository.go +++ b/services/packages/alpine/repository.go @@ -290,7 +290,7 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package privPem, _ := pem.Decode([]byte(priv)) if privPem == nil { - return fmt.Errorf("failed to decode private key pem") + return errors.New("failed to decode private key pem") } privKey, err := x509.ParsePKCS1PrivateKey(privPem.Bytes) diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go index 6731d9a1ac..438bb10837 100644 --- a/services/packages/arch/repository.go +++ b/services/packages/arch/repository.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" packages_model "code.gitea.io/gitea/models/packages" @@ -26,9 +27,9 @@ import ( "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" - "github.com/keybase/go-crypto/openpgp" - "github.com/keybase/go-crypto/openpgp/armor" - "github.com/keybase/go-crypto/openpgp/packet" + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/packet" ) const ( @@ -235,6 +236,28 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package return packages_service.DeletePackageFile(ctx, pf) } + vpfs := make(map[int64]*entryOptions) + for _, pf := range pfs { + current := &entryOptions{ + File: pf, + } + current.Version, err = packages_model.GetVersionByID(ctx, pf.VersionID) + if err != nil { + return err + } + + // here we compare the versions but not using SearchLatestVersions because we shouldn't allow "downgrading" to a older version by "latest" one. + // https://wiki.archlinux.org/title/Downgrading_packages : randomly downgrading can mess up dependencies: + // If a downgrade involves a soname change, all dependencies may need downgrading or rebuilding too. + if old, ok := vpfs[current.Version.PackageID]; ok { + if compareVersions(old.Version.Version, current.Version.Version) == -1 { + vpfs[current.Version.PackageID] = current + } + } else { + vpfs[current.Version.PackageID] = current + } + } + indexContent, _ := packages_module.NewHashedBuffer() defer indexContent.Close() @@ -243,15 +266,7 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package cache := make(map[int64]*packages_model.Package) - for _, pf := range pfs { - opts := &entryOptions{ - File: pf, - } - - opts.Version, err = packages_model.GetVersionByID(ctx, pf.VersionID) - if err != nil { - return err - } + for _, opts := range vpfs { if err := json.Unmarshal([]byte(opts.Version.MetadataJSON), &opts.VersionMetadata); err != nil { return err } @@ -263,12 +278,12 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package } cache[opts.Package.ID] = opts.Package } - opts.Blob, err = packages_model.GetBlobByID(ctx, pf.BlobID) + opts.Blob, err = packages_model.GetBlobByID(ctx, opts.File.BlobID) if err != nil { return err } - sig, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertySignature) + sig, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, opts.File.ID, arch_module.PropertySignature) if err != nil { return err } @@ -277,7 +292,7 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package } opts.Signature = sig[0].Value - meta, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyMetadata) + meta, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, opts.File.ID, arch_module.PropertyMetadata) if err != nil { return err } @@ -358,8 +373,8 @@ func writeDescription(tw *tar.Writer, opts *entryOptions) error { {"MD5SUM", opts.Blob.HashMD5}, {"SHA256SUM", opts.Blob.HashSHA256}, {"PGPSIG", opts.Signature}, - {"CSIZE", fmt.Sprintf("%d", opts.Blob.Size)}, - {"ISIZE", fmt.Sprintf("%d", opts.FileMetadata.InstalledSize)}, + {"CSIZE", strconv.FormatInt(opts.Blob.Size, 10)}, + {"ISIZE", strconv.FormatInt(opts.FileMetadata.InstalledSize, 10)}, {"NAME", opts.Package.Name}, {"BASE", opts.FileMetadata.Base}, {"ARCH", opts.FileMetadata.Architecture}, @@ -368,7 +383,7 @@ func writeDescription(tw *tar.Writer, opts *entryOptions) error { {"URL", opts.VersionMetadata.ProjectURL}, {"LICENSE", strings.Join(opts.VersionMetadata.Licenses, "\n")}, {"GROUPS", strings.Join(opts.FileMetadata.Groups, "\n")}, - {"BUILDDATE", fmt.Sprintf("%d", opts.FileMetadata.BuildDate)}, + {"BUILDDATE", strconv.FormatInt(opts.FileMetadata.BuildDate, 10)}, {"PACKAGER", opts.FileMetadata.Packager}, {"PROVIDES", strings.Join(opts.FileMetadata.Provides, "\n")}, {"REPLACES", strings.Join(opts.FileMetadata.Replaces, "\n")}, diff --git a/services/packages/arch/vercmp.go b/services/packages/arch/vercmp.go new file mode 100644 index 0000000000..d44aa530f0 --- /dev/null +++ b/services/packages/arch/vercmp.go @@ -0,0 +1,108 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "strings" + "unicode" +) + +// https://gitlab.archlinux.org/pacman/pacman/-/blob/d55b47e5512808b67bc944feb20c2bcc6c1a4c45/lib/libalpm/version.c + +import ( + "strconv" +) + +func parseEVR(evr string) (epoch, version, release string) { + if before, after, f := strings.Cut(evr, ":"); f { + epoch = before + evr = after + } else { + epoch = "0" + } + + if before, after, f := strings.Cut(evr, "-"); f { + version = before + release = after + } else { + version = evr + release = "1" + } + return epoch, version, release +} + +func compareSegments(a, b []string) int { + lenA, lenB := len(a), len(b) + l := min(lenA, lenB) + for i := range l { + if r := compare(a[i], b[i]); r != 0 { + return r + } + } + if lenA == lenB { + return 0 + } else if l == lenA { + return -1 + } + return 1 +} + +func compare(a, b string) int { + if a == b { + return 0 + } + + aNumeric := isNumeric(a) + bNumeric := isNumeric(b) + + if aNumeric && bNumeric { + aInt, _ := strconv.Atoi(a) + bInt, _ := strconv.Atoi(b) + switch { + case aInt < bInt: + return -1 + case aInt > bInt: + return 1 + default: + return 0 + } + } + + if aNumeric { + return 1 + } + if bNumeric { + return -1 + } + + return strings.Compare(a, b) +} + +func isNumeric(s string) bool { + for _, c := range s { + if !unicode.IsDigit(c) { + return false + } + } + return true +} + +func compareVersions(a, b string) int { + if a == b { + return 0 + } + + epochA, versionA, releaseA := parseEVR(a) + epochB, versionB, releaseB := parseEVR(b) + + if res := compareSegments([]string{epochA}, []string{epochB}); res != 0 { + return res + } + + if res := compareSegments(strings.Split(versionA, "."), strings.Split(versionB, ".")); res != 0 { + return res + } + + return compareSegments([]string{releaseA}, []string{releaseB}) +} diff --git a/services/packages/arch/vercmp_test.go b/services/packages/arch/vercmp_test.go new file mode 100644 index 0000000000..2014a6d429 --- /dev/null +++ b/services/packages/arch/vercmp_test.go @@ -0,0 +1,27 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCompareVersions(t *testing.T) { + // https://man.archlinux.org/man/vercmp.8.en + checks := [][]string{ + {"1.0a", "1.0b", "1.0beta", "1.0p", "1.0pre", "1.0rc", "1.0", "1.0.a", "1.0.1"}, + {"1", "1.0", "1.1", "1.1.1", "1.2", "2.0", "3.0.0"}, + } + for _, check := range checks { + for i := 0; i < len(check)-1; i++ { + require.Equal(t, -1, compareVersions(check[i], check[i+1])) + require.Equal(t, 1, compareVersions(check[i+1], check[i])) + } + } + require.Equal(t, 1, compareVersions("1.0-2", "1.0")) + require.Equal(t, 0, compareVersions("0:1.0-1", "1.0")) + require.Equal(t, 1, compareVersions("1:1.0-1", "2.0")) +} diff --git a/services/packages/auth.go b/services/packages/auth.go index 4526a8e303..6e87643e29 100644 --- a/services/packages/auth.go +++ b/services/packages/auth.go @@ -4,6 +4,7 @@ package packages import ( + "errors" "fmt" "net/http" "strings" @@ -58,7 +59,7 @@ func ParseAuthorizationRequest(req *http.Request) (*PackageMeta, error) { parts := strings.SplitN(h, " ", 2) if len(parts) != 2 { log.Error("split token failed: %s", h) - return nil, fmt.Errorf("split token failed") + return nil, errors.New("split token failed") } return ParseAuthorizationToken(parts[1]) @@ -77,7 +78,7 @@ func ParseAuthorizationToken(tokenStr string) (*PackageMeta, error) { c, ok := token.Claims.(*packageClaims) if !token.Valid || !ok { - return nil, fmt.Errorf("invalid token claim") + return nil, errors.New("invalid token claim") } return &c.PackageMeta, nil diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index e8a8313625..605335d0f1 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -11,7 +11,6 @@ import ( "io" "path" "strconv" - "time" packages_model "code.gitea.io/gitea/models/packages" repo_model "code.gitea.io/gitea/models/repo" @@ -79,7 +78,7 @@ func RebuildIndex(ctx context.Context, doer, owner *user_model.User) error { "Rebuild Cargo Index", func(t *files_service.TemporaryUploadRepository) error { // Remove all existing content but the Cargo config - files, err := t.LsFiles() + files, err := t.LsFiles(ctx) if err != nil { return err } @@ -90,7 +89,7 @@ func RebuildIndex(ctx context.Context, doer, owner *user_model.User) error { break } } - if err := t.RemoveFilesFromIndex(files...); err != nil { + if err := t.RemoveFilesFromIndex(ctx, files...); err != nil { return err } @@ -205,7 +204,7 @@ func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUplo return nil } - return writeObjectToIndex(t, BuildPackagePath(p.LowerName), b) + return writeObjectToIndex(ctx, t, BuildPackagePath(p.LowerName), b) } func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) { @@ -214,7 +213,7 @@ func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.Use if errors.Is(err, util.ErrNotExist) { repo, err = repo_service.CreateRepositoryDirectly(ctx, doer, owner, repo_service.CreateRepoOptions{ Name: IndexRepositoryName, - }) + }, true) if err != nil { return nil, fmt.Errorf("CreateRepository: %w", err) } @@ -248,34 +247,34 @@ func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, "Initialize Cargo Config", func(t *files_service.TemporaryUploadRepository) error { var b bytes.Buffer - err := json.NewEncoder(&b).Encode(BuildConfig(owner, setting.Service.RequireSignInView || owner.Visibility != structs.VisibleTypePublic || repo.IsPrivate)) + err := json.NewEncoder(&b).Encode(BuildConfig(owner, setting.Service.RequireSignInViewStrict || owner.Visibility != structs.VisibleTypePublic || repo.IsPrivate)) if err != nil { return err } - return writeObjectToIndex(t, ConfigFileName, &b) + return writeObjectToIndex(ctx, t, ConfigFileName, &b) }, ) } // This is a shorter version of CreateOrUpdateRepoFile which allows to perform multiple actions on a git repository func alterRepositoryContent(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, commitMessage string, fn func(*files_service.TemporaryUploadRepository) error) error { - t, err := files_service.NewTemporaryUploadRepository(ctx, repo) + t, err := files_service.NewTemporaryUploadRepository(repo) if err != nil { return err } defer t.Close() var lastCommitID string - if err := t.Clone(repo.DefaultBranch, true); err != nil { + if err := t.Clone(ctx, repo.DefaultBranch, true); err != nil { if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { return err } - if err := t.Init(repo.ObjectFormatName); err != nil { + if err := t.Init(ctx, repo.ObjectFormatName); err != nil { return err } } else { - if err := t.SetDefaultIndex(); err != nil { + if err := t.SetDefaultIndex(ctx); err != nil { return err } @@ -291,25 +290,30 @@ func alterRepositoryContent(ctx context.Context, doer *user_model.User, repo *re return err } - treeHash, err := t.WriteTree() + treeHash, err := t.WriteTree(ctx) if err != nil { return err } - now := time.Now() - commitHash, err := t.CommitTreeWithDate(lastCommitID, doer, doer, treeHash, commitMessage, false, now, now) + commitOpts := &files_service.CommitTreeUserOptions{ + ParentCommitID: lastCommitID, + TreeHash: treeHash, + CommitMessage: commitMessage, + DoerUser: doer, + } + commitHash, err := t.CommitTree(ctx, commitOpts) if err != nil { return err } - return t.Push(doer, commitHash, repo.DefaultBranch) + return t.Push(ctx, doer, commitHash, repo.DefaultBranch) } -func writeObjectToIndex(t *files_service.TemporaryUploadRepository, path string, r io.Reader) error { - hash, err := t.HashObject(r) +func writeObjectToIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, path string, r io.Reader) error { + hash, err := t.HashObjectAndWrite(ctx, r) if err != nil { return err } - return t.AddObjectToIndex("100644", hash, path) + return t.AddObjectToIndex(ctx, "100644", hash, path) } diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go index b7ba2b6ac4..959babe7cd 100644 --- a/services/packages/cleanup/cleanup.go +++ b/services/packages/cleanup/cleanup.go @@ -32,127 +32,136 @@ func CleanupTask(ctx context.Context, olderThan time.Duration) error { return CleanupExpiredData(ctx, olderThan) } -func ExecuteCleanupRules(outerCtx context.Context) error { - ctx, committer, err := db.TxContext(outerCtx) +func executeCleanupOneRulePackage(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package) (versionDeleted bool, err error) { + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: optional.Some(false), + Sort: packages_model.SortCreatedDesc, + }) if err != nil { - return err + return false, fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) } - defer committer.Close() - - err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { - select { - case <-outerCtx.Done(): - return db.ErrCancelledf("While processing package cleanup rules") - default: + if pcr.KeepCount > 0 { + if pcr.KeepCount < len(pvs) { + pvs = pvs[pcr.KeepCount:] + } else { + pvs = nil } - - if err := pcr.CompiledPattern(); err != nil { - return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) + } + for _, pv := range pvs { + if pcr.Type == packages_model.TypeContainer { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { + return false, fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) + } else if skip { + log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) + continue + } } - - olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) - - packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) - if err != nil { - return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + log.Debug("Rule[%d]: keep '%s/%s' (remove days) %v", pcr.ID, p.Name, pv.Version, pv.CreatedUnix.FormatDate()) + continue } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) + continue + } + log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + log.Error("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %v", pcr.ID, err) + continue + } + versionDeleted = true + } + return versionDeleted, nil +} - anyVersionDeleted := false - for _, p := range packages { - pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ - PackageID: p.ID, - IsInternal: optional.Some(false), - Sort: packages_model.SortCreatedDesc, - Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), - }) - if err != nil { - return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) - } - versionDeleted := false - for _, pv := range pvs { - if pcr.Type == packages_model.TypeContainer { - if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { - return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) - } else if skip { - log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) - continue - } - } +func executeCleanupOneRule(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + if err := pcr.CompiledPattern(); err != nil { + return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) + } - toMatch := pv.LowerVersion - if pcr.MatchFullName { - toMatch = p.LowerName + "/" + pv.LowerVersion - } + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) + } - if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { - log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) - continue - } - if pv.CreatedUnix.AsLocalTime().After(olderThan) { - log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) - continue - } - if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { - log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) - continue + anyVersionDeleted := false + for _, p := range packages { + versionDeleted := false + err = db.WithTx(ctx, func(ctx context.Context) (err error) { + versionDeleted, err = executeCleanupOneRulePackage(ctx, pcr, p) + return err + }) + if err != nil { + log.Error("CleanupRule [%d]: executeCleanupOneRulePackage(%d) failed: %v", pcr.ID, p.ID, err) + continue + } + anyVersionDeleted = anyVersionDeleted || versionDeleted + if versionDeleted { + if pcr.Type == packages_model.TypeCargo { + owner, err := user_model.GetUserByID(ctx, pcr.OwnerID) + if err != nil { + return fmt.Errorf("GetUserByID failed: %w", err) } - - log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) - - if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { - return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) + if err := cargo_service.UpdatePackageIndexIfExists(ctx, owner, owner, p.ID); err != nil { + return fmt.Errorf("CleanupRule [%d]: cargo.UpdatePackageIndexIfExists failed: %w", pcr.ID, err) } + } + } + } - versionDeleted = true - anyVersionDeleted = true + if anyVersionDeleted { + switch pcr.Type { + case packages_model.TypeDebian: + if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err) + } + case packages_model.TypeAlpine: + if err := alpine_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: alpine.BuildAllRepositoryFiles failed: %w", pcr.ID, err) } + case packages_model.TypeRpm: + if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err) + } + case packages_model.TypeArch: + release, err := arch_service.AquireRegistryLock(ctx, pcr.OwnerID) + if err != nil { + return err + } + defer release() - if versionDeleted { - if pcr.Type == packages_model.TypeCargo { - owner, err := user_model.GetUserByID(ctx, pcr.OwnerID) - if err != nil { - return fmt.Errorf("GetUserByID failed: %w", err) - } - if err := cargo_service.UpdatePackageIndexIfExists(ctx, owner, owner, p.ID); err != nil { - return fmt.Errorf("CleanupRule [%d]: cargo.UpdatePackageIndexIfExists failed: %w", pcr.ID, err) - } - } + if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err) } } + } + return nil +} - if anyVersionDeleted { - switch pcr.Type { - case packages_model.TypeDebian: - if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { - return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err) - } - case packages_model.TypeAlpine: - if err := alpine_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { - return fmt.Errorf("CleanupRule [%d]: alpine.BuildAllRepositoryFiles failed: %w", pcr.ID, err) - } - case packages_model.TypeRpm: - if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { - return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err) - } - case packages_model.TypeArch: - release, err := arch_service.AquireRegistryLock(ctx, pcr.OwnerID) - if err != nil { - return err - } - defer release() +func ExecuteCleanupRules(ctx context.Context) error { + return packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + select { + case <-ctx.Done(): + return db.ErrCancelledf("While processing package cleanup rules") + default: + } - if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { - return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err) - } - } + err := executeCleanupOneRule(ctx, pcr) + if err != nil { + log.Error("CleanupRule [%d]: executeCleanupOneRule failed: %v", pcr.ID, err) } return nil }) - if err != nil { - return err - } - - return committer.Commit() } func CleanupExpiredData(outerCtx context.Context, olderThan time.Duration) error { diff --git a/services/packages/container/blob_uploader.go b/services/packages/container/blob_uploader.go index bae2e2d6af..27bc4a5421 100644 --- a/services/packages/container/blob_uploader.go +++ b/services/packages/container/blob_uploader.go @@ -12,7 +12,7 @@ import ( packages_model "code.gitea.io/gitea/models/packages" packages_module "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/tempdir" ) var ( @@ -30,8 +30,12 @@ type BlobUploader struct { reading bool } -func buildFilePath(id string) string { - return util.FilePathJoinAbs(setting.Packages.ChunkedUploadPath, id) +func uploadPathTempDir() *tempdir.TempDir { + return setting.AppDataTempDir("package-upload") +} + +func buildFilePath(uploadPath *tempdir.TempDir, id string) string { + return uploadPath.JoinPath(id) } // NewBlobUploader creates a new blob uploader for the given id @@ -48,7 +52,12 @@ func NewBlobUploader(ctx context.Context, id string) (*BlobUploader, error) { } } - f, err := os.OpenFile(buildFilePath(model.ID), os.O_RDWR|os.O_CREATE, 0o666) + uploadPath := uploadPathTempDir() + _, err = uploadPath.MkdirAllSub("") + if err != nil { + return nil, err + } + f, err := os.OpenFile(buildFilePath(uploadPath, model.ID), os.O_RDWR|os.O_CREATE, 0o666) if err != nil { return nil, err } @@ -118,13 +127,13 @@ func (u *BlobUploader) Read(p []byte) (int, error) { return u.file.Read(p) } -// Remove deletes the data and the model of a blob upload +// RemoveBlobUploadByID Remove deletes the data and the model of a blob upload func RemoveBlobUploadByID(ctx context.Context, id string) error { if err := packages_model.DeleteBlobUploadByID(ctx, id); err != nil { return err } - err := os.Remove(buildFilePath(id)) + err := os.Remove(buildFilePath(uploadPathTempDir(), id)) if err != nil && !os.IsNotExist(err) { return err } diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go index 3f5f43bbc0..263562a396 100644 --- a/services/packages/container/cleanup.go +++ b/services/packages/container/cleanup.go @@ -13,7 +13,7 @@ import ( container_module "code.gitea.io/gitea/modules/packages/container" packages_service "code.gitea.io/gitea/services/packages" - digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/go-digest" ) // Cleanup removes expired container data @@ -57,7 +57,7 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e Type: packages_model.TypeContainer, Version: packages_model.SearchValue{ ExactMatch: true, - Value: container_model.UploadVersion, + Value: container_module.UploadVersion, }, IsInternal: optional.Some(true), HasFiles: optional.Some(false), diff --git a/services/packages/container/common.go b/services/packages/container/common.go index 5a14ed5b7a..02cbff2286 100644 --- a/services/packages/container/common.go +++ b/services/packages/container/common.go @@ -5,11 +5,17 @@ package container import ( "context" + "io" "strings" packages_model "code.gitea.io/gitea/models/packages" + container_service "code.gitea.io/gitea/models/packages/container" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" + + "github.com/opencontainers/image-spec/specs-go/v1" ) // UpdateRepositoryNames updates the repository name property for all packages of the specific owner @@ -22,7 +28,7 @@ func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwner newOwnerName = strings.ToLower(newOwnerName) for _, p := range ps { - if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil { + if err := packages_model.DeletePropertiesByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil { return err } @@ -33,3 +39,26 @@ func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwner return nil } + +func ParseManifestMetadata(ctx context.Context, rd io.Reader, ownerID int64, imageName string) (*v1.Manifest, *packages_model.PackageFileDescriptor, *container_module.Metadata, error) { + var manifest v1.Manifest + if err := json.NewDecoder(rd).Decode(&manifest); err != nil { + return nil, nil, nil, err + } + configDescriptor, err := container_service.GetContainerBlob(ctx, &container_service.BlobSearchOptions{ + OwnerID: ownerID, + Image: imageName, + Digest: manifest.Config.Digest.String(), + }) + if err != nil { + return nil, nil, nil, err + } + + configReader, err := packages.NewContentStore().OpenBlob(packages.BlobHash256Key(configDescriptor.Blob.HashSHA256)) + if err != nil { + return nil, nil, nil, err + } + defer configReader.Close() + metadata, err := container_module.ParseImageConfig(manifest.Config.MediaType, configReader) + return &manifest, configDescriptor, metadata, err +} diff --git a/services/packages/debian/repository.go b/services/packages/debian/repository.go index 13e98a820e..34b52b45cf 100644 --- a/services/packages/debian/repository.go +++ b/services/packages/debian/repository.go @@ -23,10 +23,10 @@ import ( "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" - "github.com/keybase/go-crypto/openpgp" - "github.com/keybase/go-crypto/openpgp/armor" - "github.com/keybase/go-crypto/openpgp/clearsign" - "github.com/keybase/go-crypto/openpgp/packet" + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/clearsign" + "github.com/ProtonMail/go-crypto/openpgp/packet" "github.com/ulikunitz/xz" ) diff --git a/services/packages/package_update.go b/services/packages/package_update.go new file mode 100644 index 0000000000..4a22ee7a62 --- /dev/null +++ b/services/packages/package_update.go @@ -0,0 +1,79 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package packages + +import ( + "context" + "fmt" + + org_model "code.gitea.io/gitea/models/organization" + packages_model "code.gitea.io/gitea/models/packages" + access_model "code.gitea.io/gitea/models/perm/access" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unit" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/util" +) + +func LinkToRepository(ctx context.Context, pkg *packages_model.Package, repo *repo_model.Repository, doer *user_model.User) error { + if pkg.OwnerID != repo.OwnerID { + return util.ErrPermissionDenied + } + if pkg.RepoID > 0 { + return util.ErrInvalidArgument + } + + perms, err := access_model.GetUserRepoPermission(ctx, repo, doer) + if err != nil { + return fmt.Errorf("error getting permissions for user %d on repository %d: %w", doer.ID, repo.ID, err) + } + if !perms.CanWrite(unit.TypePackages) { + return util.ErrPermissionDenied + } + + if err := packages_model.SetRepositoryLink(ctx, pkg.ID, repo.ID); err != nil { + return fmt.Errorf("error while linking package '%v' to repo '%v' : %w", pkg.Name, repo.FullName(), err) + } + return nil +} + +func UnlinkFromRepository(ctx context.Context, pkg *packages_model.Package, doer *user_model.User) error { + if pkg.RepoID == 0 { + return util.ErrInvalidArgument + } + + repo, err := repo_model.GetRepositoryByID(ctx, pkg.RepoID) + if err != nil && !repo_model.IsErrRepoNotExist(err) { + return fmt.Errorf("error getting repository %d: %w", pkg.RepoID, err) + } + if err == nil { + perms, err := access_model.GetUserRepoPermission(ctx, repo, doer) + if err != nil { + return fmt.Errorf("error getting permissions for user %d on repository %d: %w", doer.ID, repo.ID, err) + } + if !perms.CanWrite(unit.TypePackages) { + return util.ErrPermissionDenied + } + } + + user, err := user_model.GetUserByID(ctx, pkg.OwnerID) + if err != nil { + return err + } + if !doer.IsAdmin { + if !user.IsOrganization() { + if doer.ID != pkg.OwnerID { + return fmt.Errorf("no permission to unlink package '%v' from its repository, or packages are disabled", pkg.Name) + } + } else { + isOrgAdmin, err := org_model.OrgFromUser(user).IsOrgAdmin(ctx, doer.ID) + if err != nil { + return err + } else if !isOrgAdmin { + return fmt.Errorf("no permission to unlink package '%v' from its repository, or packages are disabled", pkg.Name) + } + } + } + return packages_model.UnlinkRepository(ctx, pkg.ID) +} diff --git a/services/packages/packages.go b/services/packages/packages.go index bd1d460fd3..517334cbc7 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -563,8 +563,8 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro return packages_model.DeleteFileByID(ctx, pf.ID) } -// GetFileStreamByPackageNameAndVersion returns the content of the specific package file -func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { +// OpenFileForDownloadByPackageNameAndVersion returns the content of the specific package file and increases the download counter. +func OpenFileForDownloadByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey) pv, err := packages_model.GetVersionByNameAndVersion(ctx, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) @@ -576,32 +576,38 @@ func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, return nil, nil, nil, err } - return GetFileStreamByPackageVersion(ctx, pv, pfi) + return OpenFileForDownloadByPackageVersion(ctx, pv, pfi) } -// GetFileStreamByPackageVersion returns the content of the specific package file -func GetFileStreamByPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { +// OpenFileForDownloadByPackageVersion returns the content of the specific package file and increases the download counter. +func OpenFileForDownloadByPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfi *PackageFileInfo) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, pfi.Filename, pfi.CompositeKey) if err != nil { return nil, nil, nil, err } - return GetPackageFileStream(ctx, pf) + return OpenFileForDownload(ctx, pf) } -// GetPackageFileStream returns the content of the specific package file -func GetPackageFileStream(ctx context.Context, pf *packages_model.PackageFile) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { +// OpenFileForDownload returns the content of the specific package file and increases the download counter. +func OpenFileForDownload(ctx context.Context, pf *packages_model.PackageFile) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) if err != nil { return nil, nil, nil, err } - return GetPackageBlobStream(ctx, pf, pb, nil) + return OpenBlobForDownload(ctx, pf, pb, nil) } -// GetPackageBlobStream returns the content of the specific package blob +func OpenBlobStream(pb *packages_model.PackageBlob) (io.ReadSeekCloser, error) { + cs := packages_module.NewContentStore() + key := packages_module.BlobHash256Key(pb.HashSHA256) + return cs.OpenBlob(key) +} + +// OpenBlobForDownload returns the content of the specific package blob and increases the download counter. // If the storage supports direct serving and it's enabled, only the direct serving url is returned. -func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob, serveDirectReqParams url.Values) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { +func OpenBlobForDownload(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob, serveDirectReqParams url.Values) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { key := packages_module.BlobHash256Key(pb.HashSHA256) cs := packages_module.NewContentStore() @@ -617,7 +623,7 @@ func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, p } } if u == nil { - s, err = cs.Get(key) + s, err = cs.OpenBlob(key) } if err == nil { diff --git a/services/packages/rpm/repository.go b/services/packages/rpm/repository.go index a7d196c15c..fbbf8d7dad 100644 --- a/services/packages/rpm/repository.go +++ b/services/packages/rpm/repository.go @@ -408,7 +408,6 @@ func buildPrimary(ctx context.Context, pv *packages_model.PackageVersion, pfs [] files = append(files, f) } } - packageVersion := fmt.Sprintf("%s-%s", pd.FileMetadata.Version, pd.FileMetadata.Release) packages = append(packages, &Package{ Type: "rpm", Name: pd.Package.Name, @@ -437,7 +436,7 @@ func buildPrimary(ctx context.Context, pv *packages_model.PackageVersion, pfs [] Archive: pd.FileMetadata.ArchiveSize, }, Location: Location{ - Href: fmt.Sprintf("package/%s/%s/%s/%s-%s.%s.rpm", pd.Package.Name, packageVersion, pd.FileMetadata.Architecture, pd.Package.Name, packageVersion, pd.FileMetadata.Architecture), + Href: fmt.Sprintf("package/%s/%s/%s/%s-%s.%s.rpm", pd.Package.Name, pd.Version.Version, pd.FileMetadata.Architecture, pd.Package.Name, pd.Version.Version, pd.FileMetadata.Architecture), }, Format: Format{ License: pd.VersionMetadata.License, @@ -471,7 +470,7 @@ func buildPrimary(ctx context.Context, pv *packages_model.PackageVersion, pfs [] } // https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#filelists-xml -func buildFilelists(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl +func buildFilelists(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl // duplicates with buildOther type Version struct { Epoch string `xml:"epoch,attr"` Version string `xml:"ver,attr"` @@ -518,7 +517,7 @@ func buildFilelists(ctx context.Context, pv *packages_model.PackageVersion, pfs } // https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#other-xml -func buildOther(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl +func buildOther(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl // duplicates with buildFilelists type Version struct { Epoch string `xml:"epoch,attr"` Version string `xml:"ver,attr"` diff --git a/services/projects/issue.go b/services/projects/issue.go index db1621a39f..590fe960d5 100644 --- a/services/projects/issue.go +++ b/services/projects/issue.go @@ -5,12 +5,13 @@ package project import ( "context" - "fmt" + "errors" "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" project_model "code.gitea.io/gitea/models/project" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/optional" ) // MoveIssuesOnProjectColumn moves or keeps issues in a column and sorts them inside that column @@ -28,7 +29,7 @@ func MoveIssuesOnProjectColumn(ctx context.Context, doer *user_model.User, colum return err } if int(count) != len(sortedIssueIDs) { - return fmt.Errorf("all issues have to be added to a project first") + return errors.New("all issues have to be added to a project first") } issues, err := issues_model.GetIssuesByIDs(ctx, issueIDs) @@ -55,25 +56,152 @@ func MoveIssuesOnProjectColumn(ctx context.Context, doer *user_model.User, colum continue } - _, err = db.Exec(ctx, "UPDATE `project_issue` SET project_board_id=?, sorting=? WHERE issue_id=?", column.ID, sorting, issueID) + projectColumnID, err := curIssue.ProjectColumnID(ctx) if err != nil { return err } - // add timeline to issue - if _, err := issues_model.CreateComment(ctx, &issues_model.CreateCommentOptions{ - Type: issues_model.CommentTypeProjectColumn, - Doer: doer, - Repo: curIssue.Repo, - Issue: curIssue, - ProjectID: column.ProjectID, - ProjectTitle: project.Title, - ProjectColumnID: column.ID, - ProjectColumnTitle: column.Title, - }); err != nil { + if projectColumnID != column.ID { + // add timeline to issue + if _, err := issues_model.CreateComment(ctx, &issues_model.CreateCommentOptions{ + Type: issues_model.CommentTypeProjectColumn, + Doer: doer, + Repo: curIssue.Repo, + Issue: curIssue, + ProjectID: column.ProjectID, + ProjectTitle: project.Title, + ProjectColumnID: column.ID, + ProjectColumnTitle: column.Title, + }); err != nil { + return err + } + } + + _, err = db.Exec(ctx, "UPDATE `project_issue` SET project_board_id=?, sorting=? WHERE issue_id=?", column.ID, sorting, issueID) + if err != nil { return err } } return nil }) } + +// LoadIssuesFromProject load issues assigned to each project column inside the given project +func LoadIssuesFromProject(ctx context.Context, project *project_model.Project, opts *issues_model.IssuesOptions) (map[int64]issues_model.IssueList, error) { + issueList, err := issues_model.Issues(ctx, opts.Copy(func(o *issues_model.IssuesOptions) { + o.ProjectID = project.ID + o.SortType = "project-column-sorting" + })) + if err != nil { + return nil, err + } + + if err := issueList.LoadComments(ctx); err != nil { + return nil, err + } + + defaultColumn, err := project.MustDefaultColumn(ctx) + if err != nil { + return nil, err + } + + issueColumnMap, err := issues_model.LoadProjectIssueColumnMap(ctx, project.ID, defaultColumn.ID) + if err != nil { + return nil, err + } + + results := make(map[int64]issues_model.IssueList) + for _, issue := range issueList { + projectColumnID, ok := issueColumnMap[issue.ID] + if !ok { + continue + } + if _, ok := results[projectColumnID]; !ok { + results[projectColumnID] = make(issues_model.IssueList, 0) + } + results[projectColumnID] = append(results[projectColumnID], issue) + } + return results, nil +} + +// NumClosedIssues return counter of closed issues assigned to a project +func loadNumClosedIssues(ctx context.Context, p *project_model.Project) error { + cnt, err := db.GetEngine(ctx).Table("project_issue"). + Join("INNER", "issue", "project_issue.issue_id=issue.id"). + Where("project_issue.project_id=? AND issue.is_closed=?", p.ID, true). + Cols("issue_id"). + Count() + if err != nil { + return err + } + p.NumClosedIssues = cnt + return nil +} + +// NumOpenIssues return counter of open issues assigned to a project +func loadNumOpenIssues(ctx context.Context, p *project_model.Project) error { + cnt, err := db.GetEngine(ctx).Table("project_issue"). + Join("INNER", "issue", "project_issue.issue_id=issue.id"). + Where("project_issue.project_id=? AND issue.is_closed=?", p.ID, false). + Cols("issue_id"). + Count() + if err != nil { + return err + } + p.NumOpenIssues = cnt + return nil +} + +func LoadIssueNumbersForProjects(ctx context.Context, projects []*project_model.Project, doer *user_model.User) error { + for _, project := range projects { + if err := LoadIssueNumbersForProject(ctx, project, doer); err != nil { + return err + } + } + return nil +} + +func LoadIssueNumbersForProject(ctx context.Context, project *project_model.Project, doer *user_model.User) error { + // for repository project, just get the numbers + if project.OwnerID == 0 { + if err := loadNumClosedIssues(ctx, project); err != nil { + return err + } + if err := loadNumOpenIssues(ctx, project); err != nil { + return err + } + project.NumIssues = project.NumClosedIssues + project.NumOpenIssues + return nil + } + + if err := project.LoadOwner(ctx); err != nil { + return err + } + + // for user or org projects, we need to check access permissions + opts := issues_model.IssuesOptions{ + ProjectID: project.ID, + Doer: doer, + AllPublic: doer == nil, + Owner: project.Owner, + } + + var err error + project.NumOpenIssues, err = issues_model.CountIssues(ctx, opts.Copy(func(o *issues_model.IssuesOptions) { + o.IsClosed = optional.Some(false) + })) + if err != nil { + return err + } + + project.NumClosedIssues, err = issues_model.CountIssues(ctx, opts.Copy(func(o *issues_model.IssuesOptions) { + o.IsClosed = optional.Some(true) + })) + if err != nil { + return err + } + + project.NumIssues = project.NumClosedIssues + project.NumOpenIssues + + return nil +} diff --git a/services/projects/issue_test.go b/services/projects/issue_test.go new file mode 100644 index 0000000000..e76d31e757 --- /dev/null +++ b/services/projects/issue_test.go @@ -0,0 +1,210 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package project + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + issues_model "code.gitea.io/gitea/models/issues" + org_model "code.gitea.io/gitea/models/organization" + project_model "code.gitea.io/gitea/models/project" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + + "github.com/stretchr/testify/assert" +) + +func Test_Projects(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + userAdmin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + org3 := unittest.AssertExistsAndLoadBean(t, &org_model.Organization{ID: 3}) + user4 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 4}) + + t.Run("User projects", func(t *testing.T) { + pi1 := project_model.ProjectIssue{ + ProjectID: 4, + IssueID: 1, + ProjectColumnID: 4, + } + err := db.Insert(db.DefaultContext, &pi1) + assert.NoError(t, err) + defer func() { + _, err = db.DeleteByID[project_model.ProjectIssue](db.DefaultContext, pi1.ID) + assert.NoError(t, err) + }() + + pi2 := project_model.ProjectIssue{ + ProjectID: 4, + IssueID: 4, + ProjectColumnID: 4, + } + err = db.Insert(db.DefaultContext, &pi2) + assert.NoError(t, err) + defer func() { + _, err = db.DeleteByID[project_model.ProjectIssue](db.DefaultContext, pi2.ID) + assert.NoError(t, err) + }() + + projects, err := db.Find[project_model.Project](db.DefaultContext, project_model.SearchOptions{ + OwnerID: user2.ID, + }) + assert.NoError(t, err) + assert.Len(t, projects, 3) + assert.EqualValues(t, 4, projects[0].ID) + + t.Run("Authenticated user", func(t *testing.T) { + columnIssues, err := LoadIssuesFromProject(db.DefaultContext, projects[0], &issues_model.IssuesOptions{ + Owner: user2, + Doer: user2, + }) + assert.NoError(t, err) + assert.Len(t, columnIssues, 1) // 4 has 2 issues, 6 will not contains here because 0 issues + assert.Len(t, columnIssues[4], 2) // user2 can visit both issues, one from public repository one from private repository + }) + + t.Run("Anonymous user", func(t *testing.T) { + columnIssues, err := LoadIssuesFromProject(db.DefaultContext, projects[0], &issues_model.IssuesOptions{ + AllPublic: true, + }) + assert.NoError(t, err) + assert.Len(t, columnIssues, 1) + assert.Len(t, columnIssues[4], 1) // anonymous user can only visit public repo issues + }) + + t.Run("Authenticated user with no permission to the private repo", func(t *testing.T) { + columnIssues, err := LoadIssuesFromProject(db.DefaultContext, projects[0], &issues_model.IssuesOptions{ + Owner: user2, + Doer: user4, + }) + assert.NoError(t, err) + assert.Len(t, columnIssues, 1) + assert.Len(t, columnIssues[4], 1) // user4 can only visit public repo issues + }) + }) + + t.Run("Org projects", func(t *testing.T) { + project1 := project_model.Project{ + Title: "project in an org", + OwnerID: org3.ID, + Type: project_model.TypeOrganization, + TemplateType: project_model.TemplateTypeBasicKanban, + } + err := project_model.NewProject(db.DefaultContext, &project1) + assert.NoError(t, err) + defer func() { + err := project_model.DeleteProjectByID(db.DefaultContext, project1.ID) + assert.NoError(t, err) + }() + + column1 := project_model.Column{ + Title: "column 1", + ProjectID: project1.ID, + } + err = project_model.NewColumn(db.DefaultContext, &column1) + assert.NoError(t, err) + + column2 := project_model.Column{ + Title: "column 2", + ProjectID: project1.ID, + } + err = project_model.NewColumn(db.DefaultContext, &column2) + assert.NoError(t, err) + + // issue 6 belongs to private repo 3 under org 3 + issue6 := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 6}) + err = issues_model.IssueAssignOrRemoveProject(db.DefaultContext, issue6, user2, project1.ID, column1.ID) + assert.NoError(t, err) + + // issue 16 belongs to public repo 16 under org 3 + issue16 := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 16}) + err = issues_model.IssueAssignOrRemoveProject(db.DefaultContext, issue16, user2, project1.ID, column1.ID) + assert.NoError(t, err) + + projects, err := db.Find[project_model.Project](db.DefaultContext, project_model.SearchOptions{ + OwnerID: org3.ID, + }) + assert.NoError(t, err) + assert.Len(t, projects, 1) + assert.Equal(t, project1.ID, projects[0].ID) + + t.Run("Authenticated user", func(t *testing.T) { + columnIssues, err := LoadIssuesFromProject(db.DefaultContext, projects[0], &issues_model.IssuesOptions{ + Owner: org3.AsUser(), + Doer: userAdmin, + }) + assert.NoError(t, err) + assert.Len(t, columnIssues, 1) // column1 has 2 issues, 6 will not contains here because 0 issues + assert.Len(t, columnIssues[column1.ID], 2) // user2 can visit both issues, one from public repository one from private repository + }) + + t.Run("Anonymous user", func(t *testing.T) { + columnIssues, err := LoadIssuesFromProject(db.DefaultContext, projects[0], &issues_model.IssuesOptions{ + AllPublic: true, + }) + assert.NoError(t, err) + assert.Len(t, columnIssues, 1) + assert.Len(t, columnIssues[column1.ID], 1) // anonymous user can only visit public repo issues + }) + + t.Run("Authenticated user with no permission to the private repo", func(t *testing.T) { + columnIssues, err := LoadIssuesFromProject(db.DefaultContext, projects[0], &issues_model.IssuesOptions{ + Owner: org3.AsUser(), + Doer: user2, + }) + assert.NoError(t, err) + assert.Len(t, columnIssues, 1) + assert.Len(t, columnIssues[column1.ID], 1) // user4 can only visit public repo issues + }) + }) + + t.Run("Repository projects", func(t *testing.T) { + repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + projects, err := db.Find[project_model.Project](db.DefaultContext, project_model.SearchOptions{ + RepoID: repo1.ID, + }) + assert.NoError(t, err) + assert.Len(t, projects, 1) + assert.EqualValues(t, 1, projects[0].ID) + + t.Run("Authenticated user", func(t *testing.T) { + columnIssues, err := LoadIssuesFromProject(db.DefaultContext, projects[0], &issues_model.IssuesOptions{ + RepoIDs: []int64{repo1.ID}, + Doer: userAdmin, + }) + assert.NoError(t, err) + assert.Len(t, columnIssues, 3) + assert.Len(t, columnIssues[1], 2) + assert.Len(t, columnIssues[2], 1) + assert.Len(t, columnIssues[3], 1) + }) + + t.Run("Anonymous user", func(t *testing.T) { + columnIssues, err := LoadIssuesFromProject(db.DefaultContext, projects[0], &issues_model.IssuesOptions{ + AllPublic: true, + }) + assert.NoError(t, err) + assert.Len(t, columnIssues, 3) + assert.Len(t, columnIssues[1], 2) + assert.Len(t, columnIssues[2], 1) + assert.Len(t, columnIssues[3], 1) + }) + + t.Run("Authenticated user with no permission to the private repo", func(t *testing.T) { + columnIssues, err := LoadIssuesFromProject(db.DefaultContext, projects[0], &issues_model.IssuesOptions{ + RepoIDs: []int64{repo1.ID}, + Doer: user2, + }) + assert.NoError(t, err) + assert.Len(t, columnIssues, 3) + assert.Len(t, columnIssues[1], 2) + assert.Len(t, columnIssues[2], 1) + assert.Len(t, columnIssues[3], 1) + }) + }) +} diff --git a/services/projects/main_test.go b/services/projects/main_test.go new file mode 100644 index 0000000000..d39c82a140 --- /dev/null +++ b/services/projects/main_test.go @@ -0,0 +1,17 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package project + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" + + _ "code.gitea.io/gitea/models/actions" + _ "code.gitea.io/gitea/models/activities" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m) +} diff --git a/services/pull/check.go b/services/pull/check.go index e1adc3ca3b..5d8990aa00 100644 --- a/services/pull/check.go +++ b/services/pull/check.go @@ -10,6 +10,7 @@ import ( "fmt" "strconv" "strings" + "time" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" @@ -25,6 +26,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" asymkey_service "code.gitea.io/gitea/services/asymkey" notify_service "code.gitea.io/gitea/services/notify" @@ -34,27 +36,88 @@ import ( var prPatchCheckerQueue *queue.WorkerPoolQueue[string] var ( - ErrIsClosed = errors.New("pull is closed") - ErrUserNotAllowedToMerge = ErrDisallowedToMerge{} - ErrHasMerged = errors.New("has already been merged") - ErrIsWorkInProgress = errors.New("work in progress PRs cannot be merged") - ErrIsChecking = errors.New("cannot merge while conflict checking is in progress") - ErrNotMergeableState = errors.New("not in mergeable state") - ErrDependenciesLeft = errors.New("is blocked by an open dependency") + ErrIsClosed = errors.New("pull is closed") + ErrNoPermissionToMerge = errors.New("no permission to merge") + ErrNotReadyToMerge = errors.New("not ready to merge") + ErrHasMerged = errors.New("has already been merged") + ErrIsWorkInProgress = errors.New("work in progress PRs cannot be merged") + ErrIsChecking = errors.New("cannot merge while conflict checking is in progress") + ErrNotMergeableState = errors.New("not in mergeable state") + ErrDependenciesLeft = errors.New("is blocked by an open dependency") ) -// AddToTaskQueue adds itself to pull request test task queue. -func AddToTaskQueue(ctx context.Context, pr *issues_model.PullRequest) { +func markPullRequestStatusAsChecking(ctx context.Context, pr *issues_model.PullRequest) bool { pr.Status = issues_model.PullRequestStatusChecking err := pr.UpdateColsIfNotMerged(ctx, "status") if err != nil { - log.Error("AddToTaskQueue(%-v).UpdateCols.(add to queue): %v", pr, err) + log.Error("UpdateColsIfNotMerged failed, pr: %-v, err: %v", pr, err) + return false + } + pr, err = issues_model.GetPullRequestByID(ctx, pr.ID) + if err != nil { + log.Error("GetPullRequestByID failed, pr: %-v, err: %v", pr, err) + return false + } + return pr.Status == issues_model.PullRequestStatusChecking +} + +var AddPullRequestToCheckQueue = realAddPullRequestToCheckQueue + +func realAddPullRequestToCheckQueue(prID int64) { + err := prPatchCheckerQueue.Push(strconv.FormatInt(prID, 10)) + if err != nil && !errors.Is(err, queue.ErrAlreadyInQueue) { + log.Error("Error adding %v to the pull requests check queue: %v", prID, err) + } +} + +func StartPullRequestCheckImmediately(ctx context.Context, pr *issues_model.PullRequest) { + if !markPullRequestStatusAsChecking(ctx, pr) { return } - log.Trace("Adding %-v to the test pull requests queue", pr) - err = prPatchCheckerQueue.Push(strconv.FormatInt(pr.ID, 10)) - if err != nil && err != queue.ErrAlreadyInQueue { - log.Error("Error adding %-v to the test pull requests queue: %v", pr, err) + AddPullRequestToCheckQueue(pr.ID) +} + +// StartPullRequestCheckDelayable will delay the check if the pull request was not updated recently. +// When the "base" branch gets updated, all PRs targeting that "base" branch need to re-check whether +// they are mergeable. +// When there are too many stale PRs, each "base" branch update will consume a lot of system resources. +// So we can delay the checks for PRs that were not updated recently, only mark their status as +// "checking", and then next time when these PRs are updated or viewed, the real checks will run. +func StartPullRequestCheckDelayable(ctx context.Context, pr *issues_model.PullRequest) { + if !markPullRequestStatusAsChecking(ctx, pr) { + return + } + + if setting.Repository.PullRequest.DelayCheckForInactiveDays >= 0 { + if err := pr.LoadIssue(ctx); err != nil { + return + } + duration := 24 * time.Hour * time.Duration(setting.Repository.PullRequest.DelayCheckForInactiveDays) + if pr.Issue.UpdatedUnix.AddDuration(duration) <= timeutil.TimeStampNow() { + return + } + } + + AddPullRequestToCheckQueue(pr.ID) +} + +func StartPullRequestCheckOnView(ctx context.Context, pr *issues_model.PullRequest) { + // TODO: its correctness totally depends on the "unique queue" feature and the global lock. + // So duplicate "start" requests will be ignored if there is already a task in the queue or one is running. + // Ideally in the future we should decouple the "unique queue" feature from the "start" request. + if pr.Status == issues_model.PullRequestStatusChecking { + if setting.IsInTesting { + // In testing mode, there might be an "immediate" queue, which is not a real queue, everything is executed in the same goroutine + // So we can't use the global lock here, otherwise it will cause a deadlock. + AddPullRequestToCheckQueue(pr.ID) + } else { + // When a PR check starts, the task is popped from the queue and the task handler acquires the global lock + // So we need to acquire the global lock here to prevent from duplicate tasks + _, _ = globallock.TryLockAndDo(ctx, getPullWorkingLockKey(pr.ID), func(ctx context.Context) error { + AddPullRequestToCheckQueue(pr.ID) // the queue is a unique queue and won't add the same task again + return nil + }) + } } } @@ -84,7 +147,7 @@ func CheckPullMergeable(stdCtx context.Context, doer *user_model.User, perm *acc log.Error("Error whilst checking if %-v is allowed to merge %-v: %v", doer, pr, err) return err } else if !allowedMerge { - return ErrUserNotAllowedToMerge + return ErrNoPermissionToMerge } if mergeCheckType == MergeCheckTypeManually { @@ -105,7 +168,7 @@ func CheckPullMergeable(stdCtx context.Context, doer *user_model.User, perm *acc } if err := CheckPullBranchProtections(ctx, pr, false); err != nil { - if !IsErrDisallowedToMerge(err) { + if !errors.Is(err, ErrNotReadyToMerge) { log.Error("Error whilst checking pull branch protection for %-v: %v", pr, err) return err } @@ -172,10 +235,10 @@ func isSignedIfRequired(ctx context.Context, pr *issues_model.PullRequest, doer return sign, err } -// checkAndUpdateStatus checks if pull request is possible to leaving checking status, +// markPullRequestAsMergeable checks if pull request is possible to leaving checking status, // and set to be either conflict or mergeable. -func checkAndUpdateStatus(ctx context.Context, pr *issues_model.PullRequest) { - // If status has not been changed to conflict by testPatch then we are mergeable +func markPullRequestAsMergeable(ctx context.Context, pr *issues_model.PullRequest) { + // If status has not been changed to conflict by testPullRequestTmpRepoBranchMergeable then we are mergeable if pr.Status == issues_model.PullRequestStatusChecking { pr.Status = issues_model.PullRequestStatusMergeable } @@ -206,9 +269,9 @@ func getMergeCommit(ctx context.Context, pr *issues_model.PullRequest) (*git.Com prHeadRef := pr.GetGitRefName() // Check if the pull request is merged into BaseBranch - if _, _, err := git.NewCommand(ctx, "merge-base", "--is-ancestor"). + if _, _, err := git.NewCommand("merge-base", "--is-ancestor"). AddDynamicArguments(prHeadRef, pr.BaseBranch). - RunStdString(&git.RunOpts{Dir: pr.BaseRepo.RepoPath()}); err != nil { + RunStdString(ctx, &git.RunOpts{Dir: pr.BaseRepo.RepoPath()}); err != nil { if strings.Contains(err.Error(), "exit status 1") { // prHeadRef is not an ancestor of the base branch return nil, nil @@ -234,9 +297,9 @@ func getMergeCommit(ctx context.Context, pr *issues_model.PullRequest) (*git.Com objectFormat := git.ObjectFormatFromName(pr.BaseRepo.ObjectFormatName) // Get the commit from BaseBranch where the pull request got merged - mergeCommit, _, err := git.NewCommand(ctx, "rev-list", "--ancestry-path", "--merges", "--reverse"). - AddDynamicArguments(prHeadCommitID + ".." + pr.BaseBranch). - RunStdString(&git.RunOpts{Dir: pr.BaseRepo.RepoPath()}) + mergeCommit, _, err := git.NewCommand("rev-list", "--ancestry-path", "--merges", "--reverse"). + AddDynamicArguments(prHeadCommitID+".."+pr.BaseBranch). + RunStdString(ctx, &git.RunOpts{Dir: pr.BaseRepo.RepoPath()}) if err != nil { return nil, fmt.Errorf("git rev-list --ancestry-path --merges --reverse: %w", err) } else if len(mergeCommit) < objectFormat.FullLength() { @@ -310,6 +373,10 @@ func manuallyMerged(ctx context.Context, pr *issues_model.PullRequest) bool { // InitializePullRequests checks and tests untested patches of pull requests. func InitializePullRequests(ctx context.Context) { + // If we prefer to delay the checks, then no need to do any check during startup, there should be not much difference + if setting.Repository.PullRequest.DelayCheckForInactiveDays >= 0 { + return + } prs, err := issues_model.GetPullRequestIDsByCheckStatus(ctx, issues_model.PullRequestStatusChecking) if err != nil { log.Error("Find Checking PRs: %v", err) @@ -320,24 +387,12 @@ func InitializePullRequests(ctx context.Context) { case <-ctx.Done(): return default: - log.Trace("Adding PR[%d] to the pull requests patch checking queue", prID) - if err := prPatchCheckerQueue.Push(strconv.FormatInt(prID, 10)); err != nil { - log.Error("Error adding PR[%d] to the pull requests patch checking queue %v", prID, err) - } + AddPullRequestToCheckQueue(prID) } } } -// handle passed PR IDs and test the PRs -func handler(items ...string) []string { - for _, s := range items { - id, _ := strconv.ParseInt(s, 10, 64) - testPR(id) - } - return nil -} - -func testPR(id int64) { +func checkPullRequestMergeable(id int64) { ctx := graceful.GetManager().HammerContext() releaser, err := globallock.Lock(ctx, getPullWorkingLockKey(id)) if err != nil { @@ -351,7 +406,7 @@ func testPR(id int64) { pr, err := issues_model.GetPullRequestByID(ctx, id) if err != nil { - log.Error("Unable to GetPullRequestByID[%d] for testPR: %v", id, err) + log.Error("Unable to GetPullRequestByID[%d] for checkPullRequestMergeable: %v", id, err) return } @@ -370,15 +425,15 @@ func testPR(id int64) { return } - if err := TestPatch(pr); err != nil { - log.Error("testPatch[%-v]: %v", pr, err) + if err := testPullRequestBranchMergeable(pr); err != nil { + log.Error("testPullRequestTmpRepoBranchMergeable[%-v]: %v", pr, err) pr.Status = issues_model.PullRequestStatusError if err := pr.UpdateCols(ctx, "status"); err != nil { log.Error("update pr [%-v] status to PullRequestStatusError failed: %v", pr, err) } return } - checkAndUpdateStatus(ctx, pr) + markPullRequestAsMergeable(ctx, pr) } // CheckPRsForBaseBranch check all pulls with baseBrannch @@ -387,20 +442,24 @@ func CheckPRsForBaseBranch(ctx context.Context, baseRepo *repo_model.Repository, if err != nil { return err } - for _, pr := range prs { - AddToTaskQueue(ctx, pr) + StartPullRequestCheckImmediately(ctx, pr) } - return nil } // Init runs the task queue to test all the checking status pull requests func Init() error { - prPatchCheckerQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "pr_patch_checker", handler) + prPatchCheckerQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "pr_patch_checker", func(items ...string) []string { + for _, s := range items { + id, _ := strconv.ParseInt(s, 10, 64) + checkPullRequestMergeable(id) + } + return nil + }) if prPatchCheckerQueue == nil { - return fmt.Errorf("unable to create pr_patch_checker queue") + return errors.New("unable to create pr_patch_checker queue") } go graceful.GetManager().RunWithCancel(prPatchCheckerQueue) diff --git a/services/pull/check_test.go b/services/pull/check_test.go index dcf5f7b93a..fa3a676ef1 100644 --- a/services/pull/check_test.go +++ b/services/pull/check_test.go @@ -5,7 +5,6 @@ package pull import ( - "context" "strconv" "testing" "time" @@ -33,11 +32,11 @@ func TestPullRequest_AddToTaskQueue(t *testing.T) { cfg, err := setting.GetQueueSettings(setting.CfgProvider, "pr_patch_checker") assert.NoError(t, err) - prPatchCheckerQueue, err = queue.NewWorkerPoolQueueWithContext(context.Background(), "pr_patch_checker", cfg, testHandler, true) + prPatchCheckerQueue, err = queue.NewWorkerPoolQueueWithContext(t.Context(), "pr_patch_checker", cfg, testHandler, true) assert.NoError(t, err) pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ID: 2}) - AddToTaskQueue(db.DefaultContext, pr) + StartPullRequestCheckImmediately(db.DefaultContext, pr) assert.Eventually(t, func() bool { pr = unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ID: 2}) @@ -52,7 +51,7 @@ func TestPullRequest_AddToTaskQueue(t *testing.T) { select { case id := <-idChan: - assert.EqualValues(t, pr.ID, id) + assert.Equal(t, pr.ID, id) case <-time.After(time.Second): assert.FailNow(t, "Timeout: nothing was added to pullRequestQueue") } diff --git a/services/pull/commit_status.go b/services/pull/commit_status.go index aa1ad7cd66..7952ca6fe3 100644 --- a/services/pull/commit_status.go +++ b/services/pull/commit_status.go @@ -10,94 +10,59 @@ import ( "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/commitstatus" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/structs" "github.com/gobwas/glob" "github.com/pkg/errors" ) // MergeRequiredContextsCommitStatus returns a commit status state for given required contexts -func MergeRequiredContextsCommitStatus(commitStatuses []*git_model.CommitStatus, requiredContexts []string) structs.CommitStatusState { - // matchedCount is the number of `CommitStatus.Context` that match any context of `requiredContexts` - matchedCount := 0 - returnedStatus := structs.CommitStatusSuccess - - if len(requiredContexts) > 0 { - requiredContextsGlob := make(map[string]glob.Glob, len(requiredContexts)) - for _, ctx := range requiredContexts { - if gp, err := glob.Compile(ctx); err != nil { - log.Error("glob.Compile %s failed. Error: %v", ctx, err) - } else { - requiredContextsGlob[ctx] = gp - } - } - - for _, gp := range requiredContextsGlob { - var targetStatus structs.CommitStatusState - for _, commitStatus := range commitStatuses { - if gp.Match(commitStatus.Context) { - targetStatus = commitStatus.State - matchedCount++ - break - } - } - - // If required rule not match any action, then it is pending - if targetStatus == "" { - if structs.CommitStatusPending.NoBetterThan(returnedStatus) { - returnedStatus = structs.CommitStatusPending - } - break - } - - if targetStatus.NoBetterThan(returnedStatus) { - returnedStatus = targetStatus - } - } +func MergeRequiredContextsCommitStatus(commitStatuses []*git_model.CommitStatus, requiredContexts []string) commitstatus.CommitStatusState { + if len(commitStatuses) == 0 { + return commitstatus.CommitStatusPending } - if matchedCount == 0 && returnedStatus == structs.CommitStatusSuccess { - status := git_model.CalcCommitStatus(commitStatuses) - if status != nil { - return status.State - } - return structs.CommitStatusSuccess + if len(requiredContexts) == 0 { + return git_model.CalcCommitStatus(commitStatuses).State } - return returnedStatus -} - -// IsCommitStatusContextSuccess returns true if all required status check contexts succeed. -func IsCommitStatusContextSuccess(commitStatuses []*git_model.CommitStatus, requiredContexts []string) bool { - // If no specific context is required, require that last commit status is a success - if len(requiredContexts) == 0 { - status := git_model.CalcCommitStatus(commitStatuses) - if status == nil || status.State != structs.CommitStatusSuccess { - return false + requiredContextsGlob := make(map[string]glob.Glob, len(requiredContexts)) + for _, ctx := range requiredContexts { + if gp, err := glob.Compile(ctx); err != nil { + log.Error("glob.Compile %s failed. Error: %v", ctx, err) + } else { + requiredContextsGlob[ctx] = gp } - return true } - for _, ctx := range requiredContexts { - var found bool + requiredCommitStatuses := make([]*git_model.CommitStatus, 0, len(commitStatuses)) + allRequiredContextsMatched := true + for _, gp := range requiredContextsGlob { + requiredContextMatched := false for _, commitStatus := range commitStatuses { - if commitStatus.Context == ctx { - if commitStatus.State != structs.CommitStatusSuccess { - return false - } - - found = true - break + if gp.Match(commitStatus.Context) { + requiredCommitStatuses = append(requiredCommitStatuses, commitStatus) + requiredContextMatched = true } } - if !found { - return false - } + allRequiredContextsMatched = allRequiredContextsMatched && requiredContextMatched + } + if len(requiredCommitStatuses) == 0 { + return commitstatus.CommitStatusPending + } + + returnedStatus := git_model.CalcCommitStatus(requiredCommitStatuses).State + if allRequiredContextsMatched { + return returnedStatus + } + + if returnedStatus == commitstatus.CommitStatusFailure { + return commitstatus.CommitStatusFailure } - return true + // even if part of success, return pending + return commitstatus.CommitStatusPending } // IsPullCommitStatusPass returns if all required status checks PASS @@ -118,7 +83,7 @@ func IsPullCommitStatusPass(ctx context.Context, pr *issues_model.PullRequest) ( } // GetPullRequestCommitStatusState returns pull request merged commit status state -func GetPullRequestCommitStatusState(ctx context.Context, pr *issues_model.PullRequest) (structs.CommitStatusState, error) { +func GetPullRequestCommitStatusState(ctx context.Context, pr *issues_model.PullRequest) (commitstatus.CommitStatusState, error) { // Ensure HeadRepo is loaded if err := pr.LoadHeadRepo(ctx); err != nil { return "", errors.Wrap(err, "LoadHeadRepo") @@ -131,10 +96,10 @@ func GetPullRequestCommitStatusState(ctx context.Context, pr *issues_model.PullR } defer closer.Close() - if pr.Flow == issues_model.PullRequestFlowGithub && !headGitRepo.IsBranchExist(pr.HeadBranch) { + if pr.Flow == issues_model.PullRequestFlowGithub && !gitrepo.IsBranchExist(ctx, pr.HeadRepo, pr.HeadBranch) { return "", errors.New("Head branch does not exist, can not merge") } - if pr.Flow == issues_model.PullRequestFlowAGit && !git.IsReferenceExist(ctx, headGitRepo.Path, pr.GetGitRefName()) { + if pr.Flow == issues_model.PullRequestFlowAGit && !gitrepo.IsReferenceExist(ctx, pr.HeadRepo, pr.GetGitRefName()) { return "", errors.New("Head branch does not exist, can not merge") } @@ -152,7 +117,7 @@ func GetPullRequestCommitStatusState(ctx context.Context, pr *issues_model.PullR return "", errors.Wrap(err, "LoadBaseRepo") } - commitStatuses, _, err := git_model.GetLatestCommitStatus(ctx, pr.BaseRepo.ID, sha, db.ListOptionsAll) + commitStatuses, err := git_model.GetLatestCommitStatus(ctx, pr.BaseRepo.ID, sha, db.ListOptionsAll) if err != nil { return "", errors.Wrap(err, "GetLatestCommitStatus") } diff --git a/services/pull/commit_status_test.go b/services/pull/commit_status_test.go index 592acdd55c..a58e788c04 100644 --- a/services/pull/commit_status_test.go +++ b/services/pull/commit_status_test.go @@ -8,58 +8,85 @@ import ( "testing" git_model "code.gitea.io/gitea/models/git" - "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/commitstatus" "github.com/stretchr/testify/assert" ) func TestMergeRequiredContextsCommitStatus(t *testing.T) { - testCases := [][]*git_model.CommitStatus{ + cases := []struct { + commitStatuses []*git_model.CommitStatus + requiredContexts []string + expected commitstatus.CommitStatusState + }{ { - {Context: "Build 1", State: structs.CommitStatusSuccess}, - {Context: "Build 2", State: structs.CommitStatusSuccess}, - {Context: "Build 3", State: structs.CommitStatusSuccess}, + commitStatuses: []*git_model.CommitStatus{}, + requiredContexts: []string{}, + expected: commitstatus.CommitStatusPending, }, { - {Context: "Build 1", State: structs.CommitStatusSuccess}, - {Context: "Build 2", State: structs.CommitStatusSuccess}, - {Context: "Build 2t", State: structs.CommitStatusPending}, + commitStatuses: []*git_model.CommitStatus{ + {Context: "Build xxx", State: commitstatus.CommitStatusSkipped}, + }, + requiredContexts: []string{"Build*"}, + expected: commitstatus.CommitStatusSuccess, }, { - {Context: "Build 1", State: structs.CommitStatusSuccess}, - {Context: "Build 2", State: structs.CommitStatusSuccess}, - {Context: "Build 2t", State: structs.CommitStatusFailure}, + commitStatuses: []*git_model.CommitStatus{ + {Context: "Build 1", State: commitstatus.CommitStatusSkipped}, + {Context: "Build 2", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 3", State: commitstatus.CommitStatusSuccess}, + }, + requiredContexts: []string{"Build*"}, + expected: commitstatus.CommitStatusSuccess, }, { - {Context: "Build 1", State: structs.CommitStatusSuccess}, - {Context: "Build 2", State: structs.CommitStatusSuccess}, - {Context: "Build 2t", State: structs.CommitStatusSuccess}, + commitStatuses: []*git_model.CommitStatus{ + {Context: "Build 1", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2t", State: commitstatus.CommitStatusPending}, + }, + requiredContexts: []string{"Build*", "Build 2t*"}, + expected: commitstatus.CommitStatusPending, }, { - {Context: "Build 1", State: structs.CommitStatusSuccess}, - {Context: "Build 2", State: structs.CommitStatusSuccess}, - {Context: "Build 2t", State: structs.CommitStatusSuccess}, + commitStatuses: []*git_model.CommitStatus{ + {Context: "Build 1", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2t", State: commitstatus.CommitStatusFailure}, + }, + requiredContexts: []string{"Build*", "Build 2t*"}, + expected: commitstatus.CommitStatusFailure, + }, + { + commitStatuses: []*git_model.CommitStatus{ + {Context: "Build 1", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2t", State: commitstatus.CommitStatusFailure}, + }, + requiredContexts: []string{"Build*"}, + expected: commitstatus.CommitStatusFailure, + }, + { + commitStatuses: []*git_model.CommitStatus{ + {Context: "Build 1", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2t", State: commitstatus.CommitStatusSuccess}, + }, + requiredContexts: []string{"Build*", "Build 2t*", "Build 3*"}, + expected: commitstatus.CommitStatusPending, + }, + { + commitStatuses: []*git_model.CommitStatus{ + {Context: "Build 1", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2", State: commitstatus.CommitStatusSuccess}, + {Context: "Build 2t", State: commitstatus.CommitStatusSuccess}, + }, + requiredContexts: []string{"Build*", "Build *", "Build 2t*", "Build 1*"}, + expected: commitstatus.CommitStatusSuccess, }, } - testCasesRequiredContexts := [][]string{ - {"Build*"}, - {"Build*", "Build 2t*"}, - {"Build*", "Build 2t*"}, - {"Build*", "Build 2t*", "Build 3*"}, - {"Build*", "Build *", "Build 2t*", "Build 1*"}, - } - - testCasesExpected := []structs.CommitStatusState{ - structs.CommitStatusSuccess, - structs.CommitStatusPending, - structs.CommitStatusFailure, - structs.CommitStatusPending, - structs.CommitStatusSuccess, - } - - for i, commitStatuses := range testCases { - if MergeRequiredContextsCommitStatus(commitStatuses, testCasesRequiredContexts[i]) != testCasesExpected[i] { - assert.Fail(t, "Test case failed", "Test case %d failed", i+1) - } + for i, c := range cases { + assert.Equal(t, c.expected, MergeRequiredContextsCommitStatus(c.commitStatuses, c.requiredContexts), "case %d", i) } } diff --git a/services/pull/merge.go b/services/pull/merge.go index 9c909ef795..cd9aeb2ad1 100644 --- a/services/pull/merge.go +++ b/services/pull/merge.go @@ -6,12 +6,15 @@ package pull import ( "context" + "errors" "fmt" + "maps" "os" "path/filepath" "regexp" "strconv" "strings" + "unicode" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" @@ -59,7 +62,7 @@ func getMergeMessage(ctx context.Context, baseGitRepo *git.Repository, pr *issue issueReference = "!" } - reviewedOn := fmt.Sprintf("Reviewed-on: %s", httplib.MakeAbsoluteURL(ctx, pr.Issue.Link())) + reviewedOn := "Reviewed-on: " + httplib.MakeAbsoluteURL(ctx, pr.Issue.Link()) reviewedBy := pr.GetApprovers(ctx) if mergeStyle != "" { @@ -93,9 +96,7 @@ func getMergeMessage(ctx context.Context, baseGitRepo *git.Repository, pr *issue vars["HeadRepoOwnerName"] = pr.HeadRepo.OwnerName vars["HeadRepoName"] = pr.HeadRepo.Name } - for extraKey, extraValue := range extraVars { - vars[extraKey] = extraValue - } + maps.Copy(vars, extraVars) refs, err := pr.ResolveCrossReferences(ctx) if err == nil { closeIssueIndexes := make([]string, 0, len(refs)) @@ -160,6 +161,41 @@ func GetDefaultMergeMessage(ctx context.Context, baseGitRepo *git.Repository, pr return getMergeMessage(ctx, baseGitRepo, pr, mergeStyle, nil) } +func AddCommitMessageTailer(message, tailerKey, tailerValue string) string { + tailerLine := tailerKey + ": " + tailerValue + message = strings.ReplaceAll(message, "\r\n", "\n") + message = strings.ReplaceAll(message, "\r", "\n") + if strings.Contains(message, "\n"+tailerLine+"\n") || strings.HasSuffix(message, "\n"+tailerLine) { + return message + } + + if !strings.HasSuffix(message, "\n") { + message += "\n" + } + pos1 := strings.LastIndexByte(message[:len(message)-1], '\n') + pos2 := -1 + if pos1 != -1 { + pos2 = strings.IndexByte(message[pos1:], ':') + if pos2 != -1 { + pos2 += pos1 + } + } + var lastLineKey string + if pos1 != -1 && pos2 != -1 { + lastLineKey = message[pos1+1 : pos2] + } + + isLikelyTailerLine := lastLineKey != "" && unicode.IsUpper(rune(lastLineKey[0])) && strings.Contains(message, "-") + for i := 0; isLikelyTailerLine && i < len(lastLineKey); i++ { + r := rune(lastLineKey[i]) + isLikelyTailerLine = unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' + } + if !strings.HasSuffix(message, "\n\n") && !isLikelyTailerLine { + message += "\n" + } + return message + tailerLine +} + // ErrInvalidMergeStyle represents an error if merging with disabled merge strategy type ErrInvalidMergeStyle struct { ID int64 @@ -211,7 +247,15 @@ func Merge(ctx context.Context, pr *issues_model.PullRequest, doer *user_model.U } defer releaser() defer func() { - go AddTestPullRequestTask(doer, pr.BaseRepo.ID, pr.BaseBranch, false, "", "") + go AddTestPullRequestTask(TestPullRequestOptions{ + RepoID: pr.BaseRepo.ID, + Doer: doer, + Branch: pr.BaseBranch, + IsSync: false, + IsForcePush: false, + OldCommitID: "", + NewCommitID: "", + }) }() _, err = doMergeAndPush(ctx, pr, doer, mergeStyle, expectedHeadCommitID, message, repo_module.PushTriggerPRMergeToBase) @@ -281,7 +325,7 @@ func handleCloseCrossReferences(ctx context.Context, pr *issues_model.PullReques } // doMergeAndPush performs the merge operation without changing any pull information in database and pushes it up to the base repository -func doMergeAndPush(ctx context.Context, pr *issues_model.PullRequest, doer *user_model.User, mergeStyle repo_model.MergeStyle, expectedHeadCommitID, message string, pushTrigger repo_module.PushTrigger) (string, error) { //nolint:unparam +func doMergeAndPush(ctx context.Context, pr *issues_model.PullRequest, doer *user_model.User, mergeStyle repo_model.MergeStyle, expectedHeadCommitID, message string, pushTrigger repo_module.PushTrigger) (string, error) { //nolint:unparam // non-error result is never used // Clone base repo. mergeCtx, cancel, err := createTemporaryRepoForMerge(ctx, pr, doer, expectedHeadCommitID) if err != nil { @@ -356,12 +400,12 @@ func doMergeAndPush(ctx context.Context, pr *issues_model.PullRequest, doer *use ) mergeCtx.env = append(mergeCtx.env, repo_module.EnvPushTrigger+"="+string(pushTrigger)) - pushCmd := git.NewCommand(ctx, "push", "origin").AddDynamicArguments(baseBranch + ":" + git.BranchPrefix + pr.BaseBranch) + pushCmd := git.NewCommand("push", "origin").AddDynamicArguments(baseBranch + ":" + git.BranchPrefix + pr.BaseBranch) // Push back to upstream. // This cause an api call to "/api/internal/hook/post-receive/...", // If it's merge, all db transaction and operations should be there but not here to prevent deadlock. - if err := pushCmd.Run(mergeCtx.RunOpts()); err != nil { + if err := pushCmd.Run(ctx, mergeCtx.RunOpts()); err != nil { if strings.Contains(mergeCtx.errbuf.String(), "non-fast-forward") { return "", &git.ErrPushOutOfDate{ StdOut: mergeCtx.outbuf.String(), @@ -386,13 +430,16 @@ func doMergeAndPush(ctx context.Context, pr *issues_model.PullRequest, doer *use } func commitAndSignNoAuthor(ctx *mergeContext, message string) error { - cmdCommit := git.NewCommand(ctx, "commit").AddOptionFormat("--message=%s", message) - if ctx.signKeyID == "" { + cmdCommit := git.NewCommand("commit").AddOptionFormat("--message=%s", message) + if ctx.signKey == nil { cmdCommit.AddArguments("--no-gpg-sign") } else { - cmdCommit.AddOptionFormat("-S%s", ctx.signKeyID) + if ctx.signKey.Format != "" { + cmdCommit.AddConfig("gpg.format", ctx.signKey.Format) + } + cmdCommit.AddOptionFormat("-S%s", ctx.signKey.KeyID) } - if err := cmdCommit.Run(ctx.RunOpts()); err != nil { + if err := cmdCommit.Run(ctx, ctx.RunOpts()); err != nil { log.Error("git commit %-v: %v\n%s\n%s", ctx.pr, err, ctx.outbuf.String(), ctx.errbuf.String()) return fmt.Errorf("git commit %v: %w\n%s\n%s", ctx.pr, err, ctx.outbuf.String(), ctx.errbuf.String()) } @@ -453,7 +500,7 @@ func (err ErrMergeDivergingFastForwardOnly) Error() string { } func runMergeCommand(ctx *mergeContext, mergeStyle repo_model.MergeStyle, cmd *git.Command) error { - if err := cmd.Run(ctx.RunOpts()); err != nil { + if err := cmd.Run(ctx, ctx.RunOpts()); err != nil { // Merge will leave a MERGE_HEAD file in the .git folder if there is a conflict if _, statErr := os.Stat(filepath.Join(ctx.tmpBasePath, ".git", "MERGE_HEAD")); statErr == nil { // We have a merge conflict error @@ -509,25 +556,6 @@ func IsUserAllowedToMerge(ctx context.Context, pr *issues_model.PullRequest, p a return false, nil } -// ErrDisallowedToMerge represents an error that a branch is protected and the current user is not allowed to modify it. -type ErrDisallowedToMerge struct { - Reason string -} - -// IsErrDisallowedToMerge checks if an error is an ErrDisallowedToMerge. -func IsErrDisallowedToMerge(err error) bool { - _, ok := err.(ErrDisallowedToMerge) - return ok -} - -func (err ErrDisallowedToMerge) Error() string { - return fmt.Sprintf("not allowed to merge [reason: %s]", err.Reason) -} - -func (err ErrDisallowedToMerge) Unwrap() error { - return util.ErrPermissionDenied -} - // CheckPullBranchProtections checks whether the PR is ready to be merged (reviews and status checks) func CheckPullBranchProtections(ctx context.Context, pr *issues_model.PullRequest, skipProtectedFilesCheck bool) (err error) { if err = pr.LoadBaseRepo(ctx); err != nil { @@ -547,31 +575,21 @@ func CheckPullBranchProtections(ctx context.Context, pr *issues_model.PullReques return err } if !isPass { - return ErrDisallowedToMerge{ - Reason: "Not all required status checks successful", - } + return util.ErrorWrap(ErrNotReadyToMerge, "Not all required status checks successful") } if !issues_model.HasEnoughApprovals(ctx, pb, pr) { - return ErrDisallowedToMerge{ - Reason: "Does not have enough approvals", - } + return util.ErrorWrap(ErrNotReadyToMerge, "Does not have enough approvals") } if issues_model.MergeBlockedByRejectedReview(ctx, pb, pr) { - return ErrDisallowedToMerge{ - Reason: "There are requested changes", - } + return util.ErrorWrap(ErrNotReadyToMerge, "There are requested changes") } if issues_model.MergeBlockedByOfficialReviewRequests(ctx, pb, pr) { - return ErrDisallowedToMerge{ - Reason: "There are official review requests", - } + return util.ErrorWrap(ErrNotReadyToMerge, "There are official review requests") } if issues_model.MergeBlockedByOutdatedBranch(pb, pr) { - return ErrDisallowedToMerge{ - Reason: "The head branch is behind the base branch", - } + return util.ErrorWrap(ErrNotReadyToMerge, "The head branch is behind the base branch") } if skipProtectedFilesCheck { @@ -579,9 +597,7 @@ func CheckPullBranchProtections(ctx context.Context, pr *issues_model.PullReques } if pb.MergeBlockedByProtectedFiles(pr.ChangedProtectedFiles) { - return ErrDisallowedToMerge{ - Reason: "Changed protected files", - } + return util.ErrorWrap(ErrNotReadyToMerge, "Changed protected files") } return nil @@ -613,13 +629,13 @@ func MergedManually(ctx context.Context, pr *issues_model.PullRequest, doer *use objectFormat := git.ObjectFormatFromName(pr.BaseRepo.ObjectFormatName) if len(commitID) != objectFormat.FullLength() { - return fmt.Errorf("Wrong commit ID") + return errors.New("Wrong commit ID") } commit, err := baseGitRepo.GetCommit(commitID) if err != nil { if git.IsErrNotExist(err) { - return fmt.Errorf("Wrong commit ID") + return errors.New("Wrong commit ID") } return err } @@ -630,14 +646,14 @@ func MergedManually(ctx context.Context, pr *issues_model.PullRequest, doer *use return err } if !ok { - return fmt.Errorf("Wrong commit ID") + return errors.New("Wrong commit ID") } var merged bool if merged, err = SetMerged(ctx, pr, commitID, timeutil.TimeStamp(commit.Author.When.Unix()), doer, issues_model.PullRequestStatusManuallyMerged); err != nil { return err } else if !merged { - return fmt.Errorf("SetMerged failed") + return errors.New("SetMerged failed") } return nil }) @@ -700,7 +716,7 @@ func SetMerged(ctx context.Context, pr *issues_model.PullRequest, mergedCommitID return false, fmt.Errorf("ChangeIssueStatus: %w", err) } - // We need to save all of the data used to compute this merge as it may have already been changed by TestPatch. FIXME: need to set some state to prevent TestPatch from running whilst we are merging. + // We need to save all of the data used to compute this merge as it may have already been changed by testPullRequestBranchMergeable. FIXME: need to set some state to prevent testPullRequestBranchMergeable from running whilst we are merging. if cnt, err := db.GetEngine(ctx).Where("id = ?", pr.ID). And("has_merged = ?", false). Cols("has_merged, status, merge_base, merged_commit_id, merger_id, merged_unix, conflicted_files"). diff --git a/services/pull/merge_ff_only.go b/services/pull/merge_ff_only.go index f57c732104..6c3a68b95b 100644 --- a/services/pull/merge_ff_only.go +++ b/services/pull/merge_ff_only.go @@ -11,7 +11,7 @@ import ( // doMergeStyleFastForwardOnly merges the tracking into the current HEAD - which is assumed to be staging branch (equal to the pr.BaseBranch) func doMergeStyleFastForwardOnly(ctx *mergeContext) error { - cmd := git.NewCommand(ctx, "merge", "--ff-only").AddDynamicArguments(trackingBranch) + cmd := git.NewCommand("merge", "--ff-only").AddDynamicArguments(trackingBranch) if err := runMergeCommand(ctx, repo_model.MergeStyleFastForwardOnly, cmd); err != nil { log.Error("%-v Unable to merge tracking into base: %v", ctx.pr, err) return err diff --git a/services/pull/merge_merge.go b/services/pull/merge_merge.go index bf56c071db..118d21c7cd 100644 --- a/services/pull/merge_merge.go +++ b/services/pull/merge_merge.go @@ -11,7 +11,7 @@ import ( // doMergeStyleMerge merges the tracking branch into the current HEAD - which is assumed to be the staging branch (equal to the pr.BaseBranch) func doMergeStyleMerge(ctx *mergeContext, message string) error { - cmd := git.NewCommand(ctx, "merge", "--no-ff", "--no-commit").AddDynamicArguments(trackingBranch) + cmd := git.NewCommand("merge", "--no-ff", "--no-commit").AddDynamicArguments(trackingBranch) if err := runMergeCommand(ctx, repo_model.MergeStyleMerge, cmd); err != nil { log.Error("%-v Unable to merge tracking into base: %v", ctx.pr, err) return err diff --git a/services/pull/merge_prepare.go b/services/pull/merge_prepare.go index 2e1cc8cf85..31a1e13734 100644 --- a/services/pull/merge_prepare.go +++ b/services/pull/merge_prepare.go @@ -23,11 +23,11 @@ import ( ) type mergeContext struct { - *prContext + *prTmpRepoContext doer *user_model.User sig *git.Signature committer *git.Signature - signKeyID string // empty for no-sign, non-empty to sign + signKey *git.SigningKey env []string } @@ -68,12 +68,12 @@ func createTemporaryRepoForMerge(ctx context.Context, pr *issues_model.PullReque } mergeCtx = &mergeContext{ - prContext: prCtx, - doer: doer, + prTmpRepoContext: prCtx, + doer: doer, } if expectedHeadCommitID != "" { - trackingCommitID, _, err := git.NewCommand(ctx, "show-ref", "--hash").AddDynamicArguments(git.BranchPrefix + trackingBranch).RunStdString(&git.RunOpts{Dir: mergeCtx.tmpBasePath}) + trackingCommitID, _, err := git.NewCommand("show-ref", "--hash").AddDynamicArguments(git.BranchPrefix+trackingBranch).RunStdString(ctx, &git.RunOpts{Dir: mergeCtx.tmpBasePath}) if err != nil { defer cancel() log.Error("failed to get sha of head branch in %-v: show-ref[%s] --hash refs/heads/tracking: %v", mergeCtx.pr, mergeCtx.tmpBasePath, err) @@ -99,9 +99,9 @@ func createTemporaryRepoForMerge(ctx context.Context, pr *issues_model.PullReque mergeCtx.committer = mergeCtx.sig // Determine if we should sign - sign, keyID, signer, _ := asymkey_service.SignMerge(ctx, mergeCtx.pr, mergeCtx.doer, mergeCtx.tmpBasePath, "HEAD", trackingBranch) + sign, key, signer, _ := asymkey_service.SignMerge(ctx, mergeCtx.pr, mergeCtx.doer, mergeCtx.tmpBasePath, "HEAD", trackingBranch) if sign { - mergeCtx.signKeyID = keyID + mergeCtx.signKey = key if pr.BaseRepo.GetTrustModel() == repo_model.CommitterTrustModel || pr.BaseRepo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { mergeCtx.committer = signer } @@ -151,8 +151,8 @@ func prepareTemporaryRepoForMerge(ctx *mergeContext) error { } setConfig := func(key, value string) error { - if err := git.NewCommand(ctx, "config", "--local").AddDynamicArguments(key, value). - Run(ctx.RunOpts()); err != nil { + if err := git.NewCommand("config", "--local").AddDynamicArguments(key, value). + Run(ctx, ctx.RunOpts()); err != nil { log.Error("git config [%s -> %q]: %v\n%s\n%s", key, value, err, ctx.outbuf.String(), ctx.errbuf.String()) return fmt.Errorf("git config [%s -> %q]: %w\n%s\n%s", key, value, err, ctx.outbuf.String(), ctx.errbuf.String()) } @@ -184,8 +184,8 @@ func prepareTemporaryRepoForMerge(ctx *mergeContext) error { } // Read base branch index - if err := git.NewCommand(ctx, "read-tree", "HEAD"). - Run(ctx.RunOpts()); err != nil { + if err := git.NewCommand("read-tree", "HEAD"). + Run(ctx, ctx.RunOpts()); err != nil { log.Error("git read-tree HEAD: %v\n%s\n%s", err, ctx.outbuf.String(), ctx.errbuf.String()) return fmt.Errorf("Unable to read base branch in to the index: %w\n%s\n%s", err, ctx.outbuf.String(), ctx.errbuf.String()) } @@ -221,8 +221,8 @@ func getDiffTree(ctx context.Context, repoPath, baseBranch, headBranch string, o return 0, nil, nil } - err = git.NewCommand(ctx, "diff-tree", "--no-commit-id", "--name-only", "-r", "-r", "-z", "--root").AddDynamicArguments(baseBranch, headBranch). - Run(&git.RunOpts{ + err = git.NewCommand("diff-tree", "--no-commit-id", "--name-only", "-r", "-r", "-z", "--root").AddDynamicArguments(baseBranch, headBranch). + Run(ctx, &git.RunOpts{ Dir: repoPath, Stdout: diffOutWriter, PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { @@ -272,16 +272,16 @@ func (err ErrRebaseConflicts) Error() string { // if there is a conflict it will return an ErrRebaseConflicts func rebaseTrackingOnToBase(ctx *mergeContext, mergeStyle repo_model.MergeStyle) error { // Checkout head branch - if err := git.NewCommand(ctx, "checkout", "-b").AddDynamicArguments(stagingBranch, trackingBranch). - Run(ctx.RunOpts()); err != nil { + if err := git.NewCommand("checkout", "-b").AddDynamicArguments(stagingBranch, trackingBranch). + Run(ctx, ctx.RunOpts()); err != nil { return fmt.Errorf("unable to git checkout tracking as staging in temp repo for %v: %w\n%s\n%s", ctx.pr, err, ctx.outbuf.String(), ctx.errbuf.String()) } ctx.outbuf.Reset() ctx.errbuf.Reset() // Rebase before merging - if err := git.NewCommand(ctx, "rebase").AddDynamicArguments(baseBranch). - Run(ctx.RunOpts()); err != nil { + if err := git.NewCommand("rebase").AddDynamicArguments(baseBranch). + Run(ctx, ctx.RunOpts()); err != nil { // Rebase will leave a REBASE_HEAD file in .git if there is a conflict if _, statErr := os.Stat(filepath.Join(ctx.tmpBasePath, ".git", "REBASE_HEAD")); statErr == nil { var commitSha string diff --git a/services/pull/merge_rebase.go b/services/pull/merge_rebase.go index ecf376220e..dd7c8761f0 100644 --- a/services/pull/merge_rebase.go +++ b/services/pull/merge_rebase.go @@ -16,7 +16,7 @@ import ( // getRebaseAmendMessage composes the message to amend commits in rebase merge of a pull request. func getRebaseAmendMessage(ctx *mergeContext, baseGitRepo *git.Repository) (message string, err error) { // Get existing commit message. - commitMessage, _, err := git.NewCommand(ctx, "show", "--format=%B", "-s").RunStdString(&git.RunOpts{Dir: ctx.tmpBasePath}) + commitMessage, _, err := git.NewCommand("show", "--format=%B", "-s").RunStdString(ctx, &git.RunOpts{Dir: ctx.tmpBasePath}) if err != nil { return "", err } @@ -42,7 +42,7 @@ func doMergeRebaseFastForward(ctx *mergeContext) error { return fmt.Errorf("Failed to get full commit id for HEAD: %w", err) } - cmd := git.NewCommand(ctx, "merge", "--ff-only").AddDynamicArguments(stagingBranch) + cmd := git.NewCommand("merge", "--ff-only").AddDynamicArguments(stagingBranch) if err := runMergeCommand(ctx, repo_model.MergeStyleRebase, cmd); err != nil { log.Error("Unable to merge staging into base: %v", err) return err @@ -73,7 +73,7 @@ func doMergeRebaseFastForward(ctx *mergeContext) error { } if newMessage != "" { - if err := git.NewCommand(ctx, "commit", "--amend").AddOptionFormat("--message=%s", newMessage).Run(&git.RunOpts{Dir: ctx.tmpBasePath}); err != nil { + if err := git.NewCommand("commit", "--amend").AddOptionFormat("--message=%s", newMessage).Run(ctx, &git.RunOpts{Dir: ctx.tmpBasePath}); err != nil { log.Error("Unable to amend commit message: %v", err) return err } @@ -84,7 +84,7 @@ func doMergeRebaseFastForward(ctx *mergeContext) error { // Perform rebase merge with merge commit. func doMergeRebaseMergeCommit(ctx *mergeContext, message string) error { - cmd := git.NewCommand(ctx, "merge").AddArguments("--no-ff", "--no-commit").AddDynamicArguments(stagingBranch) + cmd := git.NewCommand("merge").AddArguments("--no-ff", "--no-commit").AddDynamicArguments(stagingBranch) if err := runMergeCommand(ctx, repo_model.MergeStyleRebaseMerge, cmd); err != nil { log.Error("Unable to merge staging into base: %v", err) @@ -105,8 +105,8 @@ func doMergeStyleRebase(ctx *mergeContext, mergeStyle repo_model.MergeStyle, mes } // Checkout base branch again - if err := git.NewCommand(ctx, "checkout").AddDynamicArguments(baseBranch). - Run(ctx.RunOpts()); err != nil { + if err := git.NewCommand("checkout").AddDynamicArguments(baseBranch). + Run(ctx, ctx.RunOpts()); err != nil { log.Error("git checkout base prior to merge post staging rebase %-v: %v\n%s\n%s", ctx.pr, err, ctx.outbuf.String(), ctx.errbuf.String()) return fmt.Errorf("git checkout base prior to merge post staging rebase %v: %w\n%s\n%s", ctx.pr, err, ctx.outbuf.String(), ctx.errbuf.String()) } diff --git a/services/pull/merge_squash.go b/services/pull/merge_squash.go index 7258671888..0049c0b117 100644 --- a/services/pull/merge_squash.go +++ b/services/pull/merge_squash.go @@ -5,7 +5,6 @@ package pull import ( "fmt" - "strings" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" @@ -58,7 +57,7 @@ func doMergeStyleSquash(ctx *mergeContext, message string) error { return fmt.Errorf("getAuthorSignatureSquash: %w", err) } - cmdMerge := git.NewCommand(ctx, "merge", "--squash").AddDynamicArguments(trackingBranch) + cmdMerge := git.NewCommand("merge", "--squash").AddDynamicArguments(trackingBranch) if err := runMergeCommand(ctx, repo_model.MergeStyleSquash, cmdMerge); err != nil { log.Error("%-v Unable to merge --squash tracking into base: %v", ctx.pr, err) return err @@ -66,20 +65,21 @@ func doMergeStyleSquash(ctx *mergeContext, message string) error { if setting.Repository.PullRequest.AddCoCommitterTrailers && ctx.committer.String() != sig.String() { // add trailer - if !strings.Contains(message, fmt.Sprintf("Co-authored-by: %s", sig.String())) { - message += fmt.Sprintf("\nCo-authored-by: %s", sig.String()) - } - message += fmt.Sprintf("\nCo-committed-by: %s\n", sig.String()) + message = AddCommitMessageTailer(message, "Co-authored-by", sig.String()) + message = AddCommitMessageTailer(message, "Co-committed-by", sig.String()) // FIXME: this one should be removed, it is not really used or widely used } - cmdCommit := git.NewCommand(ctx, "commit"). + cmdCommit := git.NewCommand("commit"). AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email). AddOptionFormat("--message=%s", message) - if ctx.signKeyID == "" { + if ctx.signKey == nil { cmdCommit.AddArguments("--no-gpg-sign") } else { - cmdCommit.AddOptionFormat("-S%s", ctx.signKeyID) + if ctx.signKey.Format != "" { + cmdCommit.AddConfig("gpg.format", ctx.signKey.Format) + } + cmdCommit.AddOptionFormat("-S%s", ctx.signKey.KeyID) } - if err := cmdCommit.Run(ctx.RunOpts()); err != nil { + if err := cmdCommit.Run(ctx, ctx.RunOpts()); err != nil { log.Error("git commit %-v: %v\n%s\n%s", ctx.pr, err, ctx.outbuf.String(), ctx.errbuf.String()) return fmt.Errorf("git commit [%s:%s -> %s:%s]: %w\n%s\n%s", ctx.pr.HeadRepo.FullName(), ctx.pr.HeadBranch, ctx.pr.BaseRepo.FullName(), ctx.pr.BaseBranch, err, ctx.outbuf.String(), ctx.errbuf.String()) } diff --git a/services/pull/merge_test.go b/services/pull/merge_test.go index 6df6f55d46..91abeb9d9c 100644 --- a/services/pull/merge_test.go +++ b/services/pull/merge_test.go @@ -65,3 +65,28 @@ func Test_expandDefaultMergeMessage(t *testing.T) { }) } } + +func TestAddCommitMessageTailer(t *testing.T) { + // add tailer for empty message + assert.Equal(t, "\n\nTest-tailer: TestValue", AddCommitMessageTailer("", "Test-tailer", "TestValue")) + + // add tailer for message without newlines + assert.Equal(t, "title\n\nTest-tailer: TestValue", AddCommitMessageTailer("title", "Test-tailer", "TestValue")) + assert.Equal(t, "title\n\nNot tailer: xxx\n\nTest-tailer: TestValue", AddCommitMessageTailer("title\n\nNot tailer: xxx", "Test-tailer", "TestValue")) + assert.Equal(t, "title\n\nNotTailer: xxx\n\nTest-tailer: TestValue", AddCommitMessageTailer("title\n\nNotTailer: xxx", "Test-tailer", "TestValue")) + assert.Equal(t, "title\n\nnot-tailer: xxx\n\nTest-tailer: TestValue", AddCommitMessageTailer("title\n\nnot-tailer: xxx", "Test-tailer", "TestValue")) + + // add tailer for message with one EOL + assert.Equal(t, "title\n\nTest-tailer: TestValue", AddCommitMessageTailer("title\n", "Test-tailer", "TestValue")) + + // add tailer for message with two EOLs + assert.Equal(t, "title\n\nTest-tailer: TestValue", AddCommitMessageTailer("title\n\n", "Test-tailer", "TestValue")) + + // add tailer for message with existing tailer (won't duplicate) + assert.Equal(t, "title\n\nTest-tailer: TestValue", AddCommitMessageTailer("title\n\nTest-tailer: TestValue", "Test-tailer", "TestValue")) + assert.Equal(t, "title\n\nTest-tailer: TestValue\n", AddCommitMessageTailer("title\n\nTest-tailer: TestValue\n", "Test-tailer", "TestValue")) + + // add tailer for message with existing tailer and different value (will append) + assert.Equal(t, "title\n\nTest-tailer: v1\nTest-tailer: v2", AddCommitMessageTailer("title\n\nTest-tailer: v1", "Test-tailer", "v2")) + assert.Equal(t, "title\n\nTest-tailer: v1\nTest-tailer: v2", AddCommitMessageTailer("title\n\nTest-tailer: v1\n", "Test-tailer", "v2")) +} diff --git a/services/pull/patch.go b/services/pull/patch.go index 13623d73c6..153e0baf87 100644 --- a/services/pull/patch.go +++ b/services/pull/patch.go @@ -67,9 +67,8 @@ var patchErrorSuffices = []string{ ": does not exist in index", } -// TestPatch will test whether a simple patch will apply -func TestPatch(pr *issues_model.PullRequest) error { - ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("TestPatch: %s", pr)) +func testPullRequestBranchMergeable(pr *issues_model.PullRequest) error { + ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("testPullRequestBranchMergeable: %s", pr)) defer finished() prCtx, cancel, err := createTemporaryRepoForPR(ctx, pr) @@ -81,10 +80,10 @@ func TestPatch(pr *issues_model.PullRequest) error { } defer cancel() - return testPatch(ctx, prCtx, pr) + return testPullRequestTmpRepoBranchMergeable(ctx, prCtx, pr) } -func testPatch(ctx context.Context, prCtx *prContext, pr *issues_model.PullRequest) error { +func testPullRequestTmpRepoBranchMergeable(ctx context.Context, prCtx *prTmpRepoContext, pr *issues_model.PullRequest) error { gitRepo, err := git.OpenRepository(ctx, prCtx.tmpBasePath) if err != nil { return fmt.Errorf("OpenRepository: %w", err) @@ -92,7 +91,7 @@ func testPatch(ctx context.Context, prCtx *prContext, pr *issues_model.PullReque defer gitRepo.Close() // 1. update merge base - pr.MergeBase, _, err = git.NewCommand(ctx, "merge-base", "--", "base", "tracking").RunStdString(&git.RunOpts{Dir: prCtx.tmpBasePath}) + pr.MergeBase, _, err = git.NewCommand("merge-base", "--", "base", "tracking").RunStdString(ctx, &git.RunOpts{Dir: prCtx.tmpBasePath}) if err != nil { var err2 error pr.MergeBase, err2 = gitRepo.GetRefCommitID(git.BranchPrefix + "base") @@ -134,7 +133,7 @@ type errMergeConflict struct { } func (e *errMergeConflict) Error() string { - return fmt.Sprintf("conflict detected at: %s", e.filename) + return "conflict detected at: " + e.filename } func attemptMerge(ctx context.Context, file *unmergedFile, tmpBasePath string, filesToRemove *[]string, filesToAdd *[]git.IndexObjectInfo) error { @@ -192,7 +191,7 @@ func attemptMerge(ctx context.Context, file *unmergedFile, tmpBasePath string, f } // Need to get the objects from the object db to attempt to merge - root, _, err := git.NewCommand(ctx, "unpack-file").AddDynamicArguments(file.stage1.sha).RunStdString(&git.RunOpts{Dir: tmpBasePath}) + root, _, err := git.NewCommand("unpack-file").AddDynamicArguments(file.stage1.sha).RunStdString(ctx, &git.RunOpts{Dir: tmpBasePath}) if err != nil { return fmt.Errorf("unable to get root object: %s at path: %s for merging. Error: %w", file.stage1.sha, file.stage1.path, err) } @@ -201,7 +200,7 @@ func attemptMerge(ctx context.Context, file *unmergedFile, tmpBasePath string, f _ = util.Remove(filepath.Join(tmpBasePath, root)) }() - base, _, err := git.NewCommand(ctx, "unpack-file").AddDynamicArguments(file.stage2.sha).RunStdString(&git.RunOpts{Dir: tmpBasePath}) + base, _, err := git.NewCommand("unpack-file").AddDynamicArguments(file.stage2.sha).RunStdString(ctx, &git.RunOpts{Dir: tmpBasePath}) if err != nil { return fmt.Errorf("unable to get base object: %s at path: %s for merging. Error: %w", file.stage2.sha, file.stage2.path, err) } @@ -209,7 +208,7 @@ func attemptMerge(ctx context.Context, file *unmergedFile, tmpBasePath string, f defer func() { _ = util.Remove(base) }() - head, _, err := git.NewCommand(ctx, "unpack-file").AddDynamicArguments(file.stage3.sha).RunStdString(&git.RunOpts{Dir: tmpBasePath}) + head, _, err := git.NewCommand("unpack-file").AddDynamicArguments(file.stage3.sha).RunStdString(ctx, &git.RunOpts{Dir: tmpBasePath}) if err != nil { return fmt.Errorf("unable to get head object:%s at path: %s for merging. Error: %w", file.stage3.sha, file.stage3.path, err) } @@ -219,13 +218,13 @@ func attemptMerge(ctx context.Context, file *unmergedFile, tmpBasePath string, f }() // now git merge-file annoyingly takes a different order to the merge-tree ... - _, _, conflictErr := git.NewCommand(ctx, "merge-file").AddDynamicArguments(base, root, head).RunStdString(&git.RunOpts{Dir: tmpBasePath}) + _, _, conflictErr := git.NewCommand("merge-file").AddDynamicArguments(base, root, head).RunStdString(ctx, &git.RunOpts{Dir: tmpBasePath}) if conflictErr != nil { return &errMergeConflict{file.stage2.path} } // base now contains the merged data - hash, _, err := git.NewCommand(ctx, "hash-object", "-w", "--path").AddDynamicArguments(file.stage2.path, base).RunStdString(&git.RunOpts{Dir: tmpBasePath}) + hash, _, err := git.NewCommand("hash-object", "-w", "--path").AddDynamicArguments(file.stage2.path, base).RunStdString(ctx, &git.RunOpts{Dir: tmpBasePath}) if err != nil { return err } @@ -250,7 +249,7 @@ func AttemptThreeWayMerge(ctx context.Context, gitPath string, gitRepo *git.Repo defer cancel() // First we use read-tree to do a simple three-way merge - if _, _, err := git.NewCommand(ctx, "read-tree", "-m").AddDynamicArguments(base, ours, theirs).RunStdString(&git.RunOpts{Dir: gitPath}); err != nil { + if _, _, err := git.NewCommand("read-tree", "-m").AddDynamicArguments(base, ours, theirs).RunStdString(ctx, &git.RunOpts{Dir: gitPath}); err != nil { log.Error("Unable to run read-tree -m! Error: %v", err) return false, nil, fmt.Errorf("unable to run read-tree -m! Error: %w", err) } @@ -324,9 +323,9 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * // No conflicts detected so we need to check if the patch is empty... // a. Write the newly merged tree and check the new tree-hash var treeHash string - treeHash, _, err = git.NewCommand(ctx, "write-tree").RunStdString(&git.RunOpts{Dir: tmpBasePath}) + treeHash, _, err = git.NewCommand("write-tree").RunStdString(ctx, &git.RunOpts{Dir: tmpBasePath}) if err != nil { - lsfiles, _, _ := git.NewCommand(ctx, "ls-files", "-u").RunStdString(&git.RunOpts{Dir: tmpBasePath}) + lsfiles, _, _ := git.NewCommand("ls-files", "-u").RunStdString(ctx, &git.RunOpts{Dir: tmpBasePath}) return false, fmt.Errorf("unable to write unconflicted tree: %w\n`git ls-files -u`:\n%s", err, lsfiles) } treeHash = strings.TrimSpace(treeHash) @@ -355,23 +354,19 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * } // 3b. Create a plain patch from head to base - tmpPatchFile, err := os.CreateTemp("", "patch") + tmpPatchFile, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("patch") if err != nil { log.Error("Unable to create temporary patch file! Error: %v", err) return false, fmt.Errorf("unable to create temporary patch file! Error: %w", err) } - defer func() { - _ = util.Remove(tmpPatchFile.Name()) - }() + defer cleanup() if err := gitRepo.GetDiffBinary(pr.MergeBase+"...tracking", tmpPatchFile); err != nil { - tmpPatchFile.Close() log.Error("Unable to get patch file from %s to %s in %s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.FullName(), err) return false, fmt.Errorf("unable to get patch file from %s to %s in %s Error: %w", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.FullName(), err) } stat, err := tmpPatchFile.Stat() if err != nil { - tmpPatchFile.Close() return false, fmt.Errorf("unable to stat patch file: %w", err) } patchPath := tmpPatchFile.Name() @@ -384,10 +379,10 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * return false, nil } - log.Trace("PullRequest[%d].testPatch (patchPath): %s", pr.ID, patchPath) + log.Trace("PullRequest[%d].testPullRequestTmpRepoBranchMergeable (patchPath): %s", pr.ID, patchPath) // 4. Read the base branch in to the index of the temporary repository - _, _, err = git.NewCommand(gitRepo.Ctx, "read-tree", "base").RunStdString(&git.RunOpts{Dir: tmpBasePath}) + _, _, err = git.NewCommand("read-tree", "base").RunStdString(gitRepo.Ctx, &git.RunOpts{Dir: tmpBasePath}) if err != nil { return false, fmt.Errorf("git read-tree %s: %w", pr.BaseBranch, err) } @@ -400,7 +395,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * prConfig := prUnit.PullRequestsConfig() // 6. Prepare the arguments to apply the patch against the index - cmdApply := git.NewCommand(gitRepo.Ctx, "apply", "--check", "--cached") + cmdApply := git.NewCommand("apply", "--check", "--cached") if prConfig.IgnoreWhitespaceConflicts { cmdApply.AddArguments("--ignore-whitespace") } @@ -431,7 +426,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * // 8. Run the check command conflict = false - err = cmdApply.Run(&git.RunOpts{ + err = cmdApply.Run(gitRepo.Ctx, &git.RunOpts{ Dir: tmpBasePath, Stderr: stderrWriter, PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { @@ -454,7 +449,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * scanner := bufio.NewScanner(stderrReader) for scanner.Scan() { line := scanner.Text() - log.Trace("PullRequest[%d].testPatch: stderr: %s", pr.ID, line) + log.Trace("PullRequest[%d].testPullRequestTmpRepoBranchMergeable: stderr: %s", pr.ID, line) if strings.HasPrefix(line, prefix) { conflict = true filepath := strings.TrimSpace(strings.Split(line[len(prefix):], ":")[0]) diff --git a/services/pull/patch_unmerged.go b/services/pull/patch_unmerged.go index c60c48d923..200d2233e9 100644 --- a/services/pull/patch_unmerged.go +++ b/services/pull/patch_unmerged.go @@ -72,8 +72,8 @@ func readUnmergedLsFileLines(ctx context.Context, tmpBasePath string, outputChan }() stderr := &strings.Builder{} - err = git.NewCommand(ctx, "ls-files", "-u", "-z"). - Run(&git.RunOpts{ + err = git.NewCommand("ls-files", "-u", "-z"). + Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdout: lsFilesWriter, Stderr: stderr, diff --git a/services/pull/protected_branch.go b/services/pull/protected_branch.go new file mode 100644 index 0000000000..181bd32f44 --- /dev/null +++ b/services/pull/protected_branch.go @@ -0,0 +1,49 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package pull + +import ( + "context" + + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/gitrepo" +) + +func CreateOrUpdateProtectedBranch(ctx context.Context, repo *repo_model.Repository, + protectBranch *git_model.ProtectedBranch, whitelistOptions git_model.WhitelistOptions, +) error { + err := git_model.UpdateProtectBranch(ctx, repo, protectBranch, whitelistOptions) + if err != nil { + return err + } + + isPlainRule := !git_model.IsRuleNameSpecial(protectBranch.RuleName) + var isBranchExist bool + if isPlainRule { + // TODO: read the database directly to check if the branch exists + isBranchExist = gitrepo.IsBranchExist(ctx, repo, protectBranch.RuleName) + } + + if isBranchExist { + if err := CheckPRsForBaseBranch(ctx, repo, protectBranch.RuleName); err != nil { + return err + } + } else { + if !isPlainRule { + // FIXME: since we only need to recheck files protected rules, we could improve this + matchedBranches, err := git_model.FindAllMatchedBranches(ctx, repo.ID, protectBranch.RuleName) + if err != nil { + return err + } + for _, branchName := range matchedBranches { + if err = CheckPRsForBaseBranch(ctx, repo, branchName); err != nil { + return err + } + } + } + } + + return nil +} diff --git a/services/pull/pull.go b/services/pull/pull.go index 52abf35cec..701c4f4d32 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -6,6 +6,7 @@ package pull import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -95,7 +96,7 @@ func NewPullRequest(ctx context.Context, opts *NewPullRequestOptions) error { } defer cancel() - if err := testPatch(ctx, prCtx, pr); err != nil { + if err := testPullRequestTmpRepoBranchMergeable(ctx, prCtx, pr); err != nil { return err } @@ -175,7 +176,7 @@ func NewPullRequest(ctx context.Context, opts *NewPullRequestOptions) error { } if !pr.IsWorkInProgress(ctx) { - reviewNotifiers, err = issue_service.PullRequestCodeOwnersReview(ctx, issue, pr) + reviewNotifiers, err = issue_service.PullRequestCodeOwnersReview(ctx, pr) if err != nil { return err } @@ -313,12 +314,12 @@ func ChangeTargetBranch(ctx context.Context, pr *issues_model.PullRequest, doer pr.BaseBranch = targetBranch // Refresh patch - if err := TestPatch(pr); err != nil { + if err := testPullRequestBranchMergeable(pr); err != nil { return err } // Update target branch, PR diff and status - // This is the same as checkAndUpdateStatus in check service, but also updates base_branch + // This is the same as markPullRequestAsMergeable in check service, but also updates base_branch if pr.Status == issues_model.PullRequestStatusChecking { pr.Status = issues_model.PullRequestStatusMergeable } @@ -371,19 +372,29 @@ func checkForInvalidation(ctx context.Context, requests issues_model.PullRequest return nil } +type TestPullRequestOptions struct { + RepoID int64 + Doer *user_model.User + Branch string + IsSync bool // True means it's a pull request synchronization, false means it's triggered for pull request merging or updating + IsForcePush bool + OldCommitID string + NewCommitID string +} + // AddTestPullRequestTask adds new test tasks by given head/base repository and head/base branch, // and generate new patch for testing as needed. -func AddTestPullRequestTask(doer *user_model.User, repoID int64, branch string, isSync bool, oldCommitID, newCommitID string) { - log.Trace("AddTestPullRequestTask [head_repo_id: %d, head_branch: %s]: finding pull requests", repoID, branch) +func AddTestPullRequestTask(opts TestPullRequestOptions) { + log.Trace("AddTestPullRequestTask [head_repo_id: %d, head_branch: %s]: finding pull requests", opts.RepoID, opts.Branch) graceful.GetManager().RunWithShutdownContext(func(ctx context.Context) { // There is no sensible way to shut this down ":-(" // If you don't let it run all the way then you will lose data // TODO: graceful: AddTestPullRequestTask needs to become a queue! // GetUnmergedPullRequestsByHeadInfo() only return open and unmerged PR. - prs, err := issues_model.GetUnmergedPullRequestsByHeadInfo(ctx, repoID, branch) + prs, err := issues_model.GetUnmergedPullRequestsByHeadInfo(ctx, opts.RepoID, opts.Branch) if err != nil { - log.Error("Find pull requests [head_repo_id: %d, head_branch: %s]: %v", repoID, branch, err) + log.Error("Find pull requests [head_repo_id: %d, head_branch: %s]: %v", opts.RepoID, opts.Branch, err) return } @@ -398,26 +409,25 @@ func AddTestPullRequestTask(doer *user_model.User, repoID int64, branch string, continue } - AddToTaskQueue(ctx, pr) - comment, err := CreatePushPullComment(ctx, doer, pr, oldCommitID, newCommitID) + StartPullRequestCheckImmediately(ctx, pr) + comment, err := CreatePushPullComment(ctx, opts.Doer, pr, opts.OldCommitID, opts.NewCommitID) if err == nil && comment != nil { - notify_service.PullRequestPushCommits(ctx, doer, pr, comment) + notify_service.PullRequestPushCommits(ctx, opts.Doer, pr, comment) } } - if isSync { - requests := issues_model.PullRequestList(prs) - if err = requests.LoadAttributes(ctx); err != nil { + if opts.IsSync { + if err = prs.LoadAttributes(ctx); err != nil { log.Error("PullRequestList.LoadAttributes: %v", err) } - if invalidationErr := checkForInvalidation(ctx, requests, repoID, doer, branch); invalidationErr != nil { + if invalidationErr := checkForInvalidation(ctx, prs, opts.RepoID, opts.Doer, opts.Branch); invalidationErr != nil { log.Error("checkForInvalidation: %v", invalidationErr) } if err == nil { for _, pr := range prs { objectFormat := git.ObjectFormatFromName(pr.BaseRepo.ObjectFormatName) - if newCommitID != "" && newCommitID != objectFormat.EmptyObjectID().String() { - changed, err := checkIfPRContentChanged(ctx, pr, oldCommitID, newCommitID) + if opts.NewCommitID != "" && opts.NewCommitID != objectFormat.EmptyObjectID().String() { + changed, err := checkIfPRContentChanged(ctx, pr, opts.OldCommitID, opts.NewCommitID) if err != nil { log.Error("checkIfPRContentChanged: %v", err) } @@ -433,12 +443,12 @@ func AddTestPullRequestTask(doer *user_model.User, repoID int64, branch string, log.Error("GetFirstMatchProtectedBranchRule: %v", err) } if pb != nil && pb.DismissStaleApprovals { - if err := DismissApprovalReviews(ctx, doer, pr); err != nil { + if err := DismissApprovalReviews(ctx, opts.Doer, pr); err != nil { log.Error("DismissApprovalReviews: %v", err) } } } - if err := issues_model.MarkReviewsAsNotStale(ctx, pr.IssueID, newCommitID); err != nil { + if err := issues_model.MarkReviewsAsNotStale(ctx, pr.IssueID, opts.NewCommitID); err != nil { log.Error("MarkReviewsAsNotStale: %v", err) } divergence, err := GetDiverging(ctx, pr) @@ -452,21 +462,31 @@ func AddTestPullRequestTask(doer *user_model.User, repoID int64, branch string, } } - notify_service.PullRequestSynchronized(ctx, doer, pr) + if !pr.IsWorkInProgress(ctx) { + reviewNotifiers, err := issue_service.PullRequestCodeOwnersReview(ctx, pr) + if err != nil { + log.Error("PullRequestCodeOwnersReview: %v", err) + } + if len(reviewNotifiers) > 0 { + issue_service.ReviewRequestNotify(ctx, pr.Issue, opts.Doer, reviewNotifiers) + } + } + + notify_service.PullRequestSynchronized(ctx, opts.Doer, pr) } } } - log.Trace("AddTestPullRequestTask [base_repo_id: %d, base_branch: %s]: finding pull requests", repoID, branch) - prs, err = issues_model.GetUnmergedPullRequestsByBaseInfo(ctx, repoID, branch) + log.Trace("AddTestPullRequestTask [base_repo_id: %d, base_branch: %s]: finding pull requests", opts.RepoID, opts.Branch) + prs, err = issues_model.GetUnmergedPullRequestsByBaseInfo(ctx, opts.RepoID, opts.Branch) if err != nil { - log.Error("Find pull requests [base_repo_id: %d, base_branch: %s]: %v", repoID, branch, err) + log.Error("Find pull requests [base_repo_id: %d, base_branch: %s]: %v", opts.RepoID, opts.Branch, err) return } for _, pr := range prs { divergence, err := GetDiverging(ctx, pr) if err != nil { - if git_model.IsErrBranchNotExist(err) && !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) { + if git_model.IsErrBranchNotExist(err) && !gitrepo.IsBranchExist(ctx, pr.HeadRepo, pr.HeadBranch) { log.Warn("Cannot test PR %s/%d: head_branch %s no longer exists", pr.BaseRepo.Name, pr.IssueID, pr.HeadBranch) } else { log.Error("GetDiverging: %v", err) @@ -477,7 +497,7 @@ func AddTestPullRequestTask(doer *user_model.User, repoID int64, branch string, log.Error("UpdateCommitDivergence: %v", err) } } - AddToTaskQueue(ctx, pr) + StartPullRequestCheckDelayable(ctx, pr) } }) } @@ -504,14 +524,14 @@ func checkIfPRContentChanged(ctx context.Context, pr *issues_model.PullRequest, return false, fmt.Errorf("GetMergeBase: %w", err) } - cmd := git.NewCommand(ctx, "diff", "--name-only", "-z").AddDynamicArguments(newCommitID, oldCommitID, base) + cmd := git.NewCommand("diff", "--name-only", "-z").AddDynamicArguments(newCommitID, oldCommitID, base) stdoutReader, stdoutWriter, err := os.Pipe() if err != nil { return false, fmt.Errorf("unable to open pipe for to run diff: %w", err) } stderr := new(bytes.Buffer) - if err := cmd.Run(&git.RunOpts{ + if err := cmd.Run(ctx, &git.RunOpts{ Dir: prCtx.tmpBasePath, Stdout: stdoutWriter, Stderr: stderr, @@ -628,7 +648,7 @@ func UpdateRef(ctx context.Context, pr *issues_model.PullRequest) (err error) { return err } - _, _, err = git.NewCommand(ctx, "update-ref").AddDynamicArguments(pr.GetGitRefName(), pr.HeadCommitID).RunStdString(&git.RunOpts{Dir: pr.BaseRepo.RepoPath()}) + _, _, err = git.NewCommand("update-ref").AddDynamicArguments(pr.GetGitRefName(), pr.HeadCommitID).RunStdString(ctx, &git.RunOpts{Dir: pr.BaseRepo.RepoPath()}) if err != nil { log.Error("Unable to update ref in base repository for PR[%d] Error: %v", pr.ID, err) } @@ -636,43 +656,19 @@ func UpdateRef(ctx context.Context, pr *issues_model.PullRequest) (err error) { return err } -type errlist []error - -func (errs errlist) Error() string { - if len(errs) > 0 { - var buf strings.Builder - for i, err := range errs { - if i > 0 { - buf.WriteString(", ") - } - buf.WriteString(err.Error()) - } - return buf.String() - } - return "" -} - -// RetargetChildrenOnMerge retarget children pull requests on merge if possible -func RetargetChildrenOnMerge(ctx context.Context, doer *user_model.User, pr *issues_model.PullRequest) error { - if setting.Repository.PullRequest.RetargetChildrenOnMerge && pr.BaseRepoID == pr.HeadRepoID { - return RetargetBranchPulls(ctx, doer, pr.HeadRepoID, pr.HeadBranch, pr.BaseBranch) - } - return nil -} - -// RetargetBranchPulls change target branch for all pull requests whose base branch is the branch +// retargetBranchPulls change target branch for all pull requests whose base branch is the branch // Both branch and targetBranch must be in the same repo (for security reasons) -func RetargetBranchPulls(ctx context.Context, doer *user_model.User, repoID int64, branch, targetBranch string) error { +func retargetBranchPulls(ctx context.Context, doer *user_model.User, repoID int64, branch, targetBranch string) error { prs, err := issues_model.GetUnmergedPullRequestsByBaseInfo(ctx, repoID, branch) if err != nil { return err } - if err := issues_model.PullRequestList(prs).LoadAttributes(ctx); err != nil { + if err := prs.LoadAttributes(ctx); err != nil { return err } - var errs errlist + var errs []error for _, pr := range prs { if err = pr.Issue.LoadRepo(ctx); err != nil { errs = append(errs, err) @@ -682,40 +678,75 @@ func RetargetBranchPulls(ctx context.Context, doer *user_model.User, repoID int6 errs = append(errs, err) } } - - if len(errs) > 0 { - return errs - } - return nil + return errors.Join(errs...) } -// CloseBranchPulls close all the pull requests who's head branch is the branch -func CloseBranchPulls(ctx context.Context, doer *user_model.User, repoID int64, branch string) error { - prs, err := issues_model.GetUnmergedPullRequestsByHeadInfo(ctx, repoID, branch) +// AdjustPullsCausedByBranchDeleted close all the pull requests who's head branch is the branch +// Or Close all the plls who's base branch is the branch if setting.Repository.PullRequest.RetargetChildrenOnMerge is false. +// If it's true, Retarget all these pulls to the default branch. +func AdjustPullsCausedByBranchDeleted(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, branch string) error { + // branch as head branch + prs, err := issues_model.GetUnmergedPullRequestsByHeadInfo(ctx, repo.ID, branch) if err != nil { return err } - prs2, err := issues_model.GetUnmergedPullRequestsByBaseInfo(ctx, repoID, branch) - if err != nil { + if err := prs.LoadAttributes(ctx); err != nil { return err } - - prs = append(prs, prs2...) - if err := issues_model.PullRequestList(prs).LoadAttributes(ctx); err != nil { + prs.SetHeadRepo(repo) + if err := prs.LoadRepositories(ctx); err != nil { return err } - var errs errlist + var errs []error for _, pr := range prs { if err = issue_service.CloseIssue(ctx, pr.Issue, doer, ""); err != nil && !issues_model.IsErrIssueIsClosed(err) && !issues_model.IsErrDependenciesLeft(err) { errs = append(errs, err) } + if err == nil { + if err := issues_model.AddDeletePRBranchComment(ctx, doer, pr.BaseRepo, pr.Issue.ID, pr.HeadBranch); err != nil { + log.Error("AddDeletePRBranchComment: %v", err) + errs = append(errs, err) + } + } } - if len(errs) > 0 { - return errs + + if setting.Repository.PullRequest.RetargetChildrenOnMerge { + if err := retargetBranchPulls(ctx, doer, repo.ID, branch, repo.DefaultBranch); err != nil { + log.Error("retargetBranchPulls failed: %v", err) + errs = append(errs, err) + } + return errors.Join(errs...) } - return nil + + // branch as base branch + prs, err = issues_model.GetUnmergedPullRequestsByBaseInfo(ctx, repo.ID, branch) + if err != nil { + return err + } + + if err := prs.LoadAttributes(ctx); err != nil { + return err + } + prs.SetBaseRepo(repo) + if err := prs.LoadRepositories(ctx); err != nil { + return err + } + + errs = nil + for _, pr := range prs { + if err = issues_model.AddDeletePRBranchComment(ctx, doer, pr.BaseRepo, pr.Issue.ID, pr.BaseBranch); err != nil { + log.Error("AddDeletePRBranchComment: %v", err) + errs = append(errs, err) + } + if err == nil { + if err = issue_service.CloseIssue(ctx, pr.Issue, doer, ""); err != nil && !issues_model.IsErrIssueIsClosed(err) && !issues_model.IsErrDependenciesLeft(err) { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) } // CloseRepoBranchesPulls close all pull requests which head branches are in the given repository, but only whose base repo is not in the given repository @@ -725,14 +756,14 @@ func CloseRepoBranchesPulls(ctx context.Context, doer *user_model.User, repo *re return err } - var errs errlist + var errs []error for _, branch := range branches { - prs, err := issues_model.GetUnmergedPullRequestsByHeadInfo(ctx, repo.ID, branch.Name) + prs, err := issues_model.GetUnmergedPullRequestsByHeadInfo(ctx, repo.ID, branch) if err != nil { return err } - if err = issues_model.PullRequestList(prs).LoadAttributes(ctx); err != nil { + if err = prs.LoadAttributes(ctx); err != nil { return err } @@ -748,10 +779,7 @@ func CloseRepoBranchesPulls(ctx context.Context, doer *user_model.User, repo *re } } - if len(errs) > 0 { - return errs - } - return nil + return errors.Join(errs...) } var commitMessageTrailersPattern = regexp.MustCompile(`(?:^|\n\n)(?:[\w-]+[ \t]*:[^\n]+\n*(?:[ \t]+[^\n]+\n*)*)+$`) @@ -917,12 +945,6 @@ func GetSquashMergeCommitMessages(ctx context.Context, pr *issues_model.PullRequ return stringBuilder.String() } -// GetIssuesLastCommitStatus returns a map of issue ID to the most recent commit's latest status -func GetIssuesLastCommitStatus(ctx context.Context, issues issues_model.IssueList) (map[int64]*git_model.CommitStatus, error) { - _, lastStatus, err := GetIssuesAllCommitStatus(ctx, issues) - return lastStatus, err -} - // GetIssuesAllCommitStatus returns a map of issue ID to a list of all statuses for the most recent commit as well as a map of issue ID to only the commit's latest status func GetIssuesAllCommitStatus(ctx context.Context, issues issues_model.IssueList) (map[int64][]*git_model.CommitStatus, map[int64]*git_model.CommitStatus, error) { if err := issues.LoadPullRequests(ctx); err != nil { @@ -976,7 +998,7 @@ func getAllCommitStatus(ctx context.Context, gitRepo *git.Repository, pr *issues return nil, nil, shaErr } - statuses, _, err = git_model.GetLatestCommitStatus(ctx, pr.BaseRepo.ID, sha, db.ListOptionsAll) + statuses, err = git_model.GetLatestCommitStatus(ctx, pr.BaseRepo.ID, sha, db.ListOptionsAll) lastStatus = git_model.CalcCommitStatus(statuses) return statuses, lastStatus, err } diff --git a/services/pull/review.go b/services/pull/review.go index 78723a58ae..5c80e7b338 100644 --- a/services/pull/review.go +++ b/services/pull/review.go @@ -395,7 +395,7 @@ func DismissReview(ctx context.Context, reviewID, repoID int64, message string, } if review.Type != issues_model.ReviewTypeApprove && review.Type != issues_model.ReviewTypeReject { - return nil, fmt.Errorf("not need to dismiss this review because it's type is not Approve or change request") + return nil, errors.New("not need to dismiss this review because it's type is not Approve or change request") } // load data for notify @@ -405,7 +405,7 @@ func DismissReview(ctx context.Context, reviewID, repoID int64, message string, // Check if the review's repoID is the one we're currently expecting. if review.Issue.RepoID != repoID { - return nil, fmt.Errorf("reviews's repository is not the same as the one we expect") + return nil, errors.New("reviews's repository is not the same as the one we expect") } issue := review.Issue diff --git a/services/pull/reviewer.go b/services/pull/reviewer.go index bf0d8cb298..52f2f3401c 100644 --- a/services/pull/reviewer.go +++ b/services/pull/reviewer.go @@ -85,5 +85,5 @@ func GetReviewerTeams(ctx context.Context, repo *repo_model.Repository) ([]*orga return nil, nil } - return organization.GetTeamsWithAccessToRepoUnit(ctx, repo.OwnerID, repo.ID, perm.AccessModeRead, unit.TypePullRequests) + return organization.GetTeamsWithAccessToAnyRepoUnit(ctx, repo.OwnerID, repo.ID, perm.AccessModeRead, unit.TypePullRequests) } diff --git a/services/pull/temp_repo.go b/services/pull/temp_repo.go index e5753178b8..72406482e0 100644 --- a/services/pull/temp_repo.go +++ b/services/pull/temp_repo.go @@ -15,6 +15,7 @@ import ( issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" ) @@ -27,7 +28,7 @@ const ( stagingBranch = "staging" // this is used for a working branch ) -type prContext struct { +type prTmpRepoContext struct { context.Context tmpBasePath string pr *issues_model.PullRequest @@ -35,7 +36,7 @@ type prContext struct { errbuf *strings.Builder // any use should be preceded by a Reset and preferably after use } -func (ctx *prContext) RunOpts() *git.RunOpts { +func (ctx *prTmpRepoContext) RunOpts() *git.RunOpts { ctx.outbuf.Reset() ctx.errbuf.Reset() return &git.RunOpts{ @@ -47,7 +48,7 @@ func (ctx *prContext) RunOpts() *git.RunOpts { // createTemporaryRepoForPR creates a temporary repo with "base" for pr.BaseBranch and "tracking" for pr.HeadBranch // it also create a second base branch called "original_base" -func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) (prCtx *prContext, cancel context.CancelFunc, err error) { +func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) (prCtx *prTmpRepoContext, cancel context.CancelFunc, err error) { if err := pr.LoadHeadRepo(ctx); err != nil { log.Error("%-v LoadHeadRepo: %v", pr, err) return nil, nil, fmt.Errorf("%v LoadHeadRepo: %w", pr, err) @@ -73,23 +74,20 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) } // Clone base repo. - tmpBasePath, err := repo_module.CreateTemporaryPath("pull") + tmpBasePath, cleanup, err := repo_module.CreateTemporaryPath("pull") if err != nil { log.Error("CreateTemporaryPath[%-v]: %v", pr, err) return nil, nil, err } - prCtx = &prContext{ + cancel = cleanup + + prCtx = &prTmpRepoContext{ Context: ctx, tmpBasePath: tmpBasePath, pr: pr, outbuf: &strings.Builder{}, errbuf: &strings.Builder{}, } - cancel = func() { - if err := repo_module.RemoveTemporaryPath(tmpBasePath); err != nil { - log.Error("Error whilst removing removing temporary repo for %-v: %v", pr, err) - } - } baseRepoPath := pr.BaseRepo.RepoPath() headRepoPath := pr.HeadRepo.RepoPath() @@ -133,22 +131,22 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) return nil, nil, fmt.Errorf("Unable to add base repository to temporary repo [%s -> tmpBasePath]: %w", pr.BaseRepo.FullName(), err) } - if err := git.NewCommand(ctx, "remote", "add", "-t").AddDynamicArguments(pr.BaseBranch).AddArguments("-m").AddDynamicArguments(pr.BaseBranch).AddDynamicArguments("origin", baseRepoPath). - Run(prCtx.RunOpts()); err != nil { + if err := git.NewCommand("remote", "add", "-t").AddDynamicArguments(pr.BaseBranch).AddArguments("-m").AddDynamicArguments(pr.BaseBranch).AddDynamicArguments("origin", baseRepoPath). + Run(ctx, prCtx.RunOpts()); err != nil { log.Error("%-v Unable to add base repository as origin [%s -> %s]: %v\n%s\n%s", pr, pr.BaseRepo.FullName(), tmpBasePath, err, prCtx.outbuf.String(), prCtx.errbuf.String()) cancel() return nil, nil, fmt.Errorf("Unable to add base repository as origin [%s -> tmpBasePath]: %w\n%s\n%s", pr.BaseRepo.FullName(), err, prCtx.outbuf.String(), prCtx.errbuf.String()) } - if err := git.NewCommand(ctx, "fetch", "origin").AddArguments(fetchArgs...).AddDashesAndList(pr.BaseBranch+":"+baseBranch, pr.BaseBranch+":original_"+baseBranch). - Run(prCtx.RunOpts()); err != nil { + if err := git.NewCommand("fetch", "origin").AddArguments(fetchArgs...).AddDashesAndList(pr.BaseBranch+":"+baseBranch, pr.BaseBranch+":original_"+baseBranch). + Run(ctx, prCtx.RunOpts()); err != nil { log.Error("%-v Unable to fetch origin base branch [%s:%s -> base, original_base in %s]: %v:\n%s\n%s", pr, pr.BaseRepo.FullName(), pr.BaseBranch, tmpBasePath, err, prCtx.outbuf.String(), prCtx.errbuf.String()) cancel() return nil, nil, fmt.Errorf("Unable to fetch origin base branch [%s:%s -> base, original_base in tmpBasePath]: %w\n%s\n%s", pr.BaseRepo.FullName(), pr.BaseBranch, err, prCtx.outbuf.String(), prCtx.errbuf.String()) } - if err := git.NewCommand(ctx, "symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseBranch). - Run(prCtx.RunOpts()); err != nil { + if err := git.NewCommand("symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseBranch). + Run(ctx, prCtx.RunOpts()); err != nil { log.Error("%-v Unable to set HEAD as base branch in [%s]: %v\n%s\n%s", pr, tmpBasePath, err, prCtx.outbuf.String(), prCtx.errbuf.String()) cancel() return nil, nil, fmt.Errorf("Unable to set HEAD as base branch in tmpBasePath: %w\n%s\n%s", err, prCtx.outbuf.String(), prCtx.errbuf.String()) @@ -160,8 +158,8 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) return nil, nil, fmt.Errorf("Unable to add head base repository to temporary repo [%s -> tmpBasePath]: %w", pr.HeadRepo.FullName(), err) } - if err := git.NewCommand(ctx, "remote", "add").AddDynamicArguments(remoteRepoName, headRepoPath). - Run(prCtx.RunOpts()); err != nil { + if err := git.NewCommand("remote", "add").AddDynamicArguments(remoteRepoName, headRepoPath). + Run(ctx, prCtx.RunOpts()); err != nil { log.Error("%-v Unable to add head repository as head_repo [%s -> %s]: %v\n%s\n%s", pr, pr.HeadRepo.FullName(), tmpBasePath, err, prCtx.outbuf.String(), prCtx.errbuf.String()) cancel() return nil, nil, fmt.Errorf("Unable to add head repository as head_repo [%s -> tmpBasePath]: %w\n%s\n%s", pr.HeadRepo.FullName(), err, prCtx.outbuf.String(), prCtx.errbuf.String()) @@ -178,10 +176,10 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) } else { headBranch = pr.GetGitRefName() } - if err := git.NewCommand(ctx, "fetch").AddArguments(fetchArgs...).AddDynamicArguments(remoteRepoName, headBranch+":"+trackingBranch). - Run(prCtx.RunOpts()); err != nil { + if err := git.NewCommand("fetch").AddArguments(fetchArgs...).AddDynamicArguments(remoteRepoName, headBranch+":"+trackingBranch). + Run(ctx, prCtx.RunOpts()); err != nil { cancel() - if !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) { + if !gitrepo.IsBranchExist(ctx, pr.HeadRepo, pr.HeadBranch) { return nil, nil, git_model.ErrBranchNotExist{ BranchName: pr.HeadBranch, } diff --git a/services/pull/update.go b/services/pull/update.go index abf7ad4509..b8f84e3d65 100644 --- a/services/pull/update.go +++ b/services/pull/update.go @@ -5,6 +5,7 @@ package pull import ( "context" + "errors" "fmt" git_model "code.gitea.io/gitea/models/git" @@ -23,7 +24,7 @@ import ( func Update(ctx context.Context, pr *issues_model.PullRequest, doer *user_model.User, message string, rebase bool) error { if pr.Flow == issues_model.PullRequestFlowAGit { // TODO: update of agit flow pull request's head branch is unsupported - return fmt.Errorf("update of agit flow pull request's head branch is unsupported") + return errors.New("update of agit flow pull request's head branch is unsupported") } releaser, err := globallock.Lock(ctx, getPullWorkingLockKey(pr.ID)) @@ -40,14 +41,6 @@ func Update(ctx context.Context, pr *issues_model.PullRequest, doer *user_model. return fmt.Errorf("HeadBranch of PR %d is up to date", pr.Index) } - if rebase { - defer func() { - go AddTestPullRequestTask(doer, pr.BaseRepo.ID, pr.BaseBranch, false, "", "") - }() - - return updateHeadByRebaseOnToBase(ctx, pr, doer) - } - if err := pr.LoadBaseRepo(ctx); err != nil { log.Error("unable to load BaseRepo for %-v during update-by-merge: %v", pr, err) return fmt.Errorf("unable to load BaseRepo for PR[%d] during update-by-merge: %w", pr.ID, err) @@ -65,6 +58,22 @@ func Update(ctx context.Context, pr *issues_model.PullRequest, doer *user_model. return fmt.Errorf("unable to load HeadRepo for PR[%d] during update-by-merge: %w", pr.ID, err) } + defer func() { + go AddTestPullRequestTask(TestPullRequestOptions{ + RepoID: pr.BaseRepo.ID, + Doer: doer, + Branch: pr.BaseBranch, + IsSync: false, + IsForcePush: false, + OldCommitID: "", + NewCommitID: "", + }) + }() + + if rebase { + return updateHeadByRebaseOnToBase(ctx, pr, doer) + } + // TODO: FakePR: it is somewhat hacky, but it is the only way to "merge" at the moment // ideally in the future the "merge" functions should be refactored to decouple from the PullRequest // now use a fake reverse PR to switch head&base repos/branches @@ -81,11 +90,6 @@ func Update(ctx context.Context, pr *issues_model.PullRequest, doer *user_model. } _, err = doMergeAndPush(ctx, reversePR, doer, repo_model.MergeStyleMerge, "", message, repository.PushTriggerPRUpdateWithBase) - - defer func() { - go AddTestPullRequestTask(doer, reversePR.HeadRepo.ID, reversePR.HeadBranch, false, "", "") - }() - return err } diff --git a/services/pull/update_rebase.go b/services/pull/update_rebase.go index 3e2a7be132..9ff062f99c 100644 --- a/services/pull/update_rebase.go +++ b/services/pull/update_rebase.go @@ -27,7 +27,7 @@ func updateHeadByRebaseOnToBase(ctx context.Context, pr *issues_model.PullReques defer cancel() // Determine the old merge-base before the rebase - we use this for LFS push later on - oldMergeBase, _, _ := git.NewCommand(ctx, "merge-base").AddDashesAndList(baseBranch, trackingBranch).RunStdString(&git.RunOpts{Dir: mergeCtx.tmpBasePath}) + oldMergeBase, _, _ := git.NewCommand("merge-base").AddDashesAndList(baseBranch, trackingBranch).RunStdString(ctx, &git.RunOpts{Dir: mergeCtx.tmpBasePath}) oldMergeBase = strings.TrimSpace(oldMergeBase) // Rebase the tracking branch on to the base as the staging branch @@ -62,7 +62,7 @@ func updateHeadByRebaseOnToBase(ctx context.Context, pr *issues_model.PullReques headUser = pr.HeadRepo.Owner } - pushCmd := git.NewCommand(ctx, "push", "-f", "head_repo"). + pushCmd := git.NewCommand("push", "-f", "head_repo"). AddDynamicArguments(stagingBranch + ":" + git.BranchPrefix + pr.HeadBranch) // Push back to the head repository. @@ -71,7 +71,7 @@ func updateHeadByRebaseOnToBase(ctx context.Context, pr *issues_model.PullReques mergeCtx.outbuf.Reset() mergeCtx.errbuf.Reset() - if err := pushCmd.Run(&git.RunOpts{ + if err := pushCmd.Run(ctx, &git.RunOpts{ Env: repo_module.FullPushingEnvironment( headUser, doer, diff --git a/services/release/release.go b/services/release/release.go index 835a5943b1..0b8a74252a 100644 --- a/services/release/release.go +++ b/services/release/release.go @@ -77,7 +77,7 @@ func createTag(ctx context.Context, gitRepo *git.Repository, rel *repo_model.Rel var created bool // Only actual create when publish. if !rel.IsDraft { - if !gitRepo.IsTagExist(rel.TagName) { + if !gitrepo.IsTagExist(ctx, rel.Repo, rel.TagName) { if err := rel.LoadAttributes(ctx); err != nil { log.Error("LoadAttributes: %v", err) return false, err @@ -296,10 +296,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo } for _, attach := range attachments { if attach.ReleaseID != rel.ID { - return util.SilentWrap{ - Message: "delete attachment of release permission denied", - Err: util.ErrPermissionDenied, - } + return util.NewPermissionDeniedErrorf("delete attachment of release permission denied") } deletedUUIDs.Add(attach.UUID) } @@ -321,10 +318,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo } for _, attach := range attachments { if attach.ReleaseID != rel.ID { - return util.SilentWrap{ - Message: "update attachment of release permission denied", - Err: util.ErrPermissionDenied, - } + return util.NewPermissionDeniedErrorf("update attachment of release permission denied") } } @@ -381,8 +375,8 @@ func DeleteReleaseByID(ctx context.Context, repo *repo_model.Repository, rel *re } } - if stdout, _, err := git.NewCommand(ctx, "tag", "-d").AddDashesAndList(rel.TagName). - RunStdString(&git.RunOpts{Dir: repo.RepoPath()}); err != nil && !strings.Contains(err.Error(), "not found") { + if stdout, _, err := git.NewCommand("tag", "-d").AddDashesAndList(rel.TagName). + RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil && !strings.Contains(err.Error(), "not found") { log.Error("DeleteReleaseByID (git tag -d): %d in %v Failed:\nStdout: %s\nError: %v", rel.ID, repo, stdout, err) return fmt.Errorf("git tag -d: %w", err) } diff --git a/services/release/release_test.go b/services/release/release_test.go index 95a54832b9..36a9f667d6 100644 --- a/services/release/release_test.go +++ b/services/release/release_test.go @@ -250,9 +250,9 @@ func TestRelease_Update(t *testing.T) { assert.NoError(t, UpdateRelease(db.DefaultContext, user, gitRepo, release, []string{attach.UUID}, nil, nil)) assert.NoError(t, repo_model.GetReleaseAttachments(db.DefaultContext, release)) assert.Len(t, release.Attachments, 1) - assert.EqualValues(t, attach.UUID, release.Attachments[0].UUID) - assert.EqualValues(t, release.ID, release.Attachments[0].ReleaseID) - assert.EqualValues(t, attach.Name, release.Attachments[0].Name) + assert.Equal(t, attach.UUID, release.Attachments[0].UUID) + assert.Equal(t, release.ID, release.Attachments[0].ReleaseID) + assert.Equal(t, attach.Name, release.Attachments[0].Name) // update the attachment name assert.NoError(t, UpdateRelease(db.DefaultContext, user, gitRepo, release, nil, nil, map[string]string{ @@ -261,9 +261,9 @@ func TestRelease_Update(t *testing.T) { release.Attachments = nil assert.NoError(t, repo_model.GetReleaseAttachments(db.DefaultContext, release)) assert.Len(t, release.Attachments, 1) - assert.EqualValues(t, attach.UUID, release.Attachments[0].UUID) - assert.EqualValues(t, release.ID, release.Attachments[0].ReleaseID) - assert.EqualValues(t, "test2.txt", release.Attachments[0].Name) + assert.Equal(t, attach.UUID, release.Attachments[0].UUID) + assert.Equal(t, release.ID, release.Attachments[0].ReleaseID) + assert.Equal(t, "test2.txt", release.Attachments[0].Name) // delete the attachment assert.NoError(t, UpdateRelease(db.DefaultContext, user, gitRepo, release, nil, []string{attach.UUID}, nil)) diff --git a/services/repository/adopt.go b/services/repository/adopt.go index e37909e7ab..2bd1c55de4 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -16,7 +16,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" @@ -28,18 +27,30 @@ import ( "github.com/gobwas/glob" ) +func deleteFailedAdoptRepository(repoID int64) error { + return db.WithTx(db.DefaultContext, func(ctx context.Context) error { + if err := deleteDBRepository(ctx, repoID); err != nil { + return fmt.Errorf("deleteDBRepository: %w", err) + } + if err := git_model.DeleteRepoBranches(ctx, repoID); err != nil { + return fmt.Errorf("deleteRepoBranches: %w", err) + } + return repo_model.DeleteRepoReleases(ctx, repoID) + }) +} + // AdoptRepository adopts pre-existing repository files for the user/organization. -func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { - if !doer.IsAdmin && !u.CanCreateRepo() { +func AdoptRepository(ctx context.Context, doer, owner *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { + if !doer.CanCreateRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ - Limit: u.MaxRepoCreation, + Limit: owner.MaxRepoCreation, } } repo := &repo_model.Repository{ - OwnerID: u.ID, - Owner: u, - OwnerName: u.Name, + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, Name: opts.Name, LowerName: strings.ToLower(opts.Name), Description: opts.Description, @@ -48,75 +59,67 @@ func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts CreateR IsPrivate: opts.IsPrivate, IsFsckEnabled: !opts.IsMirror, CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, - Status: opts.Status, + Status: repo_model.RepositoryBeingMigrated, IsEmpty: !opts.AutoInit, } - if err := db.WithTx(ctx, func(ctx context.Context) error { - repoPath := repo_model.RepoPath(u.Name, repo.Name) - isExist, err := util.IsExist(repoPath) + // 1 - create the repository database operations first + err := db.WithTx(ctx, func(ctx context.Context) error { + return createRepositoryInDB(ctx, doer, owner, repo, false) + }) + if err != nil { + return nil, err + } + + // last - clean up if something goes wrong + // WARNING: Don't override all later err with local variables + defer func() { if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return err - } - if !isExist { - return repo_model.ErrRepoNotExist{ - OwnerName: u.Name, - Name: repo.Name, + // we can not use the ctx because it maybe canceled or timeout + if errDel := deleteFailedAdoptRepository(repo.ID); errDel != nil { + log.Error("Failed to delete repository %s that could not be adopted: %v", repo.FullName(), errDel) } } + }() - if err := CreateRepositoryByExample(ctx, doer, u, repo, true, false); err != nil { - return err - } - - // Re-fetch the repository from database before updating it (else it would - // override changes that were done earlier with sql) - if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { - return fmt.Errorf("getRepositoryByID: %w", err) - } - - if err := adoptRepository(ctx, repoPath, repo, opts.DefaultBranch); err != nil { - return fmt.Errorf("adoptRepository: %w", err) - } + // Re-fetch the repository from database before updating it (else it would + // override changes that were done earlier with sql) + if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { + return nil, fmt.Errorf("getRepositoryByID: %w", err) + } - if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { - return fmt.Errorf("checkDaemonExportOK: %w", err) - } + // 2 - adopt the repository from disk + if err = adoptRepository(ctx, repo, opts.DefaultBranch); err != nil { + return nil, fmt.Errorf("adoptRepository: %w", err) + } - // Initialize Issue Labels if selected - if len(opts.IssueLabels) > 0 { - if err := repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { - return fmt.Errorf("InitializeLabels: %w", err) - } - } + // 3 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } - if stdout, _, err := git.NewCommand(ctx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - return fmt.Errorf("CreateRepository(git update-server-info): %w", err) - } - return nil - }); err != nil { - return nil, err + // 4 - update repository status + repo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } - notify_service.AdoptRepository(ctx, doer, u, repo) + notify_service.AdoptRepository(ctx, doer, owner, repo) return repo, nil } -func adoptRepository(ctx context.Context, repoPath string, repo *repo_model.Repository, defaultBranch string) (err error) { - isExist, err := util.IsExist(repoPath) +func adoptRepository(ctx context.Context, repo *repo_model.Repository, defaultBranch string) (err error) { + isExist, err := gitrepo.IsRepositoryExist(ctx, repo) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) return err } if !isExist { - return fmt.Errorf("adoptRepository: path does not already exist: %s", repoPath) + return fmt.Errorf("adoptRepository: path does not already exist: %s", repo.FullName()) } - if err := repo_module.CreateDelegateHooks(repoPath); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo); err != nil { return fmt.Errorf("createDelegateHooks: %w", err) } @@ -193,8 +196,13 @@ func adoptRepository(ctx context.Context, repoPath string, repo *repo_model.Repo return fmt.Errorf("setDefaultBranch: %w", err) } } - if err = repo_module.UpdateRepository(ctx, repo, false); err != nil { - return fmt.Errorf("updateRepository: %w", err) + + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_empty", "default_branch"); err != nil { + return fmt.Errorf("UpdateRepositoryCols: %w", err) + } + + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) } return nil @@ -257,7 +265,7 @@ func checkUnadoptedRepositories(ctx context.Context, userName string, repoNamesT } return err } - repos, _, err := repo_model.GetUserRepositories(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.GetUserRepositories(ctx, repo_model.SearchRepoOptions{ Actor: ctxUser, Private: true, ListOptions: db.ListOptions{ diff --git a/services/repository/adopt_test.go b/services/repository/adopt_test.go index 123cedc1f2..86f586c748 100644 --- a/services/repository/adopt_test.go +++ b/services/repository/adopt_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -28,7 +29,7 @@ func TestCheckUnadoptedRepositories_Add(t *testing.T) { } total := 30 - for i := 0; i < total; i++ { + for range total { unadopted.add("something") } @@ -71,7 +72,7 @@ func TestListUnadoptedRepositories_ListOptions(t *testing.T) { username := "user2" unadoptedList := []string{path.Join(username, "unadopted1"), path.Join(username, "unadopted2")} for _, unadopted := range unadoptedList { - _ = os.Mkdir(path.Join(setting.RepoRootPath, unadopted+".git"), 0o755) + _ = os.Mkdir(filepath.Join(setting.RepoRootPath, unadopted+".git"), 0o755) } opts := db.ListOptions{Page: 1, PageSize: 1} @@ -89,10 +90,36 @@ func TestListUnadoptedRepositories_ListOptions(t *testing.T) { func TestAdoptRepository(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - assert.NoError(t, unittest.SyncDirs(filepath.Join(setting.RepoRootPath, "user2", "repo1.git"), filepath.Join(setting.RepoRootPath, "user2", "test-adopt.git"))) + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - _, err := AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"}) + + // a successful adopt + destDir := filepath.Join(setting.RepoRootPath, user2.Name, "test-adopt.git") + assert.NoError(t, unittest.SyncDirs(filepath.Join(setting.RepoRootPath, user2.Name, "repo1.git"), destDir)) + + adoptedRepo, err := AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"}) assert.NoError(t, err) repoTestAdopt := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: "test-adopt"}) assert.Equal(t, "sha1", repoTestAdopt.ObjectFormatName) + + // just delete the adopted repo's db records + err = deleteFailedAdoptRepository(adoptedRepo.ID) + assert.NoError(t, err) + + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: "test-adopt"}) + + // a failed adopt because some mock data + // remove the hooks directory and create a file so that we cannot create the hooks successfully + _ = os.RemoveAll(filepath.Join(destDir, "hooks", "update.d")) + assert.NoError(t, os.WriteFile(filepath.Join(destDir, "hooks", "update.d"), []byte("tests"), os.ModePerm)) + + adoptedRepo, err = AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"}) + assert.Error(t, err) + assert.Nil(t, adoptedRepo) + + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: "test-adopt"}) + + exist, err := util.IsExist(repo_model.RepoPath(user2.Name, "test-adopt")) + assert.NoError(t, err) + assert.True(t, exist) // the repository should be still in the disk } diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go index e1addbed33..a657e3884c 100644 --- a/services/repository/archiver/archiver.go +++ b/services/repository/archiver/archiver.go @@ -31,19 +31,20 @@ import ( // handle elsewhere. type ArchiveRequest struct { RepoID int64 - refName string Type git.ArchiveType CommitID string + + archiveRefShortName string // the ref short name to download the archive, for example: "master", "v1.0.0", "commit id" } // ErrUnknownArchiveFormat request archive format is not supported type ErrUnknownArchiveFormat struct { - RequestFormat string + RequestNameType string } // Error implements error func (err ErrUnknownArchiveFormat) Error() string { - return fmt.Sprintf("unknown format: %s", err.RequestFormat) + return "unknown format: " + err.RequestNameType } // Is implements error @@ -54,12 +55,12 @@ func (ErrUnknownArchiveFormat) Is(err error) bool { // RepoRefNotFoundError is returned when a requested reference (commit, tag) was not found. type RepoRefNotFoundError struct { - RefName string + RefShortName string } // Error implements error. func (e RepoRefNotFoundError) Error() string { - return fmt.Sprintf("unrecognized repository reference: %s", e.RefName) + return "unrecognized repository reference: " + e.RefShortName } func (e RepoRefNotFoundError) Is(err error) bool { @@ -67,43 +68,23 @@ func (e RepoRefNotFoundError) Is(err error) bool { return ok } -func ParseFileName(uri string) (ext string, tp git.ArchiveType, err error) { - switch { - case strings.HasSuffix(uri, ".zip"): - ext = ".zip" - tp = git.ZIP - case strings.HasSuffix(uri, ".tar.gz"): - ext = ".tar.gz" - tp = git.TARGZ - case strings.HasSuffix(uri, ".bundle"): - ext = ".bundle" - tp = git.BUNDLE - default: - return "", 0, ErrUnknownArchiveFormat{RequestFormat: uri} - } - return ext, tp, nil -} - // NewRequest creates an archival request, based on the URI. The // resulting ArchiveRequest is suitable for being passed to Await() // if it's determined that the request still needs to be satisfied. -func NewRequest(repoID int64, repo *git.Repository, refName string, fileType git.ArchiveType) (*ArchiveRequest, error) { - if fileType < git.ZIP || fileType > git.BUNDLE { - return nil, ErrUnknownArchiveFormat{RequestFormat: fileType.String()} - } - - r := &ArchiveRequest{ - RepoID: repoID, - refName: refName, - Type: fileType, +func NewRequest(repoID int64, repo *git.Repository, archiveRefExt string) (*ArchiveRequest, error) { + // here the archiveRefShortName is not a clear ref, it could be a tag, branch or commit id + archiveRefShortName, archiveType := git.SplitArchiveNameType(archiveRefExt) + if archiveType == git.ArchiveUnknown { + return nil, ErrUnknownArchiveFormat{archiveRefExt} } // Get corresponding commit. - commitID, err := repo.ConvertToGitID(r.refName) + commitID, err := repo.ConvertToGitID(archiveRefShortName) if err != nil { - return nil, RepoRefNotFoundError{RefName: r.refName} + return nil, RepoRefNotFoundError{RefShortName: archiveRefShortName} } + r := &ArchiveRequest{RepoID: repoID, archiveRefShortName: archiveRefShortName, Type: archiveType} r.CommitID = commitID.String() return r, nil } @@ -111,11 +92,11 @@ func NewRequest(repoID int64, repo *git.Repository, refName string, fileType git // GetArchiveName returns the name of the caller, based on the ref used by the // caller to create this request. func (aReq *ArchiveRequest) GetArchiveName() string { - return strings.ReplaceAll(aReq.refName, "/", "-") + "." + aReq.Type.String() + return strings.ReplaceAll(aReq.archiveRefShortName, "/", "-") + "." + aReq.Type.String() } // Await awaits the completion of an ArchiveRequest. If the archive has -// already been prepared the method returns immediately. Otherwise an archiver +// already been prepared the method returns immediately. Otherwise, an archiver // process will be started and its completion awaited. On success the returned // RepoArchiver may be used to download the archive. Note that even if the // context is cancelled/times out a started archiver will still continue to run @@ -208,8 +189,8 @@ func doArchive(ctx context.Context, r *ArchiveRequest) (*repo_model.RepoArchiver rd, w := io.Pipe() defer func() { - w.Close() - rd.Close() + _ = w.Close() + _ = rd.Close() }() done := make(chan error, 1) // Ensure that there is some capacity which will ensure that the goroutine below can always finish repo, err := repo_model.GetRepositoryByID(ctx, archiver.RepoID) @@ -230,7 +211,7 @@ func doArchive(ctx context.Context, r *ArchiveRequest) (*repo_model.RepoArchiver } }() - if archiver.Type == git.BUNDLE { + if archiver.Type == git.ArchiveBundle { err = gitRepo.CreateBundle( ctx, archiver.CommitID, diff --git a/services/repository/archiver/archiver_test.go b/services/repository/archiver/archiver_test.go index 1d0c6e513d..87324ad38c 100644 --- a/services/repository/archiver/archiver_test.go +++ b/services/repository/archiver/archiver_test.go @@ -9,7 +9,6 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/services/contexttest" _ "code.gitea.io/gitea/models/actions" @@ -31,47 +30,47 @@ func TestArchive_Basic(t *testing.T) { contexttest.LoadGitRepo(t, ctx) defer ctx.Repo.GitRepo.Close() - bogusReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.ZIP) + bogusReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) - assert.EqualValues(t, firstCommit+".zip", bogusReq.GetArchiveName()) + assert.Equal(t, firstCommit+".zip", bogusReq.GetArchiveName()) // Check a series of bogus requests. // Step 1, valid commit with a bad extension. - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, 100) + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".unknown") assert.Error(t, err) assert.Nil(t, bogusReq) // Step 2, missing commit. - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "dbffff", git.ZIP) + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "dbffff.zip") assert.Error(t, err) assert.Nil(t, bogusReq) // Step 3, doesn't look like branch/tag/commit. - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "db", git.ZIP) + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "db.zip") assert.Error(t, err) assert.Nil(t, bogusReq) - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master", git.ZIP) + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master.zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) - assert.EqualValues(t, "master.zip", bogusReq.GetArchiveName()) + assert.Equal(t, "master.zip", bogusReq.GetArchiveName()) - bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive", git.ZIP) + bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive.zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) - assert.EqualValues(t, "test-archive.zip", bogusReq.GetArchiveName()) + assert.Equal(t, "test-archive.zip", bogusReq.GetArchiveName()) // Now two valid requests, firstCommit with valid extensions. - zipReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.ZIP) + zipReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) assert.NotNil(t, zipReq) - tgzReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.TARGZ) + tgzReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".tar.gz") assert.NoError(t, err) assert.NotNil(t, tgzReq) - secondReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit, git.ZIP) + secondReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".bundle") assert.NoError(t, err) assert.NotNil(t, secondReq) @@ -91,7 +90,7 @@ func TestArchive_Basic(t *testing.T) { // Sleep two seconds to make sure the queue doesn't change. time.Sleep(2 * time.Second) - zipReq2, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.ZIP) + zipReq2, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) // This zipReq should match what's sitting in the queue, as we haven't // let it release yet. From the consumer's point of view, this looks like @@ -106,12 +105,12 @@ func TestArchive_Basic(t *testing.T) { // Now we'll submit a request and TimedWaitForCompletion twice, before and // after we release it. We should trigger both the timeout and non-timeout // cases. - timedReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit, git.TARGZ) + timedReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".tar.gz") assert.NoError(t, err) assert.NotNil(t, timedReq) doArchive(db.DefaultContext, timedReq) - zipReq2, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit, git.ZIP) + zipReq2, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) // Now, we're guaranteed to have released the original zipReq from the queue. // Ensure that we don't get handed back the released entry somehow, but they @@ -129,6 +128,6 @@ func TestArchive_Basic(t *testing.T) { } func TestErrUnknownArchiveFormat(t *testing.T) { - err := ErrUnknownArchiveFormat{RequestFormat: "master"} + err := ErrUnknownArchiveFormat{RequestNameType: "xxx"} assert.ErrorIs(t, err, ErrUnknownArchiveFormat{}) } diff --git a/services/repository/avatar.go b/services/repository/avatar.go index 15e51d4a25..26bf6da465 100644 --- a/services/repository/avatar.go +++ b/services/repository/avatar.go @@ -40,7 +40,7 @@ func UploadAvatar(ctx context.Context, repo *repo_model.Repository, data []byte) // Users can upload the same image to other repo - prefix it with ID // Then repo will be removed - only it avatar file will be removed repo.Avatar = newAvatar - if err := repo_model.UpdateRepositoryCols(ctx, repo, "avatar"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "avatar"); err != nil { return fmt.Errorf("UploadAvatar: Update repository avatar: %w", err) } @@ -77,7 +77,7 @@ func DeleteAvatar(ctx context.Context, repo *repo_model.Repository) error { defer committer.Close() repo.Avatar = "" - if err := repo_model.UpdateRepositoryCols(ctx, repo, "avatar"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "avatar"); err != nil { return fmt.Errorf("DeleteAvatar: Update repository avatar: %w", err) } @@ -112,5 +112,5 @@ func generateAvatar(ctx context.Context, templateRepo, generateRepo *repo_model. return err } - return repo_model.UpdateRepositoryCols(ctx, generateRepo, "avatar") + return repo_model.UpdateRepositoryColsNoAutoTime(ctx, generateRepo, "avatar") } diff --git a/services/repository/avatar_test.go b/services/repository/avatar_test.go index bea820e85f..2dc5173eec 100644 --- a/services/repository/avatar_test.go +++ b/services/repository/avatar_test.go @@ -59,7 +59,7 @@ func TestDeleteAvatar(t *testing.T) { err = DeleteAvatar(db.DefaultContext, repo) assert.NoError(t, err) - assert.Equal(t, "", repo.Avatar) + assert.Empty(t, repo.Avatar) } func TestGenerateAvatar(t *testing.T) { diff --git a/services/repository/branch.go b/services/repository/branch.go index fc476298ca..dd00ca7dcd 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -26,9 +26,11 @@ import ( "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/queue" repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" webhook_module "code.gitea.io/gitea/modules/webhook" + actions_service "code.gitea.io/gitea/services/actions" notify_service "code.gitea.io/gitea/services/notify" release_service "code.gitea.io/gitea/services/release" files_service "code.gitea.io/gitea/services/repository/files" @@ -301,7 +303,7 @@ func SyncBranchesToDB(ctx context.Context, repoID, pusherID int64, branchNames, // For other batches, it will hit optimization 4. if len(branchNames) != len(commitIDs) { - return fmt.Errorf("branchNames and commitIDs length not match") + return errors.New("branchNames and commitIDs length not match") } return db.WithTx(ctx, func(ctx context.Context) error { @@ -408,14 +410,37 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, doer *user_m return "target_exist", nil } - if gitRepo.IsBranchExist(to) { + if gitrepo.IsBranchExist(ctx, repo, to) { return "target_exist", nil } - if !gitRepo.IsBranchExist(from) { + if !gitrepo.IsBranchExist(ctx, repo, from) { return "from_not_exist", nil } + perm, err := access_model.GetUserRepoPermission(ctx, repo, doer) + if err != nil { + return "", err + } + + isDefault := from == repo.DefaultBranch + if isDefault && !perm.IsAdmin() { + return "", repo_model.ErrUserDoesNotHaveAccessToRepo{ + UserID: doer.ID, + RepoName: repo.LowerName, + } + } + + // If from == rule name, admins are allowed to modify them. + if protectedBranch, err := git_model.GetProtectedBranchRuleByName(ctx, repo.ID, from); err != nil { + return "", err + } else if protectedBranch != nil && !perm.IsAdmin() { + return "", repo_model.ErrUserDoesNotHaveAccessToRepo{ + UserID: doer.ID, + RepoName: repo.LowerName, + } + } + if err := git_model.RenameBranch(ctx, repo, from, to, func(ctx context.Context, isDefault bool) error { err2 := gitRepo.RenameBranch(from, to) if err2 != nil { @@ -428,7 +453,7 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, doer *user_m log.Error("DeleteCronTaskByRepo: %v", err) } // cancel running cron jobs of this repository and delete old schedules - if err := actions_model.CancelPreviousJobs( + if err := actions_service.CancelPreviousJobs( ctx, repo.ID, from, @@ -489,7 +514,7 @@ func CanDeleteBranch(ctx context.Context, repo *repo_model.Repository, branchNam } // DeleteBranch delete branch -func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, gitRepo *git.Repository, branchName string) error { +func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, gitRepo *git.Repository, branchName string, pr *issues_model.PullRequest) error { err := repo.MustNotBeArchived() if err != nil { return err @@ -519,6 +544,12 @@ func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.R } } + if pr != nil { + if err := issues_model.AddDeletePRBranchComment(ctx, doer, pr.BaseRepo, pr.Issue.ID, pr.HeadBranch); err != nil { + return fmt.Errorf("DeleteBranch: %v", err) + } + } + return gitRepo.DeleteBranch(branchName, git.DeleteBranchOptions{ Force: true, }) @@ -587,12 +618,12 @@ func AddAllRepoBranchesToSyncQueue(ctx context.Context) error { return nil } -func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, newBranchName string) error { +func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newBranchName string) error { if repo.DefaultBranch == newBranchName { return nil } - if !gitRepo.IsBranchExist(newBranchName) { + if !gitrepo.IsBranchExist(ctx, repo, newBranchName) { return git_model.ErrBranchNotExist{ BranchName: newBranchName, } @@ -609,7 +640,7 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitR log.Error("DeleteCronTaskByRepo: %v", err) } // cancel running cron jobs of this repository and delete old schedules - if err := actions_model.CancelPreviousJobs( + if err := actions_service.CancelPreviousJobs( ctx, repo.ID, oldDefaultBranchName, @@ -632,7 +663,81 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitR } } + // clear divergence cache + if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil { + log.Error("DelRepoDivergenceFromCache: %v", err) + } + notify_service.ChangeDefaultBranch(ctx, repo) return nil } + +// BranchDivergingInfo contains the information about the divergence of a head branch to the base branch. +type BranchDivergingInfo struct { + // whether the base branch contains new commits which are not in the head branch + BaseHasNewCommits bool + + // behind/after are number of commits that the head branch is behind/after the base branch, it's 0 if it's unable to calculate. + // there could be a case that BaseHasNewCommits=true while the behind/after are both 0 (unable to calculate). + HeadCommitsBehind int + HeadCommitsAhead int +} + +// GetBranchDivergingInfo returns the information about the divergence of a patch branch to the base branch. +func GetBranchDivergingInfo(ctx reqctx.RequestContext, baseRepo *repo_model.Repository, baseBranch string, headRepo *repo_model.Repository, headBranch string) (*BranchDivergingInfo, error) { + headGitBranch, err := git_model.GetBranch(ctx, headRepo.ID, headBranch) + if err != nil { + return nil, err + } + if headGitBranch.IsDeleted { + return nil, git_model.ErrBranchNotExist{ + BranchName: headBranch, + } + } + baseGitBranch, err := git_model.GetBranch(ctx, baseRepo.ID, baseBranch) + if err != nil { + return nil, err + } + if baseGitBranch.IsDeleted { + return nil, git_model.ErrBranchNotExist{ + BranchName: baseBranch, + } + } + + info := &BranchDivergingInfo{} + if headGitBranch.CommitID == baseGitBranch.CommitID { + return info, nil + } + + // if the fork repo has new commits, this call will fail because they are not in the base repo + // exit status 128 - fatal: Invalid symmetric difference expression aaaaaaaaaaaa...bbbbbbbbbbbb + // so at the moment, we first check the update time, then check whether the fork branch has base's head + diff, err := git.GetDivergingCommits(ctx, baseRepo.RepoPath(), baseGitBranch.CommitID, headGitBranch.CommitID) + if err != nil { + info.BaseHasNewCommits = baseGitBranch.UpdatedUnix > headGitBranch.UpdatedUnix + if headRepo.IsFork && info.BaseHasNewCommits { + return info, nil + } + // if the base's update time is before the fork, check whether the base's head is in the fork + headGitRepo, err := gitrepo.RepositoryFromRequestContextOrOpen(ctx, headRepo) + if err != nil { + return nil, err + } + headCommit, err := headGitRepo.GetCommit(headGitBranch.CommitID) + if err != nil { + return nil, err + } + baseCommitID, err := git.NewIDFromString(baseGitBranch.CommitID) + if err != nil { + return nil, err + } + hasPreviousCommit, _ := headCommit.HasPreviousCommit(baseCommitID) + info.BaseHasNewCommits = !hasPreviousCommit + return info, nil + } + + info.HeadCommitsBehind, info.HeadCommitsAhead = diff.Behind, diff.Ahead + info.BaseHasNewCommits = info.HeadCommitsBehind > 0 + return info, nil +} diff --git a/services/repository/check.go b/services/repository/check.go index acca15daf2..ffcd5ac749 100644 --- a/services/repository/check.go +++ b/services/repository/check.go @@ -86,10 +86,10 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args git.TrustedCmdA // GitGcRepo calls 'git gc' to remove unnecessary files and optimize the local repository func GitGcRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args git.TrustedCmdArgs) error { log.Trace("Running git gc on %-v", repo) - command := git.NewCommand(ctx, "gc").AddArguments(args...) + command := git.NewCommand("gc").AddArguments(args...) var stdout string var err error - stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()}) + stdout, _, err = command.RunStdString(ctx, &git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()}) if err != nil { log.Error("Repository garbage collection failed for %-v. Stdout: %s\nError: %v", repo, stdout, err) desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) @@ -162,7 +162,7 @@ func DeleteMissingRepositories(ctx context.Context, doer *user_model.User) error default: } log.Trace("Deleting %d/%d...", repo.OwnerID, repo.ID) - if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { log.Error("Failed to DeleteRepository %-v: Error: %v", repo, err) if err2 := system_model.CreateRepositoryNotice("Failed to DeleteRepository %s [%d]: Error: %v", repo.FullName(), repo.ID, err); err2 != nil { log.Error("CreateRepositoryNotice: %v", err) diff --git a/services/repository/commitstatus/commitstatus.go b/services/repository/commitstatus/commitstatus.go index f369a303e6..fa7a89882a 100644 --- a/services/repository/commitstatus/commitstatus.go +++ b/services/repository/commitstatus/commitstatus.go @@ -14,17 +14,17 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/commitstatus" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" - api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/notify" ) func getCacheKey(repoID int64, brancheName string) string { - hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%d:%s", repoID, brancheName))) + hashBytes := sha256.Sum256(fmt.Appendf(nil, "%d:%s", repoID, brancheName)) return fmt.Sprintf("commit_status:%x", hashBytes) } @@ -47,7 +47,7 @@ func getCommitStatusCache(repoID int64, branchName string) *commitStatusCacheVal return nil } -func updateCommitStatusCache(repoID int64, branchName string, state api.CommitStatusState, targetURL string) error { +func updateCommitStatusCache(repoID int64, branchName string, state commitstatus.CommitStatusState, targetURL string) error { c := cache.GetCache() bs, err := json.Marshal(commitStatusCacheValue{ State: state.String(), @@ -127,7 +127,7 @@ func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Rep for i, repo := range repos { if cv := getCommitStatusCache(repo.ID, repo.DefaultBranch); cv != nil { results[i] = &git_model.CommitStatus{ - State: api.CommitStatusState(cv.State), + State: commitstatus.CommitStatusState(cv.State), TargetURL: cv.TargetURL, } } else { diff --git a/services/repository/contributors_graph.go b/services/repository/contributors_graph.go index b0748f8ee3..a4ae505313 100644 --- a/services/repository/contributors_graph.go +++ b/services/repository/contributors_graph.go @@ -125,13 +125,13 @@ func getExtendedCommitStats(repo *git.Repository, revision string /*, limit int _ = stdoutWriter.Close() }() - gitCmd := git.NewCommand(repo.Ctx, "log", "--shortstat", "--no-merges", "--pretty=format:---%n%aN%n%aE%n%as", "--reverse") + gitCmd := git.NewCommand("log", "--shortstat", "--no-merges", "--pretty=format:---%n%aN%n%aE%n%as", "--reverse") // AddOptionFormat("--max-count=%d", limit) gitCmd.AddDynamicArguments(baseCommit.ID.String()) var extendedCommitStats []*ExtendedCommitStats stderr := new(strings.Builder) - err = gitCmd.Run(&git.RunOpts{ + err = gitCmd.Run(repo.Ctx, &git.RunOpts{ Dir: repo.Path, Stdout: stdoutWriter, Stderr: stderr, diff --git a/services/repository/contributors_graph_test.go b/services/repository/contributors_graph_test.go index 6db93f6a64..7d32b1c931 100644 --- a/services/repository/contributors_graph_test.go +++ b/services/repository/contributors_graph_test.go @@ -38,14 +38,14 @@ func TestRepository_ContributorsGraph(t *testing.T) { keys = append(keys, k) } slices.Sort(keys) - assert.EqualValues(t, []string{ + assert.Equal(t, []string{ "ethantkoenig@gmail.com", "jimmy.praet@telenet.be", "jon@allspice.io", "total", // generated summary }, keys) - assert.EqualValues(t, &ContributorData{ + assert.Equal(t, &ContributorData{ Name: "Ethan Koenig", AvatarLink: "/assets/img/avatar_default.png", TotalCommits: 1, @@ -58,7 +58,7 @@ func TestRepository_ContributorsGraph(t *testing.T) { }, }, }, data["ethantkoenig@gmail.com"]) - assert.EqualValues(t, &ContributorData{ + assert.Equal(t, &ContributorData{ Name: "Total", AvatarLink: "", TotalCommits: 3, diff --git a/services/repository/create.go b/services/repository/create.go index 23aacd6f95..bed02e5d7e 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/models/perm" access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" + system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" @@ -28,7 +29,6 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/templates/vars" - "code.gitea.io/gitea/modules/util" ) // CreateRepoOptions contains the create repository options @@ -52,7 +52,7 @@ type CreateRepoOptions struct { ObjectFormatName string } -func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, repoPath string, opts CreateRepoOptions) error { +func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir string, opts CreateRepoOptions) error { commitTimeStr := time.Now().Format(time.RFC3339) authorSig := repo.Owner.NewGitSig() @@ -67,8 +67,8 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, ) // Clone to temporary path and do the init commit. - if stdout, _, err := git.NewCommand(ctx, "clone").AddDynamicArguments(repoPath, tmpDir). - RunStdString(&git.RunOpts{Dir: "", Env: env}); err != nil { + if stdout, _, err := git.NewCommand("clone").AddDynamicArguments(repo.RepoPath(), tmpDir). + RunStdString(ctx, &git.RunOpts{Dir: "", Env: env}); err != nil { log.Error("Failed to clone from %v into %s: stdout: %s\nError: %v", repo, tmpDir, stdout, err) return fmt.Errorf("git clone: %w", err) } @@ -100,8 +100,8 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, // .gitignore if len(opts.Gitignores) > 0 { var buf bytes.Buffer - names := strings.Split(opts.Gitignores, ",") - for _, name := range names { + names := strings.SplitSeq(opts.Gitignores, ",") + for name := range names { data, err = options.Gitignore(name) if err != nil { return fmt.Errorf("GetRepoInitFile[%s]: %w", name, err) @@ -139,24 +139,23 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, } // InitRepository initializes README and .gitignore if needed. -func initRepository(ctx context.Context, repoPath string, u *user_model.User, repo *repo_model.Repository, opts CreateRepoOptions) (err error) { - if err = repo_module.CheckInitRepository(ctx, repo.OwnerName, repo.Name, opts.ObjectFormatName); err != nil { - return err +func initRepository(ctx context.Context, u *user_model.User, repo *repo_model.Repository, opts CreateRepoOptions) (err error) { + // Init git bare new repository. + if err = git.InitRepository(ctx, repo.RepoPath(), true, repo.ObjectFormatName); err != nil { + return fmt.Errorf("git.InitRepository: %w", err) + } else if err = gitrepo.CreateDelegateHooks(ctx, repo); err != nil { + return fmt.Errorf("createDelegateHooks: %w", err) } // Initialize repository according to user's choice. if opts.AutoInit { - tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) + tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("repos-" + repo.Name) if err != nil { - return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.RepoPath(), err) + return fmt.Errorf("failed to create temp dir for repository %s: %w", repo.FullName(), err) } - defer func() { - if err := util.RemoveAll(tmpDir); err != nil { - log.Warn("Unable to remove temporary directory: %s: Error: %v", tmpDir, err) - } - }() + defer cleanup() - if err = prepareRepoCommit(ctx, repo, tmpDir, repoPath, opts); err != nil { + if err = prepareRepoCommit(ctx, repo, tmpDir, opts); err != nil { return fmt.Errorf("prepareRepoCommit: %w", err) } @@ -192,18 +191,25 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re } } - if err = UpdateRepository(ctx, repo, false); err != nil { + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_empty", "default_branch", "default_wiki_branch"); err != nil { return fmt.Errorf("updateRepository: %w", err) } + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + return nil } // CreateRepositoryDirectly creates a repository for the user/organization. -func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { - if !doer.IsAdmin && !u.CanCreateRepo() { +// if needsUpdateToReady is true, it will update the repository status to ready when success +func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User, + opts CreateRepoOptions, needsUpdateToReady bool, +) (*repo_model.Repository, error) { + if !doer.CanCreateRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ - Limit: u.MaxRepoCreation, + Limit: owner.MaxRepoCreation, } } @@ -223,9 +229,9 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt } repo := &repo_model.Repository{ - OwnerID: u.ID, - Owner: u, - OwnerName: u.Name, + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, Name: opts.Name, LowerName: strings.ToLower(opts.Name), Description: opts.Description, @@ -244,101 +250,91 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt ObjectFormatName: opts.ObjectFormatName, } - var rollbackRepo *repo_model.Repository - - if err := db.WithTx(ctx, func(ctx context.Context) error { - if err := CreateRepositoryByExample(ctx, doer, u, repo, false, false); err != nil { - return err - } - - // No need for init mirror. - if opts.IsMirror { - return nil - } + // 1 - create the repository database operations first + err := db.WithTx(ctx, func(ctx context.Context) error { + return createRepositoryInDB(ctx, doer, owner, repo, false) + }) + if err != nil { + return nil, err + } - repoPath := repo_model.RepoPath(u.Name, repo.Name) - isExist, err := util.IsExist(repoPath) + // last - clean up if something goes wrong + // WARNING: Don't override all later err with local variables + defer func() { if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return err - } - if isExist { - // repo already exists - We have two or three options. - // 1. We fail stating that the directory exists - // 2. We create the db repository to go with this data and adopt the git repo - // 3. We delete it and start afresh - // - // Previously Gitea would just delete and start afresh - this was naughty. - // So we will now fail and delegate to other functionality to adopt or delete - log.Error("Files already exist in %s and we are not going to adopt or delete.", repoPath) - return repo_model.ErrRepoFilesAlreadyExist{ - Uname: u.Name, - Name: repo.Name, - } + // we can not use the ctx because it maybe canceled or timeout + cleanupRepository(repo.ID) } + }() - if err = initRepository(ctx, repoPath, doer, repo, opts); err != nil { - if err2 := util.RemoveAll(repoPath); err2 != nil { - log.Error("initRepository: %v", err) - return fmt.Errorf( - "delete repo directory %s/%s failed(2): %v", u.Name, repo.Name, err2) - } - return fmt.Errorf("initRepository: %w", err) - } + // No need for init mirror. + if opts.IsMirror { + return repo, nil + } - // Initialize Issue Labels if selected - if len(opts.IssueLabels) > 0 { - if err = repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { - rollbackRepo = repo - rollbackRepo.OwnerID = u.ID - return fmt.Errorf("InitializeLabels: %w", err) - } + // 2 - check whether the repository with the same storage exists + var isExist bool + isExist, err = gitrepo.IsRepositoryExist(ctx, repo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) + return nil, err + } + if isExist { + log.Error("Files already exist in %s and we are not going to adopt or delete.", repo.FullName()) + // Don't return directly, we need err in defer to cleanupRepository + err = repo_model.ErrRepoFilesAlreadyExist{ + Uname: repo.OwnerName, + Name: repo.Name, } + return nil, err + } - if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { - return fmt.Errorf("checkDaemonExportOK: %w", err) - } + // 3 - init git repository in storage + if err = initRepository(ctx, doer, repo, opts); err != nil { + return nil, fmt.Errorf("initRepository: %w", err) + } - if stdout, _, err := git.NewCommand(ctx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - rollbackRepo = repo - rollbackRepo.OwnerID = u.ID - return fmt.Errorf("CreateRepository(git update-server-info): %w", err) + // 4 - Initialize Issue Labels if selected + if len(opts.IssueLabels) > 0 { + if err = repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { + return nil, fmt.Errorf("InitializeLabels: %w", err) } + } - // update licenses - var licenses []string - if len(opts.License) > 0 { - licenses = append(licenses, ConvertLicenseName(opts.License)) + // 5 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } - stdout, _, err := git.NewCommand(ctx, "rev-parse", "HEAD").RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { - log.Error("CreateRepository(git rev-parse HEAD) in %v: Stdout: %s\nError: %v", repo, stdout, err) - rollbackRepo = repo - rollbackRepo.OwnerID = u.ID - return fmt.Errorf("CreateRepository(git rev-parse HEAD): %w", err) - } - if err := repo_model.UpdateRepoLicenses(ctx, repo, stdout, licenses); err != nil { - return err - } + // 6 - update licenses + var licenses []string + if len(opts.License) > 0 { + licenses = append(licenses, opts.License) + + var stdout string + stdout, _, err = git.NewCommand("rev-parse", "HEAD").RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) + if err != nil { + log.Error("CreateRepository(git rev-parse HEAD) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return nil, fmt.Errorf("CreateRepository(git rev-parse HEAD): %w", err) } - return nil - }); err != nil { - if rollbackRepo != nil { - if errDelete := DeleteRepositoryDirectly(ctx, doer, rollbackRepo.ID); errDelete != nil { - log.Error("Rollback deleteRepository: %v", errDelete) - } + if err = repo_model.UpdateRepoLicenses(ctx, repo, stdout, licenses); err != nil { + return nil, err } + } - return nil, err + // 7 - update repository status to be ready + if needsUpdateToReady { + repo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) + } } return repo, nil } -// CreateRepositoryByExample creates a repository for the user/organization. -func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository, overwriteOrAdopt, isFork bool) (err error) { +// createRepositoryInDB creates a repository for the user/organization. +func createRepositoryInDB(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository, isFork bool) (err error) { if err = repo_model.IsUsableRepoName(repo.Name); err != nil { return err } @@ -353,20 +349,6 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re } } - repoPath := repo_model.RepoPath(u.Name, repo.Name) - isExist, err := util.IsExist(repoPath) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return err - } - if !overwriteOrAdopt && isExist { - log.Error("Files already exist in %s and we are not going to adopt or delete.", repoPath) - return repo_model.ErrRepoFilesAlreadyExist{ - Uname: u.Name, - Name: repo.Name, - } - } - if err = db.Insert(ctx, repo); err != nil { return err } @@ -386,7 +368,8 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re } units := make([]repo_model.RepoUnit, 0, len(defaultUnits)) for _, tp := range defaultUnits { - if tp == unit.TypeIssues { + switch tp { + case unit.TypeIssues: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, @@ -396,7 +379,7 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re EnableDependencies: setting.Service.DefaultEnableDependencies, }, }) - } else if tp == unit.TypePullRequests { + case unit.TypePullRequests: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, @@ -406,13 +389,13 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re AllowRebaseUpdate: true, }, }) - } else if tp == unit.TypeProjects { + case unit.TypeProjects: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, Config: &repo_model.ProjectsConfig{ProjectsMode: repo_model.ProjectsModeAll}, }) - } else { + default: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, @@ -474,3 +457,26 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re return nil } + +func cleanupRepository(repoID int64) { + if errDelete := DeleteRepositoryDirectly(db.DefaultContext, repoID); errDelete != nil { + log.Error("cleanupRepository failed: %v", errDelete) + // add system notice + if err := system_model.CreateRepositoryNotice("DeleteRepositoryDirectly failed when cleanup repository: %v", errDelete); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } +} + +func updateGitRepoAfterCreate(ctx context.Context, repo *repo_model.Repository) error { + if err := checkDaemonExportOK(ctx, repo); err != nil { + return fmt.Errorf("checkDaemonExportOK: %w", err) + } + + if stdout, _, err := git.NewCommand("update-server-info"). + RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil { + log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return fmt.Errorf("CreateRepository(git update-server-info): %w", err) + } + return nil +} diff --git a/services/repository/create_test.go b/services/repository/create_test.go new file mode 100644 index 0000000000..fe464c1441 --- /dev/null +++ b/services/repository/create_test.go @@ -0,0 +1,57 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "os" + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" +) + +func TestCreateRepositoryDirectly(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + // a successful creating repository + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + createdRepo, err := CreateRepositoryDirectly(git.DefaultContext, user2, user2, CreateRepoOptions{ + Name: "created-repo", + }, true) + assert.NoError(t, err) + assert.NotNil(t, createdRepo) + + exist, err := util.IsExist(repo_model.RepoPath(user2.Name, createdRepo.Name)) + assert.NoError(t, err) + assert.True(t, exist) + + unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: createdRepo.Name}) + + err = DeleteRepositoryDirectly(db.DefaultContext, createdRepo.ID) + assert.NoError(t, err) + + // a failed creating because some mock data + // create the repository directory so that the creation will fail after database record created. + assert.NoError(t, os.MkdirAll(repo_model.RepoPath(user2.Name, createdRepo.Name), os.ModePerm)) + + createdRepo2, err := CreateRepositoryDirectly(db.DefaultContext, user2, user2, CreateRepoOptions{ + Name: "created-repo", + }, true) + assert.Nil(t, createdRepo2) + assert.Error(t, err) + + // assert the cleanup is successful + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: createdRepo.Name}) + + exist, err = util.IsExist(repo_model.RepoPath(user2.Name, createdRepo.Name)) + assert.NoError(t, err) + assert.False(t, exist) +} diff --git a/services/repository/delete.go b/services/repository/delete.go index 2166b4dd5c..c48d6e1d56 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -14,6 +14,7 @@ import ( git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/organization" + packages_model "code.gitea.io/gitea/models/packages" access_model "code.gitea.io/gitea/models/perm/access" project_model "code.gitea.io/gitea/models/project" repo_model "code.gitea.io/gitea/models/repo" @@ -22,17 +23,33 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" actions_module "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" + actions_service "code.gitea.io/gitea/services/actions" asymkey_service "code.gitea.io/gitea/services/asymkey" + issue_service "code.gitea.io/gitea/services/issue" "xorm.io/builder" ) +func deleteDBRepository(ctx context.Context, repoID int64) error { + if cnt, err := db.GetEngine(ctx).ID(repoID).Delete(&repo_model.Repository{}); err != nil { + return err + } else if cnt != 1 { + return repo_model.ErrRepoNotExist{ + ID: repoID, + OwnerName: "", + Name: "", + } + } + return nil +} + // DeleteRepository deletes a repository for a user or organization. // make sure if you call this func to close open sessions (sqlite will otherwise get a deadlock) -func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID int64, ignoreOrgTeams ...bool) error { +func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams ...bool) error { ctx, committer, err := db.TxContext(ctx) if err != nil { return err @@ -80,14 +97,8 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID } needRewriteKeysFile := deleted > 0 - if cnt, err := sess.ID(repoID).Delete(&repo_model.Repository{}); err != nil { + if err := deleteDBRepository(ctx, repoID); err != nil { return err - } else if cnt != 1 { - return repo_model.ErrRepoNotExist{ - ID: repoID, - OwnerName: "", - Name: "", - } } if org != nil && org.IsOrganization() { @@ -124,6 +135,14 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID return err } + // CleanupEphemeralRunnersByPickedTaskOfRepo deletes ephemeral global/org/user that have started any task of this repo + // The cannot pick a second task hardening for ephemeral runners expect that task objects remain available until runner deletion + // This method will delete affected ephemeral global/org/user runners + // &actions_model.ActionRunner{RepoID: repoID} does only handle ephemeral repository runners + if err := actions_service.CleanupEphemeralRunnersByPickedTaskOfRepo(ctx, repoID); err != nil { + return fmt.Errorf("cleanupEphemeralRunners: %w", err) + } + if err := db.DeleteBeans(ctx, &access_model.Access{RepoID: repo.ID}, &activities_model.Action{RepoID: repo.ID}, @@ -158,6 +177,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID &actions_model.ActionSchedule{RepoID: repoID}, &actions_model.ActionArtifact{RepoID: repoID}, &actions_model.ActionRunnerToken{RepoID: repoID}, + &issues_model.IssuePin{RepoID: repoID}, ); err != nil { return fmt.Errorf("deleteBeans: %w", err) } @@ -174,7 +194,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID // Delete Issues and related objects var attachmentPaths []string - if attachmentPaths, err = issues_model.DeleteIssuesByRepoID(ctx, repoID); err != nil { + if attachmentPaths, err = issue_service.DeleteIssuesByRepoID(ctx, repoID); err != nil { return err } @@ -266,6 +286,11 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID return err } + // unlink packages linked to this repository + if err = packages_model.UnlinkRepositoryFromAllPackages(ctx, repoID); err != nil { + return err + } + if err = committer.Commit(); err != nil { return err } @@ -282,8 +307,13 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID // we delete the file but the database rollback, the repository will be broken. // Remove repository files. - repoPath := repo.RepoPath() - system_model.RemoveAllWithNotice(ctx, "Delete repository files", repoPath) + if err := gitrepo.DeleteRepository(ctx, repo); err != nil { + desc := fmt.Sprintf("Delete repository files [%s]: %v", repo.FullName(), err) + // Note we use the db.DefaultContext here rather than passing in a context as the context may be cancelled + if err = system_model.CreateNotice(db.DefaultContext, system_model.NoticeRepository, desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } // Remove wiki files if repo.HasWiki() { @@ -345,7 +375,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID // DeleteOwnerRepositoriesDirectly calls DeleteRepositoryDirectly for all repos of the given owner func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User) error { for { - repos, _, err := repo_model.GetUserRepositories(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.GetUserRepositories(ctx, repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ PageSize: repo_model.RepositoryListDefaultPageSize, Page: 1, @@ -361,7 +391,7 @@ func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User break } for _, repo := range repos { - if err := DeleteRepositoryDirectly(ctx, owner, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { return fmt.Errorf("unable to delete repository %s for %s[%d]. Error: %w", repo.Name, owner.Name, owner.ID, err) } } diff --git a/services/repository/files/cherry_pick.go b/services/repository/files/cherry_pick.go index 10545e9e03..6818bb343d 100644 --- a/services/repository/files/cherry_pick.go +++ b/services/repository/files/cherry_pick.go @@ -5,6 +5,7 @@ package files import ( "context" + "errors" "fmt" "strings" @@ -32,27 +33,25 @@ func (err ErrCommitIDDoesNotMatch) Error() string { return fmt.Sprintf("file CommitID does not match [given: %s, expected: %s]", err.GivenCommitID, err.CurrentCommitID) } -// CherryPick cherrypicks or reverts a commit to the given repository +// CherryPick cherry-picks or reverts a commit to the given repository func CherryPick(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, revert bool, opts *ApplyDiffPatchOptions) (*structs.FileResponse, error) { if err := opts.Validate(ctx, repo, doer); err != nil { return nil, err } message := strings.TrimSpace(opts.Message) - author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) - - t, err := NewTemporaryUploadRepository(ctx, repo) + t, err := NewTemporaryUploadRepository(repo) if err != nil { log.Error("NewTemporaryUploadRepository failed: %v", err) } defer t.Close() - if err := t.Clone(opts.OldBranch, false); err != nil { + if err := t.Clone(ctx, opts.OldBranch, false); err != nil { return nil, err } - if err := t.SetDefaultIndex(); err != nil { + if err := t.SetDefaultIndex(ctx); err != nil { return nil, err } - if err := t.RefreshIndex(); err != nil { + if err := t.RefreshIndex(ctx); err != nil { return nil, err } @@ -102,28 +101,37 @@ func CherryPick(ctx context.Context, repo *repo_model.Repository, doer *user_mod } if conflict { - return nil, fmt.Errorf("failed to merge due to conflicts") + return nil, errors.New("failed to merge due to conflicts") } - treeHash, err := t.WriteTree() + treeHash, err := t.WriteTree(ctx) if err != nil { // likely non-sensical tree due to merge conflicts... return nil, err } // Now commit the tree - var commitHash string + commitOpts := &CommitTreeUserOptions{ + ParentCommitID: "HEAD", + TreeHash: treeHash, + CommitMessage: message, + SignOff: opts.Signoff, + DoerUser: doer, + AuthorIdentity: opts.Author, + AuthorTime: nil, + CommitterIdentity: opts.Committer, + CommitterTime: nil, + } if opts.Dates != nil { - commitHash, err = t.CommitTreeWithDate("HEAD", author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) - } else { - commitHash, err = t.CommitTree("HEAD", author, committer, treeHash, message, opts.Signoff) + commitOpts.AuthorTime, commitOpts.CommitterTime = &opts.Dates.Author, &opts.Dates.Committer } + commitHash, err := t.CommitTree(ctx, commitOpts) if err != nil { return nil, err } // Then push this tree to NewBranch - if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + if err := t.Push(ctx, doer, commitHash, opts.NewBranch); err != nil { return nil, err } diff --git a/services/repository/files/commit.go b/services/repository/files/commit.go index e0dad29273..3cc326d065 100644 --- a/services/repository/files/commit.go +++ b/services/repository/files/commit.go @@ -6,10 +6,10 @@ package files import ( "context" - asymkey_model "code.gitea.io/gitea/models/asymkey" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/structs" + asymkey_service "code.gitea.io/gitea/services/asymkey" ) // CountDivergingCommits determines how many commits a branch is ahead or behind the repository's base branch @@ -24,7 +24,7 @@ func CountDivergingCommits(ctx context.Context, repo *repo_model.Repository, bra // GetPayloadCommitVerification returns the verification information of a commit func GetPayloadCommitVerification(ctx context.Context, commit *git.Commit) *structs.PayloadCommitVerification { verification := &structs.PayloadCommitVerification{} - commitVerification := asymkey_model.ParseCommitWithSignature(ctx, commit) + commitVerification := asymkey_service.ParseCommitWithSignature(ctx, commit) if commit.Signature != nil { verification.Signature = commit.Signature.Signature verification.Payload = commit.Signature.Payload diff --git a/services/repository/files/content.go b/services/repository/files/content.go index 0ab7422ce2..2c1e88bb59 100644 --- a/services/repository/files/content.go +++ b/services/repository/files/content.go @@ -5,17 +5,18 @@ package files import ( "context" - "fmt" + "io" "net/url" "path" "strings" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" ) // ContentType repo content type @@ -23,14 +24,10 @@ type ContentType string // The string representations of different content types const ( - // ContentTypeRegular regular content type (file) - ContentTypeRegular ContentType = "file" - // ContentTypeDir dir content type (dir) - ContentTypeDir ContentType = "dir" - // ContentLink link content type (symlink) - ContentTypeLink ContentType = "symlink" - // ContentTag submodule content type (submodule) - ContentTypeSubmodule ContentType = "submodule" + ContentTypeRegular ContentType = "file" // regular content type (file) + ContentTypeDir ContentType = "dir" // dir content type (dir) + ContentTypeLink ContentType = "symlink" // link content type (symlink) + ContentTypeSubmodule ContentType = "submodule" // submodule content type (submodule) ) // String gets the string of ContentType @@ -38,67 +35,52 @@ func (ct *ContentType) String() string { return string(*ct) } -// GetContentsOrList gets the meta data of a file's contents (*ContentsResponse) if treePath not a tree -// directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag -func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, treePath, ref string) (any, error) { - if repo.IsEmpty { - return make([]any, 0), nil - } - if ref == "" { - ref = repo.DefaultBranch - } - origRef := ref - - // Check that the path given in opts.treePath is valid (not a git path) - cleanTreePath := CleanUploadFileName(treePath) - if cleanTreePath == "" && treePath != "" { - return nil, ErrFilenameInvalid{ - Path: treePath, - } - } - treePath = cleanTreePath - - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return nil, err - } - defer closer.Close() +type GetContentsOrListOptions struct { + TreePath string + IncludeSingleFileContent bool // include the file's content when the tree path is a file + IncludeLfsMetadata bool + IncludeCommitMetadata bool + IncludeCommitMessage bool +} - // Get the commit object for the ref - commit, err := gitRepo.GetCommit(ref) - if err != nil { - return nil, err +// GetContentsOrList gets the metadata of a file's contents (*ContentsResponse) if treePath not a tree +// directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag +func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, opts GetContentsOrListOptions) (ret api.ContentsExtResponse, _ error) { + entry, err := prepareGetContentsEntry(refCommit, &opts.TreePath) + if repo.IsEmpty && opts.TreePath == "" { + return api.ContentsExtResponse{DirContents: make([]*api.ContentsResponse, 0)}, nil } - - entry, err := commit.GetTreeEntryByPath(treePath) if err != nil { - return nil, err + return ret, err } + // get file contents if entry.Type() != "tree" { - return GetContents(ctx, repo, treePath, origRef, false) + ret.FileContents, err = getFileContentsByEntryInternal(ctx, repo, gitRepo, refCommit, entry, opts) + return ret, err } - // We are in a directory, so we return a list of FileContentResponse objects - var fileList []*api.ContentsResponse - - gitTree, err := commit.SubTree(treePath) + // list directory contents + gitTree, err := refCommit.Commit.SubTree(opts.TreePath) if err != nil { - return nil, err + return ret, err } entries, err := gitTree.ListEntries() if err != nil { - return nil, err + return ret, err } + ret.DirContents = make([]*api.ContentsResponse, 0, len(entries)) for _, e := range entries { - subTreePath := path.Join(treePath, e.Name()) - fileContentResponse, err := GetContents(ctx, repo, subTreePath, origRef, true) + subOpts := opts + subOpts.TreePath = path.Join(opts.TreePath, e.Name()) + subOpts.IncludeSingleFileContent = false // never include file content when listing a directory + fileContentResponse, err := GetFileContents(ctx, repo, gitRepo, refCommit, subOpts) if err != nil { - return nil, err + return ret, err } - fileList = append(fileList, fileContentResponse) + ret.DirContents = append(ret.DirContents, fileContentResponse) } - return fileList, nil + return ret, nil } // GetObjectTypeFromTreeEntry check what content is behind it @@ -117,86 +99,96 @@ func GetObjectTypeFromTreeEntry(entry *git.TreeEntry) ContentType { } } -// GetContents gets the meta data on a file's contents. Ref can be a branch, commit or tag -func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref string, forList bool) (*api.ContentsResponse, error) { - if ref == "" { - ref = repo.DefaultBranch - } - origRef := ref - +func prepareGetContentsEntry(refCommit *utils.RefCommit, treePath *string) (*git.TreeEntry, error) { // Check that the path given in opts.treePath is valid (not a git path) - cleanTreePath := CleanUploadFileName(treePath) - if cleanTreePath == "" && treePath != "" { - return nil, ErrFilenameInvalid{ - Path: treePath, - } + cleanTreePath := CleanGitTreePath(*treePath) + if cleanTreePath == "" && *treePath != "" { + return nil, ErrFilenameInvalid{Path: *treePath} } - treePath = cleanTreePath + *treePath = cleanTreePath - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return nil, err + // Only allow safe ref types + refType := refCommit.RefName.RefType() + if refType != git.RefTypeBranch && refType != git.RefTypeTag && refType != git.RefTypeCommit { + return nil, util.NewNotExistErrorf("no commit found for the ref [ref: %s]", refCommit.RefName) } - defer closer.Close() - // Get the commit object for the ref - commit, err := gitRepo.GetCommit(ref) - if err != nil { - return nil, err - } - commitID := commit.ID.String() - if len(ref) >= 4 && strings.HasPrefix(commitID, ref) { - ref = commit.ID.String() - } + return refCommit.Commit.GetTreeEntryByPath(*treePath) +} - entry, err := commit.GetTreeEntryByPath(treePath) +// GetFileContents gets the metadata on a file's contents. Ref can be a branch, commit or tag +func GetFileContents(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, opts GetContentsOrListOptions) (*api.ContentsResponse, error) { + entry, err := prepareGetContentsEntry(refCommit, &opts.TreePath) if err != nil { return nil, err } + return getFileContentsByEntryInternal(ctx, repo, gitRepo, refCommit, entry, opts) +} - refType := gitRepo.GetRefType(ref) - if refType == "invalid" { - return nil, fmt.Errorf("no commit found for the ref [ref: %s]", ref) - } - - selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(treePath) + "?ref=" + url.QueryEscape(origRef)) +func getFileContentsByEntryInternal(_ context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, entry *git.TreeEntry, opts GetContentsOrListOptions) (*api.ContentsResponse, error) { + refType := refCommit.RefName.RefType() + commit := refCommit.Commit + selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(opts.TreePath) + "?ref=" + url.QueryEscape(refCommit.InputRef)) if err != nil { return nil, err } selfURLString := selfURL.String() - err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(ref, refType != git.ObjectCommit), repo.FullName(), commitID) - if err != nil { - return nil, err - } - - lastCommit, err := commit.GetCommitByPath(treePath) - if err != nil { - return nil, err - } - // All content types have these fields in populated contentsResponse := &api.ContentsResponse{ - Name: entry.Name(), - Path: treePath, - SHA: entry.ID.String(), - LastCommitSHA: lastCommit.ID.String(), - Size: entry.Size(), - URL: &selfURLString, + Name: entry.Name(), + Path: opts.TreePath, + SHA: entry.ID.String(), + Size: entry.Size(), + URL: &selfURLString, Links: &api.FileLinksResponse{ Self: &selfURLString, }, } - // Now populate the rest of the ContentsResponse based on entry type + if opts.IncludeCommitMetadata || opts.IncludeCommitMessage { + err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(refCommit.InputRef, refType != git.RefTypeCommit), repo.FullName(), refCommit.CommitID) + if err != nil { + return nil, err + } + + lastCommit, err := refCommit.Commit.GetCommitByPath(opts.TreePath) + if err != nil { + return nil, err + } + + if opts.IncludeCommitMetadata { + contentsResponse.LastCommitSHA = util.ToPointer(lastCommit.ID.String()) + // GitHub doesn't have these fields in the response, but we could follow other similar APIs to name them + // https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits + if lastCommit.Committer != nil { + contentsResponse.LastCommitterDate = util.ToPointer(lastCommit.Committer.When) + } + if lastCommit.Author != nil { + contentsResponse.LastAuthorDate = util.ToPointer(lastCommit.Author.When) + } + } + if opts.IncludeCommitMessage { + contentsResponse.LastCommitMessage = util.ToPointer(lastCommit.Message()) + } + } + + // Now populate the rest of the ContentsResponse based on the entry type if entry.IsRegular() || entry.IsExecutable() { contentsResponse.Type = string(ContentTypeRegular) - if blobResponse, err := GetBlobBySHA(ctx, repo, gitRepo, entry.ID.String()); err != nil { - return nil, err - } else if !forList { - // We don't show the content if we are getting a list of FileContentResponses - contentsResponse.Encoding = &blobResponse.Encoding - contentsResponse.Content = &blobResponse.Content + // if it is listing the repo root dir, don't waste system resources on reading content + if opts.IncludeSingleFileContent { + blobResponse, err := GetBlobBySHA(repo, gitRepo, entry.ID.String()) + if err != nil { + return nil, err + } + contentsResponse.Encoding, contentsResponse.Content = blobResponse.Encoding, blobResponse.Content + contentsResponse.LfsOid, contentsResponse.LfsSize = blobResponse.LfsOid, blobResponse.LfsSize + } else if opts.IncludeLfsMetadata { + contentsResponse.LfsOid, contentsResponse.LfsSize, err = parsePossibleLfsPointerBlob(gitRepo, entry.ID.String()) + if err != nil { + return nil, err + } } } else if entry.IsDir() { contentsResponse.Type = string(ContentTypeDir) @@ -210,7 +202,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref contentsResponse.Target = &targetFromContent } else if entry.IsSubModule() { contentsResponse.Type = string(ContentTypeSubmodule) - submodule, err := commit.GetSubModule(treePath) + submodule, err := commit.GetSubModule(opts.TreePath) if err != nil { return nil, err } @@ -220,7 +212,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref } // Handle links if entry.IsRegular() || entry.IsLink() || entry.IsExecutable() { - downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(opts.TreePath)) if err != nil { return nil, err } @@ -228,7 +220,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref contentsResponse.DownloadURL = &downloadURLString } if !entry.IsSubModule() { - htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(opts.TreePath)) if err != nil { return nil, err } @@ -248,49 +240,59 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref return contentsResponse, nil } -// GetBlobBySHA get the GitBlobResponse of a repository using a sha hash. -func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) { +func GetBlobBySHA(repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) { gitBlob, err := gitRepo.GetBlob(sha) if err != nil { return nil, err } - content := "" - if gitBlob.Size() <= setting.API.DefaultMaxBlobSize { - content, err = gitBlob.GetBlobContentBase64() - if err != nil { - return nil, err - } + ret := &api.GitBlobResponse{ + SHA: gitBlob.ID.String(), + URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), + Size: gitBlob.Size(), } - return &api.GitBlobResponse{ - SHA: gitBlob.ID.String(), - URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), - Size: gitBlob.Size(), - Encoding: "base64", - Content: content, - }, nil -} -// TryGetContentLanguage tries to get the (linguist) language of the file content -func TryGetContentLanguage(gitRepo *git.Repository, commitID, treePath string) (string, error) { - indexFilename, worktree, deleteTemporaryFile, err := gitRepo.ReadTreeToTemporaryIndex(commitID) - if err != nil { - return "", err + blobSize := gitBlob.Size() + if blobSize > setting.API.DefaultMaxBlobSize { + return ret, nil } - defer deleteTemporaryFile() + var originContent *strings.Builder + if 0 < blobSize && blobSize < lfs.MetaFileMaxSize { + originContent = &strings.Builder{} + } - filename2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{ - CachedOnly: true, - Attributes: []string{git.AttributeLinguistLanguage, git.AttributeGitlabLanguage}, - Filenames: []string{treePath}, - IndexFile: indexFilename, - WorkTree: worktree, - }) + content, err := gitBlob.GetBlobContentBase64(originContent) if err != nil { - return "", err + return nil, err + } + + ret.Encoding, ret.Content = util.ToPointer("base64"), &content + if originContent != nil { + ret.LfsOid, ret.LfsSize = parsePossibleLfsPointerBuffer(strings.NewReader(originContent.String())) } + return ret, nil +} - language := git.TryReadLanguageAttribute(filename2attribute2info[treePath]) +func parsePossibleLfsPointerBuffer(r io.Reader) (*string, *int64) { + p, _ := lfs.ReadPointer(r) + if p.IsValid() { + return &p.Oid, &p.Size + } + return nil, nil +} - return language.Value(), nil +func parsePossibleLfsPointerBlob(gitRepo *git.Repository, sha string) (*string, *int64, error) { + gitBlob, err := gitRepo.GetBlob(sha) + if err != nil { + return nil, nil, err + } + if gitBlob.Size() > lfs.MetaFileMaxSize { + return nil, nil, nil // not a LFS pointer + } + buf, err := gitBlob.GetBlobContent(lfs.MetaFileMaxSize) + if err != nil { + return nil, nil, err + } + oid, size := parsePossibleLfsPointerBuffer(strings.NewReader(buf)) + return oid, size, nil } diff --git a/services/repository/files/content_test.go b/services/repository/files/content_test.go index 7cb46c0bb6..d72f918074 100644 --- a/services/repository/files/content_test.go +++ b/services/repository/files/content_test.go @@ -7,8 +7,8 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/gitrepo" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/contexttest" _ "code.gitea.io/gitea/models/actions" @@ -20,36 +20,6 @@ func TestMain(m *testing.M) { unittest.MainTest(m) } -func getExpectedReadmeContentsResponse() *api.ContentsResponse { - treePath := "README.md" - sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" - encoding := "base64" - content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" - selfURL := "https://try.gitea.io/api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" - htmlURL := "https://try.gitea.io/user2/repo1/src/branch/master/" + treePath - gitURL := "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/" + sha - downloadURL := "https://try.gitea.io/user2/repo1/raw/branch/master/" + treePath - return &api.ContentsResponse{ - Name: treePath, - Path: treePath, - SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", - LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - Type: "file", - Size: 30, - Encoding: &encoding, - Content: &content, - URL: &selfURL, - HTMLURL: &htmlURL, - GitURL: &gitURL, - DownloadURL: &downloadURL, - Links: &api.FileLinksResponse{ - Self: &selfURL, - GitURL: &gitURL, - HTMLURL: &htmlURL, - }, - } -} - func TestGetContents(t *testing.T) { unittest.PrepareTestEnv(t) ctx, _ := contexttest.MockContext(t, "user2/repo1") @@ -58,195 +28,22 @@ func TestGetContents(t *testing.T) { contexttest.LoadRepoCommit(t, ctx) contexttest.LoadUser(t, ctx, 2) contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - treePath := "README.md" - ref := ctx.Repo.Repository.DefaultBranch - - expectedContentsResponse := getExpectedReadmeContentsResponse() - - t.Run("Get README.md contents with GetContents(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContents(ctx, ctx.Repo.Repository, treePath, ref, false) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContents(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContents(ctx, ctx.Repo.Repository, treePath, "", false) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) -} - -func TestGetContentsOrListForDir(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - treePath := "" // root dir - ref := ctx.Repo.Repository.DefaultBranch - - readmeContentsResponse := getExpectedReadmeContentsResponse() - // because will be in a list, doesn't have encoding and content - readmeContentsResponse.Encoding = nil - readmeContentsResponse.Content = nil - - expectedContentsListResponse := []*api.ContentsResponse{ - readmeContentsResponse, - } - - t.Run("Get root dir contents with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref) - assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get root dir contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, "") - assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) - assert.NoError(t, err) - }) -} - -func TestGetContentsOrListForFile(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - treePath := "README.md" - ref := ctx.Repo.Repository.DefaultBranch - - expectedContentsResponse := getExpectedReadmeContentsResponse() - - t.Run("Get README.md contents with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, "") - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) -} - -func TestGetContentsErrors(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - treePath := "README.md" - ref := repo.DefaultBranch - - t.Run("bad treePath", func(t *testing.T) { - badTreePath := "bad/tree.md" - fileContentResponse, err := GetContents(ctx, repo, badTreePath, ref, false) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: , rel_path: bad]") - assert.Nil(t, fileContentResponse) - }) - - t.Run("bad ref", func(t *testing.T) { - badRef := "bad_ref" - fileContentResponse, err := GetContents(ctx, repo, treePath, badRef, false) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") - assert.Nil(t, fileContentResponse) - }) -} - -func TestGetContentsOrListErrors(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - repo := ctx.Repo.Repository - treePath := "README.md" - ref := repo.DefaultBranch - - t.Run("bad treePath", func(t *testing.T) { - badTreePath := "bad/tree.md" - fileContentResponse, err := GetContentsOrList(ctx, repo, badTreePath, ref) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: , rel_path: bad]") - assert.Nil(t, fileContentResponse) - }) - - t.Run("bad ref", func(t *testing.T) { - badRef := "bad_ref" - fileContentResponse, err := GetContentsOrList(ctx, repo, treePath, badRef) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") - assert.Nil(t, fileContentResponse) - }) -} - -func TestGetContentsOrListOfEmptyRepos(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user30/empty") - ctx.SetPathParam("id", "52") - contexttest.LoadRepo(t, ctx, 52) - contexttest.LoadUser(t, ctx, 30) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - - t.Run("empty repo", func(t *testing.T) { - contents, err := GetContentsOrList(ctx, repo, "", "") + // GetContentsOrList's behavior is fully tested in integration tests, so we don't need to test it here. + + t.Run("GetBlobBySHA", func(t *testing.T) { + sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" + ctx.SetPathParam("id", "1") + ctx.SetPathParam("sha", sha) + gbr, err := GetBlobBySHA(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("sha")) + expectedGBR := &api.GitBlobResponse{ + Content: util.ToPointer("dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK"), + Encoding: util.ToPointer("base64"), + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Size: 180, + } assert.NoError(t, err) - assert.Empty(t, contents) + assert.Equal(t, expectedGBR, gbr) }) } - -func TestGetBlobBySHA(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" - ctx.SetPathParam("id", "1") - ctx.SetPathParam("sha", sha) - - gitRepo, err := gitrepo.OpenRepository(ctx, ctx.Repo.Repository) - if err != nil { - t.Fail() - } - - gbr, err := GetBlobBySHA(ctx, ctx.Repo.Repository, gitRepo, ctx.PathParam("sha")) - expectedGBR := &api.GitBlobResponse{ - Content: "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK", - Encoding: "base64", - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", - SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - Size: 180, - } - assert.NoError(t, err) - assert.Equal(t, expectedGBR, gbr) -} diff --git a/services/repository/files/diff.go b/services/repository/files/diff.go index bf8b938e21..50d01f9d7c 100644 --- a/services/repository/files/diff.go +++ b/services/repository/files/diff.go @@ -16,27 +16,27 @@ func GetDiffPreview(ctx context.Context, repo *repo_model.Repository, branch, tr if branch == "" { branch = repo.DefaultBranch } - t, err := NewTemporaryUploadRepository(ctx, repo) + t, err := NewTemporaryUploadRepository(repo) if err != nil { return nil, err } defer t.Close() - if err := t.Clone(branch, true); err != nil { + if err := t.Clone(ctx, branch, true); err != nil { return nil, err } - if err := t.SetDefaultIndex(); err != nil { + if err := t.SetDefaultIndex(ctx); err != nil { return nil, err } // Add the object to the database - objectHash, err := t.HashObject(strings.NewReader(content)) + objectHash, err := t.HashObjectAndWrite(ctx, strings.NewReader(content)) if err != nil { return nil, err } // Add the object to the index - if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil { + if err := t.AddObjectToIndex(ctx, "100644", objectHash, treePath); err != nil { return nil, err } - return t.DiffIndex() + return t.DiffIndex(ctx) } diff --git a/services/repository/files/diff_test.go b/services/repository/files/diff_test.go index b7bdcd8ecf..ae702e4189 100644 --- a/services/repository/files/diff_test.go +++ b/services/repository/files/diff_test.go @@ -30,14 +30,11 @@ func TestGetDiffPreview(t *testing.T) { content := "# repo1\n\nDescription for repo1\nthis is a new line" expectedDiff := &gitdiff.Diff{ - TotalAddition: 2, - TotalDeletion: 1, Files: []*gitdiff.DiffFile{ { Name: "README.md", OldName: "README.md", NameHash: "8ec9a00bfd09b3190ac6b22251dbb1aa95a0579d", - Index: 1, Addition: 2, Deletion: 1, Type: 2, @@ -50,7 +47,6 @@ func TestGetDiffPreview(t *testing.T) { Sections: []*gitdiff.DiffSection{ { FileName: "README.md", - Name: "", Lines: []*gitdiff.DiffLine{ { LeftIdx: 0, @@ -114,7 +110,6 @@ func TestGetDiffPreview(t *testing.T) { }, IsIncomplete: false, } - expectedDiff.NumFiles = len(expectedDiff.Files) t.Run("with given branch", func(t *testing.T) { diff, err := GetDiffPreview(ctx, ctx.Repo.Repository, branch, treePath, content) @@ -123,7 +118,7 @@ func TestGetDiffPreview(t *testing.T) { assert.NoError(t, err) bs, err := json.Marshal(diff) assert.NoError(t, err) - assert.EqualValues(t, string(expectedBs), string(bs)) + assert.Equal(t, string(expectedBs), string(bs)) }) t.Run("empty branch, same results", func(t *testing.T) { @@ -133,7 +128,7 @@ func TestGetDiffPreview(t *testing.T) { assert.NoError(t, err) bs, err := json.Marshal(diff) assert.NoError(t, err) - assert.EqualValues(t, expectedBs, bs) + assert.Equal(t, expectedBs, bs) }) } diff --git a/services/repository/files/file.go b/services/repository/files/file.go index d7ca8e79e5..13d171d139 100644 --- a/services/repository/files/file.go +++ b/services/repository/files/file.go @@ -5,26 +5,48 @@ package files import ( "context" + "errors" "fmt" "net/url" "strings" "time" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" ) -func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch string, treeNames []string) (*api.FilesResponse, error) { - files := []*api.ContentsResponse{} - for _, file := range treeNames { - fileContents, _ := GetContents(ctx, repo, file, branch, false) // ok if fails, then will be nil +func GetContentsListFromTreePaths(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, treePaths []string) (files []*api.ContentsResponse) { + var size int64 + for _, treePath := range treePaths { + // ok if fails, then will be nil + fileContents, _ := GetFileContents(ctx, repo, gitRepo, refCommit, GetContentsOrListOptions{ + TreePath: treePath, + IncludeSingleFileContent: true, + IncludeCommitMetadata: true, + }) + if fileContents != nil && fileContents.Content != nil && *fileContents.Content != "" { + // if content isn't empty (e.g., due to the single blob being too large), add file size to response size + size += int64(len(*fileContents.Content)) + } + if size > setting.API.DefaultMaxResponseSize { + break // stop if max response size would be exceeded + } files = append(files, fileContents) + if len(files) == setting.API.DefaultPagingNum { + break // stop if paging num reached + } } - fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil - verification := GetPayloadCommitVerification(ctx, commit) + return files +} + +func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, treeNames []string) (*api.FilesResponse, error) { + files := GetContentsListFromTreePaths(ctx, repo, gitRepo, refCommit, treeNames) + fileCommitResponse, _ := GetFileCommitResponse(repo, refCommit.Commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, refCommit.Commit) filesResponse := &api.FilesResponse{ Files: files, Commit: fileCommitResponse, @@ -33,19 +55,6 @@ func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository return filesResponse, nil } -// GetFileResponseFromCommit Constructs a FileResponse from a Commit object -func GetFileResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch, treeName string) (*api.FileResponse, error) { - fileContents, _ := GetContents(ctx, repo, treeName, branch, false) // ok if fails, then will be nil - fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil - verification := GetPayloadCommitVerification(ctx, commit) - fileResponse := &api.FileResponse{ - Content: fileContents, - Commit: fileCommitResponse, - Verification: verification, - } - return fileResponse, nil -} - // constructs a FileResponse with the file at the index from FilesResponse func GetFileResponseFromFilesResponse(filesResponse *api.FilesResponse, index int) *api.FileResponse { content := &api.ContentsResponse{} @@ -63,10 +72,10 @@ func GetFileResponseFromFilesResponse(filesResponse *api.FilesResponse, index in // GetFileCommitResponse Constructs a FileCommitResponse from a Commit object func GetFileCommitResponse(repo *repo_model.Repository, commit *git.Commit) (*api.FileCommitResponse, error) { if repo == nil { - return nil, fmt.Errorf("repo cannot be nil") + return nil, errors.New("repo cannot be nil") } if commit == nil { - return nil, fmt.Errorf("commit cannot be nil") + return nil, errors.New("commit cannot be nil") } commitURL, _ := url.Parse(repo.APIURL() + "/git/commits/" + url.PathEscape(commit.ID.String())) commitTreeURL, _ := url.Parse(repo.APIURL() + "/git/trees/" + url.PathEscape(commit.Tree.ID.String())) @@ -111,51 +120,6 @@ func GetFileCommitResponse(repo *repo_model.Repository, commit *git.Commit) (*ap return fileCommit, nil } -// GetAuthorAndCommitterUsers Gets the author and committer user objects from the IdentityOptions -func GetAuthorAndCommitterUsers(author, committer *IdentityOptions, doer *user_model.User) (authorUser, committerUser *user_model.User) { - // Committer and author are optional. If they are not the doer (not same email address) - // then we use bogus User objects for them to store their FullName and Email. - // If only one of the two are provided, we set both of them to it. - // If neither are provided, both are the doer. - if committer != nil && committer.Email != "" { - if doer != nil && strings.EqualFold(doer.Email, committer.Email) { - committerUser = doer // the committer is the doer, so will use their user object - if committer.Name != "" { - committerUser.FullName = committer.Name - } - } else { - committerUser = &user_model.User{ - FullName: committer.Name, - Email: committer.Email, - } - } - } - if author != nil && author.Email != "" { - if doer != nil && strings.EqualFold(doer.Email, author.Email) { - authorUser = doer // the author is the doer, so will use their user object - if authorUser.Name != "" { - authorUser.FullName = author.Name - } - } else { - authorUser = &user_model.User{ - FullName: author.Name, - Email: author.Email, - } - } - } - if authorUser == nil { - if committerUser != nil { - authorUser = committerUser // No valid author was given so use the committer - } else if doer != nil { - authorUser = doer // No valid author was given and no valid committer so use the doer - } - } - if committerUser == nil { - committerUser = authorUser // No valid committer so use the author as the committer (was set to a valid user above) - } - return authorUser, committerUser -} - // ErrFilenameInvalid represents a "FilenameInvalid" kind of error. type ErrFilenameInvalid struct { Path string @@ -175,15 +139,17 @@ func (err ErrFilenameInvalid) Unwrap() error { return util.ErrInvalidArgument } -// CleanUploadFileName Trims a filename and returns empty string if it is a .git directory -func CleanUploadFileName(name string) string { - // Rebase the filename +// CleanGitTreePath cleans a tree path for git, it returns an empty string the path is invalid (e.g.: contains ".git" part) +func CleanGitTreePath(name string) string { name = util.PathJoinRel(name) // Git disallows any filenames to have a .git directory in them. - for _, part := range strings.Split(name, "/") { + for part := range strings.SplitSeq(name, "/") { if strings.ToLower(part) == ".git" { return "" } } + if name == "." { + name = "" + } return name } diff --git a/services/repository/files/file_test.go b/services/repository/files/file_test.go index 52c0574883..cdb6a266ff 100644 --- a/services/repository/files/file_test.go +++ b/services/repository/files/file_test.go @@ -6,115 +6,22 @@ package files import ( "testing" - "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/gitrepo" - "code.gitea.io/gitea/modules/setting" - api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/services/contexttest" - "github.com/stretchr/testify/assert" ) func TestCleanUploadFileName(t *testing.T) { - t.Run("Clean regular file", func(t *testing.T) { - name := "this/is/test" - cleanName := CleanUploadFileName(name) - expectedCleanName := name - assert.EqualValues(t, expectedCleanName, cleanName) - }) - - t.Run("Clean a .git path", func(t *testing.T) { - name := "this/is/test/.git" - cleanName := CleanUploadFileName(name) - expectedCleanName := "" - assert.EqualValues(t, expectedCleanName, cleanName) - }) -} - -func getExpectedFileResponse() *api.FileResponse { - treePath := "README.md" - sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" - encoding := "base64" - content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" - selfURL := setting.AppURL + "api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" - htmlURL := setting.AppURL + "user2/repo1/src/branch/master/" + treePath - gitURL := setting.AppURL + "api/v1/repos/user2/repo1/git/blobs/" + sha - downloadURL := setting.AppURL + "user2/repo1/raw/branch/master/" + treePath - return &api.FileResponse{ - Content: &api.ContentsResponse{ - Name: treePath, - Path: treePath, - SHA: sha, - LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - Type: "file", - Size: 30, - Encoding: &encoding, - Content: &content, - URL: &selfURL, - HTMLURL: &htmlURL, - GitURL: &gitURL, - DownloadURL: &downloadURL, - Links: &api.FileLinksResponse{ - Self: &selfURL, - GitURL: &gitURL, - HTMLURL: &htmlURL, - }, - }, - Commit: &api.FileCommitResponse{ - CommitMeta: api.CommitMeta{ - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", - SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - }, - HTMLURL: "https://try.gitea.io/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d", - Author: &api.CommitUser{ - Identity: api.Identity{ - Name: "user1", - Email: "address1@example.com", - }, - Date: "2017-03-19T20:47:59Z", - }, - Committer: &api.CommitUser{ - Identity: api.Identity{ - Name: "Ethan Koenig", - Email: "ethantkoenig@gmail.com", - }, - Date: "2017-03-19T20:47:59Z", - }, - Parents: []*api.CommitMeta{}, - Message: "Initial commit\n", - Tree: &api.CommitMeta{ - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/2a2f1d4670728a2e10049e345bd7a276468beab6", - SHA: "2a2f1d4670728a2e10049e345bd7a276468beab6", - }, - }, - Verification: &api.PayloadCommitVerification{ - Verified: false, - Reason: "gpg.error.not_signed_commit", - Signature: "", - Payload: "", - }, + cases := []struct { + input, expected string + }{ + {"", ""}, + {".", ""}, + {"a/./b", "a/b"}, + {"a.git", "a.git"}, + {".git/b", ""}, + {"a/.git", ""}, + {"/a/../../b", "b"}, + } + for _, c := range cases { + assert.Equal(t, c.expected, CleanGitTreePath(c.input), "input: %q", c.input) } -} - -func TestGetFileResponseFromCommit(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - branch := repo.DefaultBranch - treePath := "README.md" - gitRepo, _ := gitrepo.OpenRepository(ctx, repo) - defer gitRepo.Close() - commit, _ := gitRepo.GetBranchCommit(branch) - expectedFileResponse := getExpectedFileResponse() - - fileResponse, err := GetFileResponseFromCommit(ctx, repo, commit, branch, treePath) - assert.NoError(t, err) - assert.EqualValues(t, expectedFileResponse, fileResponse) } diff --git a/services/repository/files/patch.go b/services/repository/files/patch.go index 38c17b4073..11a8744b7f 100644 --- a/services/repository/files/patch.go +++ b/services/repository/files/patch.go @@ -12,7 +12,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -45,7 +44,6 @@ type ApplyDiffPatchOptions struct { NewBranch string Message string Content string - SHA string Author *IdentityOptions Committer *IdentityOptions Dates *CommitDateOptions @@ -62,29 +60,26 @@ func (opts *ApplyDiffPatchOptions) Validate(ctx context.Context, repo *repo_mode opts.NewBranch = opts.OldBranch } - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return err - } - defer closer.Close() - // oldBranch must exist for this operation - if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil { + if exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.OldBranch); err != nil { return err + } else if !exist { + return git_model.ErrBranchNotExist{ + BranchName: opts.OldBranch, + } } // A NewBranch can be specified for the patch to be applied to. // Check to make sure the branch does not already exist, otherwise we can't proceed. // If we aren't branching to a new branch, make sure user can commit to the given branch if opts.NewBranch != opts.OldBranch { - existingBranch, err := gitRepo.GetBranch(opts.NewBranch) - if existingBranch != nil { + exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.NewBranch) + if err != nil { + return err + } else if exist { return git_model.ErrBranchAlreadyExists{ BranchName: opts.NewBranch, } } - if err != nil && !git.IsErrBranchNotExist(err) { - return err - } } else { protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, opts.OldBranch) if err != nil { @@ -126,17 +121,15 @@ func ApplyDiffPatch(ctx context.Context, repo *repo_model.Repository, doer *user message := strings.TrimSpace(opts.Message) - author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) - - t, err := NewTemporaryUploadRepository(ctx, repo) + t, err := NewTemporaryUploadRepository(repo) if err != nil { log.Error("NewTemporaryUploadRepository failed: %v", err) } defer t.Close() - if err := t.Clone(opts.OldBranch, true); err != nil { + if err := t.Clone(ctx, opts.OldBranch, true); err != nil { return nil, err } - if err := t.SetDefaultIndex(); err != nil { + if err := t.SetDefaultIndex(ctx); err != nil { return nil, err } @@ -166,12 +159,12 @@ func ApplyDiffPatch(ctx context.Context, repo *repo_model.Repository, doer *user stdout := &strings.Builder{} stderr := &strings.Builder{} - cmdApply := git.NewCommand(ctx, "apply", "--index", "--recount", "--cached", "--ignore-whitespace", "--whitespace=fix", "--binary") + cmdApply := git.NewCommand("apply", "--index", "--recount", "--cached", "--ignore-whitespace", "--whitespace=fix", "--binary") if git.DefaultFeatures().CheckVersionAtLeast("2.32") { cmdApply.AddArguments("-3") } - if err := cmdApply.Run(&git.RunOpts{ + if err := cmdApply.Run(ctx, &git.RunOpts{ Dir: t.basePath, Stdout: stdout, Stderr: stderr, @@ -181,24 +174,33 @@ func ApplyDiffPatch(ctx context.Context, repo *repo_model.Repository, doer *user } // Now write the tree - treeHash, err := t.WriteTree() + treeHash, err := t.WriteTree(ctx) if err != nil { return nil, err } // Now commit the tree - var commitHash string + commitOpts := &CommitTreeUserOptions{ + ParentCommitID: "HEAD", + TreeHash: treeHash, + CommitMessage: message, + SignOff: opts.Signoff, + DoerUser: doer, + AuthorIdentity: opts.Author, + AuthorTime: nil, + CommitterIdentity: opts.Committer, + CommitterTime: nil, + } if opts.Dates != nil { - commitHash, err = t.CommitTreeWithDate("HEAD", author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) - } else { - commitHash, err = t.CommitTree("HEAD", author, committer, treeHash, message, opts.Signoff) + commitOpts.AuthorTime, commitOpts.CommitterTime = &opts.Dates.Author, &opts.Dates.Committer } + commitHash, err := t.CommitTree(ctx, commitOpts) if err != nil { return nil, err } // Then push this tree to NewBranch - if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + if err := t.Push(ctx, doer, commitHash, opts.NewBranch); err != nil { return nil, err } diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index 138af991f9..c2f61c8223 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -6,6 +6,7 @@ package files import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -19,44 +20,45 @@ import ( "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" "code.gitea.io/gitea/services/gitdiff" ) // TemporaryUploadRepository is a type to wrap our upload repositories as a shallow clone type TemporaryUploadRepository struct { - ctx context.Context repo *repo_model.Repository gitRepo *git.Repository basePath string + cleanup func() } // NewTemporaryUploadRepository creates a new temporary upload repository -func NewTemporaryUploadRepository(ctx context.Context, repo *repo_model.Repository) (*TemporaryUploadRepository, error) { - basePath, err := repo_module.CreateTemporaryPath("upload") +func NewTemporaryUploadRepository(repo *repo_model.Repository) (*TemporaryUploadRepository, error) { + basePath, cleanup, err := repo_module.CreateTemporaryPath("upload") if err != nil { return nil, err } - t := &TemporaryUploadRepository{ctx: ctx, repo: repo, basePath: basePath} + t := &TemporaryUploadRepository{repo: repo, basePath: basePath, cleanup: cleanup} return t, nil } // Close the repository cleaning up all files func (t *TemporaryUploadRepository) Close() { defer t.gitRepo.Close() - if err := repo_module.RemoveTemporaryPath(t.basePath); err != nil { - log.Error("Failed to remove temporary path %s: %v", t.basePath, err) + if t.cleanup != nil { + t.cleanup() } } // Clone the base repository to our path and set branch as the HEAD -func (t *TemporaryUploadRepository) Clone(branch string, bare bool) error { - cmd := git.NewCommand(t.ctx, "clone", "-s", "-b").AddDynamicArguments(branch, t.repo.RepoPath(), t.basePath) +func (t *TemporaryUploadRepository) Clone(ctx context.Context, branch string, bare bool) error { + cmd := git.NewCommand("clone", "-s", "-b").AddDynamicArguments(branch, t.repo.RepoPath(), t.basePath) if bare { cmd.AddArguments("--bare") } - if _, _, err := cmd.RunStdString(nil); err != nil { + if _, _, err := cmd.RunStdString(ctx, nil); err != nil { stderr := err.Error() if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched { return git.ErrBranchNotExist{ @@ -72,7 +74,7 @@ func (t *TemporaryUploadRepository) Clone(branch string, bare bool) error { } return fmt.Errorf("Clone: %w %s", err, stderr) } - gitRepo, err := git.OpenRepository(t.ctx, t.basePath) + gitRepo, err := git.OpenRepository(ctx, t.basePath) if err != nil { return err } @@ -81,11 +83,11 @@ func (t *TemporaryUploadRepository) Clone(branch string, bare bool) error { } // Init the repository -func (t *TemporaryUploadRepository) Init(objectFormatName string) error { - if err := git.InitRepository(t.ctx, t.basePath, false, objectFormatName); err != nil { +func (t *TemporaryUploadRepository) Init(ctx context.Context, objectFormatName string) error { + if err := git.InitRepository(ctx, t.basePath, false, objectFormatName); err != nil { return err } - gitRepo, err := git.OpenRepository(t.ctx, t.basePath) + gitRepo, err := git.OpenRepository(ctx, t.basePath) if err != nil { return err } @@ -94,28 +96,28 @@ func (t *TemporaryUploadRepository) Init(objectFormatName string) error { } // SetDefaultIndex sets the git index to our HEAD -func (t *TemporaryUploadRepository) SetDefaultIndex() error { - if _, _, err := git.NewCommand(t.ctx, "read-tree", "HEAD").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { +func (t *TemporaryUploadRepository) SetDefaultIndex(ctx context.Context) error { + if _, _, err := git.NewCommand("read-tree", "HEAD").RunStdString(ctx, &git.RunOpts{Dir: t.basePath}); err != nil { return fmt.Errorf("SetDefaultIndex: %w", err) } return nil } // RefreshIndex looks at the current index and checks to see if merges or updates are needed by checking stat() information. -func (t *TemporaryUploadRepository) RefreshIndex() error { - if _, _, err := git.NewCommand(t.ctx, "update-index", "--refresh").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { +func (t *TemporaryUploadRepository) RefreshIndex(ctx context.Context) error { + if _, _, err := git.NewCommand("update-index", "--refresh").RunStdString(ctx, &git.RunOpts{Dir: t.basePath}); err != nil { return fmt.Errorf("RefreshIndex: %w", err) } return nil } // LsFiles checks if the given filename arguments are in the index -func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, error) { +func (t *TemporaryUploadRepository) LsFiles(ctx context.Context, filenames ...string) ([]string, error) { stdOut := new(bytes.Buffer) stdErr := new(bytes.Buffer) - if err := git.NewCommand(t.ctx, "ls-files", "-z").AddDashesAndList(filenames...). - Run(&git.RunOpts{ + if err := git.NewCommand("ls-files", "-z").AddDashesAndList(filenames...). + Run(ctx, &git.RunOpts{ Dir: t.basePath, Stdout: stdOut, Stderr: stdErr, @@ -126,7 +128,7 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro } fileList := make([]string, 0, len(filenames)) - for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { + for line := range bytes.SplitSeq(stdOut.Bytes(), []byte{'\000'}) { fileList = append(fileList, string(line)) } @@ -134,7 +136,7 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro } // RemoveFilesFromIndex removes the given files from the index -func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) error { +func (t *TemporaryUploadRepository) RemoveFilesFromIndex(ctx context.Context, filenames ...string) error { objFmt, err := t.gitRepo.GetObjectFormat() if err != nil { return fmt.Errorf("unable to get object format for temporary repo: %q, error: %w", t.repo.FullName(), err) @@ -150,8 +152,8 @@ func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) er } } - if err := git.NewCommand(t.ctx, "update-index", "--remove", "-z", "--index-info"). - Run(&git.RunOpts{ + if err := git.NewCommand("update-index", "--remove", "-z", "--index-info"). + Run(ctx, &git.RunOpts{ Dir: t.basePath, Stdin: stdIn, Stdout: stdOut, @@ -162,13 +164,13 @@ func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) er return nil } -// HashObject writes the provided content to the object db and returns its hash -func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) { +// HashObjectAndWrite writes the provided content to the object db and returns its hash +func (t *TemporaryUploadRepository) HashObjectAndWrite(ctx context.Context, content io.Reader) (string, error) { stdOut := new(bytes.Buffer) stdErr := new(bytes.Buffer) - if err := git.NewCommand(t.ctx, "hash-object", "-w", "--stdin"). - Run(&git.RunOpts{ + if err := git.NewCommand("hash-object", "-w", "--stdin"). + Run(ctx, &git.RunOpts{ Dir: t.basePath, Stdin: content, Stdout: stdOut, @@ -182,8 +184,8 @@ func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error } // AddObjectToIndex adds the provided object hash to the index with the provided mode and path -func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error { - if _, _, err := git.NewCommand(t.ctx, "update-index", "--add", "--replace", "--cacheinfo").AddDynamicArguments(mode, objectHash, objectPath).RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { +func (t *TemporaryUploadRepository) AddObjectToIndex(ctx context.Context, mode, objectHash, objectPath string) error { + if _, _, err := git.NewCommand("update-index", "--add", "--replace", "--cacheinfo").AddDynamicArguments(mode, objectHash, objectPath).RunStdString(ctx, &git.RunOpts{Dir: t.basePath}); err != nil { stderr := err.Error() if matched, _ := regexp.MatchString(".*Invalid path '.*", stderr); matched { return ErrFilePathInvalid{ @@ -198,8 +200,8 @@ func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPat } // WriteTree writes the current index as a tree to the object db and returns its hash -func (t *TemporaryUploadRepository) WriteTree() (string, error) { - stdout, _, err := git.NewCommand(t.ctx, "write-tree").RunStdString(&git.RunOpts{Dir: t.basePath}) +func (t *TemporaryUploadRepository) WriteTree(ctx context.Context) (string, error) { + stdout, _, err := git.NewCommand("write-tree").RunStdString(ctx, &git.RunOpts{Dir: t.basePath}) if err != nil { log.Error("Unable to write tree in temporary repo: %s(%s): Error: %v", t.repo.FullName(), t.basePath, err) return "", fmt.Errorf("Unable to write-tree in temporary repo for: %s Error: %w", t.repo.FullName(), err) @@ -208,16 +210,16 @@ func (t *TemporaryUploadRepository) WriteTree() (string, error) { } // GetLastCommit gets the last commit ID SHA of the repo -func (t *TemporaryUploadRepository) GetLastCommit() (string, error) { - return t.GetLastCommitByRef("HEAD") +func (t *TemporaryUploadRepository) GetLastCommit(ctx context.Context) (string, error) { + return t.GetLastCommitByRef(ctx, "HEAD") } // GetLastCommitByRef gets the last commit ID SHA of the repo by ref -func (t *TemporaryUploadRepository) GetLastCommitByRef(ref string) (string, error) { +func (t *TemporaryUploadRepository) GetLastCommitByRef(ctx context.Context, ref string) (string, error) { if ref == "" { ref = "HEAD" } - stdout, _, err := git.NewCommand(t.ctx, "rev-parse").AddDynamicArguments(ref).RunStdString(&git.RunOpts{Dir: t.basePath}) + stdout, _, err := git.NewCommand("rev-parse").AddDynamicArguments(ref).RunStdString(ctx, &git.RunOpts{Dir: t.basePath}) if err != nil { log.Error("Unable to get last ref for %s in temporary repo: %s(%s): Error: %v", ref, t.repo.FullName(), t.basePath, err) return "", fmt.Errorf("Unable to rev-parse %s in temporary repo for: %s Error: %w", ref, t.repo.FullName(), err) @@ -225,15 +227,53 @@ func (t *TemporaryUploadRepository) GetLastCommitByRef(ref string) (string, erro return strings.TrimSpace(stdout), nil } -// CommitTree creates a commit from a given tree for the user with provided message -func (t *TemporaryUploadRepository) CommitTree(parent string, author, committer *user_model.User, treeHash, message string, signoff bool) (string, error) { - return t.CommitTreeWithDate(parent, author, committer, treeHash, message, signoff, time.Now(), time.Now()) +type CommitTreeUserOptions struct { + ParentCommitID string + TreeHash string + CommitMessage string + SignOff bool + + DoerUser *user_model.User + + AuthorIdentity *IdentityOptions // if nil, use doer + AuthorTime *time.Time // if nil, use now + CommitterIdentity *IdentityOptions + CommitterTime *time.Time +} + +func makeGitUserSignature(doer *user_model.User, identity, other *IdentityOptions) *git.Signature { + gitSig := &git.Signature{} + if identity != nil { + gitSig.Name, gitSig.Email = identity.GitUserName, identity.GitUserEmail + } + if other != nil { + gitSig.Name = util.IfZero(gitSig.Name, other.GitUserName) + gitSig.Email = util.IfZero(gitSig.Email, other.GitUserEmail) + } + if gitSig.Name == "" { + gitSig.Name = doer.GitName() + } + if gitSig.Email == "" { + gitSig.Email = doer.GetEmail() + } + return gitSig } -// CommitTreeWithDate creates a commit from a given tree for the user with provided message -func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, committer *user_model.User, treeHash, message string, signoff bool, authorDate, committerDate time.Time) (string, error) { - authorSig := author.NewGitSig() - committerSig := committer.NewGitSig() +// CommitTree creates a commit from a given tree for the user with provided message +func (t *TemporaryUploadRepository) CommitTree(ctx context.Context, opts *CommitTreeUserOptions) (string, error) { + authorSig := makeGitUserSignature(opts.DoerUser, opts.AuthorIdentity, opts.CommitterIdentity) + committerSig := makeGitUserSignature(opts.DoerUser, opts.CommitterIdentity, opts.AuthorIdentity) + + authorDate := opts.AuthorTime + committerDate := opts.CommitterTime + if authorDate == nil && committerDate == nil { + authorDate = util.ToPointer(time.Now()) + committerDate = authorDate + } else if authorDate == nil { + authorDate = committerDate + } else if committerDate == nil { + committerDate = authorDate + } // Because this may call hooks we should pass in the environment env := append(os.Environ(), @@ -244,24 +284,27 @@ func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, co ) messageBytes := new(bytes.Buffer) - _, _ = messageBytes.WriteString(message) + _, _ = messageBytes.WriteString(opts.CommitMessage) _, _ = messageBytes.WriteString("\n") - cmdCommitTree := git.NewCommand(t.ctx, "commit-tree").AddDynamicArguments(treeHash) - if parent != "" { - cmdCommitTree.AddOptionValues("-p", parent) + cmdCommitTree := git.NewCommand("commit-tree").AddDynamicArguments(opts.TreeHash) + if opts.ParentCommitID != "" { + cmdCommitTree.AddOptionValues("-p", opts.ParentCommitID) } var sign bool - var keyID string + var key *git.SigningKey var signer *git.Signature - if parent != "" { - sign, keyID, signer, _ = asymkey_service.SignCRUDAction(t.ctx, t.repo.RepoPath(), author, t.basePath, parent) + if opts.ParentCommitID != "" { + sign, key, signer, _ = asymkey_service.SignCRUDAction(ctx, t.repo.RepoPath(), opts.DoerUser, t.basePath, opts.ParentCommitID) } else { - sign, keyID, signer, _ = asymkey_service.SignInitialCommit(t.ctx, t.repo.RepoPath(), author) + sign, key, signer, _ = asymkey_service.SignInitialCommit(ctx, t.repo.RepoPath(), opts.DoerUser) } if sign { - cmdCommitTree.AddOptionFormat("-S%s", keyID) + if key.Format != "" { + cmdCommitTree.AddConfig("gpg.format", key.Format) + } + cmdCommitTree.AddOptionFormat("-S%s", key.KeyID) if t.repo.GetTrustModel() == repo_model.CommitterTrustModel || t.repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { if committerSig.Name != authorSig.Name || committerSig.Email != authorSig.Email { // Add trailers @@ -279,7 +322,7 @@ func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, co cmdCommitTree.AddArguments("--no-gpg-sign") } - if signoff { + if opts.SignOff { // Signed-off-by _, _ = messageBytes.WriteString("\n") _, _ = messageBytes.WriteString("Signed-off-by: ") @@ -294,7 +337,7 @@ func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, co stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) if err := cmdCommitTree. - Run(&git.RunOpts{ + Run(ctx, &git.RunOpts{ Env: env, Dir: t.basePath, Stdin: messageBytes, @@ -310,10 +353,10 @@ func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, co } // Push the provided commitHash to the repository branch by the provided user -func (t *TemporaryUploadRepository) Push(doer *user_model.User, commitHash, branch string) error { +func (t *TemporaryUploadRepository) Push(ctx context.Context, doer *user_model.User, commitHash, branch string) error { // Because calls hooks we need to pass in the environment env := repo_module.PushingEnvironment(doer, t.repo) - if err := git.Push(t.ctx, t.basePath, git.PushOptions{ + if err := git.Push(ctx, t.basePath, git.PushOptions{ Remote: t.repo.RepoPath(), Branch: strings.TrimSpace(commitHash) + ":" + git.BranchPrefix + strings.TrimSpace(branch), Env: env, @@ -335,7 +378,7 @@ func (t *TemporaryUploadRepository) Push(doer *user_model.User, commitHash, bran } // DiffIndex returns a Diff of the current index to the head -func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { +func (t *TemporaryUploadRepository) DiffIndex(ctx context.Context) (*gitdiff.Diff, error) { stdoutReader, stdoutWriter, err := os.Pipe() if err != nil { return nil, fmt.Errorf("unable to open stdout pipe: %w", err) @@ -346,8 +389,8 @@ func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { }() stderr := new(bytes.Buffer) var diff *gitdiff.Diff - err = git.NewCommand(t.ctx, "diff-index", "--src-prefix=\\a/", "--dst-prefix=\\b/", "--cached", "-p", "HEAD"). - Run(&git.RunOpts{ + err = git.NewCommand("diff-index", "--src-prefix=\\a/", "--dst-prefix=\\b/", "--cached", "-p", "HEAD"). + Run(ctx, &git.RunOpts{ Timeout: 30 * time.Second, Dir: t.basePath, Stdout: stdoutWriter, @@ -356,7 +399,7 @@ func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { _ = stdoutWriter.Close() defer cancel() var diffErr error - diff, diffErr = gitdiff.ParsePatch(t.ctx, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader, "") + diff, diffErr = gitdiff.ParsePatch(ctx, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader, "") _ = stdoutReader.Close() if diffErr != nil { // if the diffErr is not nil, it will be returned as the error of "Run()" @@ -370,18 +413,13 @@ func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { return nil, fmt.Errorf("unable to run diff-index pipeline in temporary repo: %w", err) } - diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(t.ctx, t.basePath, git.TrustedCmdArgs{"--cached"}, "HEAD") - if err != nil { - return nil, err - } - return diff, nil } // GetBranchCommit Gets the commit object of the given branch func (t *TemporaryUploadRepository) GetBranchCommit(branch string) (*git.Commit, error) { if t.gitRepo == nil { - return nil, fmt.Errorf("repository has not been cloned") + return nil, errors.New("repository has not been cloned") } return t.gitRepo.GetBranchCommit(branch) } @@ -389,7 +427,7 @@ func (t *TemporaryUploadRepository) GetBranchCommit(branch string) (*git.Commit, // GetCommit Gets the commit object of the given commit ID func (t *TemporaryUploadRepository) GetCommit(commitID string) (*git.Commit, error) { if t.gitRepo == nil { - return nil, fmt.Errorf("repository has not been cloned") + return nil, errors.New("repository has not been cloned") } return t.gitRepo.GetCommit(commitID) } diff --git a/services/repository/files/tree.go b/services/repository/files/tree.go index 6775186afd..f2cbacbf1c 100644 --- a/services/repository/files/tree.go +++ b/services/repository/files/tree.go @@ -6,10 +6,16 @@ package files import ( "context" "fmt" + "html/template" "net/url" + "path" + "sort" + "strings" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/fileicon" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -88,11 +94,7 @@ func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git if len(entries) > perPage { tree.Truncated = true } - if rangeStart+perPage < len(entries) { - rangeEnd = rangeStart + perPage - } else { - rangeEnd = len(entries) - } + rangeEnd = min(rangeStart+perPage, len(entries)) tree.Entries = make([]api.GitEntry, rangeEnd-rangeStart) for e := rangeStart; e < rangeEnd; e++ { i := e - rangeStart @@ -118,3 +120,110 @@ func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git } return tree, nil } + +func entryModeString(entryMode git.EntryMode) string { + switch entryMode { + case git.EntryModeBlob: + return "blob" + case git.EntryModeExec: + return "exec" + case git.EntryModeSymlink: + return "symlink" + case git.EntryModeCommit: + return "commit" // submodule + case git.EntryModeTree: + return "tree" + } + return "unknown" +} + +type TreeViewNode struct { + EntryName string `json:"entryName"` + EntryMode string `json:"entryMode"` + EntryIcon template.HTML `json:"entryIcon"` + EntryIconOpen template.HTML `json:"entryIconOpen,omitempty"` + + SymLinkedToMode string `json:"symLinkedToMode,omitempty"` // TODO: for the EntryMode="symlink" + + FullPath string `json:"fullPath"` + SubmoduleURL string `json:"submoduleUrl,omitempty"` + Children []*TreeViewNode `json:"children,omitempty"` +} + +func (node *TreeViewNode) sortLevel() int { + return util.Iif(node.EntryMode == "tree" || node.EntryMode == "commit", 0, 1) +} + +func newTreeViewNodeFromEntry(ctx context.Context, renderedIconPool *fileicon.RenderedIconPool, commit *git.Commit, parentDir string, entry *git.TreeEntry) *TreeViewNode { + node := &TreeViewNode{ + EntryName: entry.Name(), + EntryMode: entryModeString(entry.Mode()), + FullPath: path.Join(parentDir, entry.Name()), + } + + entryInfo := fileicon.EntryInfoFromGitTreeEntry(commit, node.FullPath, entry) + node.EntryIcon = fileicon.RenderEntryIconHTML(renderedIconPool, entryInfo) + if entryInfo.EntryMode.IsDir() { + entryInfo.IsOpen = true + node.EntryIconOpen = fileicon.RenderEntryIconHTML(renderedIconPool, entryInfo) + } + + if node.EntryMode == "commit" { + if subModule, err := commit.GetSubModule(node.FullPath); err != nil { + log.Error("GetSubModule: %v", err) + } else if subModule != nil { + submoduleFile := git.NewCommitSubmoduleFile(subModule.URL, entry.ID.String()) + webLink := submoduleFile.SubmoduleWebLink(ctx) + node.SubmoduleURL = webLink.CommitWebLink + } + } + + return node +} + +// sortTreeViewNodes list directory first and with alpha sequence +func sortTreeViewNodes(nodes []*TreeViewNode) { + sort.Slice(nodes, func(i, j int) bool { + a, b := nodes[i].sortLevel(), nodes[j].sortLevel() + if a != b { + return a < b + } + return nodes[i].EntryName < nodes[j].EntryName + }) +} + +func listTreeNodes(ctx context.Context, renderedIconPool *fileicon.RenderedIconPool, commit *git.Commit, tree *git.Tree, treePath, subPath string) ([]*TreeViewNode, error) { + entries, err := tree.ListEntries() + if err != nil { + return nil, err + } + + subPathDirName, subPathRemaining, _ := strings.Cut(subPath, "/") + nodes := make([]*TreeViewNode, 0, len(entries)) + for _, entry := range entries { + node := newTreeViewNodeFromEntry(ctx, renderedIconPool, commit, treePath, entry) + nodes = append(nodes, node) + if entry.IsDir() && subPathDirName == entry.Name() { + subTreePath := treePath + "/" + node.EntryName + if subTreePath[0] == '/' { + subTreePath = subTreePath[1:] + } + subNodes, err := listTreeNodes(ctx, renderedIconPool, commit, entry.Tree(), subTreePath, subPathRemaining) + if err != nil { + log.Error("listTreeNodes: %v", err) + } else { + node.Children = subNodes + } + } + } + sortTreeViewNodes(nodes) + return nodes, nil +} + +func GetTreeViewNodes(ctx context.Context, renderedIconPool *fileicon.RenderedIconPool, commit *git.Commit, treePath, subPath string) ([]*TreeViewNode, error) { + entry, err := commit.GetTreeEntryByPath(treePath) + if err != nil { + return nil, err + } + return listTreeNodes(ctx, renderedIconPool, commit, entry.Tree(), treePath, subPath) +} diff --git a/services/repository/files/tree_test.go b/services/repository/files/tree_test.go index 0c60fddf7b..a53f342d40 100644 --- a/services/repository/files/tree_test.go +++ b/services/repository/files/tree_test.go @@ -4,9 +4,12 @@ package files import ( + "html/template" "testing" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/fileicon" + "code.gitea.io/gitea/modules/git" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/contexttest" @@ -48,5 +51,69 @@ func TestGetTreeBySHA(t *testing.T) { TotalCount: 1, } - assert.EqualValues(t, expectedTree, tree) + assert.Equal(t, expectedTree, tree) +} + +func TestGetTreeViewNodes(t *testing.T) { + unittest.PrepareTestEnv(t) + ctx, _ := contexttest.MockContext(t, "user2/repo1") + ctx.Repo.RefFullName = git.RefNameFromBranch("sub-home-md-img-check") + contexttest.LoadRepo(t, ctx, 1) + contexttest.LoadRepoCommit(t, ctx) + contexttest.LoadUser(t, ctx, 2) + contexttest.LoadGitRepo(t, ctx) + defer ctx.Repo.GitRepo.Close() + + renderedIconPool := fileicon.NewRenderedIconPool() + mockIconForFile := func(id string) template.HTML { + return template.HTML(`<svg class="svg git-entry-icon octicon-file" width="16" height="16" aria-hidden="true"><use xlink:href="#` + id + `"></use></svg>`) + } + mockIconForFolder := func(id string) template.HTML { + return template.HTML(`<svg class="svg git-entry-icon octicon-file-directory-fill" width="16" height="16" aria-hidden="true"><use xlink:href="#` + id + `"></use></svg>`) + } + mockOpenIconForFolder := func(id string) template.HTML { + return template.HTML(`<svg class="svg git-entry-icon octicon-file-directory-open-fill" width="16" height="16" aria-hidden="true"><use xlink:href="#` + id + `"></use></svg>`) + } + treeNodes, err := GetTreeViewNodes(ctx, renderedIconPool, ctx.Repo.Commit, "", "") + assert.NoError(t, err) + assert.Equal(t, []*TreeViewNode{ + { + EntryName: "docs", + EntryMode: "tree", + FullPath: "docs", + EntryIcon: mockIconForFolder(`svg-mfi-folder-docs`), + EntryIconOpen: mockOpenIconForFolder(`svg-mfi-folder-docs`), + }, + }, treeNodes) + + treeNodes, err = GetTreeViewNodes(ctx, renderedIconPool, ctx.Repo.Commit, "", "docs/README.md") + assert.NoError(t, err) + assert.Equal(t, []*TreeViewNode{ + { + EntryName: "docs", + EntryMode: "tree", + FullPath: "docs", + EntryIcon: mockIconForFolder(`svg-mfi-folder-docs`), + EntryIconOpen: mockOpenIconForFolder(`svg-mfi-folder-docs`), + Children: []*TreeViewNode{ + { + EntryName: "README.md", + EntryMode: "blob", + FullPath: "docs/README.md", + EntryIcon: mockIconForFile(`svg-mfi-readme`), + }, + }, + }, + }, treeNodes) + + treeNodes, err = GetTreeViewNodes(ctx, renderedIconPool, ctx.Repo.Commit, "docs", "README.md") + assert.NoError(t, err) + assert.Equal(t, []*TreeViewNode{ + { + EntryName: "README.md", + EntryMode: "blob", + FullPath: "docs/README.md", + EntryIcon: mockIconForFile(`svg-mfi-readme`), + }, + }, treeNodes) } diff --git a/services/repository/files/update.go b/services/repository/files/update.go index a2763105b0..e871f777e5 100644 --- a/services/repository/files/update.go +++ b/services/repository/files/update.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "path" + "slices" "strings" "time" @@ -15,20 +16,22 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" asymkey_service "code.gitea.io/gitea/services/asymkey" pull_service "code.gitea.io/gitea/services/pull" ) // IdentityOptions for a person's identity like an author or committer type IdentityOptions struct { - Name string - Email string + GitUserName string // to match "git config user.name" + GitUserEmail string // to match "git config user.email" } // CommitDateOptions store dates for GIT_AUTHOR_DATE and GIT_COMMITTER_DATE @@ -85,14 +88,32 @@ func (err ErrRepoFileDoesNotExist) Unwrap() error { return util.ErrNotExist } +type LazyReadSeeker interface { + io.ReadSeeker + io.Closer + OpenLazyReader() error +} + // ChangeRepoFiles adds, updates or removes multiple files in the given repository -func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ChangeRepoFilesOptions) (*structs.FilesResponse, error) { +func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ChangeRepoFilesOptions) (_ *structs.FilesResponse, errRet error) { + var addedLfsPointers []lfs.Pointer + defer func() { + if errRet != nil { + for _, lfsPointer := range addedLfsPointers { + _, err := git_model.RemoveLFSMetaObjectByOid(ctx, repo.ID, lfsPointer.Oid) + if err != nil { + log.Error("ChangeRepoFiles: RemoveLFSMetaObjectByOid failed: %v", err) + } + } + } + }() + err := repo.MustNotBeArchived() if err != nil { return nil, err } - // If no branch name is set, assume default branch + // If no branch name is set, assume the default branch if opts.OldBranch == "" { opts.OldBranch = repo.DefaultBranch } @@ -107,8 +128,13 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use defer closer.Close() // oldBranch must exist for this operation - if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil && !repo.IsEmpty { + if exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.OldBranch); err != nil { return nil, err + } else if !exist && !repo.IsEmpty { + return nil, git_model.ErrBranchNotExist{ + RepoID: repo.ID, + BranchName: opts.OldBranch, + } } var treePaths []string @@ -119,14 +145,14 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } // Check that the path given in opts.treePath is valid (not a git path) - treePath := CleanUploadFileName(file.TreePath) + treePath := CleanGitTreePath(file.TreePath) if treePath == "" { return nil, ErrFilenameInvalid{ Path: file.TreePath, } } // If there is a fromTreePath (we are copying it), also clean it up - fromTreePath := CleanUploadFileName(file.FromTreePath) + fromTreePath := CleanGitTreePath(file.FromTreePath) if fromTreePath == "" && file.FromTreePath != "" { return nil, ErrFilenameInvalid{ Path: file.FromTreePath, @@ -145,30 +171,28 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use // Check to make sure the branch does not already exist, otherwise we can't proceed. // If we aren't branching to a new branch, make sure user can commit to the given branch if opts.NewBranch != opts.OldBranch { - existingBranch, err := gitRepo.GetBranch(opts.NewBranch) - if existingBranch != nil { + exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.NewBranch) + if err != nil { + return nil, err + } + if exist { return nil, git_model.ErrBranchAlreadyExists{ BranchName: opts.NewBranch, } } - if err != nil && !git.IsErrBranchNotExist(err) { - return nil, err - } } else if err := VerifyBranchProtection(ctx, repo, doer, opts.OldBranch, treePaths); err != nil { return nil, err } message := strings.TrimSpace(opts.Message) - author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) - - t, err := NewTemporaryUploadRepository(ctx, repo) + t, err := NewTemporaryUploadRepository(repo) if err != nil { log.Error("NewTemporaryUploadRepository failed: %v", err) } defer t.Close() hasOldBranch := true - if err := t.Clone(opts.OldBranch, true); err != nil { + if err := t.Clone(ctx, opts.OldBranch, true); err != nil { for _, file := range opts.Files { if file.Operation == "delete" { return nil, err @@ -177,14 +201,14 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { return nil, err } - if err := t.Init(repo.ObjectFormatName); err != nil { + if err := t.Init(ctx, repo.ObjectFormatName); err != nil { return nil, err } hasOldBranch = false opts.LastCommitID = "" } if hasOldBranch { - if err := t.SetDefaultIndex(); err != nil { + if err := t.SetDefaultIndex(ctx); err != nil { return nil, err } } @@ -192,19 +216,13 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use for _, file := range opts.Files { if file.Operation == "delete" { // Get the files in the index - filesInIndex, err := t.LsFiles(file.TreePath) + filesInIndex, err := t.LsFiles(ctx, file.TreePath) if err != nil { return nil, fmt.Errorf("DeleteRepoFile: %w", err) } // Find the file we want to delete in the index - inFilelist := false - for _, indexFile := range filesInIndex { - if indexFile == file.TreePath { - inFilelist = true - break - } - } + inFilelist := slices.Contains(filesInIndex, file.TreePath) if !inFilelist { return nil, ErrRepoFileDoesNotExist{ Path: file.TreePath, @@ -220,7 +238,7 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use return nil, err // Couldn't get a commit for the branch } - // Assigned LastCommitID in opts if it hasn't been set + // Assigned LastCommitID in "opts" if it hasn't been set if opts.LastCommitID == "" { opts.LastCommitID = commit.ID.String() } else { @@ -232,22 +250,25 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } for _, file := range opts.Files { - if err := handleCheckErrors(file, commit, opts); err != nil { + if err = handleCheckErrors(file, commit, opts); err != nil { return nil, err } } } - contentStore := lfs.NewContentStore() + lfsContentStore := lfs.NewContentStore() for _, file := range opts.Files { switch file.Operation { - case "create", "update": - if err := CreateOrUpdateFile(ctx, t, file, contentStore, repo.ID, hasOldBranch); err != nil { + case "create", "update", "rename", "upload": + addedLfsPointer, err := modifyFile(ctx, t, file, lfsContentStore, repo.ID) + if err != nil { return nil, err } + if addedLfsPointer != nil { + addedLfsPointers = append(addedLfsPointers, *addedLfsPointer) + } case "delete": - // Remove the file from the index - if err := t.RemoveFilesFromIndex(file.TreePath); err != nil { + if err = t.RemoveFilesFromIndex(ctx, file.TreePath); err != nil { return nil, err } default: @@ -256,24 +277,33 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } // Now write the tree - treeHash, err := t.WriteTree() + treeHash, err := t.WriteTree(ctx) if err != nil { return nil, err } // Now commit the tree - var commitHash string + commitOpts := &CommitTreeUserOptions{ + ParentCommitID: opts.LastCommitID, + TreeHash: treeHash, + CommitMessage: message, + SignOff: opts.Signoff, + DoerUser: doer, + AuthorIdentity: opts.Author, + AuthorTime: nil, + CommitterIdentity: opts.Committer, + CommitterTime: nil, + } if opts.Dates != nil { - commitHash, err = t.CommitTreeWithDate(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) - } else { - commitHash, err = t.CommitTree(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff) + commitOpts.AuthorTime, commitOpts.CommitterTime = &opts.Dates.Author, &opts.Dates.Committer } + commitHash, err := t.CommitTree(ctx, commitOpts) if err != nil { return nil, err } // Then push this tree to NewBranch - if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + if err := t.Push(ctx, doer, commitHash, opts.NewBranch); err != nil { log.Error("%T %v", err, err) return nil, err } @@ -283,14 +313,16 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use return nil, err } - filesResponse, err := GetFilesResponseFromCommit(ctx, repo, commit, opts.NewBranch, treePaths) + // FIXME: this call seems not right, why it needs to read the file content again + // FIXME: why it uses the NewBranch as "ref", it should use the commit ID because the response is only for this commit + filesResponse, err := GetFilesResponseFromCommit(ctx, repo, gitRepo, utils.NewRefCommit(git.RefNameFromBranch(opts.NewBranch), commit), treePaths) if err != nil { return nil, err } if repo.IsEmpty { if isEmpty, err := gitRepo.IsEmpty(); err == nil && !isEmpty { - _ = repo_model.UpdateRepositoryCols(ctx, &repo_model.Repository{ID: repo.ID, IsEmpty: false, DefaultBranch: opts.NewBranch}, "is_empty", "default_branch") + _ = repo_model.UpdateRepositoryColsWithAutoTime(ctx, &repo_model.Repository{ID: repo.ID, IsEmpty: false, DefaultBranch: opts.NewBranch}, "is_empty", "default_branch") } } @@ -356,22 +388,33 @@ func (err ErrSHAOrCommitIDNotProvided) Error() string { // handles the check for various issues for ChangeRepoFiles func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRepoFilesOptions) error { - if file.Operation == "update" || file.Operation == "delete" { - fromEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) - if err != nil { - return err + // check old entry (fromTreePath/fromEntry) + if file.Operation == "update" || file.Operation == "upload" || file.Operation == "delete" || file.Operation == "rename" { + var fromEntryIDString string + { + fromEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) + if file.Operation == "upload" && git.IsErrNotExist(err) { + fromEntry = nil + } else if err != nil { + return err + } + if fromEntry != nil { + fromEntryIDString = fromEntry.ID.String() + file.Options.executable = fromEntry.IsExecutable() // FIXME: legacy hacky approach, it shouldn't prepare the "Options" in the "check" function + } } + if file.SHA != "" { - // If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error - if file.SHA != fromEntry.ID.String() { + // If the SHA given doesn't match the SHA of the fromTreePath, throw error + if file.SHA != fromEntryIDString { return pull_service.ErrSHADoesNotMatch{ Path: file.Options.treePath, GivenSHA: file.SHA, - CurrentSHA: fromEntry.ID.String(), + CurrentSHA: fromEntryIDString, } } } else if opts.LastCommitID != "" { - // If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw + // If a lastCommitID given doesn't match the branch head's commitID throw // an error, but only if we aren't creating a new branch. if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch { if changed, err := commit.FileChangedSinceCommit(file.Options.treePath, opts.LastCommitID); err != nil { @@ -389,13 +432,13 @@ func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRep // haven't been made. We throw an error if one wasn't provided. return ErrSHAOrCommitIDNotProvided{} } - file.Options.executable = fromEntry.IsExecutable() } - if file.Operation == "create" || file.Operation == "update" { - // For the path where this file will be created/updated, we need to make - // sure no parts of the path are existing files or links except for the last - // item in the path which is the file name, and that shouldn't exist IF it is - // a new file OR is being moved to a new path. + + // check new entry (treePath/treeEntry) + if file.Operation == "create" || file.Operation == "update" || file.Operation == "upload" || file.Operation == "rename" { + // For operation's target path, we need to make sure no parts of the path are existing files or links + // except for the last item in the path (which is the file name). + // And that shouldn't exist IF it is a new file OR is being moved to a new path. treePathParts := strings.Split(file.Options.treePath, "/") subTreePath := "" for index, part := range treePathParts { @@ -432,7 +475,7 @@ func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRep Type: git.EntryModeTree, } } else if file.Options.fromTreePath != file.Options.treePath || file.Operation == "create" { - // The entry shouldn't exist if we are creating new file or moving to a new path + // The entry shouldn't exist if we are creating the new file or moving to a new path return ErrRepoFileAlreadyExists{ Path: file.Options.treePath, } @@ -443,21 +486,23 @@ func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRep return nil } -// CreateOrUpdateFile handles creating or updating a file for ChangeRepoFiles -func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile, contentStore *lfs.ContentStore, repoID int64, hasOldBranch bool) error { +func modifyFile(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile, contentStore *lfs.ContentStore, repoID int64) (addedLfsPointer *lfs.Pointer, _ error) { + if rd, ok := file.ContentReader.(LazyReadSeeker); ok { + if err := rd.OpenLazyReader(); err != nil { + return nil, fmt.Errorf("OpenLazyReader: %w", err) + } + defer rd.Close() + } + // Get the two paths (might be the same if not moving) from the index if they exist - filesInIndex, err := t.LsFiles(file.TreePath, file.FromTreePath) + filesInIndex, err := t.LsFiles(ctx, file.TreePath, file.FromTreePath) if err != nil { - return fmt.Errorf("UpdateRepoFile: %w", err) + return nil, fmt.Errorf("LsFiles: %w", err) } // If is a new file (not updating) then the given path shouldn't exist if file.Operation == "create" { - for _, indexFile := range filesInIndex { - if indexFile == file.TreePath { - return ErrRepoFileAlreadyExists{ - Path: file.TreePath, - } - } + if slices.Contains(filesInIndex, file.TreePath) { + return nil, ErrRepoFileAlreadyExists{Path: file.TreePath} } } @@ -465,79 +510,178 @@ func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file if file.Options.fromTreePath != file.Options.treePath && len(filesInIndex) > 0 { for _, indexFile := range filesInIndex { if indexFile == file.Options.fromTreePath { - if err := t.RemoveFilesFromIndex(file.FromTreePath); err != nil { - return err + if err = t.RemoveFilesFromIndex(ctx, file.FromTreePath); err != nil { + return nil, err } } } } - treeObjectContentReader := file.ContentReader - var lfsMetaObject *git_model.LFSMetaObject - if setting.LFS.StartServer && hasOldBranch { - // Check there is no way this can return multiple infos - filename2attribute2info, err := t.gitRepo.CheckAttribute(git.CheckAttributeOpts{ - Attributes: []string{"filter"}, - Filenames: []string{file.Options.treePath}, - CachedOnly: true, - }) + var writeObjectRet *writeRepoObjectRet + switch file.Operation { + case "create", "update", "upload": + writeObjectRet, err = writeRepoObjectForModify(ctx, t, file) + case "rename": + writeObjectRet, err = writeRepoObjectForRename(ctx, t, file) + default: + return nil, util.NewInvalidArgumentErrorf("unknown file modification operation: '%s'", file.Operation) + } + if err != nil { + return nil, err + } + + // Add the object to the index, the "file.Options.executable" is set in handleCheckErrors by the caller (legacy hacky approach) + if err = t.AddObjectToIndex(ctx, util.Iif(file.Options.executable, "100755", "100644"), writeObjectRet.ObjectHash, file.Options.treePath); err != nil { + return nil, err + } + + if writeObjectRet.LfsContent == nil { + return nil, nil // No LFS pointer, so nothing to do + } + defer writeObjectRet.LfsContent.Close() + + // Now we must store the content into an LFS object + lfsMetaObject, err := git_model.NewLFSMetaObject(ctx, repoID, writeObjectRet.LfsPointer) + if err != nil { + return nil, err + } + exist, err := contentStore.Exists(lfsMetaObject.Pointer) + if err != nil { + return nil, err + } + if !exist { + err = contentStore.Put(lfsMetaObject.Pointer, writeObjectRet.LfsContent) if err != nil { - return err + if _, errRemove := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); errRemove != nil { + return nil, fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, errRemove, err) + } + return nil, err } + } + return &lfsMetaObject.Pointer, nil +} - if filename2attribute2info[file.Options.treePath] != nil && filename2attribute2info[file.Options.treePath]["filter"] == "lfs" { - // OK so we are supposed to LFS this data! - pointer, err := lfs.GeneratePointer(treeObjectContentReader) +func checkIsLfsFileInGitAttributes(ctx context.Context, t *TemporaryUploadRepository, paths []string) (ret []bool, err error) { + attributesMap, err := attribute.CheckAttributes(ctx, t.gitRepo, "" /* use temp repo's working dir */, attribute.CheckAttributeOpts{ + Attributes: []string{attribute.Filter}, + Filenames: paths, + }) + if err != nil { + return nil, err + } + for _, p := range paths { + isLFSFile := attributesMap[p] != nil && attributesMap[p].Get(attribute.Filter).ToString().Value() == "lfs" + ret = append(ret, isLFSFile) + } + return ret, nil +} + +type writeRepoObjectRet struct { + ObjectHash string + LfsContent io.ReadCloser // if not nil, then the caller should store its content in LfsPointer, then close it + LfsPointer lfs.Pointer +} + +// writeRepoObjectForModify hashes the git object for create or update operations +func writeRepoObjectForModify(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile) (ret *writeRepoObjectRet, err error) { + ret = &writeRepoObjectRet{} + treeObjectContentReader := file.ContentReader + if setting.LFS.StartServer { + checkIsLfsFiles, err := checkIsLfsFileInGitAttributes(ctx, t, []string{file.Options.treePath}) + if err != nil { + return nil, err + } + if checkIsLfsFiles[0] { + // OK, so we are supposed to LFS this data! + ret.LfsPointer, err = lfs.GeneratePointer(file.ContentReader) if err != nil { - return err + return nil, err } - lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repoID} - treeObjectContentReader = strings.NewReader(pointer.StringContent()) + if _, err = file.ContentReader.Seek(0, io.SeekStart); err != nil { + return nil, err + } + ret.LfsContent = io.NopCloser(file.ContentReader) + treeObjectContentReader = strings.NewReader(ret.LfsPointer.StringContent()) } } - // Add the object to the database - objectHash, err := t.HashObject(treeObjectContentReader) + ret.ObjectHash, err = t.HashObjectAndWrite(ctx, treeObjectContentReader) if err != nil { - return err + return nil, err } + return ret, nil +} - // Add the object to the index - if file.Options.executable { - if err := t.AddObjectToIndex("100755", objectHash, file.Options.treePath); err != nil { - return err - } - } else { - if err := t.AddObjectToIndex("100644", objectHash, file.Options.treePath); err != nil { - return err +// writeRepoObjectForRename the same as writeRepoObjectForModify buf for "rename" +func writeRepoObjectForRename(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile) (ret *writeRepoObjectRet, err error) { + lastCommitID, err := t.GetLastCommit(ctx) + if err != nil { + return nil, err + } + commit, err := t.GetCommit(lastCommitID) + if err != nil { + return nil, err + } + oldEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) + if err != nil { + return nil, err + } + + ret = &writeRepoObjectRet{ObjectHash: oldEntry.ID.String()} + if !setting.LFS.StartServer { + return ret, nil + } + + checkIsLfsFiles, err := checkIsLfsFileInGitAttributes(ctx, t, []string{file.Options.fromTreePath, file.Options.treePath}) + if err != nil { + return nil, err + } + oldIsLfs, newIsLfs := checkIsLfsFiles[0], checkIsLfsFiles[1] + + // If the old and new paths are both in lfs or both not in lfs, the object hash of the old file can be used directly + // as the object doesn't change + if oldIsLfs == newIsLfs { + return ret, nil + } + + oldEntryBlobPointerBy := func(f func(r io.Reader) (lfs.Pointer, error)) (lfsPointer lfs.Pointer, err error) { + r, err := oldEntry.Blob().DataAsync() + if err != nil { + return lfsPointer, err } + defer r.Close() + return f(r) } - if lfsMetaObject != nil { - // We have an LFS object - create it - lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, lfsMetaObject.RepositoryID, lfsMetaObject.Pointer) + var treeObjectContentReader io.ReadCloser + if oldIsLfs { + // If the old is in lfs but the new isn't, read the content from lfs and add it as a normal git object + pointer, err := oldEntryBlobPointerBy(lfs.ReadPointer) if err != nil { - return err + return nil, err } - exist, err := contentStore.Exists(lfsMetaObject.Pointer) + treeObjectContentReader, err = lfs.ReadMetaObject(pointer) if err != nil { - return err + return nil, err } - if !exist { - _, err := file.ContentReader.Seek(0, io.SeekStart) - if err != nil { - return err - } - if err := contentStore.Put(lfsMetaObject.Pointer, file.ContentReader); err != nil { - if _, err2 := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); err2 != nil { - return fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, err2, err) - } - return err - } + defer treeObjectContentReader.Close() + } else { + // If the new is in lfs but the old isn't, read the content from the git object and generate a lfs pointer of it + ret.LfsPointer, err = oldEntryBlobPointerBy(lfs.GeneratePointer) + if err != nil { + return nil, err + } + ret.LfsContent, err = oldEntry.Blob().DataAsync() + if err != nil { + return nil, err } + treeObjectContentReader = io.NopCloser(strings.NewReader(ret.LfsPointer.StringContent())) } - - return nil + ret.ObjectHash, err = t.HashObjectAndWrite(ctx, treeObjectContentReader) + if err != nil { + return nil, err + } + return ret, nil } // VerifyBranchProtection verify the branch protection for modifying the given treePath on the given branch diff --git a/services/repository/files/upload.go b/services/repository/files/upload.go index cbfaf49d13..b783cbd01d 100644 --- a/services/repository/files/upload.go +++ b/services/repository/files/upload.go @@ -8,14 +8,11 @@ import ( "fmt" "os" "path" - "strings" + "sync" - git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/lfs" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/log" ) // UploadRepoFileOptions contains the uploaded repository file options @@ -27,199 +24,88 @@ type UploadRepoFileOptions struct { Message string Files []string // In UUID format. Signoff bool + Author *IdentityOptions + Committer *IdentityOptions } -type uploadInfo struct { - upload *repo_model.Upload - lfsMetaObject *git_model.LFSMetaObject +type lazyLocalFileReader struct { + *os.File + localFilename string + counter int + mu sync.Mutex } -func cleanUpAfterFailure(ctx context.Context, infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error { - for _, info := range *infos { - if info.lfsMetaObject == nil { - continue - } - if !info.lfsMetaObject.Existing { - if _, err := git_model.RemoveLFSMetaObjectByOid(ctx, t.repo.ID, info.lfsMetaObject.Oid); err != nil { - original = fmt.Errorf("%w, %v", original, err) // We wrap the original error - as this is the underlying error that required the fallback - } - } - } - return original -} - -// UploadRepoFiles uploads files to the given repository -func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UploadRepoFileOptions) error { - if len(opts.Files) == 0 { - return nil - } +var _ LazyReadSeeker = (*lazyLocalFileReader)(nil) - uploads, err := repo_model.GetUploadsByUUIDs(ctx, opts.Files) - if err != nil { - return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %w", opts.Files, err) - } +func (l *lazyLocalFileReader) Close() error { + l.mu.Lock() + defer l.mu.Unlock() - names := make([]string, len(uploads)) - infos := make([]uploadInfo, len(uploads)) - for i, upload := range uploads { - // Check file is not lfs locked, will return nil if lock setting not enabled - filepath := path.Join(opts.TreePath, upload.Name) - lfsLock, err := git_model.GetTreePathLock(ctx, repo.ID, filepath) - if err != nil { - return err - } - if lfsLock != nil && lfsLock.OwnerID != doer.ID { - u, err := user_model.GetUserByID(ctx, lfsLock.OwnerID) - if err != nil { - return err + if l.counter > 0 { + l.counter-- + if l.counter == 0 { + if err := l.File.Close(); err != nil { + return fmt.Errorf("close file %s: %w", l.localFilename, err) } - return git_model.ErrLFSFileLocked{RepoID: repo.ID, Path: filepath, UserName: u.Name} - } - - names[i] = upload.Name - infos[i] = uploadInfo{upload: upload} - } - - t, err := NewTemporaryUploadRepository(ctx, repo) - if err != nil { - return err - } - defer t.Close() - - hasOldBranch := true - if err = t.Clone(opts.OldBranch, true); err != nil { - if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { - return err - } - if err = t.Init(repo.ObjectFormatName); err != nil { - return err - } - hasOldBranch = false - opts.LastCommitID = "" - } - if hasOldBranch { - if err = t.SetDefaultIndex(); err != nil { - return err - } - } - - var filename2attribute2info map[string]map[string]string - if setting.LFS.StartServer { - filename2attribute2info, err = t.gitRepo.CheckAttribute(git.CheckAttributeOpts{ - Attributes: []string{"filter"}, - Filenames: names, - CachedOnly: true, - }) - if err != nil { - return err + l.File = nil } + return nil } + return fmt.Errorf("file %s already closed", l.localFilename) +} - // Copy uploaded files into repository. - for i := range infos { - if err := copyUploadedLFSFileIntoRepository(&infos[i], filename2attribute2info, t, opts.TreePath); err != nil { - return err - } - } +func (l *lazyLocalFileReader) OpenLazyReader() error { + l.mu.Lock() + defer l.mu.Unlock() - // Now write the tree - treeHash, err := t.WriteTree() - if err != nil { - return err + if l.File != nil { + l.counter++ + return nil } - // make author and committer the doer - author := doer - committer := doer - - // Now commit the tree - commitHash, err := t.CommitTree(opts.LastCommitID, author, committer, treeHash, opts.Message, opts.Signoff) + file, err := os.Open(l.localFilename) if err != nil { return err } + l.File = file + l.counter = 1 + return nil +} - // Now deal with LFS objects - for i := range infos { - if infos[i].lfsMetaObject == nil { - continue - } - infos[i].lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, infos[i].lfsMetaObject.RepositoryID, infos[i].lfsMetaObject.Pointer) - if err != nil { - // OK Now we need to cleanup - return cleanUpAfterFailure(ctx, &infos, t, err) - } - // Don't move the files yet - we need to ensure that - // everything can be inserted first - } - - // OK now we can insert the data into the store - there's no way to clean up the store - // once it's in there, it's in there. - contentStore := lfs.NewContentStore() - for _, info := range infos { - if err := uploadToLFSContentStore(info, contentStore); err != nil { - return cleanUpAfterFailure(ctx, &infos, t, err) - } - } - - // Then push this tree to NewBranch - if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { - return err +// UploadRepoFiles uploads files to the given repository +func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UploadRepoFileOptions) error { + if len(opts.Files) == 0 { + return nil } - return repo_model.DeleteUploads(ctx, uploads...) -} - -func copyUploadedLFSFileIntoRepository(info *uploadInfo, filename2attribute2info map[string]map[string]string, t *TemporaryUploadRepository, treePath string) error { - file, err := os.Open(info.upload.LocalPath()) + uploads, err := repo_model.GetUploadsByUUIDs(ctx, opts.Files) if err != nil { - return err + return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %w", opts.Files, err) } - defer file.Close() - var objectHash string - if setting.LFS.StartServer && filename2attribute2info[info.upload.Name] != nil && filename2attribute2info[info.upload.Name]["filter"] == "lfs" { - // Handle LFS - // FIXME: Inefficient! this should probably happen in models.Upload - pointer, err := lfs.GeneratePointer(file) - if err != nil { - return err - } - - info.lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID} - - if objectHash, err = t.HashObject(strings.NewReader(pointer.StringContent())); err != nil { - return err - } - } else if objectHash, err = t.HashObject(file); err != nil { - return err + changeOpts := &ChangeRepoFilesOptions{ + LastCommitID: opts.LastCommitID, + OldBranch: opts.OldBranch, + NewBranch: opts.NewBranch, + Message: opts.Message, + Signoff: opts.Signoff, + Author: opts.Author, + Committer: opts.Committer, + } + for _, upload := range uploads { + changeOpts.Files = append(changeOpts.Files, &ChangeRepoFile{ + Operation: "upload", + TreePath: path.Join(opts.TreePath, upload.Name), + ContentReader: &lazyLocalFileReader{localFilename: upload.LocalPath()}, + }) } - // Add the object to the index - return t.AddObjectToIndex("100644", objectHash, path.Join(treePath, info.upload.Name)) -} - -func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) error { - if info.lfsMetaObject == nil { - return nil - } - exist, err := contentStore.Exists(info.lfsMetaObject.Pointer) + _, err = ChangeRepoFiles(ctx, repo, doer, changeOpts) if err != nil { return err } - if !exist { - file, err := os.Open(info.upload.LocalPath()) - if err != nil { - return err - } - - defer file.Close() - // FIXME: Put regenerates the hash and copies the file over. - // I guess this strictly ensures the soundness of the store but this is inefficient. - if err := contentStore.Put(info.lfsMetaObject.Pointer, file); err != nil { - // OK Now we need to cleanup - // Can't clean up the store, once uploaded there they're there. - return err - } + if err := repo_model.DeleteUploads(ctx, uploads...); err != nil { + log.Error("DeleteUploads: %v", err) } return nil } diff --git a/services/repository/fork.go b/services/repository/fork.go index cff0b1a403..8bd3498b17 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -65,7 +65,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork } // Fork is prohibited, if user has reached maximum limit of repositories - if !owner.CanForkRepo() { + if !doer.CanForkRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ Limit: owner.MaxRepoCreation, } @@ -100,117 +100,106 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork IsFork: true, ForkID: opts.BaseRepo.ID, ObjectFormatName: opts.BaseRepo.ObjectFormatName, + Status: repo_model.RepositoryBeingMigrated, } - oldRepoPath := opts.BaseRepo.RepoPath() - - needsRollback := false - rollbackFn := func() { - if !needsRollback { - return - } - - repoPath := repo_model.RepoPath(owner.Name, repo.Name) - - if exists, _ := util.IsExist(repoPath); !exists { - return - } - - // As the transaction will be failed and hence database changes will be destroyed we only need - // to delete the related repository on the filesystem - if errDelete := util.RemoveAll(repoPath); errDelete != nil { - log.Error("Failed to remove fork repo") - } - } - - needsRollbackInPanic := true - defer func() { - panicErr := recover() - if panicErr == nil { - return - } - - if needsRollbackInPanic { - rollbackFn() - } - panic(panicErr) - }() - - err = db.WithTx(ctx, func(txCtx context.Context) error { - if err = CreateRepositoryByExample(txCtx, doer, owner, repo, false, true); err != nil { + // 1 - Create the repository in the database + err = db.WithTx(ctx, func(ctx context.Context) error { + if err = createRepositoryInDB(ctx, doer, owner, repo, true); err != nil { return err } - - if err = repo_model.IncrementRepoForkNum(txCtx, opts.BaseRepo.ID); err != nil { + if err = repo_model.IncrementRepoForkNum(ctx, opts.BaseRepo.ID); err != nil { return err } // copy lfs files failure should not be ignored - if err = git_model.CopyLFS(txCtx, repo, opts.BaseRepo); err != nil { - return err - } - - needsRollback = true + return git_model.CopyLFS(ctx, repo, opts.BaseRepo) + }) + if err != nil { + return nil, err + } - cloneCmd := git.NewCommand(txCtx, "clone", "--bare") - if opts.SingleBranch != "" { - cloneCmd.AddArguments("--single-branch", "--branch").AddDynamicArguments(opts.SingleBranch) - } - repoPath := repo_model.RepoPath(owner.Name, repo.Name) - if stdout, _, err := cloneCmd.AddDynamicArguments(oldRepoPath, repoPath). - RunStdBytes(&git.RunOpts{Timeout: 10 * time.Minute}); err != nil { - log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, opts.BaseRepo, stdout, err) - return fmt.Errorf("git clone: %w", err) + // last - clean up if something goes wrong + // WARNING: Don't override all later err with local variables + defer func() { + if err != nil { + // we can not use the ctx because it maybe canceled or timeout + cleanupRepository(repo.ID) } + }() - if err := repo_module.CheckDaemonExportOK(txCtx, repo); err != nil { - return fmt.Errorf("checkDaemonExportOK: %w", err) + // 2 - check whether the repository with the same storage exists + var isExist bool + isExist, err = gitrepo.IsRepositoryExist(ctx, repo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) + return nil, err + } + if isExist { + log.Error("Files already exist in %s and we are not going to adopt or delete.", repo.FullName()) + // Don't return directly, we need err in defer to cleanupRepository + err = repo_model.ErrRepoFilesAlreadyExist{ + Uname: repo.OwnerName, + Name: repo.Name, } + return nil, err + } - if stdout, _, err := git.NewCommand(txCtx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("Fork Repository (git update-server-info) failed for %v:\nStdout: %s\nError: %v", repo, stdout, err) - return fmt.Errorf("git update-server-info: %w", err) - } + // 3 - Clone the repository + cloneCmd := git.NewCommand("clone", "--bare") + if opts.SingleBranch != "" { + cloneCmd.AddArguments("--single-branch", "--branch").AddDynamicArguments(opts.SingleBranch) + } + var stdout []byte + if stdout, _, err = cloneCmd.AddDynamicArguments(opts.BaseRepo.RepoPath(), repo.RepoPath()). + RunStdBytes(ctx, &git.RunOpts{Timeout: 10 * time.Minute}); err != nil { + log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, opts.BaseRepo, stdout, err) + return nil, fmt.Errorf("git clone: %w", err) + } - if err = repo_module.CreateDelegateHooks(repoPath); err != nil { - return fmt.Errorf("createDelegateHooks: %w", err) - } + // 4 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } - gitRepo, err := gitrepo.OpenRepository(txCtx, repo) - if err != nil { - return fmt.Errorf("OpenRepository: %w", err) - } - defer gitRepo.Close() + // 5 - Create hooks + if err = gitrepo.CreateDelegateHooks(ctx, repo); err != nil { + return nil, fmt.Errorf("createDelegateHooks: %w", err) + } - _, err = repo_module.SyncRepoBranchesWithRepo(txCtx, repo, gitRepo, doer.ID) - return err - }) - needsRollbackInPanic = false + // 6 - Sync the repository branches and tags + var gitRepo *git.Repository + gitRepo, err = gitrepo.OpenRepository(ctx, repo) if err != nil { - rollbackFn() - return nil, err + return nil, fmt.Errorf("OpenRepository: %w", err) } + defer gitRepo.Close() + if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doer.ID); err != nil { + return nil, fmt.Errorf("SyncRepoBranchesWithRepo: %w", err) + } + if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { + return nil, fmt.Errorf("Sync releases from git tags failed: %v", err) + } + + // 7 - Update the repository // even if below operations failed, it could be ignored. And they will be retried - if err := repo_module.UpdateRepoSize(ctx, repo); err != nil { + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { log.Error("Failed to update size for repository: %v", err) + err = nil } - if err := repo_model.CopyLanguageStat(ctx, opts.BaseRepo, repo); err != nil { + if err = repo_model.CopyLanguageStat(ctx, opts.BaseRepo, repo); err != nil { log.Error("Copy language stat from oldRepo failed: %v", err) + err = nil } - if err := repo_model.CopyLicense(ctx, opts.BaseRepo, repo); err != nil { + if err = repo_model.CopyLicense(ctx, opts.BaseRepo, repo); err != nil { return nil, err } - gitRepo, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - log.Error("Open created git repository failed: %v", err) - } else { - defer gitRepo.Close() - if err := repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { - log.Error("Sync releases from git tags failed: %v", err) - } + // 8 - update repository status to be ready + repo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } notify_service.ForkRepository(ctx, doer, opts.BaseRepo, repo) @@ -220,7 +209,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork // ConvertForkToNormalRepository convert the provided repo from a forked repo to normal repo func ConvertForkToNormalRepository(ctx context.Context, repo *repo_model.Repository) error { - err := db.WithTx(ctx, func(ctx context.Context) error { + return db.WithTx(ctx, func(ctx context.Context) error { repo, err := repo_model.GetRepositoryByID(ctx, repo.ID) if err != nil { return err @@ -237,16 +226,8 @@ func ConvertForkToNormalRepository(ctx context.Context, repo *repo_model.Reposit repo.IsFork = false repo.ForkID = 0 - - if err := repo_module.UpdateRepository(ctx, repo, false); err != nil { - log.Error("Unable to update repository %-v whilst converting from fork. Error: %v", repo, err) - return err - } - - return nil + return repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_fork", "fork_id") }) - - return err } type findForksOptions struct { @@ -256,9 +237,11 @@ type findForksOptions struct { } func (opts findForksOptions) ToConds() builder.Cond { - return builder.Eq{"fork_id": opts.RepoID}.And( - repo_model.AccessibleRepositoryCondition(opts.Doer, unit.TypeInvalid), - ) + cond := builder.Eq{"fork_id": opts.RepoID} + if opts.Doer != nil && opts.Doer.IsAdmin { + return cond + } + return cond.And(repo_model.AccessibleRepositoryCondition(opts.Doer, unit.TypeInvalid)) } // FindForks returns all the forks of the repository diff --git a/services/repository/fork_test.go b/services/repository/fork_test.go index 452798b25b..5375f79028 100644 --- a/services/repository/fork_test.go +++ b/services/repository/fork_test.go @@ -4,13 +4,17 @@ package repository import ( + "os" "testing" + "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -35,7 +39,7 @@ func TestForkRepository(t *testing.T) { assert.False(t, repo_model.IsErrReachLimitOfRepo(err)) // change AllowForkWithoutMaximumLimit to false for the test - setting.Repository.AllowForkWithoutMaximumLimit = false + defer test.MockVariableValue(&setting.Repository.AllowForkWithoutMaximumLimit, false)() // user has reached maximum limit of repositories user.MaxRepoCreation = 0 fork2, err := ForkRepository(git.DefaultContext, user, user, ForkRepoOptions{ @@ -46,3 +50,43 @@ func TestForkRepository(t *testing.T) { assert.Nil(t, fork2) assert.True(t, repo_model.IsErrReachLimitOfRepo(err)) } + +func TestForkRepositoryCleanup(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + // a successful fork + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo10 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}) + + fork, err := ForkRepository(git.DefaultContext, user2, user2, ForkRepoOptions{ + BaseRepo: repo10, + Name: "test", + }) + assert.NoError(t, err) + assert.NotNil(t, fork) + + exist, err := util.IsExist(repo_model.RepoPath(user2.Name, "test")) + assert.NoError(t, err) + assert.True(t, exist) + + err = DeleteRepositoryDirectly(db.DefaultContext, fork.ID) + assert.NoError(t, err) + + // a failed creating because some mock data + // create the repository directory so that the creation will fail after database record created. + assert.NoError(t, os.MkdirAll(repo_model.RepoPath(user2.Name, "test"), os.ModePerm)) + + fork2, err := ForkRepository(db.DefaultContext, user2, user2, ForkRepoOptions{ + BaseRepo: repo10, + Name: "test", + }) + assert.Nil(t, fork2) + assert.Error(t, err) + + // assert the cleanup is successful + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: "test"}) + + exist, err = util.IsExist(repo_model.RepoPath(user2.Name, "test")) + assert.NoError(t, err) + assert.False(t, exist) +} diff --git a/services/repository/generate.go b/services/repository/generate.go index d5c07e9800..867b5d7855 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -17,11 +17,11 @@ import ( git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" @@ -42,10 +42,8 @@ type expansion struct { var defaultTransformers = []transformer{ {Name: "SNAKE", Transform: xstrings.ToSnakeCase}, {Name: "KEBAB", Transform: xstrings.ToKebabCase}, - {Name: "CAMEL", Transform: func(str string) string { - return xstrings.FirstRuneToLower(xstrings.ToCamelCase(str)) - }}, - {Name: "PASCAL", Transform: xstrings.ToCamelCase}, + {Name: "CAMEL", Transform: xstrings.ToCamelCase}, + {Name: "PASCAL", Transform: xstrings.ToPascalCase}, {Name: "LOWER", Transform: strings.ToLower}, {Name: "UPPER", Transform: strings.ToUpper}, {Name: "TITLE", Transform: util.ToTitleCase}, @@ -236,8 +234,8 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r return err } - if stdout, _, err := git.NewCommand(ctx, "remote", "add", "origin").AddDynamicArguments(repo.RepoPath()). - RunStdString(&git.RunOpts{Dir: tmpDir, Env: env}); err != nil { + if stdout, _, err := git.NewCommand("remote", "add", "origin").AddDynamicArguments(repo.RepoPath()). + RunStdString(ctx, &git.RunOpts{Dir: tmpDir, Env: env}); err != nil { log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err) return fmt.Errorf("git remote add: %w", err) } @@ -255,48 +253,35 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r return initRepoCommit(ctx, tmpDir, repo, repo.Owner, defaultBranch) } -func generateGitContent(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository) (err error) { - tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) +// GenerateGitContent generates git content from a template repository +func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) (err error) { + tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-" + generateRepo.Name) if err != nil { - return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.RepoPath(), err) + return fmt.Errorf("failed to create temp dir for repository %s: %w", generateRepo.FullName(), err) } + defer cleanup() - defer func() { - if err := util.RemoveAll(tmpDir); err != nil { - log.Error("RemoveAll: %v", err) - } - }() - - if err = generateRepoCommit(ctx, repo, templateRepo, generateRepo, tmpDir); err != nil { + if err = generateRepoCommit(ctx, generateRepo, templateRepo, generateRepo, tmpDir); err != nil { return fmt.Errorf("generateRepoCommit: %w", err) } // re-fetch repo - if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { + if generateRepo, err = repo_model.GetRepositoryByID(ctx, generateRepo.ID); err != nil { return fmt.Errorf("getRepositoryByID: %w", err) } // if there was no default branch supplied when generating the repo, use the default one from the template - if strings.TrimSpace(repo.DefaultBranch) == "" { - repo.DefaultBranch = templateRepo.DefaultBranch + if strings.TrimSpace(generateRepo.DefaultBranch) == "" { + generateRepo.DefaultBranch = templateRepo.DefaultBranch } - if err = gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + if err = gitrepo.SetDefaultBranch(ctx, generateRepo, generateRepo.DefaultBranch); err != nil { return fmt.Errorf("setDefaultBranch: %w", err) } - if err = UpdateRepository(ctx, repo, false); err != nil { + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, generateRepo, "default_branch"); err != nil { return fmt.Errorf("updateRepository: %w", err) } - return nil -} - -// GenerateGitContent generates git content from a template repository -func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error { - if err := generateGitContent(ctx, generateRepo, templateRepo, generateRepo); err != nil { - return err - } - if err := repo_module.UpdateRepoSize(ctx, generateRepo); err != nil { return fmt.Errorf("failed to update size for repository: %w", err) } @@ -328,58 +313,6 @@ func (gro GenerateRepoOptions) IsValid() bool { gro.IssueLabels || gro.ProtectedBranch // or other items as they are added } -// generateRepository generates a repository from a template -func generateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) { - generateRepo := &repo_model.Repository{ - OwnerID: owner.ID, - Owner: owner, - OwnerName: owner.Name, - Name: opts.Name, - LowerName: strings.ToLower(opts.Name), - Description: opts.Description, - DefaultBranch: opts.DefaultBranch, - IsPrivate: opts.Private, - IsEmpty: !opts.GitContent || templateRepo.IsEmpty, - IsFsckEnabled: templateRepo.IsFsckEnabled, - TemplateID: templateRepo.ID, - TrustModel: templateRepo.TrustModel, - ObjectFormatName: templateRepo.ObjectFormatName, - } - - if err = CreateRepositoryByExample(ctx, doer, owner, generateRepo, false, false); err != nil { - return nil, err - } - - repoPath := generateRepo.RepoPath() - isExist, err := util.IsExist(repoPath) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return nil, err - } - if isExist { - return nil, repo_model.ErrRepoFilesAlreadyExist{ - Uname: generateRepo.OwnerName, - Name: generateRepo.Name, - } - } - - if err = repo_module.CheckInitRepository(ctx, owner.Name, generateRepo.Name, generateRepo.ObjectFormatName); err != nil { - return generateRepo, err - } - - if err = repo_module.CheckDaemonExportOK(ctx, generateRepo); err != nil { - return generateRepo, fmt.Errorf("checkDaemonExportOK: %w", err) - } - - if stdout, _, err := git.NewCommand(ctx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("GenerateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", generateRepo, stdout, err) - return generateRepo, fmt.Errorf("error in GenerateRepository(git update-server-info): %w", err) - } - - return generateRepo, nil -} - var fileNameSanitizeRegexp = regexp.MustCompile(`(?i)\.\.|[<>:\"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`) // Sanitize user input to valid OS filenames diff --git a/services/repository/generate_test.go b/services/repository/generate_test.go index b0f97d0ffb..1163c392c9 100644 --- a/services/repository/generate_test.go +++ b/services/repository/generate_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var giteaTemplate = []byte(` @@ -65,3 +66,26 @@ func TestFileNameSanitize(t *testing.T) { assert.Equal(t, "_", fileNameSanitize("\u0000")) assert.Equal(t, "ç›®æ ‡", fileNameSanitize("ç›®æ ‡")) } + +func TestTransformers(t *testing.T) { + cases := []struct { + name string + expected string + }{ + {"SNAKE", "abc_def_xyz"}, + {"KEBAB", "abc-def-xyz"}, + {"CAMEL", "abcDefXyz"}, + {"PASCAL", "AbcDefXyz"}, + {"LOWER", "abc_def-xyz"}, + {"UPPER", "ABC_DEF-XYZ"}, + {"TITLE", "Abc_def-Xyz"}, + } + + input := "Abc_Def-XYZ" + assert.Len(t, defaultTransformers, len(cases)) + for i, c := range cases { + tf := defaultTransformers[i] + require.Equal(t, c.name, tf.Name) + assert.Equal(t, c.expected, tf.Transform(input), "case %s", c.name) + } +} diff --git a/services/repository/gitgraph/graph.go b/services/repository/gitgraph/graph.go new file mode 100644 index 0000000000..d06d18c1b4 --- /dev/null +++ b/services/repository/gitgraph/graph.go @@ -0,0 +1,116 @@ +// Copyright 2016 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgraph + +import ( + "bufio" + "bytes" + "context" + "os" + "strings" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" +) + +// GetCommitGraph return a list of commit (GraphItems) from all branches +func GetCommitGraph(r *git.Repository, page, maxAllowedColors int, hidePRRefs bool, branches, files []string) (*Graph, error) { + format := "DATA:%D|%H|%ad|%h|%s" + + if page == 0 { + page = 1 + } + + graphCmd := git.NewCommand("log", "--graph", "--date-order", "--decorate=full") + + if hidePRRefs { + graphCmd.AddArguments("--exclude=" + git.PullPrefix + "*") + } + + if len(branches) == 0 { + graphCmd.AddArguments("--tags", "--branches") + } + + graphCmd.AddArguments("-C", "-M", "--date=iso-strict"). + AddOptionFormat("-n %d", setting.UI.GraphMaxCommitNum*page). + AddOptionFormat("--pretty=format:%s", format) + + if len(branches) > 0 { + graphCmd.AddDynamicArguments(branches...) + } + if len(files) > 0 { + graphCmd.AddDashesAndList(files...) + } + graph := NewGraph() + + stderr := new(strings.Builder) + stdoutReader, stdoutWriter, err := os.Pipe() + if err != nil { + return nil, err + } + commitsToSkip := setting.UI.GraphMaxCommitNum * (page - 1) + + scanner := bufio.NewScanner(stdoutReader) + + if err := graphCmd.Run(r.Ctx, &git.RunOpts{ + Dir: r.Path, + Stdout: stdoutWriter, + Stderr: stderr, + PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { + _ = stdoutWriter.Close() + defer stdoutReader.Close() + parser := &Parser{} + parser.firstInUse = -1 + parser.maxAllowedColors = maxAllowedColors + if maxAllowedColors > 0 { + parser.availableColors = make([]int, maxAllowedColors) + for i := range parser.availableColors { + parser.availableColors[i] = i + 1 + } + } else { + parser.availableColors = []int{1, 2} + } + for commitsToSkip > 0 && scanner.Scan() { + line := scanner.Bytes() + dataIdx := bytes.Index(line, []byte("DATA:")) + if dataIdx < 0 { + dataIdx = len(line) + } + starIdx := bytes.IndexByte(line, '*') + if starIdx >= 0 && starIdx < dataIdx { + commitsToSkip-- + } + parser.ParseGlyphs(line[:dataIdx]) + } + + row := 0 + + // Skip initial non-commit lines + for scanner.Scan() { + line := scanner.Bytes() + if bytes.IndexByte(line, '*') >= 0 { + if err := parser.AddLineToGraph(graph, row, line); err != nil { + cancel() + return err + } + break + } + parser.ParseGlyphs(line) + } + + for scanner.Scan() { + row++ + line := scanner.Bytes() + if err := parser.AddLineToGraph(graph, row, line); err != nil { + cancel() + return err + } + } + return scanner.Err() + }, + }); err != nil { + return graph, err + } + return graph, nil +} diff --git a/services/repository/gitgraph/graph_models.go b/services/repository/gitgraph/graph_models.go new file mode 100644 index 0000000000..02b0268cd9 --- /dev/null +++ b/services/repository/gitgraph/graph_models.go @@ -0,0 +1,266 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgraph + +import ( + "bytes" + "context" + "fmt" + "strings" + "time" + + asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + asymkey_service "code.gitea.io/gitea/services/asymkey" +) + +// NewGraph creates a basic graph +func NewGraph() *Graph { + graph := &Graph{} + graph.relationCommit = &Commit{ + Row: -1, + Column: -1, + } + graph.Flows = map[int64]*Flow{} + return graph +} + +// Graph represents a collection of flows +type Graph struct { + Flows map[int64]*Flow + Commits []*Commit + MinRow int + MinColumn int + MaxRow int + MaxColumn int + relationCommit *Commit +} + +// Width returns the width of the graph +func (graph *Graph) Width() int { + return graph.MaxColumn - graph.MinColumn + 1 +} + +// Height returns the height of the graph +func (graph *Graph) Height() int { + return graph.MaxRow - graph.MinRow + 1 +} + +// AddGlyph adds glyph to flows +func (graph *Graph) AddGlyph(row, column int, flowID int64, color int, glyph byte) { + flow, ok := graph.Flows[flowID] + if !ok { + flow = NewFlow(flowID, color, row, column) + graph.Flows[flowID] = flow + } + flow.AddGlyph(row, column, glyph) + + if row < graph.MinRow { + graph.MinRow = row + } + if row > graph.MaxRow { + graph.MaxRow = row + } + if column < graph.MinColumn { + graph.MinColumn = column + } + if column > graph.MaxColumn { + graph.MaxColumn = column + } +} + +// AddCommit adds a commit at row, column on flowID with the provided data +func (graph *Graph) AddCommit(row, column int, flowID int64, data []byte) error { + commit, err := NewCommit(row, column, data) + if err != nil { + return err + } + commit.Flow = flowID + graph.Commits = append(graph.Commits, commit) + + graph.Flows[flowID].Commits = append(graph.Flows[flowID].Commits, commit) + return nil +} + +// LoadAndProcessCommits will load the git.Commits for each commit in the graph, +// the associate the commit with the user author, and check the commit verification +// before finally retrieving the latest status +func (graph *Graph) LoadAndProcessCommits(ctx context.Context, repository *repo_model.Repository, gitRepo *git.Repository) error { + var err error + var ok bool + + emails := map[string]*user_model.User{} + keyMap := map[string]bool{} + + for _, c := range graph.Commits { + if len(c.Rev) == 0 { + continue + } + c.Commit, err = gitRepo.GetCommit(c.Rev) + if err != nil { + return fmt.Errorf("GetCommit: %s Error: %w", c.Rev, err) + } + + if c.Commit.Author != nil { + email := c.Commit.Author.Email + if c.User, ok = emails[email]; !ok { + c.User, _ = user_model.GetUserByEmail(ctx, email) + emails[email] = c.User + } + } + + c.Verification = asymkey_service.ParseCommitWithSignature(ctx, c.Commit) + + _ = asymkey_model.CalculateTrustStatus(c.Verification, repository.GetTrustModel(), func(user *user_model.User) (bool, error) { + return repo_model.IsOwnerMemberCollaborator(ctx, repository, user.ID) + }, &keyMap) + + statuses, err := git_model.GetLatestCommitStatus(ctx, repository.ID, c.Commit.ID.String(), db.ListOptionsAll) + if err != nil { + log.Error("GetLatestCommitStatus: %v", err) + } else { + c.Status = git_model.CalcCommitStatus(statuses) + } + } + return nil +} + +// NewFlow creates a new flow +func NewFlow(flowID int64, color, row, column int) *Flow { + return &Flow{ + ID: flowID, + ColorNumber: color, + MinRow: row, + MinColumn: column, + MaxRow: row, + MaxColumn: column, + } +} + +// Flow represents a series of glyphs +type Flow struct { + ID int64 + ColorNumber int + Glyphs []Glyph + Commits []*Commit + MinRow int + MinColumn int + MaxRow int + MaxColumn int +} + +// Color16 wraps the color numbers around mod 16 +func (flow *Flow) Color16() int { + return flow.ColorNumber % 16 +} + +// AddGlyph adds glyph at row and column +func (flow *Flow) AddGlyph(row, column int, glyph byte) { + if row < flow.MinRow { + flow.MinRow = row + } + if row > flow.MaxRow { + flow.MaxRow = row + } + if column < flow.MinColumn { + flow.MinColumn = column + } + if column > flow.MaxColumn { + flow.MaxColumn = column + } + + flow.Glyphs = append(flow.Glyphs, Glyph{ + row, + column, + glyph, + }) +} + +// Glyph represents a co-ordinate and glyph +type Glyph struct { + Row int + Column int + Glyph byte +} + +// RelationCommit represents an empty relation commit +var RelationCommit = &Commit{ + Row: -1, +} + +func parseGitTime(timeStr string) time.Time { + t, err := time.Parse(time.RFC3339, timeStr) + if err != nil { + return time.Unix(0, 0) + } + return t +} + +// NewCommit creates a new commit from a provided line +func NewCommit(row, column int, line []byte) (*Commit, error) { + data := bytes.SplitN(line, []byte("|"), 5) + if len(data) < 5 { + return nil, fmt.Errorf("malformed data section on line %d with commit: %s", row, string(line)) + } + return &Commit{ + Row: row, + Column: column, + // 0 matches git log --pretty=format:%d => ref names, like the --decorate option of git-log(1) + Refs: newRefsFromRefNames(data[0]), + // 1 matches git log --pretty=format:%H => commit hash + Rev: string(data[1]), + // 2 matches git log --pretty=format:%ad => author date (format respects --date= option) + Date: parseGitTime(string(data[2])), + // 3 matches git log --pretty=format:%h => abbreviated commit hash + ShortRev: string(data[3]), + // 4 matches git log --pretty=format:%s => subject + Subject: string(data[4]), + }, nil +} + +func newRefsFromRefNames(refNames []byte) []git.Reference { + refBytes := bytes.Split(refNames, []byte{',', ' '}) + refs := make([]git.Reference, 0, len(refBytes)) + for _, refNameBytes := range refBytes { + if len(refNameBytes) == 0 { + continue + } + refName := string(refNameBytes) + if after, ok := strings.CutPrefix(refName, "tag: "); ok { + refName = after + } else { + refName = strings.TrimPrefix(refName, "HEAD -> ") + } + refs = append(refs, git.Reference{ + Name: refName, + }) + } + return refs +} + +// Commit represents a commit at co-ordinate X, Y with the data +type Commit struct { + Commit *git.Commit + User *user_model.User + Verification *asymkey_model.CommitVerification + Status *git_model.CommitStatus + Flow int64 + Row int + Column int + Refs []git.Reference + Rev string + Date time.Time + ShortRev string + Subject string +} + +// OnlyRelation returns whether this a relation only commit +func (c *Commit) OnlyRelation() bool { + return c.Row == -1 +} diff --git a/services/repository/gitgraph/graph_test.go b/services/repository/gitgraph/graph_test.go new file mode 100644 index 0000000000..93fa1aec6a --- /dev/null +++ b/services/repository/gitgraph/graph_test.go @@ -0,0 +1,695 @@ +// Copyright 2016 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgraph + +import ( + "bytes" + "fmt" + "slices" + "strings" + "testing" + + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/assert" +) + +func BenchmarkGetCommitGraph(b *testing.B) { + currentRepo, err := git.OpenRepository(git.DefaultContext, ".") + if err != nil || currentRepo == nil { + b.Error("Could not open repository") + } + defer currentRepo.Close() + + for b.Loop() { + graph, err := GetCommitGraph(currentRepo, 1, 0, false, nil, nil) + if err != nil { + b.Error("Could get commit graph") + } + + if len(graph.Commits) < 100 { + b.Error("Should get 100 log lines.") + } + } +} + +func BenchmarkParseCommitString(b *testing.B) { + testString := "* DATA:|4e61bacab44e9b4730e44a6615d04098dd3a8eaf|2016-12-20 21:10:41 +0100|4e61bac|Add route for graph" + + parser := &Parser{} + parser.Reset() + for b.Loop() { + parser.Reset() + graph := NewGraph() + if err := parser.AddLineToGraph(graph, 0, []byte(testString)); err != nil { + b.Error("could not parse teststring") + } + if graph.Flows[1].Commits[0].Rev != "4e61bacab44e9b4730e44a6615d04098dd3a8eaf" { + b.Error("Did not get expected data") + } + } +} + +func BenchmarkParseGlyphs(b *testing.B) { + parser := &Parser{} + parser.Reset() + tgBytes := []byte(testglyphs) + var tg []byte + for b.Loop() { + parser.Reset() + tg = tgBytes + idx := bytes.Index(tg, []byte("\n")) + for idx > 0 { + parser.ParseGlyphs(tg[:idx]) + tg = tg[idx+1:] + idx = bytes.Index(tg, []byte("\n")) + } + } +} + +func TestReleaseUnusedColors(t *testing.T) { + testcases := []struct { + availableColors []int + oldColors []int + firstInUse int // these values have to be either be correct or suggest less is + firstAvailable int // available than possibly is - i.e. you cannot say 10 is available when it + }{ + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + oldColors: []int{1, 1, 1, 1, 1}, + firstAvailable: -1, + firstInUse: 1, + }, + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + oldColors: []int{1, 2, 3, 4}, + firstAvailable: 6, + firstInUse: 0, + }, + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + oldColors: []int{6, 0, 3, 5, 3, 4, 0, 0}, + firstAvailable: 6, + firstInUse: 0, + }, + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7}, + oldColors: []int{6, 1, 3, 5, 3, 4, 2, 7}, + firstAvailable: -1, + firstInUse: 0, + }, + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7}, + oldColors: []int{6, 0, 3, 5, 3, 4, 2, 7}, + firstAvailable: -1, + firstInUse: 0, + }, + } + for _, testcase := range testcases { + parser := &Parser{} + parser.Reset() + parser.availableColors = append([]int{}, testcase.availableColors...) + parser.oldColors = append(parser.oldColors, testcase.oldColors...) + parser.firstAvailable = testcase.firstAvailable + parser.firstInUse = testcase.firstInUse + parser.releaseUnusedColors() + + if parser.firstAvailable == -1 { + // All in use + for _, color := range parser.availableColors { + found := slices.Contains(parser.oldColors, color) + if !found { + t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", + testcase.availableColors, + testcase.oldColors, + testcase.firstAvailable, + testcase.firstInUse, + parser.availableColors, + parser.oldColors, + parser.firstAvailable, + parser.firstInUse, + color) + } + } + } else if parser.firstInUse != -1 { + // Some in use + for i := parser.firstInUse; i != parser.firstAvailable; i = (i + 1) % len(parser.availableColors) { + color := parser.availableColors[i] + found := slices.Contains(parser.oldColors, color) + if !found { + t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", + testcase.availableColors, + testcase.oldColors, + testcase.firstAvailable, + testcase.firstInUse, + parser.availableColors, + parser.oldColors, + parser.firstAvailable, + parser.firstInUse, + color) + } + } + for i := parser.firstAvailable; i != parser.firstInUse; i = (i + 1) % len(parser.availableColors) { + color := parser.availableColors[i] + found := slices.Contains(parser.oldColors, color) + if found { + t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should not be available but is", + testcase.availableColors, + testcase.oldColors, + testcase.firstAvailable, + testcase.firstInUse, + parser.availableColors, + parser.oldColors, + parser.firstAvailable, + parser.firstInUse, + color) + } + } + } else { + // None in use + for _, color := range parser.oldColors { + if color != 0 { + t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should not be available but is", + testcase.availableColors, + testcase.oldColors, + testcase.firstAvailable, + testcase.firstInUse, + parser.availableColors, + parser.oldColors, + parser.firstAvailable, + parser.firstInUse, + color) + } + } + } + } +} + +func TestParseGlyphs(t *testing.T) { + parser := &Parser{} + parser.Reset() + tgBytes := []byte(testglyphs) + tg := tgBytes + idx := bytes.Index(tg, []byte("\n")) + row := 0 + for idx > 0 { + parser.ParseGlyphs(tg[:idx]) + tg = tg[idx+1:] + idx = bytes.Index(tg, []byte("\n")) + if parser.flows[0] != 1 { + t.Errorf("First column flow should be 1 but was %d", parser.flows[0]) + } + colorToFlow := map[int]int64{} + flowToColor := map[int64]int{} + + for i, flow := range parser.flows { + if flow == 0 { + continue + } + color := parser.colors[i] + + if fColor, in := flowToColor[flow]; in && fColor != color { + t.Errorf("Row %d column %d flow %d has color %d but should be %d", row, i, flow, color, fColor) + } + flowToColor[flow] = color + if cFlow, in := colorToFlow[color]; in && cFlow != flow { + t.Errorf("Row %d column %d flow %d has color %d but conflicts with flow %d", row, i, flow, color, cFlow) + } + colorToFlow[color] = flow + } + row++ + } + assert.Len(t, parser.availableColors, 9) +} + +func TestCommitStringParsing(t *testing.T) { + dataFirstPart := "* DATA:|4e61bacab44e9b4730e44a6615d04098dd3a8eaf|2016-12-20 21:10:41 +0100|4e61bac|" + tests := []struct { + shouldPass bool + testName string + commitMessage string + }{ + {true, "normal", "not a fancy message"}, + {true, "extra pipe", "An extra pipe: |"}, + {true, "extra 'Data:'", "DATA: might be trouble"}, + } + + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + testString := fmt.Sprintf("%s%s", dataFirstPart, test.commitMessage) + idx := strings.Index(testString, "DATA:") + commit, err := NewCommit(0, 0, []byte(testString[idx+5:])) + if err != nil && test.shouldPass { + t.Errorf("Could not parse %s", testString) + return + } + + assert.Equal(t, test.commitMessage, commit.Subject) + }) + } +} + +var testglyphs = `* +* +* +* +* +* +* +* +|\ +* | +* | +* | +* | +* | +| * +* | +| * +| |\ +* | | +| | * +| | |\ +* | | \ +|\ \ \ \ +| * | | | +| |\| | | +* | | | | +|/ / / / +| | | * +| * | | +| * | | +| * | | +* | | | +* | | | +* | | | +* | | | +* | | | +|\ \ \ \ +| | * | | +| | |\| | +| | | * | +| | | | * +* | | | | +* | | | | +* | | | | +* | | | | +* | | | | +|\ \ \ \ \ +| * | | | | +|/| | | | | +| | |/ / / +| |/| | | +| | | | * +| * | | | +|/| | | | +| * | | | +|/| | | | +| | |/ / +| |/| | +| * | | +| * | | +| |\ \ \ +| | * | | +| |/| | | +| | | |/ +| | |/| +| * | | +| * | | +| * | | +| | * | +| | |\ \ +| | | * | +| | |/| | +| | | * | +| | | |\ \ +| | | | * | +| | | |/| | +| | * | | | +| | * | | | +| | |\ \ \ \ +| | | * | | | +| | |/| | | | +| | | | | * | +| | | | |/ / +* | | | / / +|/ / / / / +* | | | | +|\ \ \ \ \ +| * | | | | +|/| | | | | +| * | | | | +| * | | | | +| |\ \ \ \ \ +| | | * \ \ \ +| | | |\ \ \ \ +| | | | * | | | +| | | |/| | | | +| | | | | |/ / +| | | | |/| | +* | | | | | | +* | | | | | | +* | | | | | | +| | | | * | | +* | | | | | | +| | * | | | | +| |/| | | | | +* | | | | | | +| |/ / / / / +|/| | | | | +| | | | * | +| | | |/ / +| | |/| | +| * | | | +| | | | * +| | * | | +| | |\ \ \ +| | | * | | +| | |/| | | +| | | |/ / +| | | * | +| | * | | +| | |\ \ \ +| | | * | | +| | |/| | | +| | | |/ / +| | | * | +* | | | | +|\ \ \ \ \ +| * \ \ \ \ +| |\ \ \ \ \ +| | | |/ / / +| | |/| | | +| | | | * | +| | | | * | +* | | | | | +* | | | | | +|/ / / / / +| | | * | +* | | | | +* | | | | +* | | | | +* | | | | +|\ \ \ \ \ +| * | | | | +|/| | | | | +| | * | | | +| | |\ \ \ \ +| | | * | | | +| | |/| | | | +| |/| | |/ / +| | | |/| | +| | | | | * +| |_|_|_|/ +|/| | | | +| | * | | +| |/ / / +* | | | +* | | | +| | * | +* | | | +* | | | +| * | | +| | * | +| * | | +* | | | +|\ \ \ \ +| * | | | +|/| | | | +| |/ / / +| * | | +| |\ \ \ +| | * | | +| |/| | | +| | |/ / +| | * | +| | |\ \ +| | | * | +| | |/| | +* | | | | +* | | | | +|\ \ \ \ \ +| * | | | | +|/| | | | | +| | * | | | +| | * | | | +| | * | | | +| |/ / / / +| * | | | +| |\ \ \ \ +| | * | | | +| |/| | | | +* | | | | | +* | | | | | +* | | | | | +* | | | | | +* | | | | | +| | | | * | +* | | | | | +|\ \ \ \ \ \ +| * | | | | | +|/| | | | | | +| | | | | * | +| | | | |/ / +* | | | | | +|\ \ \ \ \ \ +* | | | | | | +* | | | | | | +| | | | * | | +* | | | | | | +* | | | | | | +|\ \ \ \ \ \ \ +| | |_|_|/ / / +| |/| | | | | +| | | | * | | +| | | | * | | +| | | | * | | +| | | | * | | +| | | | * | | +| | | | * | | +| | | |/ / / +| | | * | | +| | | * | | +| | | * | | +| | |/| | | +| | | * | | +| | |/| | | +| | | |/ / +| | * | | +| |/| | | +| | | * | +| | |/ / +| | * | +| * | | +| |\ \ \ +| * | | | +| | * | | +| |/| | | +| | |/ / +| | * | +| | |\ \ +| | * | | +* | | | | +|\| | | | +| * | | | +| * | | | +| * | | | +| | * | | +| * | | | +| |\| | | +| * | | | +| | * | | +| | * | | +| * | | | +| * | | | +| * | | | +| * | | | +| * | | | +| * | | | +| * | | | +| * | | | +| | * | | +| * | | | +| * | | | +| * | | | +| * | | | +| | * | | +* | | | | +|\| | | | +| | * | | +| * | | | +| |\| | | +| | * | | +| | * | | +| | * | | +| | | * | +* | | | | +|\| | | | +| | * | | +| | |/ / +| * | | +| * | | +| |\| | +* | | | +|\| | | +| | * | +| | * | +| | * | +| * | | +| | * | +| * | | +| | * | +| | * | +| | * | +| * | | +| * | | +| * | | +| * | | +| * | | +| * | | +| * | | +* | | | +|\| | | +| * | | +| |\| | +| | * | +| | |\ \ +* | | | | +|\| | | | +| * | | | +| |\| | | +| | * | | +| | | * | +| | |/ / +* | | | +* | | | +|\| | | +| * | | +| |\| | +| | * | +| | * | +| | * | +| | | * +* | | | +|\| | | +| * | | +| * | | +| | | * +| | | |\ +* | | | | +| |_|_|/ +|/| | | +| * | | +| |\| | +| | * | +| | * | +| | * | +| | * | +| | * | +| * | | +* | | | +|\| | | +| * | | +|/| | | +| |/ / +| * | +| |\ \ +| * | | +| * | | +* | | | +|\| | | +| | * | +| * | | +| * | | +| * | | +* | | | +|\| | | +| * | | +| * | | +| | * | +| | |\ \ +| | |/ / +| |/| | +| * | | +* | | | +|\| | | +| * | | +* | | | +|\| | | +| * | | +| |\ \ \ +| * | | | +| * | | | +| | | * | +| * | | | +| * | | | +| | |/ / +| |/| | +| | * | +* | | | +|\| | | +| * | | +| * | | +| * | | +| * | | +| * | | +| |\ \ \ +* | | | | +|\| | | | +| * | | | +| * | | | +* | | | | +* | | | | +|\| | | | +| | | | * +| | | | |\ +| |_|_|_|/ +|/| | | | +| * | | | +* | | | | +* | | | | +|\| | | | +| * | | | +| |\ \ \ \ +| | | |/ / +| | |/| | +| * | | | +| * | | | +| * | | | +| * | | | +| | * | | +| | | * | +| | |/ / +| |/| | +* | | | +|\| | | +| * | | +| * | | +| * | | +| * | | +| * | | +* | | | +|\| | | +| * | | +| * | | +* | | | +| * | | +| * | | +| * | | +* | | | +* | | | +* | | | +|\| | | +| * | | +* | | | +* | | | +* | | | +* | | | +| | | * +* | | | +|\| | | +| * | | +| * | | +| * | | +` diff --git a/services/repository/gitgraph/parser.go b/services/repository/gitgraph/parser.go new file mode 100644 index 0000000000..f6bf9b0b90 --- /dev/null +++ b/services/repository/gitgraph/parser.go @@ -0,0 +1,336 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgraph + +import ( + "bytes" + "fmt" +) + +// Parser represents a git graph parser. It is stateful containing the previous +// glyphs, detected flows and color assignments. +type Parser struct { + glyphs []byte + oldGlyphs []byte + flows []int64 + oldFlows []int64 + maxFlow int64 + colors []int + oldColors []int + availableColors []int + nextAvailable int + firstInUse int + firstAvailable int + maxAllowedColors int +} + +// Reset resets the internal parser state. +func (parser *Parser) Reset() { + parser.glyphs = parser.glyphs[0:0] + parser.oldGlyphs = parser.oldGlyphs[0:0] + parser.flows = parser.flows[0:0] + parser.oldFlows = parser.oldFlows[0:0] + parser.maxFlow = 0 + parser.colors = parser.colors[0:0] + parser.oldColors = parser.oldColors[0:0] + parser.availableColors = parser.availableColors[0:0] + parser.availableColors = append(parser.availableColors, 1, 2) + parser.nextAvailable = 0 + parser.firstInUse = -1 + parser.firstAvailable = 0 + parser.maxAllowedColors = 0 +} + +// AddLineToGraph adds the line as a row to the graph +func (parser *Parser) AddLineToGraph(graph *Graph, row int, line []byte) error { + idx := bytes.Index(line, []byte("DATA:")) + if idx < 0 { + parser.ParseGlyphs(line) + } else { + parser.ParseGlyphs(line[:idx]) + } + + var err error + commitDone := false + + for column, glyph := range parser.glyphs { + if glyph == ' ' { + continue + } + + flowID := parser.flows[column] + + graph.AddGlyph(row, column, flowID, parser.colors[column], glyph) + + if glyph == '*' { + if commitDone { + if err != nil { + err = fmt.Errorf("double commit on line %d: %s. %w", row, string(line), err) + } else { + err = fmt.Errorf("double commit on line %d: %s", row, string(line)) + } + } + commitDone = true + if idx < 0 { + if err != nil { + err = fmt.Errorf("missing data section on line %d with commit: %s. %w", row, string(line), err) + } else { + err = fmt.Errorf("missing data section on line %d with commit: %s", row, string(line)) + } + continue + } + err2 := graph.AddCommit(row, column, flowID, line[idx+5:]) + if err != nil && err2 != nil { + err = fmt.Errorf("%v %w", err2, err) + continue + } else if err2 != nil { + err = err2 + continue + } + } + } + if !commitDone { + graph.Commits = append(graph.Commits, RelationCommit) + } + return err +} + +func (parser *Parser) releaseUnusedColors() { + if parser.firstInUse > -1 { + // Here we step through the old colors, searching for them in the + // "in-use" section of availableColors (that is, the colors between + // firstInUse and firstAvailable) + // Ensure that the benchmarks are not worsened with proposed changes + stepstaken := 0 + position := parser.firstInUse + for _, color := range parser.oldColors { + if color == 0 { + continue + } + found := false + i := position + for j := stepstaken; i != parser.firstAvailable && j < len(parser.availableColors); j++ { + colorToCheck := parser.availableColors[i] + if colorToCheck == color { + found = true + break + } + i = (i + 1) % len(parser.availableColors) + } + if !found { + // Duplicate color + continue + } + // Swap them around + parser.availableColors[position], parser.availableColors[i] = parser.availableColors[i], parser.availableColors[position] + stepstaken++ + position = (parser.firstInUse + stepstaken) % len(parser.availableColors) + if position == parser.firstAvailable || stepstaken == len(parser.availableColors) { + break + } + } + if stepstaken == len(parser.availableColors) { + parser.firstAvailable = -1 + } else { + parser.firstAvailable = position + if parser.nextAvailable == -1 { + parser.nextAvailable = parser.firstAvailable + } + } + } +} + +// ParseGlyphs parses the provided glyphs and sets the internal state +func (parser *Parser) ParseGlyphs(glyphs []byte) { + // Clean state for parsing this row + parser.glyphs, parser.oldGlyphs = parser.oldGlyphs, parser.glyphs + parser.glyphs = parser.glyphs[0:0] + parser.flows, parser.oldFlows = parser.oldFlows, parser.flows + parser.flows = parser.flows[0:0] + parser.colors, parser.oldColors = parser.oldColors, parser.colors + + // Ensure we have enough flows and colors + parser.colors = parser.colors[0:0] + for range glyphs { + parser.flows = append(parser.flows, 0) + parser.colors = append(parser.colors, 0) + } + + // Copy the provided glyphs in to state.glyphs for safekeeping + parser.glyphs = append(parser.glyphs, glyphs...) + + // release unused colors + parser.releaseUnusedColors() + + for i := len(glyphs) - 1; i >= 0; i-- { + glyph := glyphs[i] + switch glyph { + case '|': + fallthrough + case '*': + parser.setUpFlow(i) + case '/': + parser.setOutFlow(i) + case '\\': + parser.setInFlow(i) + case '_': + parser.setRightFlow(i) + case '.': + fallthrough + case '-': + parser.setLeftFlow(i) + case ' ': + // no-op + default: + parser.newFlow(i) + } + } +} + +func (parser *Parser) takePreviousFlow(i, j int) { + if j < len(parser.oldFlows) && parser.oldFlows[j] > 0 { + parser.flows[i] = parser.oldFlows[j] + parser.oldFlows[j] = 0 + parser.colors[i] = parser.oldColors[j] + parser.oldColors[j] = 0 + } else { + parser.newFlow(i) + } +} + +func (parser *Parser) takeCurrentFlow(i, j int) { + if j < len(parser.flows) && parser.flows[j] > 0 { + parser.flows[i] = parser.flows[j] + parser.colors[i] = parser.colors[j] + } else { + parser.newFlow(i) + } +} + +func (parser *Parser) newFlow(i int) { + parser.maxFlow++ + parser.flows[i] = parser.maxFlow + + // Now give this flow a color + if parser.nextAvailable == -1 { + next := len(parser.availableColors) + if parser.maxAllowedColors < 1 || next < parser.maxAllowedColors { + parser.nextAvailable = next + parser.firstAvailable = next + parser.availableColors = append(parser.availableColors, next+1) + } + } + parser.colors[i] = parser.availableColors[parser.nextAvailable] + if parser.firstInUse == -1 { + parser.firstInUse = parser.nextAvailable + } + parser.availableColors[parser.firstAvailable], parser.availableColors[parser.nextAvailable] = parser.availableColors[parser.nextAvailable], parser.availableColors[parser.firstAvailable] + + parser.nextAvailable = (parser.nextAvailable + 1) % len(parser.availableColors) + parser.firstAvailable = (parser.firstAvailable + 1) % len(parser.availableColors) + + if parser.nextAvailable == parser.firstInUse { + parser.nextAvailable = parser.firstAvailable + } + if parser.nextAvailable == parser.firstInUse { + parser.nextAvailable = -1 + parser.firstAvailable = -1 + } +} + +// setUpFlow handles '|' or '*' +func (parser *Parser) setUpFlow(i int) { + // In preference order: + // + // Previous Row: '\? ' ' |' ' /' + // Current Row: ' | ' ' |' ' | ' + if i > 0 && i-1 < len(parser.oldGlyphs) && parser.oldGlyphs[i-1] == '\\' { + parser.takePreviousFlow(i, i-1) + } else if i < len(parser.oldGlyphs) && (parser.oldGlyphs[i] == '|' || parser.oldGlyphs[i] == '*') { + parser.takePreviousFlow(i, i) + } else if i+1 < len(parser.oldGlyphs) && parser.oldGlyphs[i+1] == '/' { + parser.takePreviousFlow(i, i+1) + } else { + parser.newFlow(i) + } +} + +// setOutFlow handles '/' +func (parser *Parser) setOutFlow(i int) { + // In preference order: + // + // Previous Row: ' |/' ' |_' ' |' ' /' ' _' '\' + // Current Row: '/| ' '/| ' '/ ' '/ ' '/ ' '/' + if i+2 < len(parser.oldGlyphs) && + (parser.oldGlyphs[i+1] == '|' || parser.oldGlyphs[i+1] == '*') && + (parser.oldGlyphs[i+2] == '/' || parser.oldGlyphs[i+2] == '_') && + i+1 < len(parser.glyphs) && + (parser.glyphs[i+1] == '|' || parser.glyphs[i+1] == '*') { + parser.takePreviousFlow(i, i+2) + } else if i+1 < len(parser.oldGlyphs) && + (parser.oldGlyphs[i+1] == '|' || parser.oldGlyphs[i+1] == '*' || + parser.oldGlyphs[i+1] == '/' || parser.oldGlyphs[i+1] == '_') { + parser.takePreviousFlow(i, i+1) + if parser.oldGlyphs[i+1] == '/' { + parser.glyphs[i] = '|' + } + } else if i < len(parser.oldGlyphs) && parser.oldGlyphs[i] == '\\' { + parser.takePreviousFlow(i, i) + } else { + parser.newFlow(i) + } +} + +// setInFlow handles '\' +func (parser *Parser) setInFlow(i int) { + // In preference order: + // + // Previous Row: '| ' '-. ' '| ' '\ ' '/' '---' + // Current Row: '|\' ' \' ' \' ' \' '\' ' \ ' + if i > 0 && i-1 < len(parser.oldGlyphs) && + (parser.oldGlyphs[i-1] == '|' || parser.oldGlyphs[i-1] == '*') && + (parser.glyphs[i-1] == '|' || parser.glyphs[i-1] == '*') { + parser.newFlow(i) + } else if i > 0 && i-1 < len(parser.oldGlyphs) && + (parser.oldGlyphs[i-1] == '|' || parser.oldGlyphs[i-1] == '*' || + parser.oldGlyphs[i-1] == '.' || parser.oldGlyphs[i-1] == '\\') { + parser.takePreviousFlow(i, i-1) + if parser.oldGlyphs[i-1] == '\\' { + parser.glyphs[i] = '|' + } + } else if i < len(parser.oldGlyphs) && parser.oldGlyphs[i] == '/' { + parser.takePreviousFlow(i, i) + } else { + parser.newFlow(i) + } +} + +// setRightFlow handles '_' +func (parser *Parser) setRightFlow(i int) { + // In preference order: + // + // Current Row: '__' '_/' '_|_' '_|/' + if i+1 < len(parser.glyphs) && + (parser.glyphs[i+1] == '_' || parser.glyphs[i+1] == '/') { + parser.takeCurrentFlow(i, i+1) + } else if i+2 < len(parser.glyphs) && + (parser.glyphs[i+1] == '|' || parser.glyphs[i+1] == '*') && + (parser.glyphs[i+2] == '_' || parser.glyphs[i+2] == '/') { + parser.takeCurrentFlow(i, i+2) + } else { + parser.newFlow(i) + } +} + +// setLeftFlow handles '----.' +func (parser *Parser) setLeftFlow(i int) { + if parser.glyphs[i] == '.' { + parser.newFlow(i) + } else if i+1 < len(parser.glyphs) && + (parser.glyphs[i+1] == '-' || parser.glyphs[i+1] == '.') { + parser.takeCurrentFlow(i, i+1) + } else { + parser.newFlow(i) + } +} diff --git a/services/repository/hooks.go b/services/repository/hooks.go index 97e9e290a3..c13b272550 100644 --- a/services/repository/hooks.go +++ b/services/repository/hooks.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" - repo_module "code.gitea.io/gitea/modules/repository" "xorm.io/builder" ) @@ -32,11 +31,11 @@ func SyncRepositoryHooks(ctx context.Context) error { default: } - if err := repo_module.CreateDelegateHooks(repo.RepoPath()); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo); err != nil { return fmt.Errorf("SyncRepositoryHook: %w", err) } if repo.HasWiki() { - if err := repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo.WikiStorageRepo()); err != nil { return fmt.Errorf("SyncRepositoryHook: %w", err) } } diff --git a/services/repository/init.go b/services/repository/init.go index c719e11786..1eeeb4aa4f 100644 --- a/services/repository/init.go +++ b/services/repository/init.go @@ -33,18 +33,21 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi committerName := sig.Name committerEmail := sig.Email - if stdout, _, err := git.NewCommand(ctx, "add", "--all"). - RunStdString(&git.RunOpts{Dir: tmpPath}); err != nil { + if stdout, _, err := git.NewCommand("add", "--all"). + RunStdString(ctx, &git.RunOpts{Dir: tmpPath}); err != nil { log.Error("git add --all failed: Stdout: %s\nError: %v", stdout, err) return fmt.Errorf("git add --all: %w", err) } - cmd := git.NewCommand(ctx, "commit", "--message=Initial commit"). + cmd := git.NewCommand("commit", "--message=Initial commit"). AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email) - sign, keyID, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u) + sign, key, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u) if sign { - cmd.AddOptionFormat("-S%s", keyID) + if key.Format != "" { + cmd.AddConfig("gpg.format", key.Format) + } + cmd.AddOptionFormat("-S%s", key.KeyID) if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { // need to set the committer to the KeyID owner @@ -61,7 +64,7 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi ) if stdout, _, err := cmd. - RunStdString(&git.RunOpts{Dir: tmpPath, Env: env}); err != nil { + RunStdString(ctx, &git.RunOpts{Dir: tmpPath, Env: env}); err != nil { log.Error("Failed to commit: %v: Stdout: %s\nError: %v", cmd.LogString(), stdout, err) return fmt.Errorf("git commit: %w", err) } @@ -70,8 +73,8 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi defaultBranch = setting.Repository.DefaultBranch } - if stdout, _, err := git.NewCommand(ctx, "push", "origin").AddDynamicArguments("HEAD:" + defaultBranch). - RunStdString(&git.RunOpts{Dir: tmpPath, Env: repo_module.InternalPushingEnvironment(u, repo)}); err != nil { + if stdout, _, err := git.NewCommand("push", "origin").AddDynamicArguments("HEAD:"+defaultBranch). + RunStdString(ctx, &git.RunOpts{Dir: tmpPath, Env: repo_module.InternalPushingEnvironment(u, repo)}); err != nil { log.Error("Failed to push back to HEAD: Stdout: %s\nError: %v", stdout, err) return fmt.Errorf("git push: %w", err) } diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index ee0b8f6b89..78ff8c853e 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -5,7 +5,6 @@ package repository_test import ( "bytes" - "context" "testing" "time" @@ -36,7 +35,7 @@ func TestGarbageCollectLFSMetaObjects(t *testing.T) { lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent) // gc - err = repo_service.GarbageCollectLFSMetaObjects(context.Background(), repo_service.GarbageCollectLFSMetaObjectsOptions{ + err = repo_service.GarbageCollectLFSMetaObjects(t.Context(), repo_service.GarbageCollectLFSMetaObjectsOptions{ AutoFix: true, OlderThan: time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour), UpdatedLessRecentlyThan: time.Now().Add(7 * 24 * time.Hour).Add(3 * 24 * time.Hour), diff --git a/services/repository/license.go b/services/repository/license.go index 2453be3c87..8622911fa2 100644 --- a/services/repository/license.go +++ b/services/repository/license.go @@ -14,7 +14,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/graceful" - "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/queue" @@ -25,7 +24,6 @@ import ( var ( classifier *licenseclassifier.Classifier LicenseFileName = "LICENSE" - licenseAliases map[string]string // licenseUpdaterQueue represents a queue to handle update repo licenses licenseUpdaterQueue *queue.WorkerPoolQueue[*LicenseUpdaterOptions] @@ -38,34 +36,6 @@ func AddRepoToLicenseUpdaterQueue(opts *LicenseUpdaterOptions) error { return licenseUpdaterQueue.Push(opts) } -func loadLicenseAliases() error { - if licenseAliases != nil { - return nil - } - - data, err := options.AssetFS().ReadFile("license", "etc", "license-aliases.json") - if err != nil { - return err - } - err = json.Unmarshal(data, &licenseAliases) - if err != nil { - return err - } - return nil -} - -func ConvertLicenseName(name string) string { - if err := loadLicenseAliases(); err != nil { - return name - } - - v, ok := licenseAliases[name] - if ok { - return v - } - return name -} - func InitLicenseClassifier() error { // threshold should be 0.84~0.86 or the test will be failed classifier = licenseclassifier.NewClassifier(.85) @@ -74,20 +44,13 @@ func InitLicenseClassifier() error { return err } - existLicense := make(container.Set[string]) - if len(licenseFiles) > 0 { - for _, licenseFile := range licenseFiles { - licenseName := ConvertLicenseName(licenseFile) - if existLicense.Contains(licenseName) { - continue - } - existLicense.Add(licenseName) - data, err := options.License(licenseFile) - if err != nil { - return err - } - classifier.AddContent("License", licenseFile, licenseName, data) + for _, licenseFile := range licenseFiles { + licenseName := licenseFile + data, err := options.License(licenseFile) + if err != nil { + return err } + classifier.AddContent("License", licenseName, licenseName, data) } return nil } diff --git a/services/repository/license_test.go b/services/repository/license_test.go index 9d3e0f36e3..eb897f3c03 100644 --- a/services/repository/license_test.go +++ b/services/repository/license_test.go @@ -4,13 +4,13 @@ package repository import ( - "fmt" "strings" "testing" repo_module "code.gitea.io/gitea/modules/repository" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_detectLicense(t *testing.T) { @@ -33,9 +33,7 @@ func Test_detectLicense(t *testing.T) { }, } - repo_module.LoadRepoConfig() - err := loadLicenseAliases() - assert.NoError(t, err) + require.NoError(t, repo_module.LoadRepoConfig()) for _, licenseName := range repo_module.Licenses { license, err := repo_module.GetLicense(licenseName, &repo_module.LicenseValues{ Owner: "Gitea", @@ -46,14 +44,13 @@ func Test_detectLicense(t *testing.T) { assert.NoError(t, err) tests = append(tests, DetectLicenseTest{ - name: fmt.Sprintf("single license test: %s", licenseName), + name: "single license test: " + licenseName, arg: string(license), - want: []string{ConvertLicenseName(licenseName)}, + want: []string{licenseName}, }) } - err = InitLicenseClassifier() - assert.NoError(t, err) + require.NoError(t, InitLicenseClassifier()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { license, err := detectLicense(strings.NewReader(tt.arg)) diff --git a/services/repository/merge_upstream.go b/services/repository/merge_upstream.go index 85ca8f7e31..8d6f11372c 100644 --- a/services/repository/merge_upstream.go +++ b/services/repository/merge_upstream.go @@ -4,35 +4,38 @@ package repository import ( - "context" + "errors" "fmt" - git_model "code.gitea.io/gitea/models/git" issue_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/pull" ) -type UpstreamDivergingInfo struct { - BaseIsNewer bool - CommitsBehind int - CommitsAhead int -} - -func MergeUpstream(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, branch string) (mergeStyle string, err error) { +// MergeUpstream merges the base repository's default branch into the fork repository's current branch. +func MergeUpstream(ctx reqctx.RequestContext, doer *user_model.User, repo *repo_model.Repository, branch string, ffOnly bool) (mergeStyle string, err error) { if err = repo.MustNotBeArchived(); err != nil { return "", err } if err = repo.GetBaseRepo(ctx); err != nil { return "", err } + divergingInfo, err := GetUpstreamDivergingInfo(ctx, repo, branch) + if err != nil { + return "", err + } + if !divergingInfo.BaseBranchHasNewCommits { + return "up-to-date", nil + } + err = git.Push(ctx, repo.BaseRepo.RepoPath(), git.PushOptions{ Remote: repo.RepoPath(), - Branch: fmt.Sprintf("%s:%s", branch, branch), + Branch: fmt.Sprintf("%s:%s", divergingInfo.BaseBranchName, branch), Env: repo_module.PushingEnvironment(doer, repo), }) if err == nil { @@ -42,6 +45,11 @@ func MergeUpstream(ctx context.Context, doer *user_model.User, repo *repo_model. return "", err } + // If ff_only is requested and fast-forward failed, return error + if ffOnly { + return "", util.NewInvalidArgumentErrorf("fast-forward merge not possible: branch has diverged") + } + // TODO: FakePR: it is somewhat hacky, but it is the only way to "merge" at the moment // ideally in the future the "merge" functions should be refactored to decouple from the PullRequest fakeIssue := &issue_model.Issue{ @@ -64,7 +72,7 @@ func MergeUpstream(ctx context.Context, doer *user_model.User, repo *repo_model. BaseRepoID: repo.BaseRepo.ID, BaseRepo: repo.BaseRepo, HeadBranch: branch, // maybe HeadCommitID is not needed - BaseBranch: branch, + BaseBranch: divergingInfo.BaseBranchName, } fakeIssue.PullRequest = fakePR err = pull.Update(ctx, fakePR, doer, "merge upstream", false) @@ -74,42 +82,47 @@ func MergeUpstream(ctx context.Context, doer *user_model.User, repo *repo_model. return "merge", nil } -func GetUpstreamDivergingInfo(ctx context.Context, repo *repo_model.Repository, branch string) (*UpstreamDivergingInfo, error) { - if !repo.IsFork { +// UpstreamDivergingInfo is also used in templates, so it needs to search for all references before changing it. +type UpstreamDivergingInfo struct { + BaseBranchName string + BaseBranchHasNewCommits bool + HeadBranchCommitsBehind int +} + +// GetUpstreamDivergingInfo returns the information about the divergence between the fork repository's branch and the base repository's default branch. +func GetUpstreamDivergingInfo(ctx reqctx.RequestContext, forkRepo *repo_model.Repository, forkBranch string) (*UpstreamDivergingInfo, error) { + if !forkRepo.IsFork { return nil, util.NewInvalidArgumentErrorf("repo is not a fork") } - if repo.IsArchived { + if forkRepo.IsArchived { return nil, util.NewInvalidArgumentErrorf("repo is archived") } - if err := repo.GetBaseRepo(ctx); err != nil { - return nil, err - } - - forkBranch, err := git_model.GetBranch(ctx, repo.ID, branch) - if err != nil { - return nil, err - } - - baseBranch, err := git_model.GetBranch(ctx, repo.BaseRepo.ID, branch) - if err != nil { + if err := forkRepo.GetBaseRepo(ctx); err != nil { return nil, err } - info := &UpstreamDivergingInfo{} - if forkBranch.CommitID == baseBranch.CommitID { - return info, nil + // Do the best to follow the GitHub's behavior, suppose there is a `branch-a` in fork repo: + // * if `branch-a` exists in base repo: try to sync `base:branch-a` to `fork:branch-a` + // * if `branch-a` doesn't exist in base repo: try to sync `base:main` to `fork:branch-a` + info, err := GetBranchDivergingInfo(ctx, forkRepo.BaseRepo, forkBranch, forkRepo, forkBranch) + if err == nil { + return &UpstreamDivergingInfo{ + BaseBranchName: forkBranch, + BaseBranchHasNewCommits: info.BaseHasNewCommits, + HeadBranchCommitsBehind: info.HeadCommitsBehind, + }, nil } - - // TODO: if the fork repo has new commits, this call will fail: - // exit status 128 - fatal: Invalid symmetric difference expression aaaaaaaaaaaa...bbbbbbbbbbbb - // so at the moment, we are not able to handle this case, should be improved in the future - diff, err := git.GetDivergingCommits(ctx, repo.BaseRepo.RepoPath(), baseBranch.CommitID, forkBranch.CommitID) - if err != nil { - info.BaseIsNewer = baseBranch.UpdatedUnix > forkBranch.UpdatedUnix - return info, nil + if errors.Is(err, util.ErrNotExist) { + info, err = GetBranchDivergingInfo(ctx, forkRepo.BaseRepo, forkRepo.BaseRepo.DefaultBranch, forkRepo, forkBranch) + if err == nil { + return &UpstreamDivergingInfo{ + BaseBranchName: forkRepo.BaseRepo.DefaultBranch, + BaseBranchHasNewCommits: info.BaseHasNewCommits, + HeadBranchCommitsBehind: info.HeadCommitsBehind, + }, nil + } } - info.CommitsBehind, info.CommitsAhead = diff.Behind, diff.Ahead - return info, nil + return nil, err } diff --git a/services/repository/migrate.go b/services/repository/migrate.go index 6f3a87afa3..0a3dc45339 100644 --- a/services/repository/migrate.go +++ b/services/repository/migrate.go @@ -13,8 +13,10 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/organization" repo_model "code.gitea.io/gitea/models/repo" + unit_model "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migration" @@ -116,14 +118,8 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, repo.Owner = u } - if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { - return repo, fmt.Errorf("checkDaemonExportOK: %w", err) - } - - if stdout, _, err := git.NewCommand(ctx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("MigrateRepositoryGitData(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - return repo, fmt.Errorf("error in MigrateRepositoryGitData(git update-server-info): %w", err) + if err := updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) } gitRepo, err := git.OpenRepository(ctx, repoPath) @@ -140,12 +136,12 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, if !repo.IsEmpty { if len(repo.DefaultBranch) == 0 { // Try to get HEAD branch and set it as default branch. - headBranch, err := gitRepo.GetHEADBranch() + headBranchName, err := git.GetDefaultBranch(ctx, repoPath) if err != nil { return repo, fmt.Errorf("GetHEADBranch: %w", err) } - if headBranch != nil { - repo.DefaultBranch = headBranch.Name + if headBranchName != "" { + repo.DefaultBranch = headBranchName } } @@ -153,9 +149,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err) } + // if releases migration are not requested, we will sync all tags here + // otherwise, the releases sync will be done out of this function if !opts.Releases { - // note: this will greatly improve release (tag) sync - // for pull-mirrors with many tags repo.IsMirror = opts.Mirror if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { log.Error("Failed to synchronize tags to releases for repository: %v", err) @@ -224,15 +220,19 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } repo.IsMirror = true - if err = UpdateRepository(ctx, repo, false); err != nil { + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "num_watches", "is_empty", "default_branch", "default_wiki_branch", "is_mirror"); err != nil { return nil, err } + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + // this is necessary for sync local tags from remote configName := fmt.Sprintf("remote.%s.fetch", mirrorModel.GetRemoteName()) - if stdout, _, err := git.NewCommand(ctx, "config"). + if stdout, _, err := git.NewCommand("config"). AddOptionValues("--add", configName, `+refs/tags/*:refs/tags/*`). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + RunStdString(ctx, &git.RunOpts{Dir: repoPath}); err != nil { log.Error("MigrateRepositoryGitData(git config --add <remote> +refs/tags/*:refs/tags/*) in %v: Stdout: %s\nError: %v", repo, stdout, err) return repo, fmt.Errorf("error in MigrateRepositoryGitData(git config --add <remote> +refs/tags/*:refs/tags/*): %w", err) } @@ -245,15 +245,28 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } + var enableRepoUnits []repo_model.RepoUnit + if opts.Releases && !unit_model.TypeReleases.UnitGlobalDisabled() { + enableRepoUnits = append(enableRepoUnits, repo_model.RepoUnit{RepoID: repo.ID, Type: unit_model.TypeReleases}) + } + if opts.Wiki && !unit_model.TypeWiki.UnitGlobalDisabled() { + enableRepoUnits = append(enableRepoUnits, repo_model.RepoUnit{RepoID: repo.ID, Type: unit_model.TypeWiki}) + } + if len(enableRepoUnits) > 0 { + err = UpdateRepositoryUnits(ctx, repo, enableRepoUnits, nil) + if err != nil { + return nil, err + } + } return repo, committer.Commit() } // cleanUpMigrateGitConfig removes mirror info which prevents "push --all". // This also removes possible user credentials. func cleanUpMigrateGitConfig(ctx context.Context, repoPath string) error { - cmd := git.NewCommand(ctx, "remote", "rm", "origin") + cmd := git.NewCommand("remote", "rm", "origin") // if the origin does not exist - _, _, err := cmd.RunStdString(&git.RunOpts{ + _, _, err := cmd.RunStdString(ctx, &git.RunOpts{ Dir: repoPath, }) if err != nil && !git.IsRemoteNotExistError(err) { @@ -264,17 +277,16 @@ func cleanUpMigrateGitConfig(ctx context.Context, repoPath string) error { // CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo_model.Repository, error) { - repoPath := repo.RepoPath() - if err := repo_module.CreateDelegateHooks(repoPath); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo); err != nil { return repo, fmt.Errorf("createDelegateHooks: %w", err) } if repo.HasWiki() { - if err := repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo.WikiStorageRepo()); err != nil { return repo, fmt.Errorf("createDelegateHooks.(wiki): %w", err) } } - _, _, err := git.NewCommand(ctx, "remote", "rm", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err := git.NewCommand("remote", "rm", "origin").RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil && !git.IsRemoteNotExistError(err) { return repo, fmt.Errorf("CleanUpMigrateInfo: %w", err) } diff --git a/services/repository/push.go b/services/repository/push.go index 06ad65e48f..af3c873d15 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -23,6 +23,7 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" issue_service "code.gitea.io/gitea/services/issue" notify_service "code.gitea.io/gitea/services/notify" pull_service "code.gitea.io/gitea/services/pull" @@ -65,7 +66,7 @@ func PushUpdates(opts []*repo_module.PushUpdateOptions) error { for _, opt := range opts { if opt.IsNewRef() && opt.IsDelRef() { - return fmt.Errorf("Old and new revisions are both NULL") + return errors.New("Old and new revisions are both NULL") } } @@ -133,23 +134,26 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { } else { // is new tag newCommit, err := gitRepo.GetCommit(opts.NewCommitID) if err != nil { - return fmt.Errorf("gitRepo.GetCommit(%s) in %s/%s[%d]: %w", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err) + // in case there is dirty data, for example, the "github.com/git/git" repository has tags pointing to non-existing commits + if !errors.Is(err, util.ErrNotExist) { + log.Error("Unable to get tag commit: gitRepo.GetCommit(%s) in %s/%s[%d]: %v", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err) + } + } else { + commits := repo_module.NewPushCommits() + commits.HeadCommit = repo_module.CommitToPushCommit(newCommit) + commits.CompareURL = repo.ComposeCompareURL(objectFormat.EmptyObjectID().String(), opts.NewCommitID) + + notify_service.PushCommits( + ctx, pusher, repo, + &repo_module.PushUpdateOptions{ + RefFullName: opts.RefFullName, + OldCommitID: objectFormat.EmptyObjectID().String(), + NewCommitID: opts.NewCommitID, + }, commits) + + addTags = append(addTags, tagName) + notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) } - - commits := repo_module.NewPushCommits() - commits.HeadCommit = repo_module.CommitToPushCommit(newCommit) - commits.CompareURL = repo.ComposeCompareURL(objectFormat.EmptyObjectID().String(), opts.NewCommitID) - - notify_service.PushCommits( - ctx, pusher, repo, - &repo_module.PushUpdateOptions{ - RefFullName: opts.RefFullName, - OldCommitID: objectFormat.EmptyObjectID().String(), - NewCommitID: opts.NewCommitID, - }, commits) - - addTags = append(addTags, tagName) - notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) } } else if opts.RefFullName.IsBranch() { if pusher == nil || pusher.ID != opts.PusherID { @@ -163,59 +167,25 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { } } - branch := opts.RefFullName.BranchName() if !opts.IsDelRef() { + branch := opts.RefFullName.BranchName() + log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) - go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID) newCommit, err := gitRepo.GetCommit(opts.NewCommitID) if err != nil { return fmt.Errorf("gitRepo.GetCommit(%s) in %s/%s[%d]: %w", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err) } - refName := opts.RefName() - // Push new branch. var l []*git.Commit if opts.IsNewRef() { - if repo.IsEmpty { // Change default branch and empty status only if pushed ref is non-empty branch. - repo.DefaultBranch = refName - repo.IsEmpty = false - if repo.DefaultBranch != setting.Repository.DefaultBranch { - if err := gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { - return err - } - } - // Update the is empty and default_branch columns - if err := repo_model.UpdateRepositoryCols(ctx, repo, "default_branch", "is_empty"); err != nil { - return fmt.Errorf("UpdateRepositoryCols: %w", err) - } - } - - l, err = newCommit.CommitsBeforeLimit(10) - if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeLimit: %w", err) - } - notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) + l, err = pushNewBranch(ctx, repo, pusher, opts, newCommit) } else { - l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) - if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeUntil: %w", err) - } - - isForcePush, err := newCommit.IsForcePush(opts.OldCommitID) - if err != nil { - log.Error("IsForcePush %s:%s failed: %v", repo.FullName(), branch, err) - } - - if isForcePush { - log.Trace("Push %s is a force push", opts.NewCommitID) - - cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) - } else { - // TODO: increment update the commit count cache but not remove - cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) - } + l, err = pushUpdateBranch(ctx, repo, pusher, opts, newCommit) + } + if err != nil { + return err } // delete cache for divergence @@ -232,36 +202,11 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { commits := repo_module.GitToPushCommits(l) commits.HeadCommit = repo_module.CommitToPushCommit(newCommit) - if err := issue_service.UpdateIssuesCommit(ctx, pusher, repo, commits.Commits, refName); err != nil { + if err := issue_service.UpdateIssuesCommit(ctx, pusher, repo, commits.Commits, opts.RefName()); err != nil { log.Error("updateIssuesCommit: %v", err) } - oldCommitID := opts.OldCommitID - if oldCommitID == objectFormat.EmptyObjectID().String() && len(commits.Commits) > 0 { - oldCommit, err := gitRepo.GetCommit(commits.Commits[len(commits.Commits)-1].Sha1) - if err != nil && !git.IsErrNotExist(err) { - log.Error("unable to GetCommit %s from %-v: %v", oldCommitID, repo, err) - } - if oldCommit != nil { - for i := 0; i < oldCommit.ParentCount(); i++ { - commitID, _ := oldCommit.ParentID(i) - if !commitID.IsZero() { - oldCommitID = commitID.String() - break - } - } - } - } - - if oldCommitID == objectFormat.EmptyObjectID().String() && repo.DefaultBranch != branch { - oldCommitID = repo.DefaultBranch - } - - if oldCommitID != objectFormat.EmptyObjectID().String() { - commits.CompareURL = repo.ComposeCompareURL(oldCommitID, opts.NewCommitID) - } else { - commits.CompareURL = "" - } + commits.CompareURL = getCompareURL(repo, gitRepo, objectFormat, commits.Commits, opts) if len(commits.Commits) > setting.UI.FeedMaxCommitNum { commits.Commits = commits.Commits[:setting.UI.FeedMaxCommitNum] @@ -274,11 +219,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { log.Error("repo_module.CacheRef %s/%s failed: %v", repo.ID, branch, err) } } else { - notify_service.DeleteRef(ctx, pusher, repo, opts.RefFullName) - if err = pull_service.CloseBranchPulls(ctx, pusher, repo.ID, branch); err != nil { - // close all related pulls - log.Error("close related pull request failed: %v", err) - } + pushDeleteBranch(ctx, repo, pusher, opts) } // Even if user delete a branch on a repository which he didn't watch, he will be watch that. @@ -289,8 +230,11 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { log.Trace("Non-tag and non-branch commits pushed.") } } - if err := PushUpdateAddDeleteTags(ctx, repo, gitRepo, addTags, delTags); err != nil { - return fmt.Errorf("PushUpdateAddDeleteTags: %w", err) + + if len(addTags)+len(delTags) > 0 { + if err := PushUpdateAddDeleteTags(ctx, repo, gitRepo, pusher, addTags, delTags); err != nil { + return fmt.Errorf("PushUpdateAddDeleteTags: %w", err) + } } // Change repository last updated time. @@ -301,18 +245,114 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { return nil } +func getCompareURL(repo *repo_model.Repository, gitRepo *git.Repository, objectFormat git.ObjectFormat, commits []*repo_module.PushCommit, opts *repo_module.PushUpdateOptions) string { + oldCommitID := opts.OldCommitID + if oldCommitID == objectFormat.EmptyObjectID().String() && len(commits) > 0 { + oldCommit, err := gitRepo.GetCommit(commits[len(commits)-1].Sha1) + if err != nil && !git.IsErrNotExist(err) { + log.Error("unable to GetCommit %s from %-v: %v", oldCommitID, repo, err) + } + if oldCommit != nil { + for i := 0; i < oldCommit.ParentCount(); i++ { + commitID, _ := oldCommit.ParentID(i) + if !commitID.IsZero() { + oldCommitID = commitID.String() + break + } + } + } + } + + if oldCommitID == objectFormat.EmptyObjectID().String() && repo.DefaultBranch != opts.RefFullName.BranchName() { + oldCommitID = repo.DefaultBranch + } + + if oldCommitID != objectFormat.EmptyObjectID().String() { + return repo.ComposeCompareURL(oldCommitID, opts.NewCommitID) + } + return "" +} + +func pushNewBranch(ctx context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions, newCommit *git.Commit) ([]*git.Commit, error) { + if repo.IsEmpty { // Change default branch and empty status only if pushed ref is non-empty branch. + repo.DefaultBranch = opts.RefName() + repo.IsEmpty = false + if repo.DefaultBranch != setting.Repository.DefaultBranch { + if err := gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + return nil, err + } + } + // Update the is empty and default_branch columns + if err := repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "default_branch", "is_empty"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) + } + } + + l, err := newCommit.CommitsBeforeLimit(10) + if err != nil { + return nil, fmt.Errorf("newCommit.CommitsBeforeLimit: %w", err) + } + notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) + return l, nil +} + +func pushUpdateBranch(_ context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions, newCommit *git.Commit) ([]*git.Commit, error) { + l, err := newCommit.CommitsBeforeUntil(opts.OldCommitID) + if err != nil { + return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %w", err) + } + + branch := opts.RefFullName.BranchName() + + isForcePush, err := newCommit.IsForcePush(opts.OldCommitID) + if err != nil { + log.Error("IsForcePush %s:%s failed: %v", repo.FullName(), branch, err) + } + + // only update branch can trigger pull request task because the pull request hasn't been created yet when creating a branch + go pull_service.AddTestPullRequestTask(pull_service.TestPullRequestOptions{ + RepoID: repo.ID, + Doer: pusher, + Branch: branch, + IsSync: true, + IsForcePush: isForcePush, + OldCommitID: opts.OldCommitID, + NewCommitID: opts.NewCommitID, + }) + + if isForcePush { + log.Trace("Push %s is a force push", opts.NewCommitID) + + cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) + } else { + // TODO: increment update the commit count cache but not remove + cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) + } + + return l, nil +} + +func pushDeleteBranch(ctx context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions) { + notify_service.DeleteRef(ctx, pusher, repo, opts.RefFullName) + + if err := pull_service.AdjustPullsCausedByBranchDeleted(ctx, pusher, repo, opts.RefFullName.BranchName()); err != nil { + // close all related pulls + log.Error("close related pull request failed: %v", err) + } +} + // PushUpdateAddDeleteTags updates a number of added and delete tags -func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, addTags, delTags []string) error { +func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, pusher *user_model.User, addTags, delTags []string) error { return db.WithTx(ctx, func(ctx context.Context) error { - if err := repo_model.PushUpdateDeleteTagsContext(ctx, repo, delTags); err != nil { + if err := repo_model.PushUpdateDeleteTags(ctx, repo, delTags); err != nil { return err } - return pushUpdateAddTags(ctx, repo, gitRepo, addTags) + return pushUpdateAddTags(ctx, repo, gitRepo, pusher, addTags) }) } // pushUpdateAddTags updates a number of add tags -func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tags []string) error { +func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, pusher *user_model.User, tags []string) error { if len(tags) == 0 { return nil } @@ -338,14 +378,12 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo newReleases := make([]*repo_model.Release, 0, len(lowerTags)-len(relMap)) - emailToUser := make(map[string]*user_model.User) - for i, lowerTag := range lowerTags { tag, err := gitRepo.GetTag(tags[i]) if err != nil { return fmt.Errorf("GetTag: %w", err) } - commit, err := tag.Commit(gitRepo) + commit, err := gitRepo.GetTagCommit(tag.Name) if err != nil { return fmt.Errorf("Commit: %w", err) } @@ -357,29 +395,12 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo if sig == nil { sig = commit.Committer } - var author *user_model.User - createdAt := time.Unix(1, 0) + createdAt := time.Unix(1, 0) if sig != nil { - var ok bool - author, ok = emailToUser[sig.Email] - if !ok { - author, err = user_model.GetUserByEmail(ctx, sig.Email) - if err != nil && !user_model.IsErrUserNotExist(err) { - return fmt.Errorf("GetUserByEmail: %w", err) - } - if author != nil { - emailToUser[sig.Email] = author - } - } createdAt = sig.When } - commitsCount, err := commit.CommitsCount() - if err != nil { - return fmt.Errorf("CommitsCount: %w", err) - } - rel, has := relMap[lowerTag] parts := strings.SplitN(tag.Message, "\n", 2) @@ -395,31 +416,26 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo LowerTagName: lowerTag, Target: "", Sha1: commit.ID.String(), - NumCommits: commitsCount, + NumCommits: -1, // the commits count will be updated when the UI needs it Note: note, IsDraft: false, IsPrerelease: false, IsTag: true, + PublisherID: pusher.ID, CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), } - if author != nil { - rel.PublisherID = author.ID - } newReleases = append(newReleases, rel) } else { rel.Sha1 = commit.ID.String() rel.CreatedUnix = timeutil.TimeStamp(createdAt.Unix()) - rel.NumCommits = commitsCount if rel.IsTag { rel.Title = parts[0] rel.Note = note - if author != nil { - rel.PublisherID = author.ID - } } else { rel.IsDraft = false } + rel.PublisherID = pusher.ID if err = repo_model.UpdateRelease(ctx, rel); err != nil { return fmt.Errorf("Update: %w", err) } diff --git a/services/repository/repository.go b/services/repository/repository.go index 59b4491132..e574dc6c01 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -5,23 +5,29 @@ package repository import ( "context" + "errors" "fmt" + "os" + "path/filepath" + "strings" + activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/organization" - packages_model "code.gitea.io/gitea/models/packages" + access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" - system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/graceful" + issue_indexer "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/queue" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" pull_service "code.gitea.io/gitea/services/pull" ) @@ -41,7 +47,7 @@ type WebSearchResults struct { // CreateRepository creates a repository for the user/organization. func CreateRepository(ctx context.Context, doer, owner *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { - repo, err := CreateRepositoryDirectly(ctx, doer, owner, opts) + repo, err := CreateRepositoryDirectly(ctx, doer, owner, opts, true) if err != nil { // No need to rollback here we should do this in CreateRepository... return nil, err @@ -63,11 +69,7 @@ func DeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_mod notify_service.DeleteRepository(ctx, doer, repo) } - if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { - return err - } - - return packages_model.UnlinkRepositoryFromAllPackages(ctx, repo.ID) + return DeleteRepositoryDirectly(ctx, repo.ID) } // PushCreateRepo creates a repository when a new repository is pushed to an appropriate namespace @@ -77,10 +79,10 @@ func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoN if ok, err := organization.CanCreateOrgRepo(ctx, owner.ID, authUser.ID); err != nil { return nil, err } else if !ok { - return nil, fmt.Errorf("cannot push-create repository for org") + return nil, errors.New("cannot push-create repository for org") } } else if authUser.ID != owner.ID { - return nil, fmt.Errorf("cannot push-create repository for another user") + return nil, errors.New("cannot push-create repository for another user") } } @@ -99,15 +101,13 @@ func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoN func Init(ctx context.Context) error { licenseUpdaterQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_license_updater", repoLicenseUpdater) if licenseUpdaterQueue == nil { - return fmt.Errorf("unable to create repo_license_updater queue") + return errors.New("unable to create repo_license_updater queue") } go graceful.GetManager().RunWithCancel(licenseUpdaterQueue) if err := repo_module.LoadRepoConfig(); err != nil { return err } - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repository uploads", setting.Repository.Upload.TempPath) - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repositories", repo_module.LocalCopyPath()) if err := initPushQueue(); err != nil { return err } @@ -116,42 +116,107 @@ func Init(ctx context.Context) error { // UpdateRepository updates a repository func UpdateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - if err = repo_module.UpdateRepository(ctx, repo, visibilityChanged); err != nil { - return fmt.Errorf("updateRepository: %w", err) - } - - return committer.Commit() + return db.WithTx(ctx, func(ctx context.Context) error { + if err = updateRepository(ctx, repo, visibilityChanged); err != nil { + return fmt.Errorf("updateRepository: %w", err) + } + return nil + }) } -func UpdateRepositoryVisibility(ctx context.Context, repo *repo_model.Repository, isPrivate bool) (err error) { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } +func MakeRepoPublic(ctx context.Context, repo *repo_model.Repository) (err error) { + return db.WithTx(ctx, func(ctx context.Context) error { + repo.IsPrivate = false + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_private"); err != nil { + return err + } - defer committer.Close() + if err = repo.LoadOwner(ctx); err != nil { + return fmt.Errorf("LoadOwner: %w", err) + } + if repo.Owner.IsOrganization() { + // Organization repository need to recalculate access table when visibility is changed. + if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { + return fmt.Errorf("recalculateTeamAccesses: %w", err) + } + } - repo.IsPrivate = isPrivate + // Create/Remove git-daemon-export-ok for git-daemon... + if err := checkDaemonExportOK(ctx, repo); err != nil { + return err + } - if err = repo_module.UpdateRepository(ctx, repo, true); err != nil { - return fmt.Errorf("UpdateRepositoryVisibility: %w", err) - } + forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) + if err != nil { + return fmt.Errorf("getRepositoriesByForkID: %w", err) + } - return committer.Commit() -} + if repo.Owner.Visibility != structs.VisibleTypePrivate { + for i := range forkRepos { + if err = MakeRepoPublic(ctx, forkRepos[i]); err != nil { + return fmt.Errorf("MakeRepoPublic[%d]: %w", forkRepos[i].ID, err) + } + } + } -func MakeRepoPublic(ctx context.Context, repo *repo_model.Repository) (err error) { - return UpdateRepositoryVisibility(ctx, repo, false) + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + + return nil + }) } func MakeRepoPrivate(ctx context.Context, repo *repo_model.Repository) (err error) { - return UpdateRepositoryVisibility(ctx, repo, true) + return db.WithTx(ctx, func(ctx context.Context) error { + repo.IsPrivate = true + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_private"); err != nil { + return err + } + + if err = repo.LoadOwner(ctx); err != nil { + return fmt.Errorf("LoadOwner: %w", err) + } + if repo.Owner.IsOrganization() { + // Organization repository need to recalculate access table when visibility is changed. + if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { + return fmt.Errorf("recalculateTeamAccesses: %w", err) + } + } + + // If repo has become private, we need to set its actions to private. + _, err = db.GetEngine(ctx).Where("repo_id = ?", repo.ID).Cols("is_private").Update(&activities_model.Action{ + IsPrivate: true, + }) + if err != nil { + return err + } + + if err = repo_model.ClearRepoStars(ctx, repo.ID); err != nil { + return err + } + + // Create/Remove git-daemon-export-ok for git-daemon... + if err := checkDaemonExportOK(ctx, repo); err != nil { + return err + } + + forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) + if err != nil { + return fmt.Errorf("getRepositoriesByForkID: %w", err) + } + for i := range forkRepos { + if err = MakeRepoPrivate(ctx, forkRepos[i]); err != nil { + return fmt.Errorf("MakeRepoPrivate[%d]: %w", forkRepos[i].ID, err) + } + } + + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + + return nil + }) } // LinkedRepository returns the linked repo if any @@ -177,3 +242,97 @@ func LinkedRepository(ctx context.Context, a *repo_model.Attachment) (*repo_mode } return nil, -1, nil } + +// checkDaemonExportOK creates/removes git-daemon-export-ok for git-daemon... +func checkDaemonExportOK(ctx context.Context, repo *repo_model.Repository) error { + if err := repo.LoadOwner(ctx); err != nil { + return err + } + + // Create/Remove git-daemon-export-ok for git-daemon... + daemonExportFile := filepath.Join(repo.RepoPath(), `git-daemon-export-ok`) + + isExist, err := util.IsExist(daemonExportFile) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err) + return err + } + + isPublic := !repo.IsPrivate && repo.Owner.Visibility == structs.VisibleTypePublic + if !isPublic && isExist { + if err = util.Remove(daemonExportFile); err != nil { + log.Error("Failed to remove %s: %v", daemonExportFile, err) + } + } else if isPublic && !isExist { + if f, err := os.Create(daemonExportFile); err != nil { + log.Error("Failed to create %s: %v", daemonExportFile, err) + } else { + f.Close() + } + } + + return nil +} + +// updateRepository updates a repository with db context +func updateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { + repo.LowerName = strings.ToLower(repo.Name) + + e := db.GetEngine(ctx) + + if _, err = e.ID(repo.ID).NoAutoTime().AllCols().Update(repo); err != nil { + return fmt.Errorf("update: %w", err) + } + + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + + if visibilityChanged { + if err = repo.LoadOwner(ctx); err != nil { + return fmt.Errorf("LoadOwner: %w", err) + } + if repo.Owner.IsOrganization() { + // Organization repository need to recalculate access table when visibility is changed. + if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { + return fmt.Errorf("recalculateTeamAccesses: %w", err) + } + } + + // If repo has become private, we need to set its actions to private. + if repo.IsPrivate { + _, err = e.Where("repo_id = ?", repo.ID).Cols("is_private").Update(&activities_model.Action{ + IsPrivate: true, + }) + if err != nil { + return err + } + + if err = repo_model.ClearRepoStars(ctx, repo.ID); err != nil { + return err + } + } + + // Create/Remove git-daemon-export-ok for git-daemon... + if err := checkDaemonExportOK(ctx, repo); err != nil { + return err + } + + forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) + if err != nil { + return fmt.Errorf("getRepositoriesByForkID: %w", err) + } + for i := range forkRepos { + forkRepos[i].IsPrivate = repo.IsPrivate || repo.Owner.Visibility == structs.VisibleTypePrivate + if err = updateRepository(ctx, forkRepos[i], true); err != nil { + return fmt.Errorf("updateRepository[%d]: %w", forkRepos[i].ID, err) + } + } + + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + } + + return nil +} diff --git a/services/repository/repository_test.go b/services/repository/repository_test.go index 892a11a23e..8f9fdf8fa1 100644 --- a/services/repository/repository_test.go +++ b/services/repository/repository_test.go @@ -6,6 +6,7 @@ package repository import ( "testing" + activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" @@ -40,3 +41,23 @@ func TestLinkedRepository(t *testing.T) { }) } } + +func TestUpdateRepositoryVisibilityChanged(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + // Get sample repo and change visibility + repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 9) + assert.NoError(t, err) + repo.IsPrivate = true + + // Update it + err = updateRepository(db.DefaultContext, repo, true) + assert.NoError(t, err) + + // Check visibility of action has become private + act := activities_model.Action{} + _, err = db.GetEngine(db.DefaultContext).ID(3).Get(&act) + + assert.NoError(t, err) + assert.True(t, act.IsPrivate) +} diff --git a/services/repository/setting.go b/services/repository/setting.go index b82f24271e..e0c787dd2d 100644 --- a/services/repository/setting.go +++ b/services/repository/setting.go @@ -7,7 +7,6 @@ import ( "context" "slices" - actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" @@ -29,7 +28,7 @@ func UpdateRepositoryUnits(ctx context.Context, repo *repo_model.Repository, uni } if slices.Contains(deleteUnitTypes, unit.TypeActions) { - if err := actions_model.CleanRepoScheduleTasks(ctx, repo); err != nil { + if err := actions_service.CleanRepoScheduleTasks(ctx, repo); err != nil { log.Error("CleanRepoScheduleTasks: %v", err) } } diff --git a/services/repository/template.go b/services/repository/template.go index 36a680c8e2..6906a60083 100644 --- a/services/repository/template.go +++ b/services/repository/template.go @@ -5,12 +5,17 @@ package repository import ( "context" + "fmt" + "strings" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" notify_service "code.gitea.io/gitea/services/notify" ) @@ -63,70 +68,124 @@ func GenerateProtectedBranch(ctx context.Context, templateRepo, generateRepo *re // GenerateRepository generates a repository from a template func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) { - if !doer.IsAdmin && !owner.CanCreateRepo() { + if !doer.CanCreateRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ Limit: owner.MaxRepoCreation, } } - var generateRepo *repo_model.Repository - if err = db.WithTx(ctx, func(ctx context.Context) error { - generateRepo, err = generateRepository(ctx, doer, owner, templateRepo, opts) + generateRepo := &repo_model.Repository{ + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, + Name: opts.Name, + LowerName: strings.ToLower(opts.Name), + Description: opts.Description, + DefaultBranch: opts.DefaultBranch, + IsPrivate: opts.Private, + IsEmpty: !opts.GitContent || templateRepo.IsEmpty, + IsFsckEnabled: templateRepo.IsFsckEnabled, + TemplateID: templateRepo.ID, + TrustModel: templateRepo.TrustModel, + ObjectFormatName: templateRepo.ObjectFormatName, + Status: repo_model.RepositoryBeingMigrated, + } + + // 1 - Create the repository in the database + if err := db.WithTx(ctx, func(ctx context.Context) error { + return createRepositoryInDB(ctx, doer, owner, generateRepo, false) + }); err != nil { + return nil, err + } + + // last - clean up the repository if something goes wrong + defer func() { if err != nil { - return err + // we can not use the ctx because it maybe canceled or timeout + cleanupRepository(generateRepo.ID) } + }() - // Git Content - if opts.GitContent && !templateRepo.IsEmpty { - if err = GenerateGitContent(ctx, templateRepo, generateRepo); err != nil { - return err - } + // 2 - check whether the repository with the same storage exists + isExist, err := gitrepo.IsRepositoryExist(ctx, generateRepo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", generateRepo.FullName(), err) + return nil, err + } + if isExist { + // Don't return directly, we need err in defer to cleanupRepository + err = repo_model.ErrRepoFilesAlreadyExist{ + Uname: generateRepo.OwnerName, + Name: generateRepo.Name, } + return nil, err + } - // Topics - if opts.Topics { - if err = repo_model.GenerateTopics(ctx, templateRepo, generateRepo); err != nil { - return err - } + // 3 -Init git bare new repository. + if err = git.InitRepository(ctx, generateRepo.RepoPath(), true, generateRepo.ObjectFormatName); err != nil { + return nil, fmt.Errorf("git.InitRepository: %w", err) + } else if err = gitrepo.CreateDelegateHooks(ctx, generateRepo); err != nil { + return nil, fmt.Errorf("createDelegateHooks: %w", err) + } + + // 4 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, generateRepo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } + + // 5 - generate the repository contents according to the template + // Git Content + if opts.GitContent && !templateRepo.IsEmpty { + if err = GenerateGitContent(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Git Hooks - if opts.GitHooks { - if err = GenerateGitHooks(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Topics + if opts.Topics { + if err = repo_model.GenerateTopics(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Webhooks - if opts.Webhooks { - if err = GenerateWebhooks(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Git Hooks + if opts.GitHooks { + if err = GenerateGitHooks(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Avatar - if opts.Avatar && len(templateRepo.Avatar) > 0 { - if err = generateAvatar(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Webhooks + if opts.Webhooks { + if err = GenerateWebhooks(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Issue Labels - if opts.IssueLabels { - if err = GenerateIssueLabels(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Avatar + if opts.Avatar && len(templateRepo.Avatar) > 0 { + if err = generateAvatar(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - if opts.ProtectedBranch { - if err = GenerateProtectedBranch(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Issue Labels + if opts.IssueLabels { + if err = GenerateIssueLabels(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - return nil - }); err != nil { - return nil, err + if opts.ProtectedBranch { + if err = GenerateProtectedBranch(ctx, templateRepo, generateRepo); err != nil { + return nil, err + } + } + + // 6 - update repository status to be ready + generateRepo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, generateRepo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } notify_service.CreateRepository(ctx, doer, owner, generateRepo) diff --git a/services/repository/transfer.go b/services/repository/transfer.go index 9ef28ddeb9..5ad63cca67 100644 --- a/services/repository/transfer.go +++ b/services/repository/transfer.go @@ -17,29 +17,31 @@ import ( project_model "code.gitea.io/gitea/models/project" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" ) -func getRepoWorkingLockKey(repoID int64) string { - return fmt.Sprintf("repo_working_%d", repoID) +type LimitReachedError struct{ Limit int } + +func (LimitReachedError) Error() string { + return "Repository limit has been reached" } -// TransferOwnership transfers all corresponding setting from old user to new one. -func TransferOwnership(ctx context.Context, doer, newOwner *user_model.User, repo *repo_model.Repository, teams []*organization.Team) error { - if err := repo.LoadOwner(ctx); err != nil { - return err - } - for _, team := range teams { - if newOwner.ID != team.OrgID { - return fmt.Errorf("team %d does not belong to organization", team.ID) - } - } +func IsRepositoryLimitReached(err error) bool { + _, ok := err.(LimitReachedError) + return ok +} - oldOwner := repo.Owner +func getRepoWorkingLockKey(repoID int64) string { + return fmt.Sprintf("repo_working_%d", repoID) +} +// AcceptTransferOwnership transfers all corresponding setting from old user to new one. +func AcceptTransferOwnership(ctx context.Context, repo *repo_model.Repository, doer *user_model.User) error { releaser, err := globallock.Lock(ctx, getRepoWorkingLockKey(repo.ID)) if err != nil { log.Error("lock.Lock(): %v", err) @@ -47,29 +49,49 @@ func TransferOwnership(ctx context.Context, doer, newOwner *user_model.User, rep } defer releaser() - if err := transferOwnership(ctx, doer, newOwner.Name, repo); err != nil { - return err - } - releaser() - - newRepo, err := repo_model.GetRepositoryByID(ctx, repo.ID) + repoTransfer, err := repo_model.GetPendingRepositoryTransfer(ctx, repo) if err != nil { return err } - for _, team := range teams { - if err := addRepositoryToTeam(ctx, team, newRepo); err != nil { + oldOwnerName := repo.OwnerName + + if err := db.WithTx(ctx, func(ctx context.Context) error { + if err := repoTransfer.LoadAttributes(ctx); err != nil { return err } + + if !doer.CanCreateRepoIn(repoTransfer.Recipient) { + limit := util.Iif(repoTransfer.Recipient.MaxRepoCreation >= 0, repoTransfer.Recipient.MaxRepoCreation, setting.Repository.MaxCreationLimit) + return LimitReachedError{Limit: limit} + } + + if !repoTransfer.CanUserAcceptOrRejectTransfer(ctx, doer) { + return util.ErrPermissionDenied + } + + if err := repo.LoadOwner(ctx); err != nil { + return err + } + for _, team := range repoTransfer.Teams { + if repoTransfer.Recipient.ID != team.OrgID { + return fmt.Errorf("team %d does not belong to organization", team.ID) + } + } + + return transferOwnership(ctx, repoTransfer.Doer, repoTransfer.Recipient.Name, repo, repoTransfer.Teams) + }); err != nil { + return err } + releaser() - notify_service.TransferRepository(ctx, doer, repo, oldOwner.Name) + notify_service.TransferRepository(ctx, doer, repo, oldOwnerName) return nil } // transferOwnership transfers all corresponding repository items from old user to new one. -func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName string, repo *repo_model.Repository) (err error) { +func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName string, repo *repo_model.Repository, teams []*organization.Team) (err error) { repoRenamed := false wikiRenamed := false oldOwnerName := doer.Name @@ -138,7 +160,7 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName repo.OwnerName = newOwner.Name // Update repository. - if _, err := sess.ID(repo.ID).Update(repo); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "owner_id", "owner_name"); err != nil { return fmt.Errorf("update owner: %w", err) } @@ -174,15 +196,13 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName collaboration.UserID = 0 } - // Remove old team-repository relations. if oldOwner.IsOrganization() { + // Remove old team-repository relations. if err := organization.RemoveOrgRepo(ctx, oldOwner.ID, repo.ID); err != nil { return fmt.Errorf("removeOrgRepo: %w", err) } - } - // Remove project's issues that belong to old organization's projects - if oldOwner.IsOrganization() { + // Remove project's issues that belong to old organization's projects projects, err := project_model.GetAllProjectsIDsByOwnerIDAndType(ctx, oldOwner.ID, project_model.TypeOrganization) if err != nil { return fmt.Errorf("Unable to find old org projects: %w", err) @@ -225,15 +245,13 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName return fmt.Errorf("watchRepo: %w", err) } - // Remove watch for organization. if oldOwner.IsOrganization() { + // Remove watch for organization. if err := repo_model.WatchRepo(ctx, oldOwner, repo, false); err != nil { return fmt.Errorf("watchRepo [false]: %w", err) } - } - // Delete labels that belong to the old organization and comments that added these labels - if oldOwner.IsOrganization() { + // Delete labels that belong to the old organization and comments that added these labels if _, err := sess.Exec(`DELETE FROM issue_label WHERE issue_label.id IN ( SELECT il_too.id FROM ( SELECT il_too_too.id @@ -261,7 +279,6 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName // Rename remote repository to new path and delete local copy. dir := user_model.UserPath(newOwner.Name) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { return fmt.Errorf("Failed to create dir %s: %w", dir, err) } @@ -273,7 +290,6 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName // Rename remote wiki repository to new path and delete local copy. wikiPath := repo_model.WikiPath(oldOwner.Name, repo.Name) - if isExist, err := util.IsExist(wikiPath); err != nil { log.Error("Unable to check if %s exists. Error: %v", wikiPath, err) return err @@ -288,7 +304,7 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName return fmt.Errorf("deleteRepositoryTransfer: %w", err) } repo.Status = repo_model.RepositoryReady - if err := repo_model.UpdateRepositoryCols(ctx, repo, "status"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "status"); err != nil { return err } @@ -301,6 +317,17 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName return fmt.Errorf("repo_model.NewRedirect: %w", err) } + newRepo, err := repo_model.GetRepositoryByID(ctx, repo.ID) + if err != nil { + return err + } + + for _, team := range teams { + if err := addRepositoryToTeam(ctx, team, newRepo); err != nil { + return err + } + } + return committer.Commit() } @@ -321,13 +348,13 @@ func changeRepositoryName(ctx context.Context, repo *repo_model.Repository, newR return fmt.Errorf("IsRepositoryExist: %w", err) } else if has { return repo_model.ErrRepoAlreadyExist{ - Uname: repo.Owner.Name, + Uname: repo.OwnerName, Name: newRepoName, } } - newRepoPath := repo_model.RepoPath(repo.Owner.Name, newRepoName) - if err = util.Rename(repo.RepoPath(), newRepoPath); err != nil { + if err = gitrepo.RenameRepository(ctx, repo, + repo_model.StorageRepo(repo_model.RelativePath(repo.OwnerName, newRepoName))); err != nil { return fmt.Errorf("rename repository directory: %w", err) } @@ -343,17 +370,9 @@ func changeRepositoryName(ctx context.Context, repo *repo_model.Repository, newR } } - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - if err := repo_model.NewRedirect(ctx, repo.Owner.ID, repo.ID, oldRepoName, newRepoName); err != nil { - return err - } - - return committer.Commit() + return db.WithTx(ctx, func(ctx context.Context) error { + return repo_model.NewRedirect(ctx, repo.Owner.ID, repo.ID, oldRepoName, newRepoName) + }) } // ChangeRepositoryName changes all corresponding setting from old repository name to new one. @@ -387,70 +406,147 @@ func ChangeRepositoryName(ctx context.Context, doer *user_model.User, repo *repo // StartRepositoryTransfer transfer a repo from one owner to a new one. // it make repository into pending transfer state, if doer can not create repo for new owner. func StartRepositoryTransfer(ctx context.Context, doer, newOwner *user_model.User, repo *repo_model.Repository, teams []*organization.Team) error { + releaser, err := globallock.Lock(ctx, getRepoWorkingLockKey(repo.ID)) + if err != nil { + return fmt.Errorf("lock.Lock: %w", err) + } + defer releaser() + if err := repo_model.TestRepositoryReadyForTransfer(repo.Status); err != nil { return err } - // Admin is always allowed to transfer || user transfer repo back to his account - if doer.IsAdmin || doer.ID == newOwner.ID { - return TransferOwnership(ctx, doer, newOwner, repo, teams) + if !doer.CanForkRepoIn(newOwner) { + limit := util.Iif(newOwner.MaxRepoCreation >= 0, newOwner.MaxRepoCreation, setting.Repository.MaxCreationLimit) + return LimitReachedError{Limit: limit} } - if user_model.IsUserBlockedBy(ctx, doer, newOwner.ID) { - return user_model.ErrBlockedUser - } + var isDirectTransfer bool + oldOwnerName := repo.OwnerName - // If new owner is an org and user can create repos he can transfer directly too - if newOwner.IsOrganization() { - allowed, err := organization.CanCreateOrgRepo(ctx, newOwner.ID, doer.ID) - if err != nil { - return err + if err := db.WithTx(ctx, func(ctx context.Context) error { + // Admin is always allowed to transfer || user transfer repo back to his account, + // then it will transfer directly without acceptance. + if doer.IsAdmin || doer.ID == newOwner.ID { + isDirectTransfer = true + return transferOwnership(ctx, doer, newOwner.Name, repo, teams) } - if allowed { - return TransferOwnership(ctx, doer, newOwner, repo, teams) + + if user_model.IsUserBlockedBy(ctx, doer, newOwner.ID) { + return user_model.ErrBlockedUser } - } - // In case the new owner would not have sufficient access to the repo, give access rights for read - hasAccess, err := access_model.HasAnyUnitAccess(ctx, newOwner.ID, repo) - if err != nil { - return err - } - if !hasAccess { - if err := AddOrUpdateCollaborator(ctx, repo, newOwner, perm.AccessModeRead); err != nil { + // If new owner is an org and user can create repos he can transfer directly too + if newOwner.IsOrganization() { + allowed, err := organization.CanCreateOrgRepo(ctx, newOwner.ID, doer.ID) + if err != nil { + return err + } + if allowed { + isDirectTransfer = true + return transferOwnership(ctx, doer, newOwner.Name, repo, teams) + } + } + + // In case the new owner would not have sufficient access to the repo, give access rights for read + hasAccess, err := access_model.HasAnyUnitAccess(ctx, newOwner.ID, repo) + if err != nil { return err } - } + if !hasAccess { + if err := AddOrUpdateCollaborator(ctx, repo, newOwner, perm.AccessModeRead); err != nil { + return err + } + } - // Make repo as pending for transfer - repo.Status = repo_model.RepositoryPendingTransfer - if err := repo_model.CreatePendingRepositoryTransfer(ctx, doer, newOwner, repo.ID, teams); err != nil { + // Make repo as pending for transfer + repo.Status = repo_model.RepositoryPendingTransfer + return repo_model.CreatePendingRepositoryTransfer(ctx, doer, newOwner, repo.ID, teams) + }); err != nil { return err } - // notify users who are able to accept / reject transfer - notify_service.RepoPendingTransfer(ctx, doer, newOwner, repo) + if isDirectTransfer { + notify_service.TransferRepository(ctx, doer, repo, oldOwnerName) + } else { + // notify users who are able to accept / reject transfer + notify_service.RepoPendingTransfer(ctx, doer, newOwner, repo) + } return nil } -// CancelRepositoryTransfer marks the repository as ready and remove pending transfer entry, +// RejectRepositoryTransfer marks the repository as ready and remove pending transfer entry, // thus cancel the transfer process. -func CancelRepositoryTransfer(ctx context.Context, repo *repo_model.Repository) error { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err +// The accepter can reject the transfer. +func RejectRepositoryTransfer(ctx context.Context, repo *repo_model.Repository, doer *user_model.User) error { + return db.WithTx(ctx, func(ctx context.Context) error { + repoTransfer, err := repo_model.GetPendingRepositoryTransfer(ctx, repo) + if err != nil { + return err + } + + if err := repoTransfer.LoadAttributes(ctx); err != nil { + return err + } + + if !repoTransfer.CanUserAcceptOrRejectTransfer(ctx, doer) { + return util.ErrPermissionDenied + } + + repo.Status = repo_model.RepositoryReady + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "status"); err != nil { + return err + } + + return repo_model.DeleteRepositoryTransfer(ctx, repo.ID) + }) +} + +func canUserCancelTransfer(ctx context.Context, r *repo_model.RepoTransfer, u *user_model.User) bool { + if u.IsAdmin || u.ID == r.DoerID { + return true } - defer committer.Close() - repo.Status = repo_model.RepositoryReady - if err := repo_model.UpdateRepositoryCols(ctx, repo, "status"); err != nil { - return err + if err := r.LoadAttributes(ctx); err != nil { + log.Error("LoadAttributes: %v", err) + return false } - if err := repo_model.DeleteRepositoryTransfer(ctx, repo.ID); err != nil { - return err + if err := r.Repo.LoadOwner(ctx); err != nil { + log.Error("LoadOwner: %v", err) + return false } - return committer.Commit() + if !r.Repo.Owner.IsOrganization() { + return r.Repo.OwnerID == u.ID + } + + perm, err := access_model.GetUserRepoPermission(ctx, r.Repo, u) + if err != nil { + log.Error("GetUserRepoPermission: %v", err) + return false + } + return perm.IsOwner() +} + +// CancelRepositoryTransfer cancels the repository transfer process. The sender or +// the users who have admin permission of the original repository can cancel the transfer +func CancelRepositoryTransfer(ctx context.Context, repoTransfer *repo_model.RepoTransfer, doer *user_model.User) error { + return db.WithTx(ctx, func(ctx context.Context) error { + if err := repoTransfer.LoadAttributes(ctx); err != nil { + return err + } + + if !canUserCancelTransfer(ctx, repoTransfer, doer) { + return util.ErrPermissionDenied + } + + repoTransfer.Repo.Status = repo_model.RepositoryReady + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repoTransfer.Repo, "status"); err != nil { + return err + } + + return repo_model.DeleteRepositoryTransfer(ctx, repoTransfer.RepoID) + }) } diff --git a/services/repository/transfer_test.go b/services/repository/transfer_test.go index 91722fb8ae..80a073e9f9 100644 --- a/services/repository/transfer_test.go +++ b/services/repository/transfer_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package repository @@ -14,11 +14,14 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/feed" notify_service "code.gitea.io/gitea/services/notify" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var notifySync sync.Once @@ -34,23 +37,26 @@ func TestTransferOwnership(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) - repo.Owner = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}) - assert.NoError(t, TransferOwnership(db.DefaultContext, doer, doer, repo, nil)) + assert.NoError(t, repo.LoadOwner(db.DefaultContext)) + repoTransfer := unittest.AssertExistsAndLoadBean(t, &repo_model.RepoTransfer{ID: 1}) + assert.NoError(t, repoTransfer.LoadAttributes(db.DefaultContext)) + assert.NoError(t, AcceptTransferOwnership(db.DefaultContext, repo, doer)) transferredRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) - assert.EqualValues(t, 2, transferredRepo.OwnerID) + assert.EqualValues(t, 1, transferredRepo.OwnerID) // repo_transfer.yml id=1 + unittest.AssertNotExistsBean(t, &repo_model.RepoTransfer{ID: 1}) exist, err := util.IsExist(repo_model.RepoPath("org3", "repo3")) assert.NoError(t, err) assert.False(t, exist) - exist, err = util.IsExist(repo_model.RepoPath("user2", "repo3")) + exist, err = util.IsExist(repo_model.RepoPath("user1", "repo3")) assert.NoError(t, err) assert.True(t, exist) unittest.AssertExistsAndLoadBean(t, &activities_model.Action{ OpType: activities_model.ActionTransferRepo, - ActUserID: 2, + ActUserID: 1, RepoID: 3, Content: "org3/repo3", }) @@ -61,10 +67,10 @@ func TestTransferOwnership(t *testing.T) { func TestStartRepositoryTransferSetPermission(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) recipient := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 5}) - repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) - repo.Owner = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}) + assert.NoError(t, repo.LoadOwner(db.DefaultContext)) hasAccess, err := access_model.HasAnyUnitAccess(db.DefaultContext, recipient.ID, repo) assert.NoError(t, err) @@ -82,7 +88,7 @@ func TestStartRepositoryTransferSetPermission(t *testing.T) { func TestRepositoryTransfer(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) transfer, err := repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) @@ -90,7 +96,7 @@ func TestRepositoryTransfer(t *testing.T) { assert.NotNil(t, transfer) // Cancel transfer - assert.NoError(t, CancelRepositoryTransfer(db.DefaultContext, repo)) + assert.NoError(t, CancelRepositoryTransfer(db.DefaultContext, transfer, doer)) transfer, err = repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) assert.Error(t, err) @@ -113,10 +119,49 @@ func TestRepositoryTransfer(t *testing.T) { assert.Error(t, err) assert.True(t, repo_model.IsErrRepoTransferInProgress(err)) - // Unknown user - err = repo_model.CreatePendingRepositoryTransfer(db.DefaultContext, doer, &user_model.User{ID: 1000, LowerName: "user1000"}, repo.ID, nil) + repo2 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}) + // Unknown user, transfer non-existent transfer repo id = 2 + err = repo_model.CreatePendingRepositoryTransfer(db.DefaultContext, doer, &user_model.User{ID: 1000, LowerName: "user1000"}, repo2.ID, nil) assert.Error(t, err) - // Cancel transfer - assert.NoError(t, CancelRepositoryTransfer(db.DefaultContext, repo)) + // Reject transfer + err = RejectRepositoryTransfer(db.DefaultContext, repo2, doer) + assert.True(t, repo_model.IsErrNoPendingTransfer(err)) +} + +// Test transfer rejections +func TestRepositoryTransferRejection(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + // Set limit to 0 repositories so no repositories can be transferred + defer test.MockVariableValue(&setting.Repository.MaxCreationLimit, 0)() + + // Admin case + doerAdmin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 5}) + + transfer, err := repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) + require.NoError(t, err) + require.NotNil(t, transfer) + require.NoError(t, transfer.LoadRecipient(db.DefaultContext)) + + require.True(t, doerAdmin.CanCreateRepoIn(transfer.Recipient)) // admin is not subject to limits + + // Administrator should not be affected by the limits so transfer should be successful + assert.NoError(t, AcceptTransferOwnership(db.DefaultContext, repo, doerAdmin)) + + // Non admin user case + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 10}) + repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 21}) + + transfer, err = repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) + require.NoError(t, err) + require.NotNil(t, transfer) + require.NoError(t, transfer.LoadRecipient(db.DefaultContext)) + + require.False(t, doer.CanCreateRepoIn(transfer.Recipient)) // regular user is subject to limits + + // Cannot accept because of the limit + err = AcceptTransferOwnership(db.DefaultContext, repo, doer) + assert.Error(t, err) + assert.True(t, IsRepositoryLimitReached(err)) } diff --git a/services/secrets/secrets.go b/services/secrets/secrets.go index 031c474dd7..ec6a3cb062 100644 --- a/services/secrets/secrets.go +++ b/services/secrets/secrets.go @@ -10,7 +10,7 @@ import ( secret_model "code.gitea.io/gitea/models/secret" ) -func CreateOrUpdateSecret(ctx context.Context, ownerID, repoID int64, name, data string) (*secret_model.Secret, bool, error) { +func CreateOrUpdateSecret(ctx context.Context, ownerID, repoID int64, name, data, description string) (*secret_model.Secret, bool, error) { if err := ValidateName(name); err != nil { return nil, false, err } @@ -25,14 +25,14 @@ func CreateOrUpdateSecret(ctx context.Context, ownerID, repoID int64, name, data } if len(s) == 0 { - s, err := secret_model.InsertEncryptedSecret(ctx, ownerID, repoID, name, data) + s, err := secret_model.InsertEncryptedSecret(ctx, ownerID, repoID, name, data, description) if err != nil { return nil, false, err } return s, true, nil } - if err := secret_model.UpdateSecret(ctx, s[0].ID, data); err != nil { + if err := secret_model.UpdateSecret(ctx, s[0].ID, data, description); err != nil { return nil, false, err } diff --git a/services/task/task.go b/services/task/task.go index c90ee91270..ee5fa1f348 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -5,6 +5,7 @@ package task import ( "context" + "errors" "fmt" admin_model "code.gitea.io/gitea/models/admin" @@ -41,7 +42,7 @@ func Run(ctx context.Context, t *admin_model.Task) error { func Init() error { taskQueue = queue.CreateSimpleQueue(graceful.GetManager().ShutdownContext(), "task", handler) if taskQueue == nil { - return fmt.Errorf("unable to create task queue") + return errors.New("unable to create task queue") } go graceful.GetManager().RunWithCancel(taskQueue) return nil @@ -110,7 +111,7 @@ func CreateMigrateTask(ctx context.Context, doer, u *user_model.User, opts base. IsPrivate: opts.Private || setting.Repository.ForcePrivate, IsMirror: opts.Mirror, Status: repo_model.RepositoryBeingMigrated, - }) + }, false) if err != nil { task.EndTime = timeutil.TimeStampNow() task.Status = structs.TaskStatusFailed diff --git a/services/user/block.go b/services/user/block.go index c24ce5273c..7727780dfc 100644 --- a/services/user/block.go +++ b/services/user/block.go @@ -117,10 +117,10 @@ func BlockUser(ctx context.Context, doer, blocker, blockee *user_model.User, not } // cancel each other repository transfers - if err := cancelRepositoryTransfers(ctx, blocker, blockee); err != nil { + if err := cancelRepositoryTransfers(ctx, doer, blocker, blockee); err != nil { return err } - if err := cancelRepositoryTransfers(ctx, blockee, blocker); err != nil { + if err := cancelRepositoryTransfers(ctx, doer, blockee, blocker); err != nil { return err } @@ -192,7 +192,7 @@ func unwatchRepos(ctx context.Context, watcher, repoOwner *user_model.User) erro } } -func cancelRepositoryTransfers(ctx context.Context, sender, recipient *user_model.User) error { +func cancelRepositoryTransfers(ctx context.Context, doer, sender, recipient *user_model.User) error { transfers, err := repo_model.GetPendingRepositoryTransfers(ctx, &repo_model.PendingRepositoryTransferOptions{ SenderID: sender.ID, RecipientID: recipient.ID, @@ -202,12 +202,7 @@ func cancelRepositoryTransfers(ctx context.Context, sender, recipient *user_mode } for _, transfer := range transfers { - repo, err := repo_model.GetRepositoryByID(ctx, transfer.RepoID) - if err != nil { - return err - } - - if err := repo_service.CancelRepositoryTransfer(ctx, repo); err != nil { + if err := repo_service.CancelRepositoryTransfer(ctx, transfer, doer); err != nil { return err } } diff --git a/services/user/update.go b/services/user/update.go index 4a39f4f783..d7354542bf 100644 --- a/services/user/update.go +++ b/services/user/update.go @@ -15,6 +15,26 @@ import ( "code.gitea.io/gitea/modules/structs" ) +type UpdateOptionField[T any] struct { + FieldValue T + FromSync bool +} + +func UpdateOptionFieldFromValue[T any](value T) optional.Option[UpdateOptionField[T]] { + return optional.Some(UpdateOptionField[T]{FieldValue: value}) +} + +func UpdateOptionFieldFromSync[T any](value T) optional.Option[UpdateOptionField[T]] { + return optional.Some(UpdateOptionField[T]{FieldValue: value, FromSync: true}) +} + +func UpdateOptionFieldFromPtr[T any](value *T) optional.Option[UpdateOptionField[T]] { + if value == nil { + return optional.None[UpdateOptionField[T]]() + } + return UpdateOptionFieldFromValue(*value) +} + type UpdateOptions struct { KeepEmailPrivate optional.Option[bool] FullName optional.Option[string] @@ -32,7 +52,7 @@ type UpdateOptions struct { DiffViewStyle optional.Option[string] AllowCreateOrganization optional.Option[bool] IsActive optional.Option[bool] - IsAdmin optional.Option[bool] + IsAdmin optional.Option[UpdateOptionField[bool]] EmailNotificationsPreference optional.Option[string] SetLastLogin bool RepoAdminChangeTeamAccess optional.Option[bool] @@ -111,13 +131,18 @@ func UpdateUser(ctx context.Context, u *user_model.User, opts *UpdateOptions) er cols = append(cols, "is_restricted") } if opts.IsAdmin.Has() { - if !opts.IsAdmin.Value() && user_model.IsLastAdminUser(ctx, u) { - return user_model.ErrDeleteLastAdminUser{UID: u.ID} + if opts.IsAdmin.Value().FieldValue /* true */ { + u.IsAdmin = opts.IsAdmin.Value().FieldValue // set IsAdmin=true + cols = append(cols, "is_admin") + } else if !user_model.IsLastAdminUser(ctx, u) /* not the last admin */ { + u.IsAdmin = opts.IsAdmin.Value().FieldValue // it's safe to change it from false to true (not the last admin) + cols = append(cols, "is_admin") + } else /* IsAdmin=false but this is the last admin user */ { //nolint:gocritic // make it easier to read + if !opts.IsAdmin.Value().FromSync { + return user_model.ErrDeleteLastAdminUser{UID: u.ID} + } + // else: syncing from external-source, this user is the last admin, so skip the "IsAdmin=false" change } - - u.IsAdmin = opts.IsAdmin.Value() - - cols = append(cols, "is_admin") } if opts.Visibility.Has() { diff --git a/services/user/update_test.go b/services/user/update_test.go index fc24a6c212..27513e8040 100644 --- a/services/user/update_test.go +++ b/services/user/update_test.go @@ -22,7 +22,11 @@ func TestUpdateUser(t *testing.T) { admin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) assert.Error(t, UpdateUser(db.DefaultContext, admin, &UpdateOptions{ - IsAdmin: optional.Some(false), + IsAdmin: UpdateOptionFieldFromValue(false), + })) + + assert.NoError(t, UpdateUser(db.DefaultContext, admin, &UpdateOptions{ + IsAdmin: UpdateOptionFieldFromSync(false), })) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 28}) @@ -38,7 +42,7 @@ func TestUpdateUser(t *testing.T) { MaxRepoCreation: optional.Some(10), IsRestricted: optional.Some(true), IsActive: optional.Some(false), - IsAdmin: optional.Some(true), + IsAdmin: UpdateOptionFieldFromValue(true), Visibility: optional.Some(structs.VisibleTypePrivate), KeepActivityPrivate: optional.Some(true), Language: optional.Some("lang"), @@ -60,7 +64,7 @@ func TestUpdateUser(t *testing.T) { assert.Equal(t, opts.MaxRepoCreation.Value(), user.MaxRepoCreation) assert.Equal(t, opts.IsRestricted.Value(), user.IsRestricted) assert.Equal(t, opts.IsActive.Value(), user.IsActive) - assert.Equal(t, opts.IsAdmin.Value(), user.IsAdmin) + assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin) assert.Equal(t, opts.Visibility.Value(), user.Visibility) assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate) assert.Equal(t, opts.Language.Value(), user.Language) @@ -80,7 +84,7 @@ func TestUpdateUser(t *testing.T) { assert.Equal(t, opts.MaxRepoCreation.Value(), user.MaxRepoCreation) assert.Equal(t, opts.IsRestricted.Value(), user.IsRestricted) assert.Equal(t, opts.IsActive.Value(), user.IsActive) - assert.Equal(t, opts.IsAdmin.Value(), user.IsAdmin) + assert.Equal(t, opts.IsAdmin.Value().FieldValue, user.IsAdmin) assert.Equal(t, opts.Visibility.Value(), user.Visibility) assert.Equal(t, opts.KeepActivityPrivate.Value(), user.KeepActivityPrivate) assert.Equal(t, opts.Language.Value(), user.Language) diff --git a/services/user/user.go b/services/user/user.go index 1aeebff142..c7252430de 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/agit" asymkey_service "code.gitea.io/gitea/services/asymkey" @@ -177,8 +178,8 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { PageSize: repo_model.RepositoryListDefaultPageSize, Page: 1, }, - UserID: u.ID, - IncludePrivate: true, + UserID: u.ID, + IncludeVisibility: structs.VisibleTypePrivate, }) if err != nil { return fmt.Errorf("unable to find org list for %s[%d]. Error: %w", u.Name, u.ID, err) diff --git a/services/user/user_test.go b/services/user/user_test.go index 162a735cd4..28a0df8628 100644 --- a/services/user/user_test.go +++ b/services/user/user_test.go @@ -150,7 +150,7 @@ func TestRenameUser(t *testing.T) { redirectUID, err := user_model.LookupUserRedirect(db.DefaultContext, oldUsername) assert.NoError(t, err) - assert.EqualValues(t, user.ID, redirectUID) + assert.Equal(t, user.ID, redirectUID) unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerID: user.ID, OwnerName: user.Name}) }) diff --git a/services/versioned_migration/migration.go b/services/versioned_migration/migration.go new file mode 100644 index 0000000000..b66d853531 --- /dev/null +++ b/services/versioned_migration/migration.go @@ -0,0 +1,24 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package versioned_migration + +import ( + "context" + + "code.gitea.io/gitea/models/migrations" + "code.gitea.io/gitea/modules/globallock" + + "xorm.io/xorm" +) + +func Migrate(ctx context.Context, x *xorm.Engine) error { + // only one instance can do the migration at the same time if there are multiple instances + release, err := globallock.Lock(ctx, "gitea_versioned_migration") + if err != nil { + return err + } + defer release() + + return migrations.Migrate(x) +} diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index 4707602cdf..e8e6ed19c1 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "crypto/tls" "encoding/hex" + "errors" "fmt" "io" "net/http" @@ -18,6 +19,7 @@ import ( "sync" "time" + user_model "code.gitea.io/gitea/models/user" webhook_model "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/hostmatcher" @@ -40,7 +42,7 @@ func newDefaultRequest(ctx context.Context, w *webhook_model.Webhook, t *webhook case http.MethodPost: switch w.ContentType { case webhook_model.ContentTypeJSON: - req, err = http.NewRequest("POST", w.URL, strings.NewReader(t.PayloadContent)) + req, err = http.NewRequest(http.MethodPost, w.URL, strings.NewReader(t.PayloadContent)) if err != nil { return nil, nil, err } @@ -51,7 +53,7 @@ func newDefaultRequest(ctx context.Context, w *webhook_model.Webhook, t *webhook "payload": []string{t.PayloadContent}, } - req, err = http.NewRequest("POST", w.URL, strings.NewReader(forms.Encode())) + req, err = http.NewRequest(http.MethodPost, w.URL, strings.NewReader(forms.Encode())) if err != nil { return nil, nil, err } @@ -68,7 +70,7 @@ func newDefaultRequest(ctx context.Context, w *webhook_model.Webhook, t *webhook vals := u.Query() vals["payload"] = []string{t.PayloadContent} u.RawQuery = vals.Encode() - req, err = http.NewRequest("GET", u.String(), nil) + req, err = http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return nil, nil, err } @@ -80,7 +82,7 @@ func newDefaultRequest(ctx context.Context, w *webhook_model.Webhook, t *webhook return nil, nil, err } url := fmt.Sprintf("%s/%s", w.URL, url.PathEscape(txnID)) - req, err = http.NewRequest("PUT", url, strings.NewReader(t.PayloadContent)) + req, err = http.NewRequest(http.MethodPut, url, strings.NewReader(t.PayloadContent)) if err != nil { return nil, nil, err } @@ -92,10 +94,10 @@ func newDefaultRequest(ctx context.Context, w *webhook_model.Webhook, t *webhook } body = []byte(t.PayloadContent) - return req, body, addDefaultHeaders(req, []byte(w.Secret), t, body) + return req, body, addDefaultHeaders(req, []byte(w.Secret), w, t, body) } -func addDefaultHeaders(req *http.Request, secret []byte, t *webhook_model.HookTask, payloadContent []byte) error { +func addDefaultHeaders(req *http.Request, secret []byte, w *webhook_model.Webhook, t *webhook_model.HookTask, payloadContent []byte) error { var signatureSHA1 string var signatureSHA256 string if len(secret) > 0 { @@ -112,10 +114,27 @@ func addDefaultHeaders(req *http.Request, secret []byte, t *webhook_model.HookTa event := t.EventType.Event() eventType := string(t.EventType) + targetType := "default" + if w.IsSystemWebhook { + targetType = "system" + } else if w.RepoID != 0 { + targetType = "repository" + } else if w.OwnerID != 0 { + owner, err := user_model.GetUserByID(req.Context(), w.OwnerID) + if owner != nil && err == nil { + if owner.IsOrganization() { + targetType = "organization" + } else { + targetType = "user" + } + } + } + req.Header.Add("X-Gitea-Delivery", t.UUID) req.Header.Add("X-Gitea-Event", event) req.Header.Add("X-Gitea-Event-Type", eventType) req.Header.Add("X-Gitea-Signature", signatureSHA256) + req.Header.Add("X-Gitea-Hook-Installation-Target-Type", targetType) req.Header.Add("X-Gogs-Delivery", t.UUID) req.Header.Add("X-Gogs-Event", event) req.Header.Add("X-Gogs-Event-Type", eventType) @@ -125,6 +144,7 @@ func addDefaultHeaders(req *http.Request, secret []byte, t *webhook_model.HookTa req.Header["X-GitHub-Delivery"] = []string{t.UUID} req.Header["X-GitHub-Event"] = []string{event} req.Header["X-GitHub-Event-Type"] = []string{eventType} + req.Header["X-GitHub-Hook-Installation-Target-Type"] = []string{targetType} return nil } @@ -309,7 +329,7 @@ func Init() error { hookQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "webhook_sender", handler) if hookQueue == nil { - return fmt.Errorf("unable to create webhook_sender queue") + return errors.New("unable to create webhook_sender queue") } go graceful.GetManager().RunWithCancel(hookQueue) diff --git a/services/webhook/deliver_test.go b/services/webhook/deliver_test.go index d0cfc1598f..1d32d7b772 100644 --- a/services/webhook/deliver_test.go +++ b/services/webhook/deliver_test.go @@ -4,7 +4,6 @@ package webhook import ( - "context" "io" "net/http" "net/http/httptest" @@ -65,7 +64,7 @@ func TestWebhookProxy(t *testing.T) { } for _, tt := range tests { t.Run(tt.req, func(t *testing.T) { - req, err := http.NewRequest("POST", tt.req, nil) + req, err := http.NewRequest(http.MethodPost, tt.req, nil) require.NoError(t, err) u, err := webhookProxy(allowedHostMatcher)(req) @@ -92,7 +91,7 @@ func TestWebhookDeliverAuthorizationHeader(t *testing.T) { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "/webhook", r.URL.Path) assert.Equal(t, "Bearer s3cr3t-t0ken", r.Header.Get("Authorization")) - w.WriteHeader(200) + w.WriteHeader(http.StatusOK) done <- struct{}{} })) t.Cleanup(s.Close) @@ -118,7 +117,7 @@ func TestWebhookDeliverAuthorizationHeader(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, hookTask) - assert.NoError(t, Deliver(context.Background(), hookTask)) + assert.NoError(t, Deliver(t.Context(), hookTask)) select { case <-done: case <-time.After(5 * time.Second): @@ -139,7 +138,7 @@ func TestWebhookDeliverHookTask(t *testing.T) { case "/webhook/66d222a5d6349e1311f551e50722d837e30fce98": // Version 1 assert.Equal(t, "push", r.Header.Get("X-GitHub-Event")) - assert.Equal(t, "", r.Header.Get("Content-Type")) + assert.Empty(t, r.Header.Get("Content-Type")) body, err := io.ReadAll(r.Body) assert.NoError(t, err) assert.Equal(t, `{"data": 42}`, string(body)) @@ -153,11 +152,11 @@ func TestWebhookDeliverHookTask(t *testing.T) { assert.Len(t, body, 2147) default: - w.WriteHeader(404) + w.WriteHeader(http.StatusNotFound) t.Fatalf("unexpected url path %s", r.URL.Path) return } - w.WriteHeader(200) + w.WriteHeader(http.StatusOK) done <- struct{}{} })) t.Cleanup(s.Close) @@ -185,7 +184,7 @@ func TestWebhookDeliverHookTask(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, hookTask) - assert.NoError(t, Deliver(context.Background(), hookTask)) + assert.NoError(t, Deliver(t.Context(), hookTask)) select { case <-done: case <-time.After(5 * time.Second): @@ -211,7 +210,7 @@ func TestWebhookDeliverHookTask(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, hookTask) - assert.NoError(t, Deliver(context.Background(), hookTask)) + assert.NoError(t, Deliver(t.Context(), hookTask)) select { case <-done: case <-time.After(5 * time.Second): @@ -280,7 +279,7 @@ func TestWebhookDeliverSpecificTypes(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, hookTask) - assert.NoError(t, Deliver(context.Background(), hookTask)) + assert.NoError(t, Deliver(t.Context(), hookTask)) select { case gotBody := <-cases[typ].gotBody: diff --git a/services/webhook/dingtalk.go b/services/webhook/dingtalk.go index e382f5a9df..5bbc610fe5 100644 --- a/services/webhook/dingtalk.go +++ b/services/webhook/dingtalk.go @@ -30,7 +30,7 @@ func (dc dingtalkConvertor) Create(p *api.CreatePayload) (DingtalkPayload, error refName := git.RefName(p.Ref).ShortName() title := fmt.Sprintf("[%s] %s %s created", p.Repo.FullName, p.RefType, refName) - return createDingtalkPayload(title, title, fmt.Sprintf("view ref %s", refName), p.Repo.HTMLURL+"/src/"+util.PathEscapeSegments(refName)), nil + return createDingtalkPayload(title, title, "view ref "+refName, p.Repo.HTMLURL+"/src/"+util.PathEscapeSegments(refName)), nil } // Delete implements PayloadConvertor Delete method @@ -39,14 +39,14 @@ func (dc dingtalkConvertor) Delete(p *api.DeletePayload) (DingtalkPayload, error refName := git.RefName(p.Ref).ShortName() title := fmt.Sprintf("[%s] %s %s deleted", p.Repo.FullName, p.RefType, refName) - return createDingtalkPayload(title, title, fmt.Sprintf("view ref %s", refName), p.Repo.HTMLURL+"/src/"+util.PathEscapeSegments(refName)), nil + return createDingtalkPayload(title, title, "view ref "+refName, p.Repo.HTMLURL+"/src/"+util.PathEscapeSegments(refName)), nil } // Fork implements PayloadConvertor Fork method func (dc dingtalkConvertor) Fork(p *api.ForkPayload) (DingtalkPayload, error) { title := fmt.Sprintf("%s is forked to %s", p.Forkee.FullName, p.Repo.FullName) - return createDingtalkPayload(title, title, fmt.Sprintf("view forked repo %s", p.Repo.FullName), p.Repo.HTMLURL), nil + return createDingtalkPayload(title, title, "view forked repo "+p.Repo.FullName, p.Repo.HTMLURL), nil } // Push implements PayloadConvertor Push method @@ -170,6 +170,24 @@ func (dc dingtalkConvertor) Package(p *api.PackagePayload) (DingtalkPayload, err return createDingtalkPayload(text, text, "view package", p.Package.HTMLURL), nil } +func (dc dingtalkConvertor) Status(p *api.CommitStatusPayload) (DingtalkPayload, error) { + text, _ := getStatusPayloadInfo(p, noneLinkFormatter, true) + + return createDingtalkPayload(text, text, "Status Changed", p.TargetURL), nil +} + +func (dingtalkConvertor) WorkflowRun(p *api.WorkflowRunPayload) (DingtalkPayload, error) { + text, _ := getWorkflowRunPayloadInfo(p, noneLinkFormatter, true) + + return createDingtalkPayload(text, text, "Workflow Run", p.WorkflowRun.HTMLURL), nil +} + +func (dingtalkConvertor) WorkflowJob(p *api.WorkflowJobPayload) (DingtalkPayload, error) { + text, _ := getWorkflowJobPayloadInfo(p, noneLinkFormatter, true) + + return createDingtalkPayload(text, text, "Workflow Job", p.WorkflowJob.HTMLURL), nil +} + func createDingtalkPayload(title, text, singleTitle, singleURL string) DingtalkPayload { return DingtalkPayload{ MsgType: "actionCard", @@ -190,3 +208,7 @@ func newDingtalkRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_ var pc payloadConvertor[DingtalkPayload] = dingtalkConvertor{} return newJSONRequest(pc, w, t, true) } + +func init() { + RegisterWebhookRequester(webhook_module.DINGTALK, newDingtalkRequest) +} diff --git a/services/webhook/dingtalk_test.go b/services/webhook/dingtalk_test.go index 25f47347d0..763d23048a 100644 --- a/services/webhook/dingtalk_test.go +++ b/services/webhook/dingtalk_test.go @@ -4,7 +4,6 @@ package webhook import ( - "context" "net/url" "testing" @@ -236,7 +235,7 @@ func TestDingTalkJSONPayload(t *testing.T) { PayloadVersion: 2, } - req, reqBody, err := newDingtalkRequest(context.Background(), hook, task) + req, reqBody, err := newDingtalkRequest(t.Context(), hook, task) require.NotNil(t, req) require.NotNil(t, reqBody) require.NoError(t, err) diff --git a/services/webhook/discord.go b/services/webhook/discord.go index c562d98168..0426964181 100644 --- a/services/webhook/discord.go +++ b/services/webhook/discord.go @@ -101,6 +101,13 @@ var ( redColor = color("ff3232") ) +// https://discord.com/developers/docs/resources/message#embed-object-embed-limits +// Discord has some limits in place for the embeds. +// According to some tests, there is no consistent limit for different character sets. +// For example: 4096 ASCII letters are allowed, but only 2490 emoji characters are allowed. +// To keep it simple, we currently truncate at 2000. +const discordDescriptionCharactersLimit = 2000 + type discordConvertor struct { Username string AvatarURL string @@ -265,6 +272,24 @@ func (d discordConvertor) Package(p *api.PackagePayload) (DiscordPayload, error) return d.createPayload(p.Sender, text, "", p.Package.HTMLURL, color), nil } +func (d discordConvertor) Status(p *api.CommitStatusPayload) (DiscordPayload, error) { + text, color := getStatusPayloadInfo(p, noneLinkFormatter, false) + + return d.createPayload(p.Sender, text, "", p.TargetURL, color), nil +} + +func (d discordConvertor) WorkflowRun(p *api.WorkflowRunPayload) (DiscordPayload, error) { + text, color := getWorkflowRunPayloadInfo(p, noneLinkFormatter, false) + + return d.createPayload(p.Sender, text, "", p.WorkflowRun.HTMLURL, color), nil +} + +func (d discordConvertor) WorkflowJob(p *api.WorkflowJobPayload) (DiscordPayload, error) { + text, color := getWorkflowJobPayloadInfo(p, noneLinkFormatter, false) + + return d.createPayload(p.Sender, text, "", p.WorkflowJob.HTMLURL, color), nil +} + func newDiscordRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_model.HookTask) (*http.Request, []byte, error) { meta := &DiscordMeta{} if err := json.Unmarshal([]byte(w.Meta), meta); err != nil { @@ -277,12 +302,16 @@ func newDiscordRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_m return newJSONRequest(pc, w, t, true) } +func init() { + RegisterWebhookRequester(webhook_module.DISCORD, newDiscordRequest) +} + func parseHookPullRequestEventType(event webhook_module.HookEventType) (string, error) { switch event { case webhook_module.HookEventPullRequestReviewApproved: return "approved", nil case webhook_module.HookEventPullRequestReviewRejected: - return "rejected", nil + return "requested changes", nil case webhook_module.HookEventPullRequestReviewComment: return "comment", nil default: @@ -297,7 +326,7 @@ func (d discordConvertor) createPayload(s *api.User, title, text, url string, co Embeds: []DiscordEmbed{ { Title: title, - Description: text, + Description: util.TruncateRunes(text, discordDescriptionCharactersLimit), URL: url, Color: color, Author: DiscordEmbedAuthor{ diff --git a/services/webhook/discord_test.go b/services/webhook/discord_test.go index 36b99d452e..7f503e3374 100644 --- a/services/webhook/discord_test.go +++ b/services/webhook/discord_test.go @@ -4,7 +4,6 @@ package webhook import ( - "context" "testing" webhook_model "code.gitea.io/gitea/models/webhook" @@ -303,7 +302,7 @@ func TestDiscordJSONPayload(t *testing.T) { PayloadVersion: 2, } - req, reqBody, err := newDiscordRequest(context.Background(), hook, task) + req, reqBody, err := newDiscordRequest(t.Context(), hook, task) require.NotNil(t, req) require.NotNil(t, reqBody) require.NoError(t, err) diff --git a/services/webhook/feishu.go b/services/webhook/feishu.go index 7ca7d1cf5f..b6ee80c44c 100644 --- a/services/webhook/feishu.go +++ b/services/webhook/feishu.go @@ -5,9 +5,13 @@ package webhook import ( "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" "fmt" "net/http" "strings" + "time" webhook_model "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/git" @@ -16,10 +20,12 @@ import ( ) type ( - // FeishuPayload represents + // FeishuPayload represents the payload for Feishu webhook FeishuPayload struct { - MsgType string `json:"msg_type"` // text / post / image / share_chat / interactive / file /audio / media - Content struct { + Timestamp int64 `json:"timestamp,omitempty"` // Unix timestamp for signature verification + Sign string `json:"sign,omitempty"` // Signature for verification + MsgType string `json:"msg_type"` // text / post / image / share_chat / interactive / file /audio / media + Content struct { Text string `json:"text"` } `json:"content"` } @@ -166,7 +172,49 @@ func (fc feishuConvertor) Package(p *api.PackagePayload) (FeishuPayload, error) return newFeishuTextPayload(text), nil } +func (fc feishuConvertor) Status(p *api.CommitStatusPayload) (FeishuPayload, error) { + text, _ := getStatusPayloadInfo(p, noneLinkFormatter, true) + + return newFeishuTextPayload(text), nil +} + +func (feishuConvertor) WorkflowRun(p *api.WorkflowRunPayload) (FeishuPayload, error) { + text, _ := getWorkflowRunPayloadInfo(p, noneLinkFormatter, true) + + return newFeishuTextPayload(text), nil +} + +func (feishuConvertor) WorkflowJob(p *api.WorkflowJobPayload) (FeishuPayload, error) { + text, _ := getWorkflowJobPayloadInfo(p, noneLinkFormatter, true) + + return newFeishuTextPayload(text), nil +} + +// feishuGenSign generates a signature for Feishu webhook +// https://open.feishu.cn/document/client-docs/bot-v3/add-custom-bot +func feishuGenSign(secret string, timestamp int64) string { + // key="{timestamp}\n{secret}", then hmac-sha256, then base64 encode + stringToSign := fmt.Sprintf("%d\n%s", timestamp, secret) + h := hmac.New(sha256.New, []byte(stringToSign)) + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} + func newFeishuRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_model.HookTask) (*http.Request, []byte, error) { - var pc payloadConvertor[FeishuPayload] = feishuConvertor{} - return newJSONRequest(pc, w, t, true) + payload, err := newPayload(feishuConvertor{}, []byte(t.PayloadContent), t.EventType) + if err != nil { + return nil, nil, err + } + + // Add timestamp and signature if secret is provided + if w.Secret != "" { + timestamp := time.Now().Unix() + payload.Timestamp = timestamp + payload.Sign = feishuGenSign(w.Secret, timestamp) + } + + return prepareJSONRequest(payload, w, t, false /* no default headers */) +} + +func init() { + RegisterWebhookRequester(webhook_module.FEISHU, newFeishuRequest) } diff --git a/services/webhook/feishu_test.go b/services/webhook/feishu_test.go index ef18333fd4..7e200ea132 100644 --- a/services/webhook/feishu_test.go +++ b/services/webhook/feishu_test.go @@ -4,7 +4,6 @@ package webhook import ( - "context" "testing" webhook_model "code.gitea.io/gitea/models/webhook" @@ -169,6 +168,7 @@ func TestFeishuJSONPayload(t *testing.T) { URL: "https://feishu.example.com/", Meta: `{}`, HTTPMethod: "POST", + Secret: "secret", } task := &webhook_model.HookTask{ HookID: hook.ID, @@ -177,17 +177,20 @@ func TestFeishuJSONPayload(t *testing.T) { PayloadVersion: 2, } - req, reqBody, err := newFeishuRequest(context.Background(), hook, task) + req, reqBody, err := newFeishuRequest(t.Context(), hook, task) require.NotNil(t, req) require.NotNil(t, reqBody) require.NoError(t, err) assert.Equal(t, "POST", req.Method) assert.Equal(t, "https://feishu.example.com/", req.URL.String()) - assert.Equal(t, "sha256=", req.Header.Get("X-Hub-Signature-256")) assert.Equal(t, "application/json", req.Header.Get("Content-Type")) var body FeishuPayload err = json.NewDecoder(req.Body).Decode(&body) assert.NoError(t, err) assert.Equal(t, "[test/repo:test] \r\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1\r\n[2020558](http://localhost:3000/test/repo/commit/2020558fe2e34debb818a514715839cabd25e778) commit message - user1", body.Content.Text) + assert.Equal(t, feishuGenSign(hook.Secret, body.Timestamp), body.Sign) + + // a separate sign test, the result is generated by official python code, so the algo must be correct + assert.Equal(t, "rWZ84lcag1x9aBFhn1gtV4ZN+4gme3pilfQNMk86vKg=", feishuGenSign("a", 1)) } diff --git a/services/webhook/general.go b/services/webhook/general.go index dde43bb349..be457e46f5 100644 --- a/services/webhook/general.go +++ b/services/webhook/general.go @@ -9,7 +9,9 @@ import ( "net/url" "strings" + user_model "code.gitea.io/gitea/models/user" webhook_model "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -37,19 +39,20 @@ func getPullRequestInfo(p *api.PullRequestPayload) (title, link, by, operator, o for i, user := range assignList { assignStringList[i] = user.UserName } - if p.Action == api.HookIssueAssigned { + switch p.Action { + case api.HookIssueAssigned: operateResult = fmt.Sprintf("%s assign this to %s", p.Sender.UserName, assignList[len(assignList)-1].UserName) - } else if p.Action == api.HookIssueUnassigned { - operateResult = fmt.Sprintf("%s unassigned this for someone", p.Sender.UserName) - } else if p.Action == api.HookIssueMilestoned { + case api.HookIssueUnassigned: + operateResult = p.Sender.UserName + " unassigned this for someone" + case api.HookIssueMilestoned: operateResult = fmt.Sprintf("%s/milestone/%d", p.Repository.HTMLURL, p.PullRequest.Milestone.ID) } link = p.PullRequest.HTMLURL - by = fmt.Sprintf("PullRequest by %s", p.PullRequest.Poster.UserName) + by = "PullRequest by " + p.PullRequest.Poster.UserName if len(assignStringList) > 0 { - assignees = fmt.Sprintf("Assignees: %s", strings.Join(assignStringList, ", ")) + assignees = "Assignees: " + strings.Join(assignStringList, ", ") } - operator = fmt.Sprintf("Operator: %s", p.Sender.UserName) + operator = "Operator: " + p.Sender.UserName return title, link, by, operator, operateResult, assignees } @@ -62,19 +65,20 @@ func getIssuesInfo(p *api.IssuePayload) (issueTitle, link, by, operator, operate for i, user := range assignList { assignStringList[i] = user.UserName } - if p.Action == api.HookIssueAssigned { + switch p.Action { + case api.HookIssueAssigned: operateResult = fmt.Sprintf("%s assign this to %s", p.Sender.UserName, assignList[len(assignList)-1].UserName) - } else if p.Action == api.HookIssueUnassigned { - operateResult = fmt.Sprintf("%s unassigned this for someone", p.Sender.UserName) - } else if p.Action == api.HookIssueMilestoned { + case api.HookIssueUnassigned: + operateResult = p.Sender.UserName + " unassigned this for someone" + case api.HookIssueMilestoned: operateResult = fmt.Sprintf("%s/milestone/%d", p.Repository.HTMLURL, p.Issue.Milestone.ID) } link = p.Issue.HTMLURL - by = fmt.Sprintf("Issue by %s", p.Issue.Poster.UserName) + by = "Issue by " + p.Issue.Poster.UserName if len(assignStringList) > 0 { - assignees = fmt.Sprintf("Assignees: %s", strings.Join(assignStringList, ", ")) + assignees = "Assignees: " + strings.Join(assignStringList, ", ") } - operator = fmt.Sprintf("Operator: %s", p.Sender.UserName) + operator = "Operator: " + p.Sender.UserName return issueTitle, link, by, operator, operateResult, assignees } @@ -83,11 +87,11 @@ func getIssuesCommentInfo(p *api.IssueCommentPayload) (title, link, by, operator title = fmt.Sprintf("[Comment-%s #%d]: %s\n%s", p.Repository.FullName, p.Issue.Index, p.Action, p.Issue.Title) link = p.Issue.HTMLURL if p.IsPull { - by = fmt.Sprintf("PullRequest by %s", p.Issue.Poster.UserName) + by = "PullRequest by " + p.Issue.Poster.UserName } else { - by = fmt.Sprintf("Issue by %s", p.Issue.Poster.UserName) + by = "Issue by " + p.Issue.Poster.UserName } - operator = fmt.Sprintf("Operator: %s", p.Sender.UserName) + operator = "Operator: " + p.Sender.UserName return title, link, by, operator } @@ -131,7 +135,7 @@ func getIssuesPayloadInfo(p *api.IssuePayload, linkFormatter linkFormatter, with text = fmt.Sprintf("[%s] Issue milestone cleared: %s", repoLink, titleLink) } if withSender { - text += fmt.Sprintf(" by %s", linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName)) + text += " by " + linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName) } if p.Action == api.HookIssueOpened || p.Action == api.HookIssueEdited { @@ -196,7 +200,7 @@ func getPullRequestPayloadInfo(p *api.PullRequestPayload, linkFormatter linkForm text = fmt.Sprintf("[%s] Pull request review request removed: %s", repoLink, titleLink) } if withSender { - text += fmt.Sprintf(" by %s", linkFormatter(setting.AppURL+p.Sender.UserName, p.Sender.UserName)) + text += " by " + linkFormatter(setting.AppURL+p.Sender.UserName, p.Sender.UserName) } return text, issueTitle, extraMarkdown, color @@ -218,7 +222,7 @@ func getReleasePayloadInfo(p *api.ReleasePayload, linkFormatter linkFormatter, w color = redColor } if withSender { - text += fmt.Sprintf(" by %s", linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName)) + text += " by " + linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName) } return text, color @@ -247,7 +251,7 @@ func getWikiPayloadInfo(p *api.WikiPayload, linkFormatter linkFormatter, withSen } if withSender { - text += fmt.Sprintf(" by %s", linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName)) + text += " by " + linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName) } return text, color, pageLink @@ -283,7 +287,7 @@ func getIssueCommentPayloadInfo(p *api.IssueCommentPayload, linkFormatter linkFo color = redColor } if withSender { - text += fmt.Sprintf(" by %s", linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName)) + text += " by " + linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName) } return text, issueTitle, color @@ -294,14 +298,92 @@ func getPackagePayloadInfo(p *api.PackagePayload, linkFormatter linkFormatter, w switch p.Action { case api.HookPackageCreated: - text = fmt.Sprintf("Package created: %s", refLink) + text = "Package created: " + refLink color = greenColor case api.HookPackageDeleted: - text = fmt.Sprintf("Package deleted: %s", refLink) + text = "Package deleted: " + refLink color = redColor } if withSender { - text += fmt.Sprintf(" by %s", linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName)) + text += " by " + linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName) + } + + return text, color +} + +func getStatusPayloadInfo(p *api.CommitStatusPayload, linkFormatter linkFormatter, withSender bool) (text string, color int) { + refLink := linkFormatter(p.TargetURL, fmt.Sprintf("%s [%s]", p.Context, base.ShortSha(p.SHA))) + + text = fmt.Sprintf("Commit Status changed: %s - %s", refLink, p.Description) + color = greenColor + if withSender { + if user_model.IsGiteaActionsUserName(p.Sender.UserName) { + text += " by " + p.Sender.FullName + } else { + text += " by " + linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName) + } + } + + return text, color +} + +func getWorkflowRunPayloadInfo(p *api.WorkflowRunPayload, linkFormatter linkFormatter, withSender bool) (text string, color int) { + description := p.WorkflowRun.Conclusion + if description == "" { + description = p.WorkflowRun.Status + } + refLink := linkFormatter(p.WorkflowRun.HTMLURL, fmt.Sprintf("%s(#%d)", p.WorkflowRun.DisplayTitle, p.WorkflowRun.ID)+"["+base.ShortSha(p.WorkflowRun.HeadSha)+"]:"+description) + + text = fmt.Sprintf("Workflow Run %s: %s", p.Action, refLink) + switch description { + case "waiting": + color = orangeColor + case "queued": + color = orangeColorLight + case "success": + color = greenColor + case "failure": + color = redColor + case "cancelled": + color = yellowColor + case "skipped": + color = purpleColor + default: + color = greyColor + } + if withSender { + text += " by " + linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName) + } + + return text, color +} + +func getWorkflowJobPayloadInfo(p *api.WorkflowJobPayload, linkFormatter linkFormatter, withSender bool) (text string, color int) { + description := p.WorkflowJob.Conclusion + if description == "" { + description = p.WorkflowJob.Status + } + refLink := linkFormatter(p.WorkflowJob.HTMLURL, fmt.Sprintf("%s(#%d)", p.WorkflowJob.Name, p.WorkflowJob.RunID)+"["+base.ShortSha(p.WorkflowJob.HeadSha)+"]:"+description) + + text = fmt.Sprintf("Workflow Job %s: %s", p.Action, refLink) + switch description { + case "waiting": + color = orangeColor + case "queued": + color = orangeColorLight + case "success": + color = greenColor + case "failure": + color = redColor + case "cancelled": + color = yellowColor + case "skipped": + color = purpleColor + default: + color = greyColor + } + if withSender { + text += " by " + linkFormatter(setting.AppURL+url.PathEscape(p.Sender.UserName), p.Sender.UserName) } return text, color diff --git a/services/webhook/general_test.go b/services/webhook/general_test.go index ef1ec7f324..ec735d785a 100644 --- a/services/webhook/general_test.go +++ b/services/webhook/general_test.go @@ -319,8 +319,8 @@ func packageTestPayload() *api.PackagePayload { AvatarURL: "http://localhost:3000/user1/avatar", }, Repository: nil, - Organization: &api.User{ - UserName: "org1", + Organization: &api.Organization{ + Name: "org1", AvatarURL: "http://localhost:3000/org1/avatar", }, Package: &api.Package{ diff --git a/services/webhook/matrix.go b/services/webhook/matrix.go index 5e9f808d8b..3e9163f78c 100644 --- a/services/webhook/matrix.go +++ b/services/webhook/matrix.go @@ -15,6 +15,7 @@ import ( "strings" webhook_model "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" @@ -24,6 +25,10 @@ import ( webhook_module "code.gitea.io/gitea/modules/webhook" ) +func init() { + RegisterWebhookRequester(webhook_module.MATRIX, newMatrixRequest) +} + func newMatrixRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_model.HookTask) (*http.Request, []byte, error) { meta := &MatrixMeta{} if err := json.Unmarshal([]byte(w.Meta), meta); err != nil { @@ -52,7 +57,7 @@ func newMatrixRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_mo } req.Header.Set("Content-Type", "application/json") - return req, body, addDefaultHeaders(req, []byte(w.Secret), t, body) // likely useless, but has always been sent historially + return req, body, addDefaultHeaders(req, []byte(w.Secret), w, t, body) // likely useless, but has always been sent historially } const matrixPayloadSizeLimit = 1024 * 64 @@ -240,6 +245,25 @@ func (m matrixConvertor) Package(p *api.PackagePayload) (MatrixPayload, error) { return m.newPayload(text) } +func (m matrixConvertor) Status(p *api.CommitStatusPayload) (MatrixPayload, error) { + refLink := htmlLinkFormatter(p.TargetURL, fmt.Sprintf("%s [%s]", p.Context, base.ShortSha(p.SHA))) + text := fmt.Sprintf("Commit Status changed: %s - %s", refLink, p.Description) + + return m.newPayload(text) +} + +func (m matrixConvertor) WorkflowRun(p *api.WorkflowRunPayload) (MatrixPayload, error) { + text, _ := getWorkflowRunPayloadInfo(p, htmlLinkFormatter, true) + + return m.newPayload(text) +} + +func (m matrixConvertor) WorkflowJob(p *api.WorkflowJobPayload) (MatrixPayload, error) { + text, _ := getWorkflowJobPayloadInfo(p, htmlLinkFormatter, true) + + return m.newPayload(text) +} + var urlRegex = regexp.MustCompile(`<a [^>]*?href="([^">]*?)">(.*?)</a>`) func getMessageBody(htmlText string) string { diff --git a/services/webhook/matrix_test.go b/services/webhook/matrix_test.go index 058f8e3c5f..d36d93c5a7 100644 --- a/services/webhook/matrix_test.go +++ b/services/webhook/matrix_test.go @@ -4,7 +4,6 @@ package webhook import ( - "context" "testing" webhook_model "code.gitea.io/gitea/models/webhook" @@ -211,7 +210,7 @@ func TestMatrixJSONPayload(t *testing.T) { PayloadVersion: 2, } - req, reqBody, err := newMatrixRequest(context.Background(), hook, task) + req, reqBody, err := newMatrixRequest(t.Context(), hook, task) require.NotNil(t, req) require.NotNil(t, reqBody) require.NoError(t, err) diff --git a/services/webhook/msteams.go b/services/webhook/msteams.go index 7ef96ffa27..450a544b42 100644 --- a/services/webhook/msteams.go +++ b/services/webhook/msteams.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "strings" webhook_model "code.gitea.io/gitea/models/webhook" @@ -73,7 +74,7 @@ func (m msteamsConvertor) Create(p *api.CreatePayload) (MSTeamsPayload, error) { "", p.Repo.HTMLURL+"/src/"+util.PathEscapeSegments(refName), greenColor, - &MSTeamsFact{fmt.Sprintf("%s:", p.RefType), refName}, + &MSTeamsFact{p.RefType + ":", refName}, ), nil } @@ -90,7 +91,7 @@ func (m msteamsConvertor) Delete(p *api.DeletePayload) (MSTeamsPayload, error) { "", p.Repo.HTMLURL+"/src/"+util.PathEscapeSegments(refName), yellowColor, - &MSTeamsFact{fmt.Sprintf("%s:", p.RefType), refName}, + &MSTeamsFact{p.RefType + ":", refName}, ), nil } @@ -148,7 +149,7 @@ func (m msteamsConvertor) Push(p *api.PushPayload) (MSTeamsPayload, error) { text, titleLink, greenColor, - &MSTeamsFact{"Commit count:", fmt.Sprintf("%d", p.TotalCommits)}, + &MSTeamsFact{"Commit count:", strconv.Itoa(p.TotalCommits)}, ), nil } @@ -163,7 +164,7 @@ func (m msteamsConvertor) Issue(p *api.IssuePayload) (MSTeamsPayload, error) { extraMarkdown, p.Issue.HTMLURL, color, - &MSTeamsFact{"Issue #:", fmt.Sprintf("%d", p.Issue.ID)}, + &MSTeamsFact{"Issue #:", strconv.FormatInt(p.Issue.ID, 10)}, ), nil } @@ -178,7 +179,7 @@ func (m msteamsConvertor) IssueComment(p *api.IssueCommentPayload) (MSTeamsPaylo p.Comment.Body, p.Comment.HTMLURL, color, - &MSTeamsFact{"Issue #:", fmt.Sprintf("%d", p.Issue.ID)}, + &MSTeamsFact{"Issue #:", strconv.FormatInt(p.Issue.ID, 10)}, ), nil } @@ -193,7 +194,7 @@ func (m msteamsConvertor) PullRequest(p *api.PullRequestPayload) (MSTeamsPayload extraMarkdown, p.PullRequest.HTMLURL, color, - &MSTeamsFact{"Pull request #:", fmt.Sprintf("%d", p.PullRequest.ID)}, + &MSTeamsFact{"Pull request #:", strconv.FormatInt(p.PullRequest.ID, 10)}, ), nil } @@ -230,7 +231,7 @@ func (m msteamsConvertor) Review(p *api.PullRequestPayload, event webhook_module text, p.PullRequest.HTMLURL, color, - &MSTeamsFact{"Pull request #:", fmt.Sprintf("%d", p.PullRequest.ID)}, + &MSTeamsFact{"Pull request #:", strconv.FormatInt(p.PullRequest.ID, 10)}, ), nil } @@ -303,6 +304,48 @@ func (m msteamsConvertor) Package(p *api.PackagePayload) (MSTeamsPayload, error) ), nil } +func (m msteamsConvertor) Status(p *api.CommitStatusPayload) (MSTeamsPayload, error) { + title, color := getStatusPayloadInfo(p, noneLinkFormatter, false) + + return createMSTeamsPayload( + p.Repo, + p.Sender, + title, + "", + p.TargetURL, + color, + &MSTeamsFact{"CommitStatus:", p.Context}, + ), nil +} + +func (msteamsConvertor) WorkflowRun(p *api.WorkflowRunPayload) (MSTeamsPayload, error) { + title, color := getWorkflowRunPayloadInfo(p, noneLinkFormatter, false) + + return createMSTeamsPayload( + p.Repo, + p.Sender, + title, + "", + p.WorkflowRun.HTMLURL, + color, + &MSTeamsFact{"WorkflowRun:", p.WorkflowRun.DisplayTitle}, + ), nil +} + +func (msteamsConvertor) WorkflowJob(p *api.WorkflowJobPayload) (MSTeamsPayload, error) { + title, color := getWorkflowJobPayloadInfo(p, noneLinkFormatter, false) + + return createMSTeamsPayload( + p.Repo, + p.Sender, + title, + "", + p.WorkflowJob.HTMLURL, + color, + &MSTeamsFact{"WorkflowJob:", p.WorkflowJob.Name}, + ), nil +} + func createMSTeamsPayload(r *api.Repository, s *api.User, title, text, actionTarget string, color int, fact *MSTeamsFact) MSTeamsPayload { facts := make([]MSTeamsFact, 0, 2) if r != nil { @@ -349,3 +392,7 @@ func newMSTeamsRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_m var pc payloadConvertor[MSTeamsPayload] = msteamsConvertor{} return newJSONRequest(pc, w, t, true) } + +func init() { + RegisterWebhookRequester(webhook_module.MSTEAMS, newMSTeamsRequest) +} diff --git a/services/webhook/msteams_test.go b/services/webhook/msteams_test.go index 01e08b918e..0d98b94bad 100644 --- a/services/webhook/msteams_test.go +++ b/services/webhook/msteams_test.go @@ -4,7 +4,6 @@ package webhook import ( - "context" "testing" webhook_model "code.gitea.io/gitea/models/webhook" @@ -336,7 +335,7 @@ func TestMSTeamsPayload(t *testing.T) { assert.Equal(t, "[test/repo] New wiki page 'index' (Wiki change comment)", pl.Summary) assert.Len(t, pl.Sections, 1) assert.Equal(t, "user1", pl.Sections[0].ActivitySubtitle) - assert.Equal(t, "", pl.Sections[0].Text) + assert.Empty(t, pl.Sections[0].Text) assert.Len(t, pl.Sections[0].Facts, 2) for _, fact := range pl.Sections[0].Facts { if fact.Name == "Repository:" { @@ -357,7 +356,7 @@ func TestMSTeamsPayload(t *testing.T) { assert.Equal(t, "[test/repo] Wiki page 'index' edited (Wiki change comment)", pl.Summary) assert.Len(t, pl.Sections, 1) assert.Equal(t, "user1", pl.Sections[0].ActivitySubtitle) - assert.Equal(t, "", pl.Sections[0].Text) + assert.Empty(t, pl.Sections[0].Text) assert.Len(t, pl.Sections[0].Facts, 2) for _, fact := range pl.Sections[0].Facts { if fact.Name == "Repository:" { @@ -439,7 +438,7 @@ func TestMSTeamsJSONPayload(t *testing.T) { PayloadVersion: 2, } - req, reqBody, err := newMSTeamsRequest(context.Background(), hook, task) + req, reqBody, err := newMSTeamsRequest(t.Context(), hook, task) require.NotNil(t, req) require.NotNil(t, reqBody) require.NoError(t, err) diff --git a/services/webhook/notifier.go b/services/webhook/notifier.go index a3d5cb34b1..672abd5c95 100644 --- a/services/webhook/notifier.go +++ b/services/webhook/notifier.go @@ -6,14 +6,18 @@ package webhook import ( "context" + actions_model "code.gitea.io/gitea/models/actions" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/organization" packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/perm" access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -290,6 +294,43 @@ func (m *webhookNotifier) NewIssue(ctx context.Context, issue *issues_model.Issu } } +func (m *webhookNotifier) DeleteIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Issue) { + permission, _ := access_model.GetUserRepoPermission(ctx, issue.Repo, doer) + if issue.IsPull { + if err := issue.LoadPullRequest(ctx); err != nil { + log.Error("LoadPullRequest: %v", err) + return + } + if err := PrepareWebhooks(ctx, EventSource{Repository: issue.Repo}, webhook_module.HookEventPullRequest, &api.PullRequestPayload{ + Action: api.HookIssueDeleted, + Index: issue.Index, + PullRequest: convert.ToAPIPullRequest(ctx, issue.PullRequest, doer), + Repository: convert.ToRepo(ctx, issue.Repo, permission), + Sender: convert.ToUser(ctx, doer, nil), + }); err != nil { + log.Error("PrepareWebhooks: %v", err) + } + } else { + if err := issue.LoadRepo(ctx); err != nil { + log.Error("issue.LoadRepo: %v", err) + return + } + if err := issue.LoadPoster(ctx); err != nil { + log.Error("issue.LoadPoster: %v", err) + return + } + if err := PrepareWebhooks(ctx, EventSource{Repository: issue.Repo}, webhook_module.HookEventIssues, &api.IssuePayload{ + Action: api.HookIssueDeleted, + Index: issue.Index, + Issue: convert.ToAPIIssue(ctx, issue.Poster, issue), + Repository: convert.ToRepo(ctx, issue.Repo, permission), + Sender: convert.ToUser(ctx, doer, nil), + }); err != nil { + log.Error("PrepareWebhooks: %v", err) + } + } +} + func (m *webhookNotifier) NewPullRequest(ctx context.Context, pull *issues_model.PullRequest, mentions []*user_model.User) { if err := pull.LoadIssue(ctx); err != nil { log.Error("pull.LoadIssue: %v", err) @@ -601,7 +642,7 @@ func (m *webhookNotifier) IssueChangeMilestone(ctx context.Context, doer *user_m func (m *webhookNotifier) PushCommits(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { apiPusher := convert.ToUser(ctx, pusher, nil) - apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo.RepoPath(), repo.HTMLURL()) + apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo) if err != nil { log.Error("commits.ToAPIPayloadCommits failed: %v", err) return @@ -763,12 +804,10 @@ func (m *webhookNotifier) PullRequestReviewRequest(ctx context.Context, doer *us func (m *webhookNotifier) CreateRef(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, refFullName git.RefName, refID string) { apiPusher := convert.ToUser(ctx, pusher, nil) apiRepo := convert.ToRepo(ctx, repo, access_model.Permission{AccessMode: perm.AccessModeNone}) - refName := refFullName.ShortName() - if err := PrepareWebhooks(ctx, EventSource{Repository: repo}, webhook_module.HookEventCreate, &api.CreatePayload{ - Ref: refName, // FIXME: should it be a full ref name? + Ref: refFullName.ShortName(), // FIXME: should it be a full ref name? But it will break the existing webhooks? Sha: refID, - RefType: refFullName.RefType(), + RefType: string(refFullName.RefType()), Repo: apiRepo, Sender: apiPusher, }); err != nil { @@ -800,11 +839,9 @@ func (m *webhookNotifier) PullRequestSynchronized(ctx context.Context, doer *use func (m *webhookNotifier) DeleteRef(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, refFullName git.RefName) { apiPusher := convert.ToUser(ctx, pusher, nil) apiRepo := convert.ToRepo(ctx, repo, access_model.Permission{AccessMode: perm.AccessModeOwner}) - refName := refFullName.ShortName() - if err := PrepareWebhooks(ctx, EventSource{Repository: repo}, webhook_module.HookEventDelete, &api.DeletePayload{ - Ref: refName, // FIXME: should it be a full ref name? - RefType: refFullName.RefType(), + Ref: refFullName.ShortName(), // FIXME: should it be a full ref name? But it will break the existing webhooks? + RefType: string(refFullName.RefType()), PusherType: api.PusherTypeUser, Repo: apiRepo, Sender: apiPusher, @@ -844,7 +881,7 @@ func (m *webhookNotifier) DeleteRelease(ctx context.Context, doer *user_model.Us func (m *webhookNotifier) SyncPushCommits(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { apiPusher := convert.ToUser(ctx, pusher, nil) - apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo.RepoPath(), repo.HTMLURL()) + apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo) if err != nil { log.Error("commits.ToAPIPayloadCommits failed: %v", err) return @@ -868,12 +905,17 @@ func (m *webhookNotifier) SyncPushCommits(ctx context.Context, pusher *user_mode func (m *webhookNotifier) CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, commit *repository.PushCommit, sender *user_model.User, status *git_model.CommitStatus) { apiSender := convert.ToUser(ctx, sender, nil) - apiCommit, err := repository.ToAPIPayloadCommit(ctx, map[string]*user_model.User{}, repo.RepoPath(), repo.HTMLURL(), commit) + apiCommit, err := repository.ToAPIPayloadCommit(ctx, map[string]*user_model.User{}, repo, commit) if err != nil { log.Error("commits.ToAPIPayloadCommits failed: %v", err) return } + // as a webhook url, target should be an absolute url. But for internal actions target url + // the target url is a url path with no host and port to make it easy to be visited + // from multiple hosts. So we need to convert it to an absolute url here. + target := httplib.MakeAbsoluteURL(ctx, status.TargetURL) + payload := api.CommitStatusPayload{ Context: status.Context, CreatedAt: status.CreatedUnix.AsTime().UTC(), @@ -881,7 +923,7 @@ func (m *webhookNotifier) CreateCommitStatus(ctx context.Context, repo *repo_mod ID: status.ID, SHA: commit.Sha1, State: status.State.String(), - TargetURL: status.TargetURL, + TargetURL: target, Commit: apiCommit, Repo: convert.ToRepo(ctx, repo, access_model.Permission{AccessMode: perm.AccessModeOwner}), @@ -924,10 +966,90 @@ func notifyPackage(ctx context.Context, sender *user_model.User, pd *packages_mo return } + var org *api.Organization + if pd.Owner.IsOrganization() { + org = convert.ToOrganization(ctx, organization.OrgFromUser(pd.Owner)) + } + if err := PrepareWebhooks(ctx, source, webhook_module.HookEventPackage, &api.PackagePayload{ - Action: action, - Package: apiPackage, - Sender: convert.ToUser(ctx, sender, nil), + Action: action, + Package: apiPackage, + Organization: org, + Sender: convert.ToUser(ctx, sender, nil), + }); err != nil { + log.Error("PrepareWebhooks: %v", err) + } +} + +func (*webhookNotifier) WorkflowJobStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, job *actions_model.ActionRunJob, task *actions_model.ActionTask) { + source := EventSource{ + Repository: repo, + Owner: repo.Owner, + } + + var org *api.Organization + if repo.Owner.IsOrganization() { + org = convert.ToOrganization(ctx, organization.OrgFromUser(repo.Owner)) + } + + status, _ := convert.ToActionsStatus(job.Status) + + convertedJob, err := convert.ToActionWorkflowJob(ctx, repo, task, job) + if err != nil { + log.Error("ToActionWorkflowJob: %v", err) + return + } + + if err := PrepareWebhooks(ctx, source, webhook_module.HookEventWorkflowJob, &api.WorkflowJobPayload{ + Action: status, + WorkflowJob: convertedJob, + Organization: org, + Repo: convert.ToRepo(ctx, repo, access_model.Permission{AccessMode: perm.AccessModeOwner}), + Sender: convert.ToUser(ctx, sender, nil), + }); err != nil { + log.Error("PrepareWebhooks: %v", err) + } +} + +func (*webhookNotifier) WorkflowRunStatusUpdate(ctx context.Context, repo *repo_model.Repository, sender *user_model.User, run *actions_model.ActionRun) { + source := EventSource{ + Repository: repo, + Owner: repo.Owner, + } + + var org *api.Organization + if repo.Owner.IsOrganization() { + org = convert.ToOrganization(ctx, organization.OrgFromUser(repo.Owner)) + } + + status := convert.ToWorkflowRunAction(run.Status) + + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + log.Error("OpenRepository: %v", err) + return + } + defer gitRepo.Close() + + convertedWorkflow, err := convert.GetActionWorkflow(ctx, gitRepo, repo, run.WorkflowID) + if err != nil { + log.Error("GetActionWorkflow: %v", err) + return + } + + convertedRun, err := convert.ToActionWorkflowRun(ctx, repo, run) + if err != nil { + log.Error("ToActionWorkflowRun: %v", err) + return + } + + if err := PrepareWebhooks(ctx, source, webhook_module.HookEventWorkflowRun, &api.WorkflowRunPayload{ + Action: status, + Workflow: convertedWorkflow, + WorkflowRun: convertedRun, + Organization: org, + Repo: convert.ToRepo(ctx, repo, access_model.Permission{AccessMode: perm.AccessModeOwner}), + Sender: convert.ToUser(ctx, sender, nil), }); err != nil { log.Error("PrepareWebhooks: %v", err) } diff --git a/services/webhook/packagist.go b/services/webhook/packagist.go index 4d809ab3a6..e6a00b0293 100644 --- a/services/webhook/packagist.go +++ b/services/webhook/packagist.go @@ -110,6 +110,18 @@ func (pc packagistConvertor) Package(_ *api.PackagePayload) (PackagistPayload, e return PackagistPayload{}, nil } +func (pc packagistConvertor) Status(_ *api.CommitStatusPayload) (PackagistPayload, error) { + return PackagistPayload{}, nil +} + +func (pc packagistConvertor) WorkflowRun(_ *api.WorkflowRunPayload) (PackagistPayload, error) { + return PackagistPayload{}, nil +} + +func (pc packagistConvertor) WorkflowJob(_ *api.WorkflowJobPayload) (PackagistPayload, error) { + return PackagistPayload{}, nil +} + func newPackagistRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_model.HookTask) (*http.Request, []byte, error) { meta := &PackagistMeta{} if err := json.Unmarshal([]byte(w.Meta), meta); err != nil { @@ -120,3 +132,7 @@ func newPackagistRequest(_ context.Context, w *webhook_model.Webhook, t *webhook } return newJSONRequest(pc, w, t, true) } + +func init() { + RegisterWebhookRequester(webhook_module.PACKAGIST, newPackagistRequest) +} diff --git a/services/webhook/packagist_test.go b/services/webhook/packagist_test.go index f47807fa6e..4e77f29edc 100644 --- a/services/webhook/packagist_test.go +++ b/services/webhook/packagist_test.go @@ -4,7 +4,6 @@ package webhook import ( - "context" "testing" webhook_model "code.gitea.io/gitea/models/webhook" @@ -164,7 +163,7 @@ func TestPackagistJSONPayload(t *testing.T) { PayloadVersion: 2, } - req, reqBody, err := newPackagistRequest(context.Background(), hook, task) + req, reqBody, err := newPackagistRequest(t.Context(), hook, task) require.NotNil(t, req) require.NotNil(t, reqBody) require.NoError(t, err) @@ -199,7 +198,7 @@ func TestPackagistEmptyPayload(t *testing.T) { PayloadVersion: 2, } - req, reqBody, err := newPackagistRequest(context.Background(), hook, task) + req, reqBody, err := newPackagistRequest(t.Context(), hook, task) require.NotNil(t, req) require.NotNil(t, reqBody) require.NoError(t, err) @@ -211,5 +210,5 @@ func TestPackagistEmptyPayload(t *testing.T) { var body PackagistPayload err = json.NewDecoder(req.Body).Decode(&body) assert.NoError(t, err) - assert.Equal(t, "", body.PackagistRepository.URL) + assert.Empty(t, body.PackagistRepository.URL) } diff --git a/services/webhook/payloader.go b/services/webhook/payloader.go index ab280a25b6..b607bf3250 100644 --- a/services/webhook/payloader.go +++ b/services/webhook/payloader.go @@ -28,6 +28,9 @@ type payloadConvertor[T any] interface { Release(*api.ReleasePayload) (T, error) Wiki(*api.WikiPayload) (T, error) Package(*api.PackagePayload) (T, error) + Status(*api.CommitStatusPayload) (T, error) + WorkflowRun(*api.WorkflowRunPayload) (T, error) + WorkflowJob(*api.WorkflowJobPayload) (T, error) } func convertUnmarshalledJSON[T, P any](convert func(P) (T, error), data []byte) (t T, err error) { @@ -77,6 +80,12 @@ func newPayload[T any](rc payloadConvertor[T], data []byte, event webhook_module return convertUnmarshalledJSON(rc.Wiki, data) case webhook_module.HookEventPackage: return convertUnmarshalledJSON(rc.Package, data) + case webhook_module.HookEventStatus: + return convertUnmarshalledJSON(rc.Status, data) + case webhook_module.HookEventWorkflowRun: + return convertUnmarshalledJSON(rc.WorkflowRun, data) + case webhook_module.HookEventWorkflowJob: + return convertUnmarshalledJSON(rc.WorkflowJob, data) } return t, fmt.Errorf("newPayload unsupported event: %s", event) } @@ -86,7 +95,10 @@ func newJSONRequest[T any](pc payloadConvertor[T], w *webhook_model.Webhook, t * if err != nil { return nil, nil, err } + return prepareJSONRequest(payload, w, t, withDefaultHeaders) +} +func prepareJSONRequest[T any](payload T, w *webhook_model.Webhook, t *webhook_model.HookTask, withDefaultHeaders bool) (*http.Request, []byte, error) { body, err := json.MarshalIndent(payload, "", " ") if err != nil { return nil, nil, err @@ -104,7 +116,7 @@ func newJSONRequest[T any](pc payloadConvertor[T], w *webhook_model.Webhook, t * req.Header.Set("Content-Type", "application/json") if withDefaultHeaders { - return req, body, addDefaultHeaders(req, []byte(w.Secret), t, body) + return req, body, addDefaultHeaders(req, []byte(w.Secret), w, t, body) } return req, body, nil } diff --git a/services/webhook/slack.go b/services/webhook/slack.go index c905e7a89f..3d645a55d0 100644 --- a/services/webhook/slack.go +++ b/services/webhook/slack.go @@ -84,9 +84,9 @@ func SlackLinkFormatter(url, text string) string { // SlackLinkToRef slack-formatter link to a repo ref func SlackLinkToRef(repoURL, ref string) string { // FIXME: SHA1 hardcoded here - url := git.RefURL(repoURL, ref) - refName := git.RefName(ref).ShortName() - return SlackLinkFormatter(url, refName) + refName := git.RefName(ref) + url := repoURL + "/src/" + refName.RefWebLinkPath() + return SlackLinkFormatter(url, refName.ShortName()) } // Create implements payloadConvertor Create method @@ -167,6 +167,24 @@ func (s slackConvertor) Package(p *api.PackagePayload) (SlackPayload, error) { return s.createPayload(text, nil), nil } +func (s slackConvertor) Status(p *api.CommitStatusPayload) (SlackPayload, error) { + text, _ := getStatusPayloadInfo(p, SlackLinkFormatter, true) + + return s.createPayload(text, nil), nil +} + +func (s slackConvertor) WorkflowRun(p *api.WorkflowRunPayload) (SlackPayload, error) { + text, _ := getWorkflowRunPayloadInfo(p, SlackLinkFormatter, true) + + return s.createPayload(text, nil), nil +} + +func (s slackConvertor) WorkflowJob(p *api.WorkflowJobPayload) (SlackPayload, error) { + text, _ := getWorkflowJobPayloadInfo(p, SlackLinkFormatter, true) + + return s.createPayload(text, nil), nil +} + // Push implements payloadConvertor Push method func (s slackConvertor) Push(p *api.PushPayload) (SlackPayload, error) { // n new commits @@ -295,6 +313,10 @@ func newSlackRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_mod return newJSONRequest(pc, w, t, true) } +func init() { + RegisterWebhookRequester(webhook_module.SLACK, newSlackRequest) +} + var slackChannel = regexp.MustCompile(`^#?[a-z0-9_-]{1,80}$`) // IsValidSlackChannel validates a channel name conforms to what slack expects: diff --git a/services/webhook/slack_test.go b/services/webhook/slack_test.go index 7ebf16aba2..839ed6f770 100644 --- a/services/webhook/slack_test.go +++ b/services/webhook/slack_test.go @@ -4,7 +4,6 @@ package webhook import ( - "context" "testing" webhook_model "code.gitea.io/gitea/models/webhook" @@ -178,7 +177,7 @@ func TestSlackJSONPayload(t *testing.T) { PayloadVersion: 2, } - req, reqBody, err := newSlackRequest(context.Background(), hook, task) + req, reqBody, err := newSlackRequest(t.Context(), hook, task) require.NotNil(t, req) require.NotNil(t, reqBody) require.NoError(t, err) diff --git a/services/webhook/telegram.go b/services/webhook/telegram.go index e54d6f2947..fdd428b45c 100644 --- a/services/webhook/telegram.go +++ b/services/webhook/telegram.go @@ -174,10 +174,28 @@ func (t telegramConvertor) Package(p *api.PackagePayload) (TelegramPayload, erro return createTelegramPayloadHTML(text), nil } +func (t telegramConvertor) Status(p *api.CommitStatusPayload) (TelegramPayload, error) { + text, _ := getStatusPayloadInfo(p, htmlLinkFormatter, true) + + return createTelegramPayloadHTML(text), nil +} + +func (telegramConvertor) WorkflowRun(p *api.WorkflowRunPayload) (TelegramPayload, error) { + text, _ := getWorkflowRunPayloadInfo(p, htmlLinkFormatter, true) + + return createTelegramPayloadHTML(text), nil +} + +func (telegramConvertor) WorkflowJob(p *api.WorkflowJobPayload) (TelegramPayload, error) { + text, _ := getWorkflowJobPayloadInfo(p, htmlLinkFormatter, true) + + return createTelegramPayloadHTML(text), nil +} + func createTelegramPayloadHTML(msgHTML string) TelegramPayload { // https://core.telegram.org/bots/api#formatting-options return TelegramPayload{ - Message: strings.TrimSpace(markup.Sanitize(msgHTML)), + Message: strings.TrimSpace(string(markup.Sanitize(msgHTML))), ParseMode: "HTML", DisableWebPreview: true, } @@ -187,3 +205,7 @@ func newTelegramRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_ var pc payloadConvertor[TelegramPayload] = telegramConvertor{} return newJSONRequest(pc, w, t, true) } + +func init() { + RegisterWebhookRequester(webhook_module.TELEGRAM, newTelegramRequest) +} diff --git a/services/webhook/telegram_test.go b/services/webhook/telegram_test.go index 7ba81f1564..3fa8e27836 100644 --- a/services/webhook/telegram_test.go +++ b/services/webhook/telegram_test.go @@ -4,7 +4,6 @@ package webhook import ( - "context" "testing" webhook_model "code.gitea.io/gitea/models/webhook" @@ -195,7 +194,7 @@ func TestTelegramJSONPayload(t *testing.T) { PayloadVersion: 2, } - req, reqBody, err := newTelegramRequest(context.Background(), hook, task) + req, reqBody, err := newTelegramRequest(t.Context(), hook, task) require.NotNil(t, req) require.NotNil(t, reqBody) require.NoError(t, err) diff --git a/services/webhook/webhook.go b/services/webhook/webhook.go index e0e8fa2fc1..182078b39d 100644 --- a/services/webhook/webhook.go +++ b/services/webhook/webhook.go @@ -27,16 +27,12 @@ import ( "github.com/gobwas/glob" ) -var webhookRequesters = map[webhook_module.HookType]func(context.Context, *webhook_model.Webhook, *webhook_model.HookTask) (req *http.Request, body []byte, err error){ - webhook_module.SLACK: newSlackRequest, - webhook_module.DISCORD: newDiscordRequest, - webhook_module.DINGTALK: newDingtalkRequest, - webhook_module.TELEGRAM: newTelegramRequest, - webhook_module.MSTEAMS: newMSTeamsRequest, - webhook_module.FEISHU: newFeishuRequest, - webhook_module.MATRIX: newMatrixRequest, - webhook_module.WECHATWORK: newWechatworkRequest, - webhook_module.PACKAGIST: newPackagistRequest, +type Requester func(context.Context, *webhook_model.Webhook, *webhook_model.HookTask) (req *http.Request, body []byte, err error) + +var webhookRequesters = map[webhook_module.HookType]Requester{} + +func RegisterWebhookRequester(hookType webhook_module.HookType, requester Requester) { + webhookRequesters[hookType] = requester } // IsValidHookTaskType returns true if a webhook registered @@ -137,14 +133,8 @@ func PrepareWebhook(ctx context.Context, w *webhook_model.Webhook, event webhook return nil } - for _, e := range w.EventCheckers() { - if event == e.Type { - if !e.Has() { - return nil - } - - break - } + if !w.HasEvent(event) { + return nil } // Avoid sending "0 new commits" to non-integration relevant webhooks (e.g. slack, discord, etc.). diff --git a/services/webhook/webhook_test.go b/services/webhook/webhook_test.go index 6bac02712b..5a805347e3 100644 --- a/services/webhook/webhook_test.go +++ b/services/webhook/webhook_test.go @@ -13,6 +13,7 @@ import ( webhook_model "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/test" webhook_module "code.gitea.io/gitea/modules/webhook" "code.gitea.io/gitea/services/convert" @@ -84,7 +85,8 @@ func TestPrepareWebhooksBranchFilterNoMatch(t *testing.T) { func TestWebhookUserMail(t *testing.T) { require.NoError(t, unittest.PrepareTestDatabase()) - setting.Service.NoReplyAddress = "no-reply.com" + defer test.MockVariableValue(&setting.Service.NoReplyAddress, "no-reply.com")() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) assert.Equal(t, user.GetPlaceholderEmail(), convert.ToUser(db.DefaultContext, user, nil).Email) assert.Equal(t, user.Email, convert.ToUser(db.DefaultContext, user, user).Email) diff --git a/services/webhook/wechatwork.go b/services/webhook/wechatwork.go index 1d8c1d7dac..1875317406 100644 --- a/services/webhook/wechatwork.go +++ b/services/webhook/wechatwork.go @@ -175,7 +175,29 @@ func (wc wechatworkConvertor) Package(p *api.PackagePayload) (WechatworkPayload, return newWechatworkMarkdownPayload(text), nil } +func (wc wechatworkConvertor) Status(p *api.CommitStatusPayload) (WechatworkPayload, error) { + text, _ := getStatusPayloadInfo(p, noneLinkFormatter, true) + + return newWechatworkMarkdownPayload(text), nil +} + +func (wc wechatworkConvertor) WorkflowRun(p *api.WorkflowRunPayload) (WechatworkPayload, error) { + text, _ := getWorkflowRunPayloadInfo(p, noneLinkFormatter, true) + + return newWechatworkMarkdownPayload(text), nil +} + +func (wc wechatworkConvertor) WorkflowJob(p *api.WorkflowJobPayload) (WechatworkPayload, error) { + text, _ := getWorkflowJobPayloadInfo(p, noneLinkFormatter, true) + + return newWechatworkMarkdownPayload(text), nil +} + func newWechatworkRequest(_ context.Context, w *webhook_model.Webhook, t *webhook_model.HookTask) (*http.Request, []byte, error) { var pc payloadConvertor[WechatworkPayload] = wechatworkConvertor{} return newJSONRequest(pc, w, t, true) } + +func init() { + RegisterWebhookRequester(webhook_module.WECHATWORK, newWechatworkRequest) +} diff --git a/services/webtheme/webtheme.go b/services/webtheme/webtheme.go index dc801e1ff7..4e89d6dbac 100644 --- a/services/webtheme/webtheme.go +++ b/services/webtheme/webtheme.go @@ -4,6 +4,7 @@ package webtheme import ( + "regexp" "sort" "strings" "sync" @@ -12,63 +13,154 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/public" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) var ( - availableThemes []string - availableThemesSet container.Set[string] - themeOnce sync.Once + availableThemes []*ThemeMetaInfo + availableThemeInternalNames container.Set[string] + themeOnce sync.Once ) +const ( + fileNamePrefix = "theme-" + fileNameSuffix = ".css" +) + +type ThemeMetaInfo struct { + FileName string + InternalName string + DisplayName string +} + +func parseThemeMetaInfoToMap(cssContent string) map[string]string { + /* + The theme meta info is stored in the CSS file's variables of `gitea-theme-meta-info` element, + which is a privately defined and is only used by backend to extract the meta info. + Not using ":root" because it is difficult to parse various ":root" blocks when importing other files, + it is difficult to control the overriding, and it's difficult to avoid user's customized overridden styles. + */ + metaInfoContent := cssContent + if pos := strings.LastIndex(metaInfoContent, "gitea-theme-meta-info"); pos >= 0 { + metaInfoContent = metaInfoContent[pos:] + } + + reMetaInfoItem := ` +( +\s*(--[-\w]+) +\s*: +\s*( +("(\\"|[^"])*") +|('(\\'|[^'])*') +|([^'";]+) +) +\s*; +\s* +) +` + reMetaInfoItem = strings.ReplaceAll(reMetaInfoItem, "\n", "") + reMetaInfoBlock := `\bgitea-theme-meta-info\s*\{(` + reMetaInfoItem + `+)\}` + re := regexp.MustCompile(reMetaInfoBlock) + matchedMetaInfoBlock := re.FindAllStringSubmatch(metaInfoContent, -1) + if len(matchedMetaInfoBlock) == 0 { + return nil + } + re = regexp.MustCompile(strings.ReplaceAll(reMetaInfoItem, "\n", "")) + matchedItems := re.FindAllStringSubmatch(matchedMetaInfoBlock[0][1], -1) + m := map[string]string{} + for _, item := range matchedItems { + v := item[3] + if after, ok := strings.CutPrefix(v, `"`); ok { + v = strings.TrimSuffix(after, `"`) + v = strings.ReplaceAll(v, `\"`, `"`) + } else if after, ok := strings.CutPrefix(v, `'`); ok { + v = strings.TrimSuffix(after, `'`) + v = strings.ReplaceAll(v, `\'`, `'`) + } + m[item[2]] = v + } + return m +} + +func defaultThemeMetaInfoByFileName(fileName string) *ThemeMetaInfo { + themeInfo := &ThemeMetaInfo{ + FileName: fileName, + InternalName: strings.TrimSuffix(strings.TrimPrefix(fileName, fileNamePrefix), fileNameSuffix), + } + themeInfo.DisplayName = themeInfo.InternalName + return themeInfo +} + +func defaultThemeMetaInfoByInternalName(fileName string) *ThemeMetaInfo { + return defaultThemeMetaInfoByFileName(fileNamePrefix + fileName + fileNameSuffix) +} + +func parseThemeMetaInfo(fileName, cssContent string) *ThemeMetaInfo { + themeInfo := defaultThemeMetaInfoByFileName(fileName) + m := parseThemeMetaInfoToMap(cssContent) + if m == nil { + return themeInfo + } + themeInfo.DisplayName = m["--theme-display-name"] + return themeInfo +} + func initThemes() { availableThemes = nil defer func() { - availableThemesSet = container.SetOf(availableThemes...) - if !availableThemesSet.Contains(setting.UI.DefaultTheme) { + availableThemeInternalNames = container.Set[string]{} + for _, theme := range availableThemes { + availableThemeInternalNames.Add(theme.InternalName) + } + if !availableThemeInternalNames.Contains(setting.UI.DefaultTheme) { setting.LogStartupProblem(1, log.ERROR, "Default theme %q is not available, please correct the '[ui].DEFAULT_THEME' setting in the config file", setting.UI.DefaultTheme) } }() cssFiles, err := public.AssetFS().ListFiles("/assets/css") if err != nil { log.Error("Failed to list themes: %v", err) - availableThemes = []string{setting.UI.DefaultTheme} + availableThemes = []*ThemeMetaInfo{defaultThemeMetaInfoByInternalName(setting.UI.DefaultTheme)} return } - var foundThemes []string - for _, name := range cssFiles { - name, ok := strings.CutPrefix(name, "theme-") - if !ok { - continue - } - name, ok = strings.CutSuffix(name, ".css") - if !ok { - continue + var foundThemes []*ThemeMetaInfo + for _, fileName := range cssFiles { + if strings.HasPrefix(fileName, fileNamePrefix) && strings.HasSuffix(fileName, fileNameSuffix) { + content, err := public.AssetFS().ReadFile("/assets/css/" + fileName) + if err != nil { + log.Error("Failed to read theme file %q: %v", fileName, err) + continue + } + foundThemes = append(foundThemes, parseThemeMetaInfo(fileName, util.UnsafeBytesToString(content))) } - foundThemes = append(foundThemes, name) } if len(setting.UI.Themes) > 0 { allowedThemes := container.SetOf(setting.UI.Themes...) for _, theme := range foundThemes { - if allowedThemes.Contains(theme) { + if allowedThemes.Contains(theme.InternalName) { availableThemes = append(availableThemes, theme) } } } else { availableThemes = foundThemes } - sort.Strings(availableThemes) + sort.Slice(availableThemes, func(i, j int) bool { + if availableThemes[i].InternalName == setting.UI.DefaultTheme { + return true + } + return availableThemes[i].DisplayName < availableThemes[j].DisplayName + }) if len(availableThemes) == 0 { setting.LogStartupProblem(1, log.ERROR, "No theme candidate in asset files, but Gitea requires there should be at least one usable theme") - availableThemes = []string{setting.UI.DefaultTheme} + availableThemes = []*ThemeMetaInfo{defaultThemeMetaInfoByInternalName(setting.UI.DefaultTheme)} } } -func GetAvailableThemes() []string { +func GetAvailableThemes() []*ThemeMetaInfo { themeOnce.Do(initThemes) return availableThemes } -func IsThemeAvailable(name string) bool { +func IsThemeAvailable(internalName string) bool { themeOnce.Do(initThemes) - return availableThemesSet.Contains(name) + return availableThemeInternalNames.Contains(internalName) } diff --git a/services/webtheme/webtheme_test.go b/services/webtheme/webtheme_test.go new file mode 100644 index 0000000000..587953ab0c --- /dev/null +++ b/services/webtheme/webtheme_test.go @@ -0,0 +1,37 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package webtheme + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseThemeMetaInfo(t *testing.T) { + m := parseThemeMetaInfoToMap(`gitea-theme-meta-info { + --k1: "v1"; + --k2: "v\"2"; + --k3: 'v3'; + --k4: 'v\'4'; + --k5: v5; +}`) + assert.Equal(t, map[string]string{ + "--k1": "v1", + "--k2": `v"2`, + "--k3": "v3", + "--k4": "v'4", + "--k5": "v5", + }, m) + + // if an auto theme imports others, the meta info should be extracted from the last one + // the meta in imported themes should be ignored to avoid incorrect overriding + m = parseThemeMetaInfoToMap(` +@media (prefers-color-scheme: dark) { gitea-theme-meta-info { --k1: foo; } } +@media (prefers-color-scheme: light) { gitea-theme-meta-info { --k1: bar; } } +gitea-theme-meta-info { + --k2: real; +}`) + assert.Equal(t, map[string]string{"--k2": "real"}, m) +} diff --git a/services/wiki/wiki.go b/services/wiki/wiki.go index 7a0419aea7..0a955406e2 100644 --- a/services/wiki/wiki.go +++ b/services/wiki/wiki.go @@ -41,9 +41,9 @@ func InitWiki(ctx context.Context, repo *repo_model.Repository) error { if err := git.InitRepository(ctx, repo.WikiPath(), true, repo.ObjectFormatName); err != nil { return fmt.Errorf("InitRepository: %w", err) - } else if err = repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil { + } else if err = gitrepo.CreateDelegateHooks(ctx, repo.WikiStorageRepo()); err != nil { return fmt.Errorf("createDelegateHooks: %w", err) - } else if _, _, err = git.NewCommand(ctx, "symbolic-ref", "HEAD").AddDynamicArguments(git.BranchPrefix + repo.DefaultWikiBranch).RunStdString(&git.RunOpts{Dir: repo.WikiPath()}); err != nil { + } else if _, _, err = git.NewCommand("symbolic-ref", "HEAD").AddDynamicArguments(git.BranchPrefix+repo.DefaultWikiBranch).RunStdString(ctx, &git.RunOpts{Dir: repo.WikiPath()}); err != nil { return fmt.Errorf("unable to set default wiki branch to %q: %w", repo.DefaultWikiBranch, err) } return nil @@ -100,17 +100,13 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return fmt.Errorf("InitWiki: %w", err) } - hasDefaultBranch := git.IsBranchExist(ctx, repo.WikiPath(), repo.DefaultWikiBranch) + hasDefaultBranch := gitrepo.IsBranchExist(ctx, repo.WikiStorageRepo(), repo.DefaultWikiBranch) - basePath, err := repo_module.CreateTemporaryPath("update-wiki") + basePath, cleanup, err := repo_module.CreateTemporaryPath("update-wiki") if err != nil { return err } - defer func() { - if err := repo_module.RemoveTemporaryPath(basePath); err != nil { - log.Error("Merge: RemoveTemporaryPath: %s", err) - } - }() + defer cleanup() cloneOpts := git.CloneRepoOptions{ Bare: true, @@ -198,7 +194,7 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model sign, signingKey, signer, _ := asymkey_service.SignWikiCommit(ctx, repo, doer) if sign { - commitTreeOpts.KeyID = signingKey + commitTreeOpts.Key = signingKey if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { committer = signer } @@ -264,15 +260,11 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return fmt.Errorf("InitWiki: %w", err) } - basePath, err := repo_module.CreateTemporaryPath("update-wiki") + basePath, cleanup, err := repo_module.CreateTemporaryPath("update-wiki") if err != nil { return err } - defer func() { - if err := repo_module.RemoveTemporaryPath(basePath); err != nil { - log.Error("Merge: RemoveTemporaryPath: %s", err) - } - }() + defer cleanup() if err := git.Clone(ctx, repo.WikiPath(), basePath, git.CloneRepoOptions{ Bare: true, @@ -324,7 +316,7 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model sign, signingKey, signer, _ := asymkey_service.SignWikiCommit(ctx, repo, doer) if sign { - commitTreeOpts.KeyID = signingKey + commitTreeOpts.Key = signingKey if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { committer = signer } @@ -373,7 +365,7 @@ func ChangeDefaultWikiBranch(ctx context.Context, repo *repo_model.Repository, n } return db.WithTx(ctx, func(ctx context.Context) error { repo.DefaultWikiBranch = newBranch - if err := repo_model.UpdateRepositoryCols(ctx, repo, "default_wiki_branch"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "default_wiki_branch"); err != nil { return fmt.Errorf("unable to update database: %w", err) } @@ -381,7 +373,7 @@ func ChangeDefaultWikiBranch(ctx context.Context, repo *repo_model.Repository, n return nil } - oldDefBranch, err := gitrepo.GetWikiDefaultBranch(ctx, repo) + oldDefBranch, err := gitrepo.GetDefaultBranch(ctx, repo.WikiStorageRepo()) if err != nil { return fmt.Errorf("unable to get default branch: %w", err) } @@ -389,7 +381,7 @@ func ChangeDefaultWikiBranch(ctx context.Context, repo *repo_model.Repository, n return nil } - gitRepo, err := gitrepo.OpenWikiRepository(ctx, repo) + gitRepo, err := gitrepo.OpenRepository(ctx, repo.WikiStorageRepo()) if errors.Is(err, util.ErrNotExist) { return nil // no git repo on storage, no need to do anything else } else if err != nil { diff --git a/services/wiki/wiki_test.go b/services/wiki/wiki_test.go index 0a18cffa25..6ea3ca9c1b 100644 --- a/services/wiki/wiki_test.go +++ b/services/wiki/wiki_test.go @@ -17,6 +17,7 @@ import ( _ "code.gitea.io/gitea/models/actions" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMain(m *testing.M) { @@ -25,7 +26,7 @@ func TestMain(m *testing.M) { func TestWebPathSegments(t *testing.T) { a := WebPathSegments("a%2Fa/b+c/d-e/f-g.-") - assert.EqualValues(t, []string{"a/a", "b c", "d e", "f-g"}, a) + assert.Equal(t, []string{"a/a", "b c", "d e", "f-g"}, a) } func TestUserTitleToWebPath(t *testing.T) { @@ -62,7 +63,7 @@ func TestWebPathToDisplayName(t *testing.T) { {"a b", "a%20b.md"}, } { _, displayName := WebPathToUserTitle(test.WebPath) - assert.EqualValues(t, test.Expected, displayName) + assert.Equal(t, test.Expected, displayName) } } @@ -79,7 +80,7 @@ func TestWebPathToGitPath(t *testing.T) { {"2000-01-02-meeting.md", "2000-01-02+meeting"}, {"2000-01-02 meeting.-.md", "2000-01-02%20meeting.-"}, } { - assert.EqualValues(t, test.Expected, WebPathToGitPath(test.WikiName)) + assert.Equal(t, test.Expected, WebPathToGitPath(test.WikiName)) } } @@ -115,9 +116,9 @@ func TestGitPathToWebPath(t *testing.T) { func TestUserWebGitPathConsistency(t *testing.T) { maxLen := 20 b := make([]byte, maxLen) - for i := 0; i < 1000; i++ { + for range 1000 { l := rand.Intn(maxLen) - for j := 0; j < l; j++ { + for j := range l { r := rand.Intn(0x80-0x20) + 0x20 b[j] = byte(r) } @@ -133,9 +134,9 @@ func TestUserWebGitPathConsistency(t *testing.T) { _, userTitle1 := WebPathToUserTitle(webPath1) gitPath1 := WebPathToGitPath(webPath1) - assert.EqualValues(t, userTitle, userTitle1, "UserTitle for userTitle: %q", userTitle) - assert.EqualValues(t, webPath, webPath1, "WebPath for userTitle: %q", userTitle) - assert.EqualValues(t, gitPath, gitPath1, "GitPath for userTitle: %q", userTitle) + assert.Equal(t, userTitle, userTitle1, "UserTitle for userTitle: %q", userTitle) + assert.Equal(t, webPath, webPath1, "WebPath for userTitle: %q", userTitle) + assert.Equal(t, gitPath, gitPath1, "GitPath for userTitle: %q", userTitle) } } @@ -165,17 +166,16 @@ func TestRepository_AddWikiPage(t *testing.T) { webPath := UserTitleToWebPath("", userTitle) assert.NoError(t, AddWikiPage(git.DefaultContext, doer, repo, webPath, wikiContent, commitMsg)) // Now need to show that the page has been added: - gitRepo, err := gitrepo.OpenWikiRepository(git.DefaultContext, repo) - if !assert.NoError(t, err) { - return - } + gitRepo, err := gitrepo.OpenRepository(git.DefaultContext, repo.WikiStorageRepo()) + require.NoError(t, err) + defer gitRepo.Close() masterTree, err := gitRepo.GetTree(repo.DefaultWikiBranch) assert.NoError(t, err) gitPath := WebPathToGitPath(webPath) entry, err := masterTree.GetTreeEntryByPath(gitPath) assert.NoError(t, err) - assert.EqualValues(t, gitPath, entry.Name(), "%s not added correctly", userTitle) + assert.Equal(t, gitPath, entry.Name(), "%s not added correctly", userTitle) }) } @@ -213,14 +213,14 @@ func TestRepository_EditWikiPage(t *testing.T) { assert.NoError(t, EditWikiPage(git.DefaultContext, doer, repo, "Home", webPath, newWikiContent, commitMsg)) // Now need to show that the page has been added: - gitRepo, err := gitrepo.OpenWikiRepository(git.DefaultContext, repo) + gitRepo, err := gitrepo.OpenRepository(git.DefaultContext, repo.WikiStorageRepo()) assert.NoError(t, err) masterTree, err := gitRepo.GetTree(repo.DefaultWikiBranch) assert.NoError(t, err) gitPath := WebPathToGitPath(webPath) entry, err := masterTree.GetTreeEntryByPath(gitPath) assert.NoError(t, err) - assert.EqualValues(t, gitPath, entry.Name(), "%s not edited correctly", newWikiName) + assert.Equal(t, gitPath, entry.Name(), "%s not edited correctly", newWikiName) if newWikiName != "Home" { _, err := masterTree.GetTreeEntryByPath("Home.md") @@ -237,10 +237,9 @@ func TestRepository_DeleteWikiPage(t *testing.T) { assert.NoError(t, DeleteWikiPage(git.DefaultContext, doer, repo, "Home")) // Now need to show that the page has been added: - gitRepo, err := gitrepo.OpenWikiRepository(git.DefaultContext, repo) - if !assert.NoError(t, err) { - return - } + gitRepo, err := gitrepo.OpenRepository(git.DefaultContext, repo.WikiStorageRepo()) + require.NoError(t, err) + defer gitRepo.Close() masterTree, err := gitRepo.GetTree(repo.DefaultWikiBranch) assert.NoError(t, err) @@ -252,10 +251,9 @@ func TestRepository_DeleteWikiPage(t *testing.T) { func TestPrepareWikiFileName(t *testing.T) { unittest.PrepareTestEnv(t) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - gitRepo, err := gitrepo.OpenWikiRepository(git.DefaultContext, repo) - if !assert.NoError(t, err) { - return - } + gitRepo, err := gitrepo.OpenRepository(git.DefaultContext, repo.WikiStorageRepo()) + require.NoError(t, err) + defer gitRepo.Close() tests := []struct { @@ -292,7 +290,7 @@ func TestPrepareWikiFileName(t *testing.T) { t.Errorf("expect to find an escaped file but we could not detect one") } } - assert.EqualValues(t, tt.wikiPath, newWikiPath) + assert.Equal(t, tt.wikiPath, newWikiPath) }) } } @@ -307,21 +305,20 @@ func TestPrepareWikiFileName_FirstPage(t *testing.T) { assert.NoError(t, err) gitRepo, err := git.OpenRepository(git.DefaultContext, tmpDir) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) + defer gitRepo.Close() existence, newWikiPath, err := prepareGitPath(gitRepo, "master", "Home") assert.False(t, existence) assert.NoError(t, err) - assert.EqualValues(t, "Home.md", newWikiPath) + assert.Equal(t, "Home.md", newWikiPath) } func TestWebPathConversion(t *testing.T) { assert.Equal(t, "path/wiki", WebPathToURLPath(WebPath("path/wiki"))) assert.Equal(t, "wiki", WebPathToURLPath(WebPath("wiki"))) - assert.Equal(t, "", WebPathToURLPath(WebPath(""))) + assert.Empty(t, WebPathToURLPath(WebPath(""))) } func TestWebPathFromRequest(t *testing.T) { |