diff options
Diffstat (limited to 'modules')
506 files changed, 8897 insertions, 7124 deletions
diff --git a/modules/actions/artifacts.go b/modules/actions/artifacts.go new file mode 100644 index 0000000000..4d074435ef --- /dev/null +++ b/modules/actions/artifacts.go @@ -0,0 +1,48 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "net/http" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/services/context" +) + +// Artifacts using the v4 backend are stored as a single combined zip file per artifact on the backend +// The v4 backend ensures ContentEncoding is set to "application/zip", which is not the case for the old backend +func IsArtifactV4(art *actions_model.ActionArtifact) bool { + return art.ArtifactName+".zip" == art.ArtifactPath && art.ContentEncoding == "application/zip" +} + +func DownloadArtifactV4ServeDirectOnly(ctx *context.Base, art *actions_model.ActionArtifact) (bool, error) { + if setting.Actions.ArtifactStorage.ServeDirect() { + u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath, nil) + if u != nil && err == nil { + ctx.Redirect(u.String(), http.StatusFound) + return true, nil + } + } + return false, nil +} + +func DownloadArtifactV4Fallback(ctx *context.Base, art *actions_model.ActionArtifact) error { + f, err := storage.ActionsArtifacts.Open(art.StoragePath) + if err != nil { + return err + } + defer f.Close() + http.ServeContent(ctx.Resp, ctx.Req, art.ArtifactName+".zip", art.CreatedUnix.AsLocalTime(), f) + return nil +} + +func DownloadArtifactV4(ctx *context.Base, art *actions_model.ActionArtifact) error { + ok, err := DownloadArtifactV4ServeDirectOnly(ctx, art) + if ok || err != nil { + return err + } + return DownloadArtifactV4Fallback(ctx, art) +} diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 0d2b0dd919..27bcafa649 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -6,6 +6,7 @@ package actions import ( "bytes" "io" + "slices" "strings" "code.gitea.io/gitea/modules/git" @@ -43,21 +44,23 @@ func IsWorkflow(path string) bool { return strings.HasPrefix(path, ".gitea/workflows") || strings.HasPrefix(path, ".github/workflows") } -func ListWorkflows(commit *git.Commit) (git.Entries, error) { - tree, err := commit.SubTree(".gitea/workflows") +func ListWorkflows(commit *git.Commit) (string, git.Entries, error) { + rpath := ".gitea/workflows" + tree, err := commit.SubTree(rpath) if _, ok := err.(git.ErrNotExist); ok { - tree, err = commit.SubTree(".github/workflows") + rpath = ".github/workflows" + tree, err = commit.SubTree(rpath) } if _, ok := err.(git.ErrNotExist); ok { - return nil, nil + return "", nil, nil } if err != nil { - return nil, err + return "", nil, err } entries, err := tree.ListEntriesRecursiveFast() if err != nil { - return nil, err + return "", nil, err } ret := make(git.Entries, 0, len(entries)) @@ -66,7 +69,7 @@ func ListWorkflows(commit *git.Commit) (git.Entries, error) { ret = append(ret, entry) } } - return ret, nil + return rpath, ret, nil } func GetContentFromEntry(entry *git.TreeEntry) ([]byte, error) { @@ -102,7 +105,7 @@ func DetectWorkflows( payload api.Payloader, detectSchedule bool, ) ([]*DetectedWorkflow, []*DetectedWorkflow, error) { - entries, err := ListWorkflows(commit) + _, entries, err := ListWorkflows(commit) if err != nil { return nil, nil, err } @@ -147,7 +150,7 @@ func DetectWorkflows( } func DetectScheduledWorkflows(gitRepo *git.Repository, commit *git.Commit) ([]*DetectedWorkflow, error) { - entries, err := ListWorkflows(commit) + _, entries, err := ListWorkflows(commit) if err != nil { return nil, err } @@ -243,6 +246,10 @@ func detectMatched(gitRepo *git.Repository, commit *git.Commit, triggedEvent web webhook_module.HookEventPackage: return matchPackageEvent(payload.(*api.PackagePayload), evt) + case // workflow_run + webhook_module.HookEventWorkflowRun: + return matchWorkflowRunEvent(payload.(*api.WorkflowRunPayload), evt) + default: log.Warn("unsupported event %q", triggedEvent) return false @@ -311,6 +318,10 @@ func matchPushEvent(commit *git.Commit, pushPayload *api.PushPayload, evt *jobpa matchTimes++ } case "paths": + if refName.IsTag() { + matchTimes++ + break + } filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err) @@ -324,6 +335,10 @@ func matchPushEvent(commit *git.Commit, pushPayload *api.PushPayload, evt *jobpa } } case "paths-ignore": + if refName.IsTag() { + matchTimes++ + break + } filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err) @@ -463,7 +478,7 @@ func matchPullRequestEvent(gitRepo *git.Repository, commit *git.Commit, prPayloa matchTimes++ } case "paths": - filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.Base.Ref) + filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err) } else { @@ -476,7 +491,7 @@ func matchPullRequestEvent(gitRepo *git.Repository, commit *git.Commit, prPayloa } } case "paths-ignore": - filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.Base.Ref) + filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err) } else { @@ -554,21 +569,12 @@ func matchPullRequestReviewEvent(prPayload *api.PullRequestPayload, evt *jobpars actions = append(actions, "submitted", "edited") } - matched := false for _, val := range vals { - for _, action := range actions { - if glob.MustCompile(val, '/').Match(action) { - matched = true - break - } - } - if matched { + if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) { + matchTimes++ break } } - if matched { - matchTimes++ - } default: log.Warn("pull request review event unsupported condition %q", cond) } @@ -603,21 +609,12 @@ func matchPullRequestReviewCommentEvent(prPayload *api.PullRequestPayload, evt * actions = append(actions, "created", "edited") } - matched := false for _, val := range vals { - for _, action := range actions { - if glob.MustCompile(val, '/').Match(action) { - matched = true - break - } - } - if matched { + if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) { + matchTimes++ break } } - if matched { - matchTimes++ - } default: log.Warn("pull request review comment event unsupported condition %q", cond) } @@ -698,3 +695,53 @@ func matchPackageEvent(payload *api.PackagePayload, evt *jobparser.Event) bool { } return matchTimes == len(evt.Acts()) } + +func matchWorkflowRunEvent(payload *api.WorkflowRunPayload, evt *jobparser.Event) bool { + // with no special filter parameters + if len(evt.Acts()) == 0 { + return true + } + + matchTimes := 0 + // all acts conditions should be satisfied + for cond, vals := range evt.Acts() { + switch cond { + case "types": + action := payload.Action + for _, val := range vals { + if glob.MustCompile(val, '/').Match(action) { + matchTimes++ + break + } + } + case "workflows": + workflow := payload.Workflow + patterns, err := workflowpattern.CompilePatterns(vals...) + if err != nil { + break + } + if !workflowpattern.Skip(patterns, []string{workflow.Name}, &workflowpattern.EmptyTraceWriter{}) { + matchTimes++ + } + case "branches": + patterns, err := workflowpattern.CompilePatterns(vals...) + if err != nil { + break + } + if !workflowpattern.Skip(patterns, []string{payload.WorkflowRun.HeadBranch}, &workflowpattern.EmptyTraceWriter{}) { + matchTimes++ + } + case "branches-ignore": + patterns, err := workflowpattern.CompilePatterns(vals...) + if err != nil { + break + } + if !workflowpattern.Filter(patterns, []string{payload.WorkflowRun.HeadBranch}, &workflowpattern.EmptyTraceWriter{}) { + matchTimes++ + } + default: + log.Warn("workflow run event unsupported condition %q", cond) + } + } + return matchTimes == len(evt.Acts()) +} diff --git a/modules/actions/workflows_test.go b/modules/actions/workflows_test.go index c8e1e553fe..e23431651d 100644 --- a/modules/actions/workflows_test.go +++ b/modules/actions/workflows_test.go @@ -125,6 +125,24 @@ func TestDetectMatched(t *testing.T) { yamlOn: "on: schedule", expected: true, }, + { + desc: "push to tag matches workflow with paths condition (should skip paths check)", + triggedEvent: webhook_module.HookEventPush, + payload: &api.PushPayload{ + Ref: "refs/tags/v1.0.0", + Before: "0000000", + Commits: []*api.PayloadCommit{ + { + ID: "abcdef123456", + Added: []string{"src/main.go"}, + Message: "Release v1.0.0", + }, + }, + }, + commit: nil, + yamlOn: "on:\n push:\n paths:\n - src/**", + expected: true, + }, } for _, tc := range testCases { diff --git a/modules/assetfs/embed.go b/modules/assetfs/embed.go new file mode 100644 index 0000000000..95176372d1 --- /dev/null +++ b/modules/assetfs/embed.go @@ -0,0 +1,375 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "bytes" + "compress/gzip" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" +) + +type EmbeddedFile interface { + io.ReadSeeker + fs.ReadDirFile + ReadDir(n int) ([]fs.DirEntry, error) +} + +type EmbeddedFileInfo interface { + fs.FileInfo + fs.DirEntry + GetGzipContent() ([]byte, bool) +} + +type decompressor interface { + io.Reader + Close() error + Reset(io.Reader) error +} + +type embeddedFileInfo struct { + fs *embeddedFS + fullName string + data []byte + + BaseName string `json:"n"` + OriginSize int64 `json:"s,omitempty"` + DataBegin int64 `json:"b,omitempty"` + DataLen int64 `json:"l,omitempty"` + Children []*embeddedFileInfo `json:"c,omitempty"` +} + +func (fi *embeddedFileInfo) GetGzipContent() ([]byte, bool) { + // when generating the bindata, if the compressed data equals or is larger than the original data, we store the original data + if fi.DataLen == fi.OriginSize { + return nil, false + } + return fi.data, true +} + +type EmbeddedFileBase struct { + info *embeddedFileInfo + dataReader io.ReadSeeker + seekPos int64 +} + +func (f *EmbeddedFileBase) ReadDir(n int) ([]fs.DirEntry, error) { + // this method is used to satisfy the "func (f ioFile) ReadDir(...)" in httpfs + l, err := f.info.fs.ReadDir(f.info.fullName) + if err != nil { + return nil, err + } + if n < 0 || n > len(l) { + return l, nil + } + return l[:n], nil +} + +type EmbeddedOriginFile struct { + EmbeddedFileBase +} + +type EmbeddedCompressedFile struct { + EmbeddedFileBase + decompressor decompressor + decompressorPos int64 +} + +type embeddedFS struct { + meta func() *EmbeddedMeta + + files map[string]*embeddedFileInfo + filesMu sync.RWMutex + + data []byte +} + +type EmbeddedMeta struct { + Root *embeddedFileInfo +} + +func NewEmbeddedFS(data []byte) fs.ReadDirFS { + efs := &embeddedFS{data: data, files: make(map[string]*embeddedFileInfo)} + efs.meta = sync.OnceValue(func() *EmbeddedMeta { + var meta EmbeddedMeta + p := bytes.LastIndexByte(data, '\n') + if p < 0 { + return &meta + } + if err := json.Unmarshal(data[p+1:], &meta); err != nil { + panic("embedded file is not valid") + } + return &meta + }) + return efs +} + +var _ fs.ReadDirFS = (*embeddedFS)(nil) + +func (e *embeddedFS) ReadDir(name string) (l []fs.DirEntry, err error) { + fi, err := e.getFileInfo(name) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fs.ErrNotExist + } + l = make([]fs.DirEntry, len(fi.Children)) + for i, child := range fi.Children { + l[i], err = e.getFileInfo(name + "/" + child.BaseName) + if err != nil { + return nil, err + } + } + return l, nil +} + +func (e *embeddedFS) getFileInfo(fullName string) (*embeddedFileInfo, error) { + // no need to do heavy "path.Clean()" because we don't want to support "foo/../bar" or absolute paths + fullName = strings.TrimPrefix(fullName, "./") + if fullName == "" { + fullName = "." + } + + e.filesMu.RLock() + fi := e.files[fullName] + e.filesMu.RUnlock() + if fi != nil { + return fi, nil + } + + fields := strings.Split(fullName, "/") + fi = e.meta().Root + if fullName != "." { + found := true + for _, field := range fields { + for _, child := range fi.Children { + if found = child.BaseName == field; found { + fi = child + break + } + } + if !found { + return nil, fs.ErrNotExist + } + } + } + + e.filesMu.Lock() + defer e.filesMu.Unlock() + if fi != nil { + fi.fs = e + fi.fullName = fullName + fi.data = e.data[fi.DataBegin : fi.DataBegin+fi.DataLen] + e.files[fullName] = fi // do not cache nil, otherwise keeping accessing random non-existing file will cause OOM + return fi, nil + } + return nil, fs.ErrNotExist +} + +func (e *embeddedFS) Open(name string) (fs.File, error) { + info, err := e.getFileInfo(name) + if err != nil { + return nil, err + } + base := EmbeddedFileBase{info: info} + base.dataReader = bytes.NewReader(base.info.data) + if info.DataLen != info.OriginSize { + decomp, err := gzip.NewReader(base.dataReader) + if err != nil { + return nil, err + } + return &EmbeddedCompressedFile{EmbeddedFileBase: base, decompressor: decomp}, nil + } + return &EmbeddedOriginFile{base}, nil +} + +var ( + _ EmbeddedFileInfo = (*embeddedFileInfo)(nil) + _ EmbeddedFile = (*EmbeddedOriginFile)(nil) + _ EmbeddedFile = (*EmbeddedCompressedFile)(nil) +) + +func (f *EmbeddedOriginFile) Read(p []byte) (n int, err error) { + return f.dataReader.Read(p) +} + +func (f *EmbeddedCompressedFile) Read(p []byte) (n int, err error) { + if f.decompressorPos > f.seekPos { + if err = f.decompressor.Reset(bytes.NewReader(f.info.data)); err != nil { + return 0, err + } + f.decompressorPos = 0 + } + if f.decompressorPos < f.seekPos { + if _, err = io.CopyN(io.Discard, f.decompressor, f.seekPos-f.decompressorPos); err != nil { + return 0, err + } + f.decompressorPos = f.seekPos + } + n, err = f.decompressor.Read(p) + f.decompressorPos += int64(n) + f.seekPos = f.decompressorPos + return n, err +} + +func (f *EmbeddedFileBase) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + f.seekPos = offset + case io.SeekCurrent: + f.seekPos += offset + case io.SeekEnd: + f.seekPos = f.info.OriginSize + offset + } + return f.seekPos, nil +} + +func (f *EmbeddedFileBase) Stat() (fs.FileInfo, error) { + return f.info, nil +} + +func (f *EmbeddedOriginFile) Close() error { + return nil +} + +func (f *EmbeddedCompressedFile) Close() error { + return f.decompressor.Close() +} + +func (fi *embeddedFileInfo) Name() string { + return fi.BaseName +} + +func (fi *embeddedFileInfo) Size() int64 { + return fi.OriginSize +} + +func (fi *embeddedFileInfo) Mode() fs.FileMode { + return util.Iif(fi.IsDir(), fs.ModeDir|0o555, 0o444) +} + +func (fi *embeddedFileInfo) ModTime() time.Time { + return getExecutableModTime() +} + +func (fi *embeddedFileInfo) IsDir() bool { + return fi.Children != nil +} + +func (fi *embeddedFileInfo) Sys() any { + return nil +} + +func (fi *embeddedFileInfo) Type() fs.FileMode { + return util.Iif(fi.IsDir(), fs.ModeDir, 0) +} + +func (fi *embeddedFileInfo) Info() (fs.FileInfo, error) { + return fi, nil +} + +// getExecutableModTime returns the modification time of the executable file. +// In bindata, we can't use the ModTime of the files because we need to make the build reproducible +var getExecutableModTime = sync.OnceValue(func() (modTime time.Time) { + exePath, err := os.Executable() + if err != nil { + return modTime + } + exePath, err = filepath.Abs(exePath) + if err != nil { + return modTime + } + exePath, err = filepath.EvalSymlinks(exePath) + if err != nil { + return modTime + } + st, err := os.Stat(exePath) + if err != nil { + return modTime + } + return st.ModTime() +}) + +func GenerateEmbedBindata(fsRootPath, outputFile string) error { + output, err := os.OpenFile(outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer output.Close() + + meta := &EmbeddedMeta{} + meta.Root = &embeddedFileInfo{} + var outputOffset int64 + var embedFiles func(parent *embeddedFileInfo, fsPath, embedPath string) error + embedFiles = func(parent *embeddedFileInfo, fsPath, embedPath string) error { + dirEntries, err := os.ReadDir(fsPath) + if err != nil { + return err + } + for _, dirEntry := range dirEntries { + if err != nil { + return err + } + if dirEntry.IsDir() { + child := &embeddedFileInfo{ + BaseName: dirEntry.Name(), + Children: []*embeddedFileInfo{}, // non-nil means it's a directory + } + parent.Children = append(parent.Children, child) + if err = embedFiles(child, filepath.Join(fsPath, dirEntry.Name()), path.Join(embedPath, dirEntry.Name())); err != nil { + return err + } + } else { + data, err := os.ReadFile(filepath.Join(fsPath, dirEntry.Name())) + if err != nil { + return err + } + var compressed bytes.Buffer + gz, _ := gzip.NewWriterLevel(&compressed, gzip.BestCompression) + if _, err = gz.Write(data); err != nil { + return err + } + if err = gz.Close(); err != nil { + return err + } + + // only use the compressed data if it is smaller than the original data + outputBytes := util.Iif(len(compressed.Bytes()) < len(data), compressed.Bytes(), data) + child := &embeddedFileInfo{ + BaseName: dirEntry.Name(), + OriginSize: int64(len(data)), + DataBegin: outputOffset, + DataLen: int64(len(outputBytes)), + } + if _, err = output.Write(outputBytes); err != nil { + return err + } + outputOffset += child.DataLen + parent.Children = append(parent.Children, child) + } + } + return nil + } + + if err = embedFiles(meta.Root, fsRootPath, ""); err != nil { + return err + } + jsonBuf, err := json.Marshal(meta) // can't use json.NewEncoder here because it writes extra EOL + if err != nil { + return err + } + _, _ = output.Write([]byte{'\n'}) + _, err = output.Write(jsonBuf) + return err +} diff --git a/modules/assetfs/embed_test.go b/modules/assetfs/embed_test.go new file mode 100644 index 0000000000..06598da4c4 --- /dev/null +++ b/modules/assetfs/embed_test.go @@ -0,0 +1,98 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "bytes" + "io/fs" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEmbed(t *testing.T) { + tmpDir := t.TempDir() + tmpDataDir := tmpDir + "/data" + _ = os.MkdirAll(tmpDataDir+"/foo/bar", 0o755) + _ = os.WriteFile(tmpDataDir+"/a.txt", []byte("a"), 0o644) + _ = os.WriteFile(tmpDataDir+"/foo/bar/b.txt", bytes.Repeat([]byte("a"), 1000), 0o644) + _ = os.WriteFile(tmpDataDir+"/foo/c.txt", []byte("c"), 0o644) + require.NoError(t, GenerateEmbedBindata(tmpDataDir, tmpDir+"/out.dat")) + + data, err := os.ReadFile(tmpDir + "/out.dat") + require.NoError(t, err) + efs := NewEmbeddedFS(data) + + // test a non-existing file + _, err = fs.ReadFile(efs, "not exist") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // test a normal file (no compression) + content, err := fs.ReadFile(efs, "a.txt") + require.NoError(t, err) + assert.Equal(t, "a", string(content)) + fi, err := fs.Stat(efs, "a.txt") + require.NoError(t, err) + _, ok := fi.(EmbeddedFileInfo).GetGzipContent() + assert.False(t, ok) + + // test a compressed file + content, err = fs.ReadFile(efs, "foo/bar/b.txt") + require.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte("a"), 1000), content) + fi, err = fs.Stat(efs, "foo/bar/b.txt") + require.NoError(t, err) + assert.False(t, fi.Mode().IsDir()) + assert.True(t, fi.Mode().IsRegular()) + gzipContent, ok := fi.(EmbeddedFileInfo).GetGzipContent() + assert.True(t, ok) + assert.Greater(t, len(gzipContent), 1) + assert.Less(t, len(gzipContent), 1000) + + // test list root directory + entries, err := fs.ReadDir(efs, ".") + require.NoError(t, err) + assert.Len(t, entries, 2) + assert.Equal(t, "a.txt", entries[0].Name()) + assert.False(t, entries[0].IsDir()) + + // test list subdirectory + entries, err = fs.ReadDir(efs, "foo") + require.NoError(t, err) + require.Len(t, entries, 2) + assert.Equal(t, "bar", entries[0].Name()) + assert.True(t, entries[0].IsDir()) + assert.Equal(t, "c.txt", entries[1].Name()) + assert.False(t, entries[1].IsDir()) + + // test directory mode + fi, err = fs.Stat(efs, "foo") + require.NoError(t, err) + assert.True(t, fi.IsDir()) + assert.True(t, fi.Mode().IsDir()) + assert.False(t, fi.Mode().IsRegular()) + + // test httpfs + hfs := http.FS(efs) + hf, err := hfs.Open("foo/bar/b.txt") + require.NoError(t, err) + hi, err := hf.Stat() + require.NoError(t, err) + fiEmbedded, ok := hi.(EmbeddedFileInfo) + require.True(t, ok) + gzipContent, ok = fiEmbedded.GetGzipContent() + assert.True(t, ok) + assert.Greater(t, len(gzipContent), 1) + assert.Less(t, len(gzipContent), 1000) + + // test httpfs directory listing + hf, err = hfs.Open("foo") + require.NoError(t, err) + dirs, err := hf.Readdir(1) + require.NoError(t, err) + assert.Len(t, dirs, 1) +} diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index 4f3811ba2b..ce55475bd9 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -52,8 +52,8 @@ func Local(name, base string, sub ...string) *Layer { } // Bindata returns a new Layer with the given name, it serves files from the given bindata asset. -func Bindata(name string, fs http.FileSystem) *Layer { - return &Layer{name: name, fs: fs} +func Bindata(name string, fs fs.FS) *Layer { + return &Layer{name: name, fs: http.FS(fs)} } // LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers. diff --git a/modules/assetfs/layered_test.go b/modules/assetfs/layered_test.go index 03a3ae0d7c..b549815ea5 100644 --- a/modules/assetfs/layered_test.go +++ b/modules/assetfs/layered_test.go @@ -52,7 +52,7 @@ func TestLayered(t *testing.T) { assert.NoError(t, err) bs, err := io.ReadAll(f) assert.NoError(t, err) - assert.EqualValues(t, "f1", string(bs)) + assert.Equal(t, "f1", string(bs)) _ = f.Close() assertRead := func(expected string, expectedErr error, elems ...string) { @@ -76,27 +76,27 @@ func TestLayered(t *testing.T) { files, err := assets.ListFiles(".", true) assert.NoError(t, err) - assert.EqualValues(t, []string{"f1", "f2", "fa"}, files) + assert.Equal(t, []string{"f1", "f2", "fa"}, files) files, err = assets.ListFiles(".", false) assert.NoError(t, err) - assert.EqualValues(t, []string{"d1", "d2", "da"}, files) + assert.Equal(t, []string{"d1", "d2", "da"}, files) files, err = assets.ListFiles(".") assert.NoError(t, err) - assert.EqualValues(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files) + assert.Equal(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files) files, err = assets.ListAllFiles(".", true) assert.NoError(t, err) - assert.EqualValues(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files) + assert.Equal(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files) files, err = assets.ListAllFiles(".", false) assert.NoError(t, err) - assert.EqualValues(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files) + assert.Equal(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files) files, err = assets.ListAllFiles(".") assert.NoError(t, err) - assert.EqualValues(t, []string{ + assert.Equal(t, []string{ "d1", "d1/f", "d2", "d2/f", "da", "da/f", "da/sub1", "da/sub2", @@ -104,6 +104,6 @@ func TestLayered(t *testing.T) { }, files) assert.Empty(t, assets.GetFileLayerName("no-such")) - assert.EqualValues(t, "l1", assets.GetFileLayerName("f1")) - assert.EqualValues(t, "l2", assets.GetFileLayerName("f2")) + assert.Equal(t, "l1", assets.GetFileLayerName("f1")) + assert.Equal(t, "l2", assets.GetFileLayerName("f2")) } diff --git a/modules/auth/httpauth/httpauth.go b/modules/auth/httpauth/httpauth.go new file mode 100644 index 0000000000..7f1f1ee152 --- /dev/null +++ b/modules/auth/httpauth/httpauth.go @@ -0,0 +1,47 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httpauth + +import ( + "encoding/base64" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +type BasicAuth struct { + Username, Password string +} + +type BearerToken struct { + Token string +} + +type ParsedAuthorizationHeader struct { + BasicAuth *BasicAuth + BearerToken *BearerToken +} + +func ParseAuthorizationHeader(header string) (ret ParsedAuthorizationHeader, _ bool) { + parts := strings.Fields(header) + if len(parts) != 2 { + return ret, false + } + if util.AsciiEqualFold(parts[0], "basic") { + s, err := base64.StdEncoding.DecodeString(parts[1]) + if err != nil { + return ret, false + } + u, p, ok := strings.Cut(string(s), ":") + if !ok { + return ret, false + } + ret.BasicAuth = &BasicAuth{Username: u, Password: p} + return ret, true + } else if util.AsciiEqualFold(parts[0], "token") || util.AsciiEqualFold(parts[0], "bearer") { + ret.BearerToken = &BearerToken{Token: parts[1]} + return ret, true + } + return ret, false +} diff --git a/modules/auth/httpauth/httpauth_test.go b/modules/auth/httpauth/httpauth_test.go new file mode 100644 index 0000000000..087b86917f --- /dev/null +++ b/modules/auth/httpauth/httpauth_test.go @@ -0,0 +1,43 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httpauth + +import ( + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseAuthorizationHeader(t *testing.T) { + type parsed = ParsedAuthorizationHeader + type basic = BasicAuth + type bearer = BearerToken + cases := []struct { + headerValue string + expected parsed + ok bool + }{ + {"", parsed{}, false}, + {"?", parsed{}, false}, + {"foo", parsed{}, false}, + {"any value", parsed{}, false}, + + {"Basic ?", parsed{}, false}, + {"Basic " + base64.StdEncoding.EncodeToString([]byte("foo")), parsed{}, false}, + {"Basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true}, + {"basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true}, + + {"token value", parsed{BearerToken: &bearer{"value"}}, true}, + {"Token value", parsed{BearerToken: &bearer{"value"}}, true}, + {"bearer value", parsed{BearerToken: &bearer{"value"}}, true}, + {"Bearer value", parsed{BearerToken: &bearer{"value"}}, true}, + {"Bearer wrong value", parsed{}, false}, + } + for _, c := range cases { + ret, ok := ParseAuthorizationHeader(c.headerValue) + assert.Equal(t, c.ok, ok, "header %q", c.headerValue) + assert.Equal(t, c.expected, ret, "header %q", c.headerValue) + } +} diff --git a/modules/auth/openid/discovery_cache_test.go b/modules/auth/openid/discovery_cache_test.go index 7d4b27c5df..f3d7dd226e 100644 --- a/modules/auth/openid/discovery_cache_test.go +++ b/modules/auth/openid/discovery_cache_test.go @@ -26,7 +26,8 @@ func (s *testDiscoveredInfo) OpLocalID() string { } func TestTimedDiscoveryCache(t *testing.T) { - dc := newTimedDiscoveryCache(1 * time.Second) + ttl := 50 * time.Millisecond + dc := newTimedDiscoveryCache(ttl) // Put some initial values dc.Put("foo", &testDiscoveredInfo{}) // openid.opEndpoint: "a", openid.opLocalID: "b", openid.claimedID: "c"}) @@ -41,8 +42,8 @@ func TestTimedDiscoveryCache(t *testing.T) { // Attempt to get a non-existent value assert.Nil(t, dc.Get("bar")) - // Sleep one second and try retrieve again - time.Sleep(1 * time.Second) + // Sleep for a while and try to retrieve again + time.Sleep(ttl * 3 / 2) assert.Nil(t, dc.Get("foo")) } diff --git a/modules/auth/password/hash/common.go b/modules/auth/password/hash/common.go index 487c0738f4..d5e2c34314 100644 --- a/modules/auth/password/hash/common.go +++ b/modules/auth/password/hash/common.go @@ -18,7 +18,7 @@ func parseIntParam(value, param, algorithmName, config string, previousErr error return parsed, previousErr // <- Keep the previous error as this function should still return an error once everything has been checked if any call failed } -func parseUIntParam(value, param, algorithmName, config string, previousErr error) (uint64, error) { //nolint:unparam +func parseUIntParam(value, param, algorithmName, config string, previousErr error) (uint64, error) { //nolint:unparam // algorithmName is always argon2 parsed, err := strconv.ParseUint(value, 10, 64) if err != nil { log.Error("invalid integer for %s representation in %s hash spec %s", param, algorithmName, config) diff --git a/modules/auth/password/password.go b/modules/auth/password/password.go index c66b62937f..a1e101dd62 100644 --- a/modules/auth/password/password.go +++ b/modules/auth/password/password.go @@ -101,7 +101,7 @@ func Generate(n int) (string, error) { buffer := make([]byte, n) maxInt := big.NewInt(int64(len(validChars))) for { - for j := 0; j < n; j++ { + for j := range n { rnd, err := rand.Int(rand.Reader, maxInt) if err != nil { return "", err diff --git a/modules/auth/password/password_test.go b/modules/auth/password/password_test.go index 6c35dc86bd..0fea593c85 100644 --- a/modules/auth/password/password_test.go +++ b/modules/auth/password/password_test.go @@ -50,7 +50,7 @@ func TestComplexity_Generate(t *testing.T) { test := func(t *testing.T, modes []string) { testComplextity(modes) - for i := 0; i < maxCount; i++ { + for range maxCount { pwd, err := Generate(pwdLen) assert.NoError(t, err) assert.Len(t, pwd, pwdLen) diff --git a/modules/auth/password/pwn/pwn.go b/modules/auth/password/pwn/pwn.go index f77ce9f40b..99a6ca6cea 100644 --- a/modules/auth/password/pwn/pwn.go +++ b/modules/auth/password/pwn/pwn.go @@ -101,7 +101,7 @@ func (c *Client) CheckPassword(pw string, padding bool) (int, error) { } defer resp.Body.Close() - for _, pair := range strings.Split(string(body), "\n") { + for pair := range strings.SplitSeq(string(body), "\n") { parts := strings.Split(pair, ":") if len(parts) != 2 { continue diff --git a/modules/avatar/avatar_test.go b/modules/avatar/avatar_test.go index a721c77868..a5a1a7c1b0 100644 --- a/modules/avatar/avatar_test.go +++ b/modules/avatar/avatar_test.go @@ -94,8 +94,8 @@ func Test_ProcessAvatarImage(t *testing.T) { assert.NotEqual(t, origin, result) decoded, err := png.Decode(bytes.NewReader(result)) assert.NoError(t, err) - assert.EqualValues(t, scaledSize, decoded.Bounds().Max.X) - assert.EqualValues(t, scaledSize, decoded.Bounds().Max.Y) + assert.Equal(t, scaledSize, decoded.Bounds().Max.X) + assert.Equal(t, scaledSize, decoded.Bounds().Max.Y) // if origin image is smaller than the default size, use the origin image origin = newImgData(1) diff --git a/modules/avatar/hash_test.go b/modules/avatar/hash_test.go index 1b8249c696..c518144b47 100644 --- a/modules/avatar/hash_test.go +++ b/modules/avatar/hash_test.go @@ -19,8 +19,8 @@ func Test_HashAvatar(t *testing.T) { var buff bytes.Buffer png.Encode(&buff, myImage) - assert.EqualValues(t, "9ddb5bac41d57e72aa876321d0c09d71090c05f94bc625303801be2f3240d2cb", avatar.HashAvatar(1, buff.Bytes())) - assert.EqualValues(t, "9a5d44e5d637b9582a976676e8f3de1dccd877c2fe3e66ca3fab1629f2f47609", avatar.HashAvatar(8, buff.Bytes())) - assert.EqualValues(t, "ed7399158672088770de6f5211ce15528ebd675e92fc4fc060c025f4b2794ccb", avatar.HashAvatar(1024, buff.Bytes())) - assert.EqualValues(t, "161178642c7d59eb25a61dddced5e6b66eae1c70880d5f148b1b497b767e72d9", avatar.HashAvatar(1024, []byte{})) + assert.Equal(t, "9ddb5bac41d57e72aa876321d0c09d71090c05f94bc625303801be2f3240d2cb", avatar.HashAvatar(1, buff.Bytes())) + assert.Equal(t, "9a5d44e5d637b9582a976676e8f3de1dccd877c2fe3e66ca3fab1629f2f47609", avatar.HashAvatar(8, buff.Bytes())) + assert.Equal(t, "ed7399158672088770de6f5211ce15528ebd675e92fc4fc060c025f4b2794ccb", avatar.HashAvatar(1024, buff.Bytes())) + assert.Equal(t, "161178642c7d59eb25a61dddced5e6b66eae1c70880d5f148b1b497b767e72d9", avatar.HashAvatar(1024, []byte{})) } diff --git a/modules/avatar/identicon/block.go b/modules/avatar/identicon/block.go index cb1803a231..fc8ce90212 100644 --- a/modules/avatar/identicon/block.go +++ b/modules/avatar/identicon/block.go @@ -24,8 +24,8 @@ func drawBlock(img *image.Paletted, x, y, size, angle int, points []int) { rotate(points, m, m, angle) } - for i := 0; i < size; i++ { - for j := 0; j < size; j++ { + for i := range size { + for j := range size { if pointInPolygon(i, j, points) { img.SetColorIndex(x+i, y+j, 1) } diff --git a/modules/avatar/identicon/identicon.go b/modules/avatar/identicon/identicon.go index 63926d5f19..ee92416a53 100644 --- a/modules/avatar/identicon/identicon.go +++ b/modules/avatar/identicon/identicon.go @@ -8,6 +8,7 @@ package identicon import ( "crypto/sha256" + "errors" "fmt" "image" "image/color" @@ -29,7 +30,7 @@ type Identicon struct { // fore all possible foreground colors. only one foreground color will be picked randomly for one image func New(size int, back color.Color, fore ...color.Color) (*Identicon, error) { if len(fore) == 0 { - return nil, fmt.Errorf("foreground is not set") + return nil, errors.New("foreground is not set") } if size < minImageSize { @@ -133,7 +134,7 @@ func drawBlocks(p *image.Paletted, size int, c, b1, b2 blockFunc, b1Angle, b2Ang // then we make it left-right mirror, so we didn't draw 3/6/9 before for x := 0; x < size/2; x++ { - for y := 0; y < size; y++ { + for y := range size { p.SetColorIndex(size-x, y, p.ColorIndexAt(x, y)) } } diff --git a/modules/badge/badge.go b/modules/badge/badge.go index b30d0b4729..d2e9bd9d1b 100644 --- a/modules/badge/badge.go +++ b/modules/badge/badge.go @@ -4,6 +4,10 @@ package badge import ( + "strings" + "sync" + "unicode" + actions_model "code.gitea.io/gitea/models/actions" ) @@ -11,94 +15,115 @@ import ( // We use 10x scale to calculate more precisely // Then scale down to normal size in tmpl file -type Label struct { - text string - width int -} - -func (l Label) Text() string { - return l.text -} - -func (l Label) Width() int { - return l.width -} - -func (l Label) TextLength() int { - return int(float64(l.width-defaultOffset) * 9.5) -} - -func (l Label) X() int { - return l.width*5 + 10 -} - -type Message struct { +type Text struct { text string width int x int } -func (m Message) Text() string { - return m.text +func (t Text) Text() string { + return t.text } -func (m Message) Width() int { - return m.width +func (t Text) Width() int { + return t.width } -func (m Message) X() int { - return m.x +func (t Text) X() int { + return t.x } -func (m Message) TextLength() int { - return int(float64(m.width-defaultOffset) * 9.5) +func (t Text) TextLength() int { + return int(float64(t.width-defaultOffset) * 10) } type Badge struct { - Color string - FontSize int - Label Label - Message Message + IDPrefix string + FontFamily string + Color string + FontSize int + Label Text + Message Text } func (b Badge) Width() int { return b.Label.width + b.Message.width } +// Style follows https://shields.io/badges const ( - defaultOffset = 9 - defaultFontSize = 11 - DefaultColor = "#9f9f9f" // Grey - defaultFontWidth = 7 // approximate speculation + StyleFlat = "flat" + StyleFlatSquare = "flat-square" ) -var StatusColorMap = map[actions_model.Status]string{ - actions_model.StatusSuccess: "#4c1", // Green - actions_model.StatusSkipped: "#dfb317", // Yellow - actions_model.StatusUnknown: "#97ca00", // Light Green - actions_model.StatusFailure: "#e05d44", // Red - actions_model.StatusCancelled: "#fe7d37", // Orange - actions_model.StatusWaiting: "#dfb317", // Yellow - actions_model.StatusRunning: "#dfb317", // Yellow - actions_model.StatusBlocked: "#dfb317", // Yellow -} +const ( + defaultOffset = 10 + defaultFontSize = 11 + DefaultColor = "#9f9f9f" // Grey + DefaultFontFamily = "DejaVu Sans,Verdana,Geneva,sans-serif" + DefaultStyle = StyleFlat +) + +var GlobalVars = sync.OnceValue(func() (ret struct { + StatusColorMap map[actions_model.Status]string + DejaVuGlyphWidthData map[rune]uint8 + AllStyles []string +}, +) { + ret.StatusColorMap = map[actions_model.Status]string{ + actions_model.StatusSuccess: "#4c1", // Green + actions_model.StatusSkipped: "#dfb317", // Yellow + actions_model.StatusUnknown: "#97ca00", // Light Green + actions_model.StatusFailure: "#e05d44", // Red + actions_model.StatusCancelled: "#fe7d37", // Orange + actions_model.StatusWaiting: "#dfb317", // Yellow + actions_model.StatusRunning: "#dfb317", // Yellow + actions_model.StatusBlocked: "#dfb317", // Yellow + } + ret.DejaVuGlyphWidthData = dejaVuGlyphWidthDataFunc() + ret.AllStyles = []string{StyleFlat, StyleFlatSquare} + return ret +}) // GenerateBadge generates badge with given template func GenerateBadge(label, message, color string) Badge { - lw := defaultFontWidth*len(label) + defaultOffset - mw := defaultFontWidth*len(message) + defaultOffset - x := lw*10 + mw*5 - 10 + lw := calculateTextWidth(label) + defaultOffset + mw := calculateTextWidth(message) + defaultOffset + + lx := lw * 5 + mx := lw*10 + mw*5 - 10 return Badge{ - Label: Label{ + FontFamily: DefaultFontFamily, + Label: Text{ text: label, width: lw, + x: lx, }, - Message: Message{ + Message: Text{ text: message, width: mw, - x: x, + x: mx, }, FontSize: defaultFontSize * 10, Color: color, } } + +func calculateTextWidth(text string) int { + width := 0 + widthData := GlobalVars().DejaVuGlyphWidthData + for _, char := range strings.TrimSpace(text) { + charWidth, ok := widthData[char] + if !ok { + // use the width of 'm' in case of missing glyph width data for a printable character + if unicode.IsPrint(char) { + charWidth = widthData['m'] + } else { + charWidth = 0 + } + } + width += int(charWidth) + } + + return width +} diff --git a/modules/badge/badge_glyph_width.go b/modules/badge/badge_glyph_width.go new file mode 100644 index 0000000000..0d950c5a70 --- /dev/null +++ b/modules/badge/badge_glyph_width.go @@ -0,0 +1,206 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package badge + +// DejaVuGlyphWidthData is generated by `sfnt.Face.GlyphAdvance(nil, <rune>, 11, font.HintingNone)` with DejaVu Sans +// v2.37 (https://github.com/dejavu-fonts/dejavu-fonts/releases/download/version_2_37/dejavu-sans-ttf-2.37.zip). +// +// Fonts defined in "DefaultFontFamily" all have similar widths (including "DejaVu Sans"), +// and these widths are fixed and don't seem to change. +// +// A devtest page "/devtest/badge-actions-svg" could be used to check the rendered images. + +func dejaVuGlyphWidthDataFunc() map[rune]uint8 { + return map[rune]uint8{ + 32: 3, + 33: 4, + 34: 5, + 35: 9, + 36: 7, + 37: 10, + 38: 9, + 39: 3, + 40: 4, + 41: 4, + 42: 6, + 43: 9, + 44: 3, + 45: 4, + 46: 3, + 47: 4, + 48: 7, + 49: 7, + 50: 7, + 51: 7, + 52: 7, + 53: 7, + 54: 7, + 55: 7, + 56: 7, + 57: 7, + 58: 4, + 59: 4, + 60: 9, + 61: 9, + 62: 9, + 63: 6, + 64: 11, + 65: 8, + 66: 8, + 67: 8, + 68: 8, + 69: 7, + 70: 6, + 71: 9, + 72: 8, + 73: 3, + 74: 3, + 75: 7, + 76: 6, + 77: 9, + 78: 8, + 79: 9, + 80: 7, + 81: 9, + 82: 8, + 83: 7, + 84: 7, + 85: 8, + 86: 8, + 87: 11, + 88: 8, + 89: 7, + 90: 8, + 91: 4, + 92: 4, + 93: 4, + 94: 9, + 95: 6, + 96: 6, + 97: 7, + 98: 7, + 99: 6, + 100: 7, + 101: 7, + 102: 4, + 103: 7, + 104: 7, + 105: 3, + 106: 3, + 107: 6, + 108: 3, + 109: 11, + 110: 7, + 111: 7, + 112: 7, + 113: 7, + 114: 5, + 115: 6, + 116: 4, + 117: 7, + 118: 7, + 119: 9, + 120: 7, + 121: 7, + 122: 6, + 123: 7, + 124: 4, + 125: 7, + 126: 9, + 161: 4, + 162: 7, + 163: 7, + 164: 7, + 165: 7, + 166: 4, + 167: 6, + 168: 6, + 169: 11, + 170: 5, + 171: 7, + 172: 9, + 174: 11, + 175: 6, + 176: 6, + 177: 9, + 178: 4, + 179: 4, + 180: 6, + 181: 7, + 182: 7, + 183: 3, + 184: 6, + 185: 4, + 186: 5, + 187: 7, + 188: 11, + 189: 11, + 190: 11, + 191: 6, + 192: 8, + 193: 8, + 194: 8, + 195: 8, + 196: 8, + 197: 8, + 198: 11, + 199: 8, + 200: 7, + 201: 7, + 202: 7, + 203: 7, + 204: 3, + 205: 3, + 206: 3, + 207: 3, + 208: 9, + 209: 8, + 210: 9, + 211: 9, + 212: 9, + 213: 9, + 214: 9, + 215: 9, + 216: 9, + 217: 8, + 218: 8, + 219: 8, + 220: 8, + 221: 7, + 222: 7, + 223: 7, + 224: 7, + 225: 7, + 226: 7, + 227: 7, + 228: 7, + 229: 7, + 230: 11, + 231: 6, + 232: 7, + 233: 7, + 234: 7, + 235: 7, + 236: 3, + 237: 3, + 238: 3, + 239: 3, + 240: 7, + 241: 7, + 242: 7, + 243: 7, + 244: 7, + 245: 7, + 246: 7, + 247: 9, + 248: 7, + 249: 7, + 250: 7, + 251: 7, + 252: 7, + 253: 7, + 254: 7, + 255: 7, + } +} diff --git a/modules/base/tool.go b/modules/base/tool.go index 1d16186bc5..ed94575e74 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -8,17 +8,12 @@ import ( "crypto/sha1" "crypto/sha256" "crypto/subtle" - "encoding/base64" "encoding/hex" - "errors" "fmt" "hash" "strconv" - "strings" "time" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -38,19 +33,6 @@ func ShortSha(sha1 string) string { return util.TruncateRunes(sha1, 10) } -// BasicAuthDecode decode basic auth string -func BasicAuthDecode(encoded string) (string, string, error) { - s, err := base64.StdEncoding.DecodeString(encoded) - if err != nil { - return "", "", err - } - - if username, password, ok := strings.Cut(string(s), ":"); ok { - return username, password, nil - } - return "", "", errors.New("invalid basic authentication") -} - // VerifyTimeLimitCode verify time limit code func VerifyTimeLimitCode(now time.Time, data string, minutes int, code string) bool { if len(code) <= 18 { @@ -64,10 +46,7 @@ func VerifyTimeLimitCode(now time.Time, data string, minutes int, code string) b // check code retCode := CreateTimeLimitCode(data, aliveTime, startTimeStr, nil) if subtle.ConstantTimeCompare([]byte(retCode), []byte(code)) != 1 { - retCode = CreateTimeLimitCode(data, aliveTime, startTimeStr, sha1.New()) // TODO: this is only for the support of legacy codes, remove this in/after 1.23 - if subtle.ConstantTimeCompare([]byte(retCode), []byte(code)) != 1 { - return false - } + return false } // check time is expired or not: startTime <= now && now < startTime + minutes @@ -143,25 +122,3 @@ func Int64sToStrings(ints []int64) []string { } return strs } - -// EntryIcon returns the octicon class for displaying files/directories -func EntryIcon(entry *git.TreeEntry) string { - switch { - case entry.IsLink(): - te, err := entry.FollowLink() - if err != nil { - log.Debug(err.Error()) - return "file-symlink-file" - } - if te.IsDir() { - return "file-directory-symlink" - } - return "file-symlink-file" - case entry.IsDir(): - return "file-directory-fill" - case entry.IsSubModule(): - return "file-submodule" - } - - return "file" -} diff --git a/modules/base/tool_test.go b/modules/base/tool_test.go index c821a55c19..b7365e40c4 100644 --- a/modules/base/tool_test.go +++ b/modules/base/tool_test.go @@ -26,25 +26,6 @@ func TestShortSha(t *testing.T) { assert.Equal(t, "veryverylo", ShortSha("veryverylong")) } -func TestBasicAuthDecode(t *testing.T) { - _, _, err := BasicAuthDecode("?") - assert.Equal(t, "illegal base64 data at input byte 0", err.Error()) - - user, pass, err := BasicAuthDecode("Zm9vOmJhcg==") - assert.NoError(t, err) - assert.Equal(t, "foo", user) - assert.Equal(t, "bar", pass) - - _, _, err = BasicAuthDecode("aW52YWxpZA==") - assert.Error(t, err) - - _, _, err = BasicAuthDecode("invalid") - assert.Error(t, err) - - _, _, err = BasicAuthDecode("YWxpY2U=") // "alice", no colon - assert.Error(t, err) -} - func TestVerifyTimeLimitCode(t *testing.T) { defer test.MockVariableValue(&setting.InstallLock, true)() initGeneralSecret := func(secret string) { @@ -86,13 +67,10 @@ JWT_SECRET = %s verifyDataCode := func(c string) bool { return VerifyTimeLimitCode(now, "data", 2, c) } - code1 := CreateTimeLimitCode("data", 2, now, sha1.New()) - code2 := CreateTimeLimitCode("data", 2, now, nil) - assert.True(t, verifyDataCode(code1)) - assert.True(t, verifyDataCode(code2)) + code := CreateTimeLimitCode("data", 2, now, nil) + assert.True(t, verifyDataCode(code)) initGeneralSecret("000_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko") - assert.False(t, verifyDataCode(code1)) - assert.False(t, verifyDataCode(code2)) + assert.False(t, verifyDataCode(code)) }) } @@ -137,5 +115,3 @@ func TestInt64sToStrings(t *testing.T) { Int64sToStrings([]int64{1, 4, 16, 64, 256}), ) } - -// TODO: Test EntryIcon diff --git a/modules/cache/cache.go b/modules/cache/cache.go index f7828e3cae..039caa9fbc 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -4,6 +4,8 @@ package cache import ( + "encoding/hex" + "errors" "fmt" "strconv" "time" @@ -22,7 +24,7 @@ func Init() error { if err != nil { return err } - for i := 0; i < 10; i++ { + for range 10 { if err = c.Ping(); err == nil { break } @@ -48,10 +50,10 @@ const ( // returns func Test() (time.Duration, error) { if defaultCache == nil { - return 0, fmt.Errorf("default cache not initialized") + return 0, errors.New("default cache not initialized") } - testData := fmt.Sprintf("%x", make([]byte, 500)) + testData := hex.EncodeToString(make([]byte, 500)) start := time.Now() @@ -63,10 +65,10 @@ func Test() (time.Duration, error) { } testVal, hit := defaultCache.Get(testCacheKey) if !hit { - return 0, fmt.Errorf("expect cache hit but got none") + return 0, errors.New("expect cache hit but got none") } if testVal != testData { - return 0, fmt.Errorf("expect cache to return same value as stored but got other") + return 0, errors.New("expect cache to return same value as stored but got other") } return time.Since(start), nil diff --git a/modules/cache/cache_redis.go b/modules/cache/cache_redis.go index c5b52a2086..7473c938af 100644 --- a/modules/cache/cache_redis.go +++ b/modules/cache/cache_redis.go @@ -11,7 +11,7 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/nosql" - "gitea.com/go-chi/cache" //nolint:depguard + "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here "github.com/redis/go-redis/v9" ) diff --git a/modules/cache/cache_test.go b/modules/cache/cache_test.go index 5408020306..d6ea2032ee 100644 --- a/modules/cache/cache_test.go +++ b/modules/cache/cache_test.go @@ -4,7 +4,7 @@ package cache import ( - "fmt" + "errors" "testing" "time" @@ -57,22 +57,22 @@ func TestGetString(t *testing.T) { createTestCache() data, err := GetString("key", func() (string, error) { - return "", fmt.Errorf("some error") + return "", errors.New("some error") }) assert.Error(t, err) - assert.Equal(t, "", data) + assert.Empty(t, data) data, err = GetString("key", func() (string, error) { return "", nil }) assert.NoError(t, err) - assert.Equal(t, "", data) + assert.Empty(t, data) data, err = GetString("key", func() (string, error) { return "some data", nil }) assert.NoError(t, err) - assert.Equal(t, "", data) + assert.Empty(t, data) Remove("key") data, err = GetString("key", func() (string, error) { @@ -82,7 +82,7 @@ func TestGetString(t *testing.T) { assert.Equal(t, "some data", data) data, err = GetString("key", func() (string, error) { - return "", fmt.Errorf("some error") + return "", errors.New("some error") }) assert.NoError(t, err) assert.Equal(t, "some data", data) @@ -93,7 +93,7 @@ func TestGetInt64(t *testing.T) { createTestCache() data, err := GetInt64("key", func() (int64, error) { - return 0, fmt.Errorf("some error") + return 0, errors.New("some error") }) assert.Error(t, err) assert.EqualValues(t, 0, data) @@ -118,7 +118,7 @@ func TestGetInt64(t *testing.T) { assert.EqualValues(t, 100, data) data, err = GetInt64("key", func() (int64, error) { - return 0, fmt.Errorf("some error") + return 0, errors.New("some error") }) assert.NoError(t, err) assert.EqualValues(t, 100, data) diff --git a/modules/cache/cache_twoqueue.go b/modules/cache/cache_twoqueue.go index 1eda2debc4..c8db686e57 100644 --- a/modules/cache/cache_twoqueue.go +++ b/modules/cache/cache_twoqueue.go @@ -10,7 +10,7 @@ import ( "code.gitea.io/gitea/modules/json" - mc "gitea.com/go-chi/cache" //nolint:depguard + mc "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here lru "github.com/hashicorp/golang-lru/v2" ) diff --git a/modules/cache/context.go b/modules/cache/context.go index 484cee659a..23f7c23a52 100644 --- a/modules/cache/context.go +++ b/modules/cache/context.go @@ -5,176 +5,39 @@ package cache import ( "context" - "sync" "time" - - "code.gitea.io/gitea/modules/log" ) -// cacheContext is a context that can be used to cache data in a request level context -// This is useful for caching data that is expensive to calculate and is likely to be -// used multiple times in a request. -type cacheContext struct { - data map[any]map[any]any - lock sync.RWMutex - created time.Time - discard bool -} - -func (cc *cacheContext) Get(tp, key any) any { - cc.lock.RLock() - defer cc.lock.RUnlock() - return cc.data[tp][key] -} - -func (cc *cacheContext) Put(tp, key, value any) { - cc.lock.Lock() - defer cc.lock.Unlock() - - if cc.discard { - return - } - - d := cc.data[tp] - if d == nil { - d = make(map[any]any) - cc.data[tp] = d - } - d[key] = value -} - -func (cc *cacheContext) Delete(tp, key any) { - cc.lock.Lock() - defer cc.lock.Unlock() - delete(cc.data[tp], key) -} - -func (cc *cacheContext) Discard() { - cc.lock.Lock() - defer cc.lock.Unlock() - cc.data = nil - cc.discard = true -} - -func (cc *cacheContext) isDiscard() bool { - cc.lock.RLock() - defer cc.lock.RUnlock() - return cc.discard -} - -// cacheContextLifetime is the max lifetime of cacheContext. -// Since cacheContext is used to cache data in a request level context, 5 minutes is enough. -// If a cacheContext is used more than 5 minutes, it's probably misuse. -const cacheContextLifetime = 5 * time.Minute - -var timeNow = time.Now +type cacheContextKeyType struct{} -func (cc *cacheContext) Expired() bool { - return timeNow().Sub(cc.created) > cacheContextLifetime -} - -var cacheContextKey = struct{}{} - -/* -Since there are both WithCacheContext and WithNoCacheContext, -it may be confusing when there is nesting. - -Some cases to explain the design: - -When: -- A, B or C means a cache context. -- A', B' or C' means a discard cache context. -- ctx means context.Backgrand(). -- A(ctx) means a cache context with ctx as the parent context. -- B(A(ctx)) means a cache context with A(ctx) as the parent context. -- With is alias of WithCacheContext. -- WithNo is alias of WithNoCacheContext. +var cacheContextKey = cacheContextKeyType{} -So: -- With(ctx) -> A(ctx) -- With(With(ctx)) -> A(ctx), not B(A(ctx)), always reuse parent cache context if possible. -- With(With(With(ctx))) -> A(ctx), not C(B(A(ctx))), ditto. -- WithNo(ctx) -> ctx, not A'(ctx), don't create new cache context if we don't have to. -- WithNo(With(ctx)) -> A'(ctx) -- WithNo(WithNo(With(ctx))) -> A'(ctx), not B'(A'(ctx)), don't create new cache context if we don't have to. -- With(WithNo(With(ctx))) -> B(A'(ctx)), not A(ctx), never reuse a discard cache context. -- WithNo(With(WithNo(With(ctx)))) -> B'(A'(ctx)) -- With(WithNo(With(WithNo(With(ctx))))) -> C(B'(A'(ctx))), so there's always only one not-discard cache context. -*/ +// contextCacheLifetime is the max lifetime of context cache. +// Since context cache is used to cache data in a request level context, 5 minutes is enough. +// If a context cache is used more than 5 minutes, it's probably abused. +const contextCacheLifetime = 5 * time.Minute func WithCacheContext(ctx context.Context) context.Context { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if !c.isDiscard() { - // reuse parent context - return ctx - } - } - // FIXME: review the use of this nolint directive - return context.WithValue(ctx, cacheContextKey, &cacheContext{ //nolint:staticcheck - data: make(map[any]map[any]any), - created: timeNow(), - }) -} - -func WithNoCacheContext(ctx context.Context) context.Context { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - // The caller want to run long-life tasks, but the parent context is a cache context. - // So we should disable and clean the cache data, or it will be kept in memory for a long time. - c.Discard() + if c := GetContextCache(ctx); c != nil { return ctx } - - return ctx -} - -func GetContextData(ctx context.Context, tp, key any) any { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if c.Expired() { - // The warning means that the cache context is misused for long-life task, - // it can be resolved with WithNoCacheContext(ctx). - log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c) - return nil - } - return c.Get(tp, key) - } - return nil + return context.WithValue(ctx, cacheContextKey, NewEphemeralCache(contextCacheLifetime)) } -func SetContextData(ctx context.Context, tp, key, value any) { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if c.Expired() { - // The warning means that the cache context is misused for long-life task, - // it can be resolved with WithNoCacheContext(ctx). - log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c) - return - } - c.Put(tp, key, value) - return - } -} - -func RemoveContextData(ctx context.Context, tp, key any) { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if c.Expired() { - // The warning means that the cache context is misused for long-life task, - // it can be resolved with WithNoCacheContext(ctx). - log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c) - return - } - c.Delete(tp, key) - } +func GetContextCache(ctx context.Context) *EphemeralCache { + c, _ := ctx.Value(cacheContextKey).(*EphemeralCache) + return c } // GetWithContextCache returns the cache value of the given key in the given context. -func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error) { - v := GetContextData(ctx, cacheGroupKey, cacheTargetID) - if vv, ok := v.(T); ok { - return vv, nil - } - t, err := f() - if err != nil { - return t, err - } - SetContextData(ctx, cacheGroupKey, cacheTargetID, t) - return t, nil +// FIXME: in some cases, the "context cache" should not be used, because it has uncontrollable behaviors +// For example, these calls: +// * GetWithContextCache(TargetID) -> OtherCodeCreateModel(TargetID) -> GetWithContextCache(TargetID) +// Will cause the second call is not able to get the correct created target. +// UNLESS it is certain that the target won't be changed during the request, DO NOT use it. +func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) { + if c := GetContextCache(ctx); c != nil { + return GetWithEphemeralCache(ctx, c, groupKey, targetKey, f) + } + return f(ctx, targetKey) } diff --git a/modules/cache/context_test.go b/modules/cache/context_test.go index c01b9e8d84..8371c2b908 100644 --- a/modules/cache/context_test.go +++ b/modules/cache/context_test.go @@ -8,71 +8,43 @@ import ( "testing" "time" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" ) func TestWithCacheContext(t *testing.T) { - ctx := WithCacheContext(context.Background()) - - v := GetContextData(ctx, "empty_field", "my_config1") + ctx := WithCacheContext(t.Context()) + c := GetContextCache(ctx) + v, _ := c.Get("empty_field", "my_config1") assert.Nil(t, v) const field = "system_setting" - v = GetContextData(ctx, field, "my_config1") + v, _ = c.Get(field, "my_config1") assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") + c.Put(field, "my_config1", 1) + v, _ = c.Get(field, "my_config1") assert.NotNil(t, v) - assert.EqualValues(t, 1, v.(int)) + assert.Equal(t, 1, v.(int)) - RemoveContextData(ctx, field, "my_config1") - RemoveContextData(ctx, field, "my_config2") // remove a non-exist key + c.Delete(field, "my_config1") + c.Delete(field, "my_config2") // remove a non-exist key - v = GetContextData(ctx, field, "my_config1") + v, _ = c.Get(field, "my_config1") assert.Nil(t, v) - vInt, err := GetWithContextCache(ctx, field, "my_config1", func() (int, error) { + vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) { return 1, nil }) assert.NoError(t, err) - assert.EqualValues(t, 1, vInt) + assert.Equal(t, 1, vInt) - v = GetContextData(ctx, field, "my_config1") + v, _ = c.Get(field, "my_config1") assert.EqualValues(t, 1, v) - now := timeNow - defer func() { - timeNow = now - }() - timeNow = func() time.Time { - return now().Add(5 * time.Minute) - } - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) -} - -func TestWithNoCacheContext(t *testing.T) { - ctx := context.Background() - - const field = "system_setting" - - v := GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) // still no cache - - ctx = WithCacheContext(ctx) - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") - assert.NotNil(t, v) - - ctx = WithNoCacheContext(ctx) - v = GetContextData(ctx, field, "my_config1") + defer test.MockVariableValue(&timeNow, func() time.Time { + return time.Now().Add(5 * time.Minute) + })() + v, _ = c.Get(field, "my_config1") assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) // still no cache } diff --git a/modules/cache/ephemeral.go b/modules/cache/ephemeral.go new file mode 100644 index 0000000000..6996010ac4 --- /dev/null +++ b/modules/cache/ephemeral.go @@ -0,0 +1,90 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cache + +import ( + "context" + "sync" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" +) + +// EphemeralCache is a cache that can be used to store data in a request level context +// This is useful for caching data that is expensive to calculate and is likely to be +// used multiple times in a request. +type EphemeralCache struct { + data map[any]map[any]any + lock sync.RWMutex + created time.Time + checkLifeTime time.Duration +} + +var timeNow = time.Now + +func NewEphemeralCache(checkLifeTime ...time.Duration) *EphemeralCache { + return &EphemeralCache{ + data: make(map[any]map[any]any), + created: timeNow(), + checkLifeTime: util.OptionalArg(checkLifeTime, 0), + } +} + +func (cc *EphemeralCache) checkExceededLifeTime(tp, key any) bool { + if cc.checkLifeTime > 0 && timeNow().Sub(cc.created) > cc.checkLifeTime { + log.Warn("EphemeralCache is expired, is highly likely to be abused for long-life tasks: %v, %v", tp, key) + return true + } + return false +} + +func (cc *EphemeralCache) Get(tp, key any) (any, bool) { + if cc.checkExceededLifeTime(tp, key) { + return nil, false + } + cc.lock.RLock() + defer cc.lock.RUnlock() + ret, ok := cc.data[tp][key] + return ret, ok +} + +func (cc *EphemeralCache) Put(tp, key, value any) { + if cc.checkExceededLifeTime(tp, key) { + return + } + + cc.lock.Lock() + defer cc.lock.Unlock() + + d := cc.data[tp] + if d == nil { + d = make(map[any]any) + cc.data[tp] = d + } + d[key] = value +} + +func (cc *EphemeralCache) Delete(tp, key any) { + if cc.checkExceededLifeTime(tp, key) { + return + } + + cc.lock.Lock() + defer cc.lock.Unlock() + delete(cc.data[tp], key) +} + +func GetWithEphemeralCache[T, K any](ctx context.Context, c *EphemeralCache, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) { + v, has := c.Get(groupKey, targetKey) + if vv, ok := v.(T); has && ok { + return vv, nil + } + t, err := f(ctx, targetKey) + if err != nil { + return t, err + } + c.Put(groupKey, targetKey, t) + return t, nil +} diff --git a/modules/cache/string_cache.go b/modules/cache/string_cache.go index 4f659616f5..3562b7a926 100644 --- a/modules/cache/string_cache.go +++ b/modules/cache/string_cache.go @@ -11,7 +11,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - chi_cache "gitea.com/go-chi/cache" //nolint:depguard + chi_cache "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here ) type GetJSONError struct { diff --git a/modules/cachegroup/cachegroup.go b/modules/cachegroup/cachegroup.go new file mode 100644 index 0000000000..06085f860f --- /dev/null +++ b/modules/cachegroup/cachegroup.go @@ -0,0 +1,12 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cachegroup + +const ( + User = "user" + EmailAvatarLink = "email_avatar_link" + UserEmailAddresses = "user_email_addresses" + GPGKeyWithSubKeys = "gpg_key_with_subkeys" + RepoUserPermission = "repo_user_permission" +) diff --git a/modules/charset/ambiguous_gen_test.go b/modules/charset/ambiguous_gen_test.go index 221c27d0e1..d3be0b1a13 100644 --- a/modules/charset/ambiguous_gen_test.go +++ b/modules/charset/ambiguous_gen_test.go @@ -14,7 +14,7 @@ import ( func TestAmbiguousCharacters(t *testing.T) { for locale, ambiguous := range AmbiguousCharacters { assert.Equal(t, locale, ambiguous.Locale) - assert.Equal(t, len(ambiguous.Confusable), len(ambiguous.With)) + assert.Len(t, ambiguous.With, len(ambiguous.Confusable)) assert.True(t, sort.SliceIsSorted(ambiguous.Confusable, func(i, j int) bool { return ambiguous.Confusable[i] < ambiguous.Confusable[j] })) diff --git a/modules/charset/charset.go b/modules/charset/charset.go index 1855446a98..597ce5120c 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -164,7 +164,7 @@ func DetectEncoding(content []byte) (string, error) { } times := 1024 / len(content) detectContent = make([]byte, 0, times*len(content)) - for i := 0; i < times; i++ { + for range times { detectContent = append(detectContent, content...) } } else { diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index 19b1303365..cd2e3b9aaa 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -242,7 +242,7 @@ func stringMustEndWith(t *testing.T, expected, value string) { func TestToUTF8WithFallbackReader(t *testing.T) { resetDefaultCharsetsOrder() - for testLen := 0; testLen < 2048; testLen++ { + for testLen := range 2048 { pattern := " test { () }\n" input := "" for len(input) < testLen { @@ -252,7 +252,7 @@ func TestToUTF8WithFallbackReader(t *testing.T) { input += "// Выключаем" rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)), ConvertOpts{}) r, _ := io.ReadAll(rd) - assert.EqualValuesf(t, input, string(r), "testing string len=%d", testLen) + assert.Equalf(t, input, string(r), "testing string len=%d", testLen) } truncatedOneByteExtension := failFastBytes diff --git a/modules/structs/commit_status.go b/modules/commitstatus/commit_status.go index dc880ef5eb..a0ab4e7186 100644 --- a/modules/structs/commit_status.go +++ b/modules/commitstatus/commit_status.go @@ -1,11 +1,11 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package structs +package commitstatus // CommitStatusState holds the state of a CommitStatus -// It can be "pending", "success", "error" and "failure" -type CommitStatusState string +// swagger:enum CommitStatusState +type CommitStatusState string //nolint:revive // export stutter const ( // CommitStatusPending is for when the CommitStatus is Pending @@ -18,35 +18,14 @@ const ( CommitStatusFailure CommitStatusState = "failure" // CommitStatusWarning is for when the CommitStatus is Warning CommitStatusWarning CommitStatusState = "warning" + // CommitStatusSkipped is for when CommitStatus is Skipped + CommitStatusSkipped CommitStatusState = "skipped" ) -var commitStatusPriorities = map[CommitStatusState]int{ - CommitStatusError: 0, - CommitStatusFailure: 1, - CommitStatusWarning: 2, - CommitStatusPending: 3, - CommitStatusSuccess: 4, -} - func (css CommitStatusState) String() string { return string(css) } -// NoBetterThan returns true if this State is no better than the given State -// This function only handles the states defined in CommitStatusPriorities -func (css CommitStatusState) NoBetterThan(css2 CommitStatusState) bool { - // NoBetterThan only handles the 5 states above - if _, exist := commitStatusPriorities[css]; !exist { - return false - } - - if _, exist := commitStatusPriorities[css2]; !exist { - return false - } - - return commitStatusPriorities[css] <= commitStatusPriorities[css2] -} - // IsPending represents if commit status state is pending func (css CommitStatusState) IsPending() bool { return css == CommitStatusPending @@ -71,3 +50,32 @@ func (css CommitStatusState) IsFailure() bool { func (css CommitStatusState) IsWarning() bool { return css == CommitStatusWarning } + +// IsSkipped represents if commit status state is skipped +func (css CommitStatusState) IsSkipped() bool { + return css == CommitStatusSkipped +} + +type CommitStatusStates []CommitStatusState //nolint:revive // export stutter + +// According to https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#get-the-combined-status-for-a-specific-reference +// > Additionally, a combined state is returned. The state is one of: +// > failure if any of the contexts report as error or failure +// > pending if there are no statuses or a context is pending +// > success if the latest status for all contexts is success +func (css CommitStatusStates) Combine() CommitStatusState { + successCnt := 0 + for _, state := range css { + switch { + case state.IsError() || state.IsFailure(): + return CommitStatusFailure + case state.IsPending(): + case state.IsSuccess() || state.IsWarning() || state.IsSkipped(): + successCnt++ + } + } + if successCnt > 0 && successCnt == len(css) { + return CommitStatusSuccess + } + return CommitStatusPending +} diff --git a/modules/commitstatus/commit_status_test.go b/modules/commitstatus/commit_status_test.go new file mode 100644 index 0000000000..10d8f20aa4 --- /dev/null +++ b/modules/commitstatus/commit_status_test.go @@ -0,0 +1,201 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package commitstatus + +import "testing" + +func TestCombine(t *testing.T) { + tests := []struct { + name string + states CommitStatusStates + expected CommitStatusState + }{ + // 0 states + { + name: "empty", + states: CommitStatusStates{}, + expected: CommitStatusPending, + }, + // 1 state + { + name: "pending", + states: CommitStatusStates{CommitStatusPending}, + expected: CommitStatusPending, + }, + { + name: "success", + states: CommitStatusStates{CommitStatusSuccess}, + expected: CommitStatusSuccess, + }, + { + name: "error", + states: CommitStatusStates{CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "failure", + states: CommitStatusStates{CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "warning", + states: CommitStatusStates{CommitStatusWarning}, + expected: CommitStatusSuccess, + }, + // 2 states + { + name: "pending and success", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess}, + expected: CommitStatusPending, + }, + { + name: "pending and error", + states: CommitStatusStates{CommitStatusPending, CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "pending and failure", + states: CommitStatusStates{CommitStatusPending, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "pending and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusWarning}, + expected: CommitStatusPending, + }, + { + name: "success and error", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "success and failure", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "success and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusWarning}, + expected: CommitStatusSuccess, + }, + { + name: "error and failure", + states: CommitStatusStates{CommitStatusError, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "error and warning", + states: CommitStatusStates{CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "failure and warning", + states: CommitStatusStates{CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + // 3 states + { + name: "pending, success and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusWarning}, + expected: CommitStatusPending, + }, + { + name: "pending, success and error", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "pending, success and failure", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "pending, error and failure", + states: CommitStatusStates{CommitStatusPending, CommitStatusError, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "success, error and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "success, failure and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "error, failure and warning", + states: CommitStatusStates{CommitStatusError, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "success, warning and skipped", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusWarning, CommitStatusSkipped}, + expected: CommitStatusSuccess, + }, + // All success + { + name: "all success", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusSuccess}, + expected: CommitStatusSuccess, + }, + // All pending + { + name: "all pending", + states: CommitStatusStates{CommitStatusPending, CommitStatusPending, CommitStatusPending}, + expected: CommitStatusPending, + }, + { + name: "all skipped", + states: CommitStatusStates{CommitStatusSkipped, CommitStatusSkipped, CommitStatusSkipped}, + expected: CommitStatusSuccess, + }, + // 4 states + { + name: "pending, success, error and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "pending, success, failure and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "pending, error, failure and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusError, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "success, error, failure and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusError, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "mixed states", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "mixed states with all success", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusPending, CommitStatusWarning}, + expected: CommitStatusPending, + }, + { + name: "all success with warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusSuccess, CommitStatusWarning}, + expected: CommitStatusSuccess, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.states.Combine() + if result != tt.expected { + t.Errorf("expected %v, got %v", tt.expected, result) + } + }) + } +} diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 29ed58db97..be9fc5f823 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -5,7 +5,6 @@ package csv import ( "bytes" - "context" "encoding/csv" "io" "strconv" @@ -100,10 +99,10 @@ j, ,\x20 for n, c := range cases { rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(decodeSlashes(t, c.csv))) assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err) - assert.EqualValues(t, c.expectedDelimiter, rd.Comma, "case %d: delimiter should be '%c', got '%c'", n, c.expectedDelimiter, rd.Comma) + assert.Equal(t, c.expectedDelimiter, rd.Comma, "case %d: delimiter should be '%c', got '%c'", n, c.expectedDelimiter, rd.Comma) rows, err := rd.ReadAll() assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err) - assert.EqualValues(t, c.expectedRows, rows, "case %d: rows should be equal", n) + assert.Equal(t, c.expectedRows, rows, "case %d: rows should be equal", n) } } @@ -231,8 +230,8 @@ John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimiters`, } for n, c := range cases { - delimiter := determineDelimiter(markup.NewRenderContext(context.Background()).WithRelativePath(c.filename), []byte(decodeSlashes(t, c.csv))) - assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + delimiter := determineDelimiter(markup.NewRenderContext(t.Context()).WithRelativePath(c.filename), []byte(decodeSlashes(t, c.csv))) + assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -297,7 +296,7 @@ abc | |123 for n, c := range cases { modifiedText := removeQuotedString(decodeSlashes(t, c.text)) - assert.EqualValues(t, c.expectedText, modifiedText, "case %d: modified text should be equal", n) + assert.Equal(t, c.expectedText, modifiedText, "case %d: modified text should be equal", n) } } @@ -452,7 +451,7 @@ jkl`, for n, c := range cases { delimiter := guessDelimiter([]byte(decodeSlashes(t, c.csv))) - assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -544,7 +543,7 @@ a|"he said, ""here I am"""`, for n, c := range cases { delimiter := guessFromBeforeAfterQuotes([]byte(decodeSlashes(t, c.csv))) - assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -580,7 +579,7 @@ func TestFormatError(t *testing.T) { assert.Error(t, err, "case %d: expected an error to be returned", n) } else { assert.NoError(t, err, "case %d: no error was expected, got error: %v", n, err) - assert.EqualValues(t, c.expectedMessage, message, "case %d: messages should be equal, expected '%s' got '%s'", n, c.expectedMessage, message) + assert.Equal(t, c.expectedMessage, message, "case %d: messages should be equal, expected '%s' got '%s'", n, c.expectedMessage, message) } } } diff --git a/modules/dump/dumper_test.go b/modules/dump/dumper_test.go index 2db3a598a4..8f06c1851d 100644 --- a/modules/dump/dumper_test.go +++ b/modules/dump/dumper_test.go @@ -103,11 +103,11 @@ func TestDumper(t *testing.T) { d.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1")) err := d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")}) assert.NoError(t, err) - assert.EqualValues(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added)) + assert.Equal(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added)) tw = &testWriter{} d = &Dumper{Writer: tw} err = d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil) assert.NoError(t, err) - assert.EqualValues(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added)) + assert.Equal(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added)) } diff --git a/modules/fileicon/basic.go b/modules/fileicon/basic.go new file mode 100644 index 0000000000..9c513ccbd9 --- /dev/null +++ b/modules/fileicon/basic.go @@ -0,0 +1,31 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import ( + "html/template" + + "code.gitea.io/gitea/modules/svg" + "code.gitea.io/gitea/modules/util" +) + +func BasicEntryIconName(entry *EntryInfo) string { + svgName := "octicon-file" + switch { + case entry.EntryMode.IsLink(): + svgName = "octicon-file-symlink-file" + if entry.SymlinkToMode.IsDir() { + svgName = "octicon-file-directory-symlink" + } + case entry.EntryMode.IsDir(): + svgName = util.Iif(entry.IsOpen, "octicon-file-directory-open-fill", "octicon-file-directory-fill") + case entry.EntryMode.IsSubModule(): + svgName = "octicon-file-submodule" + } + return svgName +} + +func BasicEntryIconHTML(entry *EntryInfo) template.HTML { + return svg.RenderHTML(BasicEntryIconName(entry)) +} diff --git a/modules/fileicon/entry.go b/modules/fileicon/entry.go new file mode 100644 index 0000000000..0326c2bfa8 --- /dev/null +++ b/modules/fileicon/entry.go @@ -0,0 +1,31 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import "code.gitea.io/gitea/modules/git" + +type EntryInfo struct { + BaseName string + EntryMode git.EntryMode + SymlinkToMode git.EntryMode + IsOpen bool +} + +func EntryInfoFromGitTreeEntry(commit *git.Commit, fullPath string, gitEntry *git.TreeEntry) *EntryInfo { + ret := &EntryInfo{BaseName: gitEntry.Name(), EntryMode: gitEntry.Mode()} + if gitEntry.IsLink() { + if res, err := git.EntryFollowLink(commit, fullPath, gitEntry); err == nil && res.TargetEntry.IsDir() { + ret.SymlinkToMode = res.TargetEntry.Mode() + } + } + return ret +} + +func EntryInfoFolder() *EntryInfo { + return &EntryInfo{EntryMode: git.EntryModeTree} +} + +func EntryInfoFolderOpen() *EntryInfo { + return &EntryInfo{EntryMode: git.EntryModeTree, IsOpen: true} +} diff --git a/modules/fileicon/material.go b/modules/fileicon/material.go new file mode 100644 index 0000000000..5361592d8a --- /dev/null +++ b/modules/fileicon/material.go @@ -0,0 +1,162 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import ( + "html/template" + "strings" + "sync" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/options" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/svg" + "code.gitea.io/gitea/modules/util" +) + +type materialIconRulesData struct { + FileNames map[string]string `json:"fileNames"` + FolderNames map[string]string `json:"folderNames"` + FileExtensions map[string]string `json:"fileExtensions"` + LanguageIDs map[string]string `json:"languageIds"` +} + +type MaterialIconProvider struct { + once sync.Once + rules *materialIconRulesData + svgs map[string]string +} + +var materialIconProvider MaterialIconProvider + +func DefaultMaterialIconProvider() *MaterialIconProvider { + materialIconProvider.once.Do(materialIconProvider.loadData) + return &materialIconProvider +} + +func (m *MaterialIconProvider) loadData() { + buf, err := options.AssetFS().ReadFile("fileicon/material-icon-rules.json") + if err != nil { + log.Error("Failed to read material icon rules: %v", err) + return + } + err = json.Unmarshal(buf, &m.rules) + if err != nil { + log.Error("Failed to unmarshal material icon rules: %v", err) + return + } + + buf, err = options.AssetFS().ReadFile("fileicon/material-icon-svgs.json") + if err != nil { + log.Error("Failed to read material icon rules: %v", err) + return + } + err = json.Unmarshal(buf, &m.svgs) + if err != nil { + log.Error("Failed to unmarshal material icon rules: %v", err) + return + } + log.Debug("Loaded material icon rules and SVG images") +} + +func (m *MaterialIconProvider) renderFileIconSVG(p *RenderedIconPool, name, svg, extraClass string) template.HTML { + // This part is a bit hacky, but it works really well. It should be safe to do so because all SVG icons are generated by us. + // Will try to refactor this in the future. + if !strings.HasPrefix(svg, "<svg") { + panic("Invalid SVG icon") + } + svgID := "svg-mfi-" + name + svgCommonAttrs := `class="svg git-entry-icon ` + extraClass + `" width="16" height="16" aria-hidden="true"` + svgHTML := template.HTML(`<svg id="` + svgID + `" ` + svgCommonAttrs + svg[4:]) + if p == nil { + return svgHTML + } + if p.IconSVGs[svgID] == "" { + p.IconSVGs[svgID] = svgHTML + } + return template.HTML(`<svg ` + svgCommonAttrs + `><use xlink:href="#` + svgID + `"></use></svg>`) +} + +func (m *MaterialIconProvider) EntryIconHTML(p *RenderedIconPool, entry *EntryInfo) template.HTML { + if m.rules == nil { + return BasicEntryIconHTML(entry) + } + + if entry.EntryMode.IsLink() { + if entry.SymlinkToMode.IsDir() { + // keep the old "octicon-xxx" class name to make some "theme plugin selector" could still work + return svg.RenderHTML("material-folder-symlink", 16, "octicon-file-directory-symlink") + } + return svg.RenderHTML("octicon-file-symlink-file") // TODO: find some better icons for them + } + + name := m.FindIconName(entry) + iconSVG := m.svgs[name] + if iconSVG == "" { + name = "file" + if entry.EntryMode.IsDir() { + name = util.Iif(entry.IsOpen, "folder-open", "folder") + } + iconSVG = m.svgs[name] + if iconSVG == "" { + setting.PanicInDevOrTesting("missing file icon for %s", name) + } + } + + // keep the old "octicon-xxx" class name to make some "theme plugin selector" could still work + extraClass := "octicon-file" + switch { + case entry.EntryMode.IsDir(): + extraClass = BasicEntryIconName(entry) + case entry.EntryMode.IsSubModule(): + extraClass = "octicon-file-submodule" + } + return m.renderFileIconSVG(p, name, iconSVG, extraClass) +} + +func (m *MaterialIconProvider) findIconNameWithLangID(s string) string { + if _, ok := m.svgs[s]; ok { + return s + } + if s, ok := m.rules.LanguageIDs[s]; ok { + if _, ok = m.svgs[s]; ok { + return s + } + } + return "" +} + +func (m *MaterialIconProvider) FindIconName(entry *EntryInfo) string { + if entry.EntryMode.IsSubModule() { + return "folder-git" + } + + fileNameLower := strings.ToLower(entry.BaseName) + if entry.EntryMode.IsDir() { + if s, ok := m.rules.FolderNames[fileNameLower]; ok { + return s + } + return util.Iif(entry.IsOpen, "folder-open", "folder") + } + + if s, ok := m.rules.FileNames[fileNameLower]; ok { + if s = m.findIconNameWithLangID(s); s != "" { + return s + } + } + + for i := len(fileNameLower) - 1; i >= 0; i-- { + if fileNameLower[i] == '.' { + ext := fileNameLower[i+1:] + if s, ok := m.rules.FileExtensions[ext]; ok { + if s = m.findIconNameWithLangID(s); s != "" { + return s + } + } + } + } + + return "file" +} diff --git a/modules/fileicon/material_test.go b/modules/fileicon/material_test.go new file mode 100644 index 0000000000..d2a769eaac --- /dev/null +++ b/modules/fileicon/material_test.go @@ -0,0 +1,27 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon_test + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/fileicon" + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m, &unittest.TestOptions{FixtureFiles: []string{}}) +} + +func TestFindIconName(t *testing.T) { + unittest.PrepareTestEnv(t) + p := fileicon.DefaultMaterialIconProvider() + assert.Equal(t, "php", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.php", EntryMode: git.EntryModeBlob})) + assert.Equal(t, "php", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.PHP", EntryMode: git.EntryModeBlob})) + assert.Equal(t, "javascript", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.js", EntryMode: git.EntryModeBlob})) + assert.Equal(t, "visualstudio", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.vba", EntryMode: git.EntryModeBlob})) +} diff --git a/modules/fileicon/render.go b/modules/fileicon/render.go new file mode 100644 index 0000000000..8ed86b9ac0 --- /dev/null +++ b/modules/fileicon/render.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import ( + "html/template" + "strings" + + "code.gitea.io/gitea/modules/setting" +) + +type RenderedIconPool struct { + IconSVGs map[string]template.HTML +} + +func NewRenderedIconPool() *RenderedIconPool { + return &RenderedIconPool{ + IconSVGs: make(map[string]template.HTML), + } +} + +func (p *RenderedIconPool) RenderToHTML() template.HTML { + if len(p.IconSVGs) == 0 { + return "" + } + sb := &strings.Builder{} + sb.WriteString(`<div class=tw-hidden>`) + for _, icon := range p.IconSVGs { + sb.WriteString(string(icon)) + } + sb.WriteString(`</div>`) + return template.HTML(sb.String()) +} + +func RenderEntryIconHTML(renderedIconPool *RenderedIconPool, entry *EntryInfo) template.HTML { + if setting.UI.FileIconTheme == "material" { + return DefaultMaterialIconProvider().EntryIconHTML(renderedIconPool, entry) + } + return BasicEntryIconHTML(entry) +} diff --git a/modules/git/attribute.go b/modules/git/attribute.go deleted file mode 100644 index 4dfa510369..0000000000 --- a/modules/git/attribute.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "code.gitea.io/gitea/modules/optional" -) - -const ( - AttributeLinguistVendored = "linguist-vendored" - AttributeLinguistGenerated = "linguist-generated" - AttributeLinguistDocumentation = "linguist-documentation" - AttributeLinguistDetectable = "linguist-detectable" - AttributeLinguistLanguage = "linguist-language" - AttributeGitlabLanguage = "gitlab-language" -) - -// true if "set"/"true", false if "unset"/"false", none otherwise -func AttributeToBool(attr map[string]string, name string) optional.Option[bool] { - switch attr[name] { - case "set", "true": - return optional.Some(true) - case "unset", "false": - return optional.Some(false) - } - return optional.None[bool]() -} - -func AttributeToString(attr map[string]string, name string) optional.Option[string] { - if value, has := attr[name]; has && value != "unspecified" { - return optional.Some(value) - } - return optional.None[string]() -} diff --git a/modules/git/attribute/attribute.go b/modules/git/attribute/attribute.go new file mode 100644 index 0000000000..adf323ef41 --- /dev/null +++ b/modules/git/attribute/attribute.go @@ -0,0 +1,114 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "strings" + + "code.gitea.io/gitea/modules/optional" +) + +type Attribute string + +const ( + LinguistVendored = "linguist-vendored" + LinguistGenerated = "linguist-generated" + LinguistDocumentation = "linguist-documentation" + LinguistDetectable = "linguist-detectable" + LinguistLanguage = "linguist-language" + GitlabLanguage = "gitlab-language" + Lockable = "lockable" + Filter = "filter" +) + +var LinguistAttributes = []string{ + LinguistVendored, + LinguistGenerated, + LinguistDocumentation, + LinguistDetectable, + LinguistLanguage, + GitlabLanguage, +} + +func (a Attribute) IsUnspecified() bool { + return a == "" || a == "unspecified" +} + +func (a Attribute) ToString() optional.Option[string] { + if !a.IsUnspecified() { + return optional.Some(string(a)) + } + return optional.None[string]() +} + +// ToBool converts the attribute value to optional boolean: true if "set"/"true", false if "unset"/"false", none otherwise +func (a Attribute) ToBool() optional.Option[bool] { + switch a { + case "set", "true": + return optional.Some(true) + case "unset", "false": + return optional.Some(false) + } + return optional.None[bool]() +} + +type Attributes struct { + m map[string]Attribute +} + +func NewAttributes() *Attributes { + return &Attributes{m: make(map[string]Attribute)} +} + +func (attrs *Attributes) Get(name string) Attribute { + if value, has := attrs.m[name]; has { + return value + } + return "" +} + +func (attrs *Attributes) GetVendored() optional.Option[bool] { + return attrs.Get(LinguistVendored).ToBool() +} + +func (attrs *Attributes) GetGenerated() optional.Option[bool] { + return attrs.Get(LinguistGenerated).ToBool() +} + +func (attrs *Attributes) GetDocumentation() optional.Option[bool] { + return attrs.Get(LinguistDocumentation).ToBool() +} + +func (attrs *Attributes) GetDetectable() optional.Option[bool] { + return attrs.Get(LinguistDetectable).ToBool() +} + +func (attrs *Attributes) GetLinguistLanguage() optional.Option[string] { + return attrs.Get(LinguistLanguage).ToString() +} + +func (attrs *Attributes) GetGitlabLanguage() optional.Option[string] { + attrStr := attrs.Get(GitlabLanguage).ToString() + if attrStr.Has() { + raw := attrStr.Value() + // gitlab-language may have additional parameters after the language + // ignore them and just use the main language + // https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type + if idx := strings.IndexByte(raw, '?'); idx >= 0 { + return optional.Some(raw[:idx]) + } + } + return attrStr +} + +func (attrs *Attributes) GetLanguage() optional.Option[string] { + // prefer linguist-language over gitlab-language + // if linguist-language is not set, use gitlab-language + // if both are not set, return none + language := attrs.GetLinguistLanguage() + if language.Value() == "" { + language = attrs.GetGitlabLanguage() + } + return language +} diff --git a/modules/git/attribute/attribute_test.go b/modules/git/attribute/attribute_test.go new file mode 100644 index 0000000000..dadb5582a3 --- /dev/null +++ b/modules/git/attribute/attribute_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Attribute(t *testing.T) { + assert.Empty(t, Attribute("").ToString().Value()) + assert.Empty(t, Attribute("unspecified").ToString().Value()) + assert.Equal(t, "python", Attribute("python").ToString().Value()) + assert.Equal(t, "Java", Attribute("Java").ToString().Value()) + + attributes := Attributes{ + m: map[string]Attribute{ + LinguistGenerated: "true", + LinguistDocumentation: "false", + LinguistDetectable: "set", + LinguistLanguage: "Python", + GitlabLanguage: "Java", + "filter": "unspecified", + "test": "", + }, + } + + assert.Empty(t, attributes.Get("test").ToString().Value()) + assert.Empty(t, attributes.Get("filter").ToString().Value()) + assert.Equal(t, "Python", attributes.Get(LinguistLanguage).ToString().Value()) + assert.Equal(t, "Java", attributes.Get(GitlabLanguage).ToString().Value()) + assert.True(t, attributes.Get(LinguistGenerated).ToBool().Value()) + assert.False(t, attributes.Get(LinguistDocumentation).ToBool().Value()) + assert.True(t, attributes.Get(LinguistDetectable).ToBool().Value()) +} diff --git a/modules/git/attribute/batch.go b/modules/git/attribute/batch.go new file mode 100644 index 0000000000..4e31fda575 --- /dev/null +++ b/modules/git/attribute/batch.go @@ -0,0 +1,216 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" +) + +// BatchChecker provides a reader for check-attribute content that can be long running +type BatchChecker struct { + attributesNum int + repo *git.Repository + stdinWriter *os.File + stdOut *nulSeparatedAttributeWriter + ctx context.Context + cancel context.CancelFunc + cmd *git.Command +} + +// NewBatchChecker creates a check attribute reader for the current repository and provided commit ID +// If treeish is empty, then it will use current working directory, otherwise it will use the provided treeish on the bare repo +func NewBatchChecker(repo *git.Repository, treeish string, attributes []string) (checker *BatchChecker, returnedErr error) { + ctx, cancel := context.WithCancel(repo.Ctx) + defer func() { + if returnedErr != nil { + cancel() + } + }() + + cmd, envs, cleanup, err := checkAttrCommand(repo, treeish, nil, attributes) + if err != nil { + return nil, err + } + defer func() { + if returnedErr != nil { + cleanup() + } + }() + + cmd.AddArguments("--stdin") + + checker = &BatchChecker{ + attributesNum: len(attributes), + repo: repo, + ctx: ctx, + cmd: cmd, + cancel: func() { + cancel() + cleanup() + }, + } + + stdinReader, stdinWriter, err := os.Pipe() + if err != nil { + return nil, err + } + checker.stdinWriter = stdinWriter + + lw := new(nulSeparatedAttributeWriter) + lw.attributes = make(chan attributeTriple, len(attributes)) + lw.closed = make(chan struct{}) + checker.stdOut = lw + + go func() { + defer func() { + _ = stdinReader.Close() + _ = lw.Close() + }() + stdErr := new(bytes.Buffer) + err := cmd.Run(ctx, &git.RunOpts{ + Env: envs, + Dir: repo.Path, + Stdin: stdinReader, + Stdout: lw, + Stderr: stdErr, + }) + + if err != nil && !git.IsErrCanceledOrKilled(err) { + log.Error("Attribute checker for commit %s exits with error: %v", treeish, err) + } + checker.cancel() + }() + + return checker, nil +} + +// CheckPath check attr for given path +func (c *BatchChecker) CheckPath(path string) (rs *Attributes, err error) { + defer func() { + if err != nil && err != c.ctx.Err() { + log.Error("Unexpected error when checking path %s in %s, error: %v", path, filepath.Base(c.repo.Path), err) + } + }() + + select { + case <-c.ctx.Done(): + return nil, c.ctx.Err() + default: + } + + if _, err = c.stdinWriter.Write([]byte(path + "\x00")); err != nil { + defer c.Close() + return nil, err + } + + reportTimeout := func() error { + stdOutClosed := false + select { + case <-c.stdOut.closed: + stdOutClosed = true + default: + } + debugMsg := fmt.Sprintf("check path %q in repo %q", path, filepath.Base(c.repo.Path)) + debugMsg += fmt.Sprintf(", stdOut: tmp=%q, pos=%d, closed=%v", string(c.stdOut.tmp), c.stdOut.pos, stdOutClosed) + if c.cmd != nil { + debugMsg += fmt.Sprintf(", process state: %q", c.cmd.ProcessState()) + } + _ = c.Close() + return fmt.Errorf("CheckPath timeout: %s", debugMsg) + } + + rs = NewAttributes() + for i := 0; i < c.attributesNum; i++ { + select { + case <-time.After(5 * time.Second): + // there is no "hang" problem now. This code is just used to catch other potential problems. + return nil, reportTimeout() + case attr, ok := <-c.stdOut.ReadAttribute(): + if !ok { + return nil, c.ctx.Err() + } + rs.m[attr.Attribute] = Attribute(attr.Value) + case <-c.ctx.Done(): + return nil, c.ctx.Err() + } + } + return rs, nil +} + +func (c *BatchChecker) Close() error { + c.cancel() + err := c.stdinWriter.Close() + return err +} + +type attributeTriple struct { + Filename string + Attribute string + Value string +} + +type nulSeparatedAttributeWriter struct { + tmp []byte + attributes chan attributeTriple + closed chan struct{} + working attributeTriple + pos int +} + +func (wr *nulSeparatedAttributeWriter) Write(p []byte) (n int, err error) { + l, read := len(p), 0 + + nulIdx := bytes.IndexByte(p, '\x00') + for nulIdx >= 0 { + wr.tmp = append(wr.tmp, p[:nulIdx]...) + switch wr.pos { + case 0: + wr.working = attributeTriple{ + Filename: string(wr.tmp), + } + case 1: + wr.working.Attribute = string(wr.tmp) + case 2: + wr.working.Value = string(wr.tmp) + } + wr.tmp = wr.tmp[:0] + wr.pos++ + if wr.pos > 2 { + wr.attributes <- wr.working + wr.pos = 0 + } + read += nulIdx + 1 + if l > read { + p = p[nulIdx+1:] + nulIdx = bytes.IndexByte(p, '\x00') + } else { + return l, nil + } + } + wr.tmp = append(wr.tmp, p...) + return l, nil +} + +func (wr *nulSeparatedAttributeWriter) ReadAttribute() <-chan attributeTriple { + return wr.attributes +} + +func (wr *nulSeparatedAttributeWriter) Close() error { + select { + case <-wr.closed: + return nil + default: + } + close(wr.attributes) + close(wr.closed) + return nil +} diff --git a/modules/git/attribute/batch_test.go b/modules/git/attribute/batch_test.go new file mode 100644 index 0000000000..30a3d805fe --- /dev/null +++ b/modules/git/attribute/batch_test.go @@ -0,0 +1,172 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "path/filepath" + "testing" + "time" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { + wr := &nulSeparatedAttributeWriter{ + attributes: make(chan attributeTriple, 5), + } + + testStr := ".gitignore\"\n\x00linguist-vendored\x00unspecified\x00" + + n, err := wr.Write([]byte(testStr)) + + assert.Len(t, testStr, n) + assert.NoError(t, err) + select { + case attr := <-wr.ReadAttribute(): + assert.Equal(t, ".gitignore\"\n", attr.Filename) + assert.Equal(t, LinguistVendored, attr.Attribute) + assert.Equal(t, "unspecified", attr.Value) + case <-time.After(100 * time.Millisecond): + assert.FailNow(t, "took too long to read an attribute from the list") + } + // Write a second attribute again + n, err = wr.Write([]byte(testStr)) + + assert.Len(t, testStr, n) + assert.NoError(t, err) + + select { + case attr := <-wr.ReadAttribute(): + assert.Equal(t, ".gitignore\"\n", attr.Filename) + assert.Equal(t, LinguistVendored, attr.Attribute) + assert.Equal(t, "unspecified", attr.Value) + case <-time.After(100 * time.Millisecond): + assert.FailNow(t, "took too long to read an attribute from the list") + } + + // Write a partial attribute + _, err = wr.Write([]byte("incomplete-file")) + assert.NoError(t, err) + _, err = wr.Write([]byte("name\x00")) + assert.NoError(t, err) + + select { + case <-wr.ReadAttribute(): + assert.FailNow(t, "There should not be an attribute ready to read") + case <-time.After(100 * time.Millisecond): + } + _, err = wr.Write([]byte("attribute\x00")) + assert.NoError(t, err) + select { + case <-wr.ReadAttribute(): + assert.FailNow(t, "There should not be an attribute ready to read") + case <-time.After(100 * time.Millisecond): + } + + _, err = wr.Write([]byte("value\x00")) + assert.NoError(t, err) + + attr := <-wr.ReadAttribute() + assert.Equal(t, "incomplete-filename", attr.Filename) + assert.Equal(t, "attribute", attr.Attribute) + assert.Equal(t, "value", attr.Value) + + _, err = wr.Write([]byte("shouldbe.vendor\x00linguist-vendored\x00set\x00shouldbe.vendor\x00linguist-generated\x00unspecified\x00shouldbe.vendor\x00linguist-language\x00unspecified\x00")) + assert.NoError(t, err) + attr = <-wr.ReadAttribute() + assert.NoError(t, err) + assert.Equal(t, attributeTriple{ + Filename: "shouldbe.vendor", + Attribute: LinguistVendored, + Value: "set", + }, attr) + attr = <-wr.ReadAttribute() + assert.NoError(t, err) + assert.Equal(t, attributeTriple{ + Filename: "shouldbe.vendor", + Attribute: LinguistGenerated, + Value: "unspecified", + }, attr) + attr = <-wr.ReadAttribute() + assert.NoError(t, err) + assert.Equal(t, attributeTriple{ + Filename: "shouldbe.vendor", + Attribute: LinguistLanguage, + Value: "unspecified", + }, attr) +} + +func expectedAttrs() *Attributes { + return &Attributes{ + m: map[string]Attribute{ + LinguistGenerated: "unspecified", + LinguistDetectable: "unspecified", + LinguistDocumentation: "unspecified", + LinguistVendored: "unspecified", + LinguistLanguage: "Python", + GitlabLanguage: "unspecified", + }, + } +} + +func Test_BatchChecker(t *testing.T) { + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + require.NoError(t, err) + defer gitRepo.Close() + + commitID := "8fee858da5796dfb37704761701bb8e800ad9ef3" + + t.Run("Create index file to run git check-attr", func(t *testing.T) { + defer test.MockVariableValue(&git.DefaultFeatures().SupportCheckAttrOnBare, false)() + checker, err := NewBatchChecker(gitRepo, commitID, LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) + + // run git check-attr on work tree + t.Run("Run git check-attr on git work tree", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "test-repo") + err := git.Clone(t.Context(), repoPath, dir, git.CloneRepoOptions{ + Shared: true, + Branch: "master", + }) + assert.NoError(t, err) + + tempRepo, err := git.OpenRepository(t.Context(), dir) + assert.NoError(t, err) + defer tempRepo.Close() + + checker, err := NewBatchChecker(tempRepo, "", LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) + + if !git.DefaultFeatures().SupportCheckAttrOnBare { + t.Skip("git version 2.40 is required to support run check-attr on bare repo") + return + } + + t.Run("Run git check-attr in bare repository", func(t *testing.T) { + checker, err := NewBatchChecker(gitRepo, commitID, LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) +} diff --git a/modules/git/attribute/checker.go b/modules/git/attribute/checker.go new file mode 100644 index 0000000000..167b31416e --- /dev/null +++ b/modules/git/attribute/checker.go @@ -0,0 +1,101 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + + "code.gitea.io/gitea/modules/git" +) + +func checkAttrCommand(gitRepo *git.Repository, treeish string, filenames, attributes []string) (*git.Command, []string, func(), error) { + cancel := func() {} + envs := []string{"GIT_FLUSH=1"} + cmd := git.NewCommand("check-attr", "-z") + if len(attributes) == 0 { + cmd.AddArguments("--all") + } + + // there is treeish, read from bare repo or temp index created by "read-tree" + if treeish != "" { + if git.DefaultFeatures().SupportCheckAttrOnBare { + cmd.AddArguments("--source") + cmd.AddDynamicArguments(treeish) + } else { + indexFilename, worktree, deleteTemporaryFile, err := gitRepo.ReadTreeToTemporaryIndex(treeish) + if err != nil { + return nil, nil, nil, err + } + + cmd.AddArguments("--cached") + envs = append(envs, + "GIT_INDEX_FILE="+indexFilename, + "GIT_WORK_TREE="+worktree, + ) + cancel = deleteTemporaryFile + } + } else { + // Read from existing index, in cases where the repo is bare and has an index, + // or the work tree contains unstaged changes that shouldn't affect the attribute check. + // It is caller's responsibility to add changed ".gitattributes" into the index if they want to respect the new changes. + cmd.AddArguments("--cached") + } + + cmd.AddDynamicArguments(attributes...) + if len(filenames) > 0 { + cmd.AddDashesAndList(filenames...) + } + return cmd, envs, cancel, nil +} + +type CheckAttributeOpts struct { + Filenames []string + Attributes []string +} + +// CheckAttributes return the attributes of the given filenames and attributes in the given treeish. +// If treeish is empty, then it will use current working directory, otherwise it will use the provided treeish on the bare repo +func CheckAttributes(ctx context.Context, gitRepo *git.Repository, treeish string, opts CheckAttributeOpts) (map[string]*Attributes, error) { + cmd, envs, cancel, err := checkAttrCommand(gitRepo, treeish, opts.Filenames, opts.Attributes) + if err != nil { + return nil, err + } + defer cancel() + + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + if err := cmd.Run(ctx, &git.RunOpts{ + Env: append(os.Environ(), envs...), + Dir: gitRepo.Path, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + return nil, fmt.Errorf("failed to run check-attr: %w\n%s\n%s", err, stdOut.String(), stdErr.String()) + } + + fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) + if len(fields)%3 != 1 { + return nil, errors.New("wrong number of fields in return from check-attr") + } + + attributesMap := make(map[string]*Attributes) + for i := 0; i < (len(fields) / 3); i++ { + filename := string(fields[3*i]) + attribute := string(fields[3*i+1]) + info := string(fields[3*i+2]) + attribute2info, ok := attributesMap[filename] + if !ok { + attribute2info = NewAttributes() + attributesMap[filename] = attribute2info + } + attribute2info.m[attribute] = Attribute(info) + } + + return attributesMap, nil +} diff --git a/modules/git/attribute/checker_test.go b/modules/git/attribute/checker_test.go new file mode 100644 index 0000000000..67fbda8918 --- /dev/null +++ b/modules/git/attribute/checker_test.go @@ -0,0 +1,84 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_Checker(t *testing.T) { + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + require.NoError(t, err) + defer gitRepo.Close() + + commitID := "8fee858da5796dfb37704761701bb8e800ad9ef3" + + t.Run("Create index file to run git check-attr", func(t *testing.T) { + defer test.MockVariableValue(&git.DefaultFeatures().SupportCheckAttrOnBare, false)() + attrs, err := CheckAttributes(t.Context(), gitRepo, commitID, CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + // run git check-attr on work tree + t.Run("Run git check-attr on git work tree", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "test-repo") + err := git.Clone(t.Context(), repoPath, dir, git.CloneRepoOptions{ + Shared: true, + Branch: "master", + }) + assert.NoError(t, err) + + tempRepo, err := git.OpenRepository(t.Context(), dir) + assert.NoError(t, err) + defer tempRepo.Close() + + attrs, err := CheckAttributes(t.Context(), tempRepo, "", CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + t.Run("Run git check-attr in bare repository using index", func(t *testing.T) { + attrs, err := CheckAttributes(t.Context(), gitRepo, "", CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + if !git.DefaultFeatures().SupportCheckAttrOnBare { + t.Skip("git version 2.40 is required to support run check-attr on bare repo without using index") + return + } + + t.Run("Run git check-attr in bare repository", func(t *testing.T) { + attrs, err := CheckAttributes(t.Context(), gitRepo, commitID, CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) +} diff --git a/modules/git/attribute/main_test.go b/modules/git/attribute/main_test.go new file mode 100644 index 0000000000..df8241bfb0 --- /dev/null +++ b/modules/git/attribute/main_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "context" + "fmt" + "os" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func testRun(m *testing.M) error { + gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + if err != nil { + return fmt.Errorf("unable to create temp dir: %w", err) + } + defer util.RemoveAll(gitHomePath) + setting.Git.HomePath = gitHomePath + + if err = git.InitFull(context.Background()); err != nil { + return fmt.Errorf("failed to call Init: %w", err) + } + + exitCode := m.Run() + if exitCode != 0 { + return fmt.Errorf("run test failed, ExitCode=%d", exitCode) + } + return nil +} + +func TestMain(m *testing.M) { + if err := testRun(m); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Test failed: %v", err) + os.Exit(1) + } +} diff --git a/modules/git/batch.go b/modules/git/batch.go index 3ec4f1ddcc..f9e1748b54 100644 --- a/modules/git/batch.go +++ b/modules/git/batch.go @@ -14,25 +14,26 @@ type Batch struct { Writer WriteCloserError } -func (repo *Repository) NewBatch(ctx context.Context) (*Batch, error) { +// NewBatch creates a new batch for the given repository, the Close must be invoked before release the batch +func NewBatch(ctx context.Context, repoPath string) (*Batch, error) { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repo.Path); err != nil { + if err := ensureValidGitRepository(ctx, repoPath); err != nil { return nil, err } var batch Batch - batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repo.Path) + batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repoPath) return &batch, nil } -func (repo *Repository) NewBatchCheck(ctx context.Context) (*Batch, error) { +func NewBatchCheck(ctx context.Context, repoPath string) (*Batch, error) { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repo.Path); err != nil { + if err := ensureValidGitRepository(ctx, repoPath); err != nil { return nil, err } var check Batch - check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repo.Path) + check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repoPath) return &check, nil } diff --git a/modules/git/batch_reader.go b/modules/git/batch_reader.go index 33e54fe75c..7bbab76bb8 100644 --- a/modules/git/batch_reader.go +++ b/modules/git/batch_reader.go @@ -29,8 +29,8 @@ type WriteCloserError interface { // This is needed otherwise the git cat-file will hang for invalid repositories. func ensureValidGitRepository(ctx context.Context, repoPath string) error { stderr := strings.Builder{} - err := NewCommand(ctx, "rev-parse"). - Run(&RunOpts{ + err := NewCommand("rev-parse"). + Run(ctx, &RunOpts{ Dir: repoPath, Stderr: &stderr, }) @@ -61,8 +61,8 @@ func catFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, go func() { stderr := strings.Builder{} - err := NewCommand(ctx, "cat-file", "--batch-check"). - Run(&RunOpts{ + err := NewCommand("cat-file", "--batch-check"). + Run(ctx, &RunOpts{ Dir: repoPath, Stdin: batchStdinReader, Stdout: batchStdoutWriter, @@ -109,8 +109,8 @@ func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufi go func() { stderr := strings.Builder{} - err := NewCommand(ctx, "cat-file", "--batch"). - Run(&RunOpts{ + err := NewCommand("cat-file", "--batch"). + Run(ctx, &RunOpts{ Dir: repoPath, Stdin: batchStdinReader, Stdout: batchStdoutWriter, diff --git a/modules/git/blame.go b/modules/git/blame.go index cad720edf4..659dec34a1 100644 --- a/modules/git/blame.go +++ b/modules/git/blame.go @@ -11,7 +11,7 @@ import ( "os" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/setting" ) // BlamePart represents block of blame - continuous lines with one sha @@ -29,12 +29,13 @@ type BlameReader struct { bufferedReader *bufio.Reader done chan error lastSha *string - ignoreRevsFile *string + ignoreRevsFile string objectFormat ObjectFormat + cleanupFuncs []func() } func (r *BlameReader) UsesIgnoreRevs() bool { - return r.ignoreRevsFile != nil + return r.ignoreRevsFile != "" } // NextPart returns next part of blame (sequential code lines with the same commit) @@ -122,40 +123,49 @@ func (r *BlameReader) Close() error { r.bufferedReader = nil _ = r.reader.Close() _ = r.output.Close() - if r.ignoreRevsFile != nil { - _ = util.Remove(*r.ignoreRevsFile) + for _, cleanup := range r.cleanupFuncs { + if cleanup != nil { + cleanup() + } } return err } // CreateBlameReader creates reader for given repository, commit and file -func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (*BlameReader, error) { - var ignoreRevsFile *string +func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (rd *BlameReader, err error) { + var ignoreRevsFileName string + var ignoreRevsFileCleanup func() + defer func() { + if err != nil && ignoreRevsFileCleanup != nil { + ignoreRevsFileCleanup() + } + }() + + cmd := NewCommandNoGlobals("blame", "--porcelain") + if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { - ignoreRevsFile = tryCreateBlameIgnoreRevsFile(commit) + ignoreRevsFileName, ignoreRevsFileCleanup, err = tryCreateBlameIgnoreRevsFile(commit) + if err != nil && !IsErrNotExist(err) { + return nil, err + } + if ignoreRevsFileName != "" { + // Possible improvement: use --ignore-revs-file /dev/stdin on unix + // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. + cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName) + } } - cmd := NewCommandContextNoGlobals(ctx, "blame", "--porcelain") - if ignoreRevsFile != nil { - // Possible improvement: use --ignore-revs-file /dev/stdin on unix - // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. - cmd.AddOptionValues("--ignore-revs-file", *ignoreRevsFile) - } cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file) + + done := make(chan error, 1) reader, stdout, err := os.Pipe() if err != nil { - if ignoreRevsFile != nil { - _ = util.Remove(*ignoreRevsFile) - } return nil, err } - - done := make(chan error, 1) - go func() { stderr := bytes.Buffer{} // TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close" - err := cmd.Run(&RunOpts{ + err := cmd.Run(ctx, &RunOpts{ UseContextTimeout: true, Dir: repoPath, Stdout: stdout, @@ -169,40 +179,40 @@ func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath }() bufferedReader := bufio.NewReader(reader) - return &BlameReader{ output: stdout, reader: reader, bufferedReader: bufferedReader, done: done, - ignoreRevsFile: ignoreRevsFile, + ignoreRevsFile: ignoreRevsFileName, objectFormat: objectFormat, + cleanupFuncs: []func(){ignoreRevsFileCleanup}, }, nil } -func tryCreateBlameIgnoreRevsFile(commit *Commit) *string { +func tryCreateBlameIgnoreRevsFile(commit *Commit) (string, func(), error) { entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs") if err != nil { - return nil + return "", nil, err } r, err := entry.Blob().DataAsync() if err != nil { - return nil + return "", nil, err } defer r.Close() - f, err := os.CreateTemp("", "gitea_git-blame-ignore-revs") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs") if err != nil { - return nil + return "", nil, err } - + filename := f.Name() _, err = io.Copy(f, r) _ = f.Close() if err != nil { - _ = util.Remove(f.Name()) - return nil + cleanup() + return "", nil, err } - return util.ToPointer(f.Name()) + return filename, cleanup, nil } diff --git a/modules/git/blame_sha256_test.go b/modules/git/blame_sha256_test.go index da451f22fc..c0a97bed3b 100644 --- a/modules/git/blame_sha256_test.go +++ b/modules/git/blame_sha256_test.go @@ -7,11 +7,14 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutputSha256(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + setting.AppDataPath = t.TempDir() + ctx, cancel := context.WithCancel(t.Context()) defer cancel() if isGogit { diff --git a/modules/git/blame_test.go b/modules/git/blame_test.go index 4220c85600..809d6fbcf7 100644 --- a/modules/git/blame_test.go +++ b/modules/git/blame_test.go @@ -7,11 +7,14 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutput(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + setting.AppDataPath = t.TempDir() + ctx, cancel := context.WithCancel(t.Context()) defer cancel() t.Run("Without .git-blame-ignore-revs", func(t *testing.T) { diff --git a/modules/git/blob.go b/modules/git/blob.go index bcecb42e16..40d8f44e79 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -7,7 +7,9 @@ package git import ( "bytes" "encoding/base64" + "errors" "io" + "strings" "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" @@ -20,22 +22,28 @@ func (b *Blob) Name() string { return b.name } -// GetBlobContent Gets the limited content of the blob as raw text -func (b *Blob) GetBlobContent(limit int64) (string, error) { +// GetBlobBytes Gets the limited content of the blob +func (b *Blob) GetBlobBytes(limit int64) ([]byte, error) { if limit <= 0 { - return "", nil + return nil, nil } dataRc, err := b.DataAsync() if err != nil { - return "", err + return nil, err } defer dataRc.Close() - buf, err := util.ReadWithLimit(dataRc, int(limit)) + return util.ReadWithLimit(dataRc, int(limit)) +} + +// GetBlobContent Gets the limited content of the blob as raw text +func (b *Blob) GetBlobContent(limit int64) (string, error) { + buf, err := b.GetBlobBytes(limit) return string(buf), err } -// GetBlobLineCount gets line count of the blob -func (b *Blob) GetBlobLineCount() (int, error) { +// GetBlobLineCount gets line count of the blob. +// It will also try to write the content to w if it's not nil, then we could pre-fetch the content without reading it again. +func (b *Blob) GetBlobLineCount(w io.Writer) (int, error) { reader, err := b.DataAsync() if err != nil { return 0, err @@ -44,59 +52,61 @@ func (b *Blob) GetBlobLineCount() (int, error) { buf := make([]byte, 32*1024) count := 1 lineSep := []byte{'\n'} - - c, err := reader.Read(buf) - if c == 0 && err == io.EOF { - return 0, nil - } for { + c, err := reader.Read(buf) + if w != nil { + if _, err := w.Write(buf[:c]); err != nil { + return count, err + } + } count += bytes.Count(buf[:c], lineSep) switch { - case err == io.EOF: + case errors.Is(err, io.EOF): return count, nil case err != nil: return count, err } - c, err = reader.Read(buf) } } -// GetBlobContentBase64 Reads the content of the blob with a base64 encode and returns the encoded string -func (b *Blob) GetBlobContentBase64() (string, error) { +// GetBlobContentBase64 Reads the content of the blob with a base64 encoding and returns the encoded string +func (b *Blob) GetBlobContentBase64(originContent *strings.Builder) (string, error) { dataRc, err := b.DataAsync() if err != nil { return "", err } defer dataRc.Close() - pr, pw := io.Pipe() - encoder := base64.NewEncoder(base64.StdEncoding, pw) - - go func() { - _, err := io.Copy(encoder, dataRc) - _ = encoder.Close() - - if err != nil { - _ = pw.CloseWithError(err) - } else { - _ = pw.Close() + base64buf := &strings.Builder{} + encoder := base64.NewEncoder(base64.StdEncoding, base64buf) + buf := make([]byte, 32*1024) +loop: + for { + n, err := dataRc.Read(buf) + if n > 0 { + if originContent != nil { + _, _ = originContent.Write(buf[:n]) + } + if _, err := encoder.Write(buf[:n]); err != nil { + return "", err + } + } + switch { + case errors.Is(err, io.EOF): + break loop + case err != nil: + return "", err } - }() - - out, err := io.ReadAll(pr) - if err != nil { - return "", err } - return string(out), nil + _ = encoder.Close() + return base64buf.String(), nil } // GuessContentType guesses the content type of the blob. func (b *Blob) GuessContentType() (typesniffer.SniffedType, error) { - r, err := b.DataAsync() + buf, err := b.GetBlobBytes(typesniffer.SniffContentSize) if err != nil { return typesniffer.SniffedType{}, err } - defer r.Close() - - return typesniffer.DetectContentTypeFromReader(r) + return typesniffer.DetectContentType(buf), nil } diff --git a/modules/git/blob_test.go b/modules/git/blob_test.go index d0804350ed..f21e8d146d 100644 --- a/modules/git/blob_test.go +++ b/modules/git/blob_test.go @@ -47,7 +47,7 @@ func Benchmark_Blob_Data(b *testing.B) { b.Fatal(err) } - for i := 0; i < b.N; i++ { + for b.Loop() { r, err := testBlob.DataAsync() if err != nil { b.Fatal(err) diff --git a/modules/git/cmdverb.go b/modules/git/cmdverb.go new file mode 100644 index 0000000000..3d6f4ae0c6 --- /dev/null +++ b/modules/git/cmdverb.go @@ -0,0 +1,36 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +const ( + CmdVerbUploadPack = "git-upload-pack" + CmdVerbUploadArchive = "git-upload-archive" + CmdVerbReceivePack = "git-receive-pack" + CmdVerbLfsAuthenticate = "git-lfs-authenticate" + CmdVerbLfsTransfer = "git-lfs-transfer" + + CmdSubVerbLfsUpload = "upload" + CmdSubVerbLfsDownload = "download" +) + +func IsAllowedVerbForServe(verb string) bool { + switch verb { + case CmdVerbUploadPack, + CmdVerbUploadArchive, + CmdVerbReceivePack, + CmdVerbLfsAuthenticate, + CmdVerbLfsTransfer: + return true + } + return false +} + +func IsAllowedVerbForServeLfs(verb string) bool { + switch verb { + case CmdVerbLfsAuthenticate, + CmdVerbLfsTransfer: + return true + } + return false +} diff --git a/modules/git/command.go b/modules/git/command.go index 2584e3cc57..22f1d02339 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -18,6 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/git/internal" //nolint:depguard // only this file can use the internal type CmdArg, other files and packages should use AddXxx functions + "code.gitea.io/gitea/modules/gtprof" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/util" @@ -43,9 +44,10 @@ const DefaultLocale = "C" type Command struct { prog string args []string - parentContext context.Context globalArgsLength int brokenArgs []string + cmd *exec.Cmd // for debug purpose only + configArgs []string } func logArgSanitize(arg string) string { @@ -54,7 +56,7 @@ func logArgSanitize(arg string) string { } else if filepath.IsAbs(arg) { base := filepath.Base(arg) dir := filepath.Dir(arg) - return filepath.Join(filepath.Base(dir), base) + return ".../" + filepath.Join(filepath.Base(dir), base) } return arg } @@ -79,9 +81,16 @@ func (c *Command) LogString() string { return strings.Join(a, " ") } +func (c *Command) ProcessState() string { + if c.cmd == nil { + return "" + } + return c.cmd.ProcessState.String() +} + // NewCommand creates and returns a new Git Command based on given command and arguments. // Each argument should be safe to be trusted. User-provided arguments should be passed to AddDynamicArguments instead. -func NewCommand(ctx context.Context, args ...internal.CmdArg) *Command { +func NewCommand(args ...internal.CmdArg) *Command { // Make an explicit copy of globalCommandArgs, otherwise append might overwrite it cargs := make([]string, 0, len(globalCommandArgs)+len(args)) for _, arg := range globalCommandArgs { @@ -93,31 +102,23 @@ func NewCommand(ctx context.Context, args ...internal.CmdArg) *Command { return &Command{ prog: GitExecutable, args: cargs, - parentContext: ctx, globalArgsLength: len(globalCommandArgs), } } -// NewCommandContextNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args +// NewCommandNoGlobals creates and returns a new Git Command based on given command and arguments only with the specified args and don't use global command args // Each argument should be safe to be trusted. User-provided arguments should be passed to AddDynamicArguments instead. -func NewCommandContextNoGlobals(ctx context.Context, args ...internal.CmdArg) *Command { +func NewCommandNoGlobals(args ...internal.CmdArg) *Command { cargs := make([]string, 0, len(args)) for _, arg := range args { cargs = append(cargs, string(arg)) } return &Command{ - prog: GitExecutable, - args: cargs, - parentContext: ctx, + prog: GitExecutable, + args: cargs, } } -// SetParentContext sets the parent context for this command -func (c *Command) SetParentContext(ctx context.Context) *Command { - c.parentContext = ctx - return c -} - // isSafeArgumentValue checks if the argument is safe to be used as a value (not an option) func isSafeArgumentValue(s string) bool { return s == "" || s[0] != '-' @@ -196,6 +197,16 @@ func (c *Command) AddDashesAndList(list ...string) *Command { return c } +func (c *Command) AddConfig(key, value string) *Command { + kv := key + "=" + value + if !isSafeArgumentValue(kv) { + c.brokenArgs = append(c.brokenArgs, key) + } else { + c.configArgs = append(c.configArgs, "-c", kv) + } + return c +} + // ToTrustedCmdArgs converts a list of strings (trusted as argument) to TrustedCmdArgs // In most cases, it shouldn't be used. Use NewCommand().AddXxx() function instead func ToTrustedCmdArgs(args []string) TrustedCmdArgs { @@ -276,11 +287,11 @@ func CommonCmdServEnvs() []string { var ErrBrokenCommand = errors.New("git command is broken") // Run runs the command with the RunOpts -func (c *Command) Run(opts *RunOpts) error { - return c.run(1, opts) +func (c *Command) Run(ctx context.Context, opts *RunOpts) error { + return c.run(ctx, 1, opts) } -func (c *Command) run(skip int, opts *RunOpts) error { +func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { if len(c.brokenArgs) != 0 { log.Error("git command is broken: %s, broken args: %s", c.LogString(), strings.Join(c.brokenArgs, " ")) return ErrBrokenCommand @@ -295,29 +306,34 @@ func (c *Command) run(skip int, opts *RunOpts) error { timeout = defaultCommandExecutionTimeout } - var desc string + cmdLogString := c.LogString() callerInfo := util.CallerFuncName(1 /* util */ + 1 /* this */ + skip /* parent */) if pos := strings.LastIndex(callerInfo, "/"); pos >= 0 { callerInfo = callerInfo[pos+1:] } // these logs are for debugging purposes only, so no guarantee of correctness or stability - desc = fmt.Sprintf("git.Run(by:%s, repo:%s): %s", callerInfo, logArgSanitize(opts.Dir), c.LogString()) + desc := fmt.Sprintf("git.Run(by:%s, repo:%s): %s", callerInfo, logArgSanitize(opts.Dir), cmdLogString) log.Debug("git.Command: %s", desc) - var ctx context.Context + _, span := gtprof.GetTracer().Start(ctx, gtprof.TraceSpanGitRun) + defer span.End() + span.SetAttributeString(gtprof.TraceAttrFuncCaller, callerInfo) + span.SetAttributeString(gtprof.TraceAttrGitCommand, cmdLogString) + var cancel context.CancelFunc var finished context.CancelFunc if opts.UseContextTimeout { - ctx, cancel, finished = process.GetManager().AddContext(c.parentContext, desc) + ctx, cancel, finished = process.GetManager().AddContext(ctx, desc) } else { - ctx, cancel, finished = process.GetManager().AddContextTimeout(c.parentContext, timeout, desc) + ctx, cancel, finished = process.GetManager().AddContextTimeout(ctx, timeout, desc) } defer finished() startTime := time.Now() - cmd := exec.CommandContext(ctx, c.prog, c.args...) + cmd := exec.CommandContext(ctx, c.prog, append(c.configArgs, c.args...)...) + c.cmd = cmd // for debug purpose only if opts.Env == nil { cmd.Env = os.Environ() } else { @@ -352,9 +368,10 @@ func (c *Command) run(skip int, opts *RunOpts) error { // We need to check if the context is canceled by the program on Windows. // This is because Windows does not have signal checking when terminating the process. // It always returns exit code 1, unlike Linux, which has many exit codes for signals. + // `err.Error()` returns "exit status 1" when using the `git check-attr` command after the context is canceled. if runtime.GOOS == "windows" && err != nil && - err.Error() == "" && + (err.Error() == "" || err.Error() == "exit status 1") && cmd.ProcessState.ExitCode() == 1 && ctx.Err() == context.Canceled { return ctx.Err() @@ -404,8 +421,8 @@ func IsErrorExitCode(err error, code int) bool { } // RunStdString runs the command with options and returns stdout/stderr as string. and store stderr to returned error (err combined with stderr). -func (c *Command) RunStdString(opts *RunOpts) (stdout, stderr string, runErr RunStdError) { - stdoutBytes, stderrBytes, err := c.runStdBytes(opts) +func (c *Command) RunStdString(ctx context.Context, opts *RunOpts) (stdout, stderr string, runErr RunStdError) { + stdoutBytes, stderrBytes, err := c.runStdBytes(ctx, opts) stdout = util.UnsafeBytesToString(stdoutBytes) stderr = util.UnsafeBytesToString(stderrBytes) if err != nil { @@ -416,11 +433,11 @@ func (c *Command) RunStdString(opts *RunOpts) (stdout, stderr string, runErr Run } // RunStdBytes runs the command with options and returns stdout/stderr as bytes. and store stderr to returned error (err combined with stderr). -func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunStdError) { - return c.runStdBytes(opts) +func (c *Command) RunStdBytes(ctx context.Context, opts *RunOpts) (stdout, stderr []byte, runErr RunStdError) { + return c.runStdBytes(ctx, opts) } -func (c *Command) runStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunStdError) { +func (c *Command) runStdBytes(ctx context.Context, opts *RunOpts) (stdout, stderr []byte, runErr RunStdError) { if opts == nil { opts = &RunOpts{} } @@ -443,7 +460,7 @@ func (c *Command) runStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS PipelineFunc: opts.PipelineFunc, } - err := c.run(2, newOpts) + err := c.run(ctx, 2, newOpts) stderr = stderrBuf.Bytes() if err != nil { return nil, stderr, &runStdError{err: err, stderr: util.UnsafeBytesToString(stderr)} diff --git a/modules/git/command_race_test.go b/modules/git/command_race_test.go index f567406822..a6aa3a1580 100644 --- a/modules/git/command_race_test.go +++ b/modules/git/command_race_test.go @@ -15,9 +15,9 @@ func TestRunWithContextNoTimeout(t *testing.T) { maxLoops := 10 // 'git --version' does not block so it must be finished before the timeout triggered. - cmd := NewCommand(context.Background(), "--version") + cmd := NewCommand("--version") for i := 0; i < maxLoops; i++ { - if err := cmd.Run(&RunOpts{}); err != nil { + if err := cmd.Run(t.Context(), &RunOpts{}); err != nil { t.Fatal(err) } } @@ -27,9 +27,9 @@ func TestRunWithContextTimeout(t *testing.T) { maxLoops := 10 // 'git hash-object --stdin' blocks on stdin so we can have the timeout triggered. - cmd := NewCommand(context.Background(), "hash-object", "--stdin") + cmd := NewCommand("hash-object", "--stdin") for i := 0; i < maxLoops; i++ { - if err := cmd.Run(&RunOpts{Timeout: 1 * time.Millisecond}); err != nil { + if err := cmd.Run(t.Context(), &RunOpts{Timeout: 1 * time.Millisecond}); err != nil { if err != context.DeadlineExceeded { t.Fatalf("Testing %d/%d: %v", i, maxLoops, err) } diff --git a/modules/git/command_test.go b/modules/git/command_test.go index 0823afd7f7..eb112707e7 100644 --- a/modules/git/command_test.go +++ b/modules/git/command_test.go @@ -4,21 +4,20 @@ package git import ( - "context" "testing" "github.com/stretchr/testify/assert" ) func TestRunWithContextStd(t *testing.T) { - cmd := NewCommand(context.Background(), "--version") - stdout, stderr, err := cmd.RunStdString(&RunOpts{}) + cmd := NewCommand("--version") + stdout, stderr, err := cmd.RunStdString(t.Context(), &RunOpts{}) assert.NoError(t, err) assert.Empty(t, stderr) assert.Contains(t, stdout, "git version") - cmd = NewCommand(context.Background(), "--no-such-arg") - stdout, stderr, err = cmd.RunStdString(&RunOpts{}) + cmd = NewCommand("--no-such-arg") + stdout, stderr, err = cmd.RunStdString(t.Context(), &RunOpts{}) if assert.Error(t, err) { assert.Equal(t, stderr, err.Stderr()) assert.Contains(t, err.Stderr(), "unknown option:") @@ -26,17 +25,17 @@ func TestRunWithContextStd(t *testing.T) { assert.Empty(t, stdout) } - cmd = NewCommand(context.Background()) + cmd = NewCommand() cmd.AddDynamicArguments("-test") - assert.ErrorIs(t, cmd.Run(&RunOpts{}), ErrBrokenCommand) + assert.ErrorIs(t, cmd.Run(t.Context(), &RunOpts{}), ErrBrokenCommand) - cmd = NewCommand(context.Background()) + cmd = NewCommand() cmd.AddDynamicArguments("--test") - assert.ErrorIs(t, cmd.Run(&RunOpts{}), ErrBrokenCommand) + assert.ErrorIs(t, cmd.Run(t.Context(), &RunOpts{}), ErrBrokenCommand) subCmd := "version" - cmd = NewCommand(context.Background()).AddDynamicArguments(subCmd) // for test purpose only, the sub-command should never be dynamic for production - stdout, stderr, err = cmd.RunStdString(&RunOpts{}) + cmd = NewCommand().AddDynamicArguments(subCmd) // for test purpose only, the sub-command should never be dynamic for production + stdout, stderr, err = cmd.RunStdString(t.Context(), &RunOpts{}) assert.NoError(t, err) assert.Empty(t, stderr) assert.Contains(t, stdout, "git version") @@ -54,9 +53,9 @@ func TestGitArgument(t *testing.T) { } func TestCommandString(t *testing.T) { - cmd := NewCommandContextNoGlobals(context.Background(), "a", "-m msg", "it's a test", `say "hello"`) - assert.EqualValues(t, cmd.prog+` a "-m msg" "it's a test" "say \"hello\""`, cmd.LogString()) + cmd := NewCommandNoGlobals("a", "-m msg", "it's a test", `say "hello"`) + assert.Equal(t, cmd.prog+` a "-m msg" "it's a test" "say \"hello\""`, cmd.LogString()) - cmd = NewCommandContextNoGlobals(context.Background(), "url: https://a:b@c/", "/root/dir-a/dir-b") - assert.EqualValues(t, cmd.prog+` "url: https://sanitized-credential@c/" dir-a/dir-b`, cmd.LogString()) + cmd = NewCommandNoGlobals("url: https://a:b@c/", "/root/dir-a/dir-b") + assert.Equal(t, cmd.prog+` "url: https://sanitized-credential@c/" .../dir-a/dir-b`, cmd.LogString()) } diff --git a/modules/git/commit.go b/modules/git/commit.go index 0a50ba4356..ed4876e7b3 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -20,7 +20,8 @@ import ( // Commit represents a git commit. type Commit struct { - Tree + Tree // FIXME: bad design, this field can be nil if the commit is from "last commit cache" + ID ObjectID // The ID of this commit object Author *Signature Committer *Signature @@ -34,7 +35,7 @@ type Commit struct { // CommitSignature represents a git commit signature part. type CommitSignature struct { Signature string - Payload string // TODO check if can be reconstruct from the rest of commit information to not have duplicate data + Payload string } // Message returns the commit message. Same as retrieving CommitMessage directly. @@ -91,12 +92,12 @@ func AddChanges(repoPath string, all bool, files ...string) error { // AddChangesWithArgs marks local changes to be ready for commit. func AddChangesWithArgs(repoPath string, globalArgs TrustedCmdArgs, all bool, files ...string) error { - cmd := NewCommandContextNoGlobals(DefaultContext, globalArgs...).AddArguments("add") + cmd := NewCommandNoGlobals(globalArgs...).AddArguments("add") if all { cmd.AddArguments("--all") } cmd.AddDashesAndList(files...) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + _, _, err := cmd.RunStdString(DefaultContext, &RunOpts{Dir: repoPath}) return err } @@ -118,7 +119,7 @@ func CommitChanges(repoPath string, opts CommitChangesOptions) error { // CommitChangesWithArgs commits local changes with given committer, author and message. // If author is nil, it will be the same as committer. func CommitChangesWithArgs(repoPath string, args TrustedCmdArgs, opts CommitChangesOptions) error { - cmd := NewCommandContextNoGlobals(DefaultContext, args...) + cmd := NewCommandNoGlobals(args...) if opts.Committer != nil { cmd.AddOptionValues("-c", "user.name="+opts.Committer.Name) cmd.AddOptionValues("-c", "user.email="+opts.Committer.Email) @@ -133,7 +134,7 @@ func CommitChangesWithArgs(repoPath string, args TrustedCmdArgs, opts CommitChan } cmd.AddOptionFormat("--message=%s", opts.Message) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + _, _, err := cmd.RunStdString(DefaultContext, &RunOpts{Dir: repoPath}) // No stderr but exit status 1 means nothing to commit. if err != nil && err.Error() == "exit status 1" { return nil @@ -143,7 +144,7 @@ func CommitChangesWithArgs(repoPath string, args TrustedCmdArgs, opts CommitChan // AllCommitsCount returns count of all commits in repository func AllCommitsCount(ctx context.Context, repoPath string, hidePRRefs bool, files ...string) (int64, error) { - cmd := NewCommand(ctx, "rev-list") + cmd := NewCommand("rev-list") if hidePRRefs { cmd.AddArguments("--exclude=" + PullPrefix + "*") } @@ -152,7 +153,7 @@ func AllCommitsCount(ctx context.Context, repoPath string, hidePRRefs bool, file cmd.AddDashesAndList(files...) } - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return 0, err } @@ -166,11 +167,13 @@ type CommitsCountOptions struct { Not string Revision []string RelPath []string + Since string + Until string } // CommitsCount returns number of total commits of until given revision. func CommitsCount(ctx context.Context, opts CommitsCountOptions) (int64, error) { - cmd := NewCommand(ctx, "rev-list", "--count") + cmd := NewCommand("rev-list", "--count") cmd.AddDynamicArguments(opts.Revision...) @@ -182,7 +185,7 @@ func CommitsCount(ctx context.Context, opts CommitsCountOptions) (int64, error) cmd.AddDashesAndList(opts.RelPath...) } - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: opts.RepoPath}) + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: opts.RepoPath}) if err != nil { return 0, err } @@ -199,8 +202,8 @@ func (c *Commit) CommitsCount() (int64, error) { } // CommitsByRange returns the specific page commits before current revision, every page's number default by CommitsRangeSize -func (c *Commit) CommitsByRange(page, pageSize int, not string) ([]*Commit, error) { - return c.repo.commitsByRange(c.ID, page, pageSize, not) +func (c *Commit) CommitsByRange(page, pageSize int, not, since, until string) ([]*Commit, error) { + return c.repo.commitsByRangeWithTime(c.ID, page, pageSize, not, since, until) } // CommitsBefore returns all the commits before current revision @@ -217,7 +220,7 @@ func (c *Commit) HasPreviousCommit(objectID ObjectID) (bool, error) { return false, nil } - _, _, err := NewCommand(c.repo.Ctx, "merge-base", "--is-ancestor").AddDynamicArguments(that, this).RunStdString(&RunOpts{Dir: c.repo.Path}) + _, _, err := NewCommand("merge-base", "--is-ancestor").AddDynamicArguments(that, this).RunStdString(c.repo.Ctx, &RunOpts{Dir: c.repo.Path}) if err == nil { return true, nil } @@ -275,8 +278,8 @@ func NewSearchCommitsOptions(searchString string, forAllRefs bool) SearchCommits var keywords, authors, committers []string var after, before string - fields := strings.Fields(searchString) - for _, k := range fields { + fields := strings.FieldsSeq(searchString) + for k := range fields { switch { case strings.HasPrefix(k, "author:"): authors = append(authors, strings.TrimPrefix(k, "author:")) @@ -358,12 +361,12 @@ func (c *Commit) GetFileContent(filename string, limit int) (string, error) { // GetBranchName gets the closest branch name (as returned by 'git name-rev --name-only') func (c *Commit) GetBranchName() (string, error) { - cmd := NewCommand(c.repo.Ctx, "name-rev") + cmd := NewCommand("name-rev") if DefaultFeatures().CheckVersionAtLeast("2.13.0") { cmd.AddArguments("--exclude", "refs/tags/*") } cmd.AddArguments("--name-only", "--no-undefined").AddDynamicArguments(c.ID.String()) - data, _, err := cmd.RunStdString(&RunOpts{Dir: c.repo.Path}) + data, _, err := cmd.RunStdString(c.repo.Ctx, &RunOpts{Dir: c.repo.Path}) if err != nil { // handle special case where git can not describe commit if strings.Contains(err.Error(), "cannot describe") { @@ -441,7 +444,7 @@ func GetCommitFileStatus(ctx context.Context, repoPath, commitID string) (*Commi }() stderr := new(bytes.Buffer) - err := NewCommand(ctx, "log", "--name-status", "-m", "--pretty=format:", "--first-parent", "--no-renames", "-z", "-1").AddDynamicArguments(commitID).Run(&RunOpts{ + err := NewCommand("log", "--name-status", "-m", "--pretty=format:", "--first-parent", "--no-renames", "-z", "-1").AddDynamicArguments(commitID).Run(ctx, &RunOpts{ Dir: repoPath, Stdout: w, Stderr: stderr, @@ -457,7 +460,7 @@ func GetCommitFileStatus(ctx context.Context, repoPath, commitID string) (*Commi // GetFullCommitID returns full length (40) of commit ID by given short SHA in a repository. func GetFullCommitID(ctx context.Context, repoPath, shortID string) (string, error) { - commitID, _, err := NewCommand(ctx, "rev-parse").AddDynamicArguments(shortID).RunStdString(&RunOpts{Dir: repoPath}) + commitID, _, err := NewCommand("rev-parse").AddDynamicArguments(shortID).RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { if strings.Contains(err.Error(), "exit status 128") { return "", ErrNotExist{shortID, ""} diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index ef2df0b133..1b45fc8a6c 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -7,8 +7,7 @@ package git import ( "context" - "fmt" - "io" + "maps" "path" "sort" @@ -40,9 +39,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath return nil, nil, err } - for pth, found := range commits { - revs[pth] = found - } + maps.Copy(revs, commits) } } else { sort.Strings(entryPaths) @@ -65,7 +62,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath log.Debug("missing commit for %s", entry.Name()) } - // If the entry if a submodule add a submodule file for this + // If the entry is a submodule add a submodule file for this if entry.IsSubModule() { subModuleURL := "" var fullPath string @@ -85,8 +82,8 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath } // Retrieve the commit for the treePath itself (see above). We basically - // get it for free during the tree traversal and it's used for listing - // pages to display information about newest commit for a given path. + // get it for free during the tree traversal, and it's used for listing + // pages to display information about the newest commit for a given path. var treeCommit *Commit var ok bool if treePath == "" { @@ -124,48 +121,25 @@ func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, return nil, err } - batchStdinWriter, batchReader, cancel, err := commit.repo.CatFileBatch(ctx) - if err != nil { - return nil, err - } - defer cancel() - commitsMap := map[string]*Commit{} commitsMap[commit.ID.String()] = commit commitCommits := map[string]*Commit{} for path, commitID := range revs { - c, ok := commitsMap[commitID] - if ok { - commitCommits[path] = c + if len(commitID) == 0 { continue } - if len(commitID) == 0 { + c, ok := commitsMap[commitID] + if ok { + commitCommits[path] = c continue } - _, err := batchStdinWriter.Write([]byte(commitID + "\n")) - if err != nil { - return nil, err - } - _, typ, size, err := ReadBatchLine(batchReader) + c, err := commit.repo.GetCommit(commitID) // Ensure the commit exists in the repository if err != nil { return nil, err } - if typ != "commit" { - if err := DiscardFull(batchReader, size+1); err != nil { - return nil, err - } - return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID) - } - c, err = CommitFromReader(commit.repo, MustIDFromString(commitID), io.LimitReader(batchReader, size)) - if err != nil { - return nil, err - } - if _, err := batchReader.Discard(1); err != nil { - return nil, err - } commitCommits[path] = c } diff --git a/modules/git/commit_info_test.go b/modules/git/commit_info_test.go index 1e331fac00..ba518ab245 100644 --- a/modules/git/commit_info_test.go +++ b/modules/git/commit_info_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "path/filepath" "testing" "time" @@ -83,7 +82,7 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) { } // FIXME: Context.TODO() - if graceful has started we should use its Shutdown context otherwise use install signals in TestMain. - commitsInfo, treeCommit, err := entries.GetCommitsInfo(context.TODO(), commit, testCase.Path) + commitsInfo, treeCommit, err := entries.GetCommitsInfo(t.Context(), commit, testCase.Path) assert.NoError(t, err, "Unable to get commit information for entries of subtree: %s in commit: %s from testcase due to error: %v", testCase.Path, testCase.CommitID, err) if err != nil { t.FailNow() @@ -159,8 +158,8 @@ func BenchmarkEntries_GetCommitsInfo(b *testing.B) { entries.Sort() b.ResetTimer() b.Run(benchmark.name, func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _, err := entries.GetCommitsInfo(context.Background(), commit, "") + for b.Loop() { + _, _, err := entries.GetCommitsInfo(b.Context(), commit, "") if err != nil { b.Fatal(err) } diff --git a/modules/git/commit_reader.go b/modules/git/commit_reader.go index 228bbaf314..eb8f4c6322 100644 --- a/modules/git/commit_reader.go +++ b/modules/git/commit_reader.go @@ -6,10 +6,44 @@ package git import ( "bufio" "bytes" + "fmt" "io" - "strings" ) +const ( + commitHeaderGpgsig = "gpgsig" + commitHeaderGpgsigSha256 = "gpgsig-sha256" +) + +func assignCommitFields(gitRepo *Repository, commit *Commit, headerKey string, headerValue []byte) error { + if len(headerValue) > 0 && headerValue[len(headerValue)-1] == '\n' { + headerValue = headerValue[:len(headerValue)-1] // remove trailing newline + } + switch headerKey { + case "tree": + objID, err := NewIDFromString(string(headerValue)) + if err != nil { + return fmt.Errorf("invalid tree ID %q: %w", string(headerValue), err) + } + commit.Tree = *NewTree(gitRepo, objID) + case "parent": + objID, err := NewIDFromString(string(headerValue)) + if err != nil { + return fmt.Errorf("invalid parent ID %q: %w", string(headerValue), err) + } + commit.Parents = append(commit.Parents, objID) + case "author": + commit.Author.Decode(headerValue) + case "committer": + commit.Committer.Decode(headerValue) + case commitHeaderGpgsig, commitHeaderGpgsigSha256: + // if there are duplicate "gpgsig" and "gpgsig-sha256" headers, then the signature must have already been invalid + // so we don't need to handle duplicate headers here + commit.Signature = &CommitSignature{Signature: string(headerValue)} + } + return nil +} + // CommitFromReader will generate a Commit from a provided reader // We need this to interpret commits from cat-file or cat-file --batch // @@ -21,90 +55,46 @@ func CommitFromReader(gitRepo *Repository, objectID ObjectID, reader io.Reader) Committer: &Signature{}, } - payloadSB := new(strings.Builder) - signatureSB := new(strings.Builder) - messageSB := new(strings.Builder) - message := false - pgpsig := false - - bufReader, ok := reader.(*bufio.Reader) - if !ok { - bufReader = bufio.NewReader(reader) - } - -readLoop: + bufReader := bufio.NewReader(reader) + inHeader := true + var payloadSB, messageSB bytes.Buffer + var headerKey string + var headerValue []byte for { line, err := bufReader.ReadBytes('\n') - if err != nil { - if err == io.EOF { - if message { - _, _ = messageSB.Write(line) - } - _, _ = payloadSB.Write(line) - break readLoop - } - return nil, err + if err != nil && err != io.EOF { + return nil, fmt.Errorf("unable to read commit %q: %w", objectID.String(), err) } - if pgpsig { - if len(line) > 0 && line[0] == ' ' { - _, _ = signatureSB.Write(line[1:]) - continue - } - pgpsig = false + if len(line) == 0 { + break } - if !message { - // This is probably not correct but is copied from go-gits interpretation... - trimmed := bytes.TrimSpace(line) - if len(trimmed) == 0 { - message = true - _, _ = payloadSB.Write(line) - continue - } - - split := bytes.SplitN(trimmed, []byte{' '}, 2) - var data []byte - if len(split) > 1 { - data = split[1] + if inHeader { + inHeader = !(len(line) == 1 && line[0] == '\n') // still in header if line is not just a newline + k, v, _ := bytes.Cut(line, []byte{' '}) + if len(k) != 0 || !inHeader { + if headerKey != "" { + if err = assignCommitFields(gitRepo, commit, headerKey, headerValue); err != nil { + return nil, fmt.Errorf("unable to parse commit %q: %w", objectID.String(), err) + } + } + headerKey = string(k) // it also resets the headerValue to empty string if not inHeader + headerValue = v + } else { + headerValue = append(headerValue, v...) } - - switch string(split[0]) { - case "tree": - commit.Tree = *NewTree(gitRepo, MustIDFromString(string(data))) + if headerKey != commitHeaderGpgsig && headerKey != commitHeaderGpgsigSha256 { _, _ = payloadSB.Write(line) - case "parent": - commit.Parents = append(commit.Parents, MustIDFromString(string(data))) - _, _ = payloadSB.Write(line) - case "author": - commit.Author = &Signature{} - commit.Author.Decode(data) - _, _ = payloadSB.Write(line) - case "committer": - commit.Committer = &Signature{} - commit.Committer.Decode(data) - _, _ = payloadSB.Write(line) - case "encoding": - _, _ = payloadSB.Write(line) - case "gpgsig": - fallthrough - case "gpgsig-sha256": // FIXME: no intertop, so only 1 exists at present. - _, _ = signatureSB.Write(data) - _ = signatureSB.WriteByte('\n') - pgpsig = true } } else { _, _ = messageSB.Write(line) _, _ = payloadSB.Write(line) } } + commit.CommitMessage = messageSB.String() - commit.Signature = &CommitSignature{ - Signature: signatureSB.String(), - Payload: payloadSB.String(), - } - if len(commit.Signature.Signature) == 0 { - commit.Signature = nil + if commit.Signature != nil { + commit.Signature.Payload = payloadSB.String() } - return commit, nil } diff --git a/modules/git/commit_sha256_test.go b/modules/git/commit_sha256_test.go index f6ca83c9ed..97ccecdacc 100644 --- a/modules/git/commit_sha256_test.go +++ b/modules/git/commit_sha256_test.go @@ -60,8 +60,7 @@ func TestGetFullCommitIDErrorSha256(t *testing.T) { } func TestCommitFromReaderSha256(t *testing.T) { - commitString := `9433b2a62b964c17a4485ae180f45f595d3e69d31b786087775e28c6b6399df0 commit 1114 -tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e + commitString := `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e parent 26e9ccc29fad747e9c5d9f4c9ddeb7eff61cc45ef6a8dc258cbeb181afc055e8 author Adam Majer <amajer@suse.de> 1698676906 +0100 committer Adam Majer <amajer@suse.de> 1698676906 +0100 @@ -97,7 +96,7 @@ signed commit` assert.NoError(t, err) require.NotNil(t, commitFromReader) assert.EqualValues(t, sha, commitFromReader.ID) - assert.EqualValues(t, `-----BEGIN PGP SIGNATURE----- + assert.Equal(t, `-----BEGIN PGP SIGNATURE----- iQIrBAABCgAtFiEES+fB08xlgTrzSdQvhkUIsBsmec8FAmU/wKoPHGFtYWplckBz dXNlLmRlAAoJEIZFCLAbJnnP4s4PQIJATa++WPzR6/H4etT7bsOGoMyguEJYyWOd @@ -112,21 +111,20 @@ VAEUo6ecdDxSpyt2naeg9pKus/BRi7P6g4B1hkk/zZstUX/QP4IQuAJbXjkvsC+X HKRr3NlRM/DygzTyj0gN74uoa0goCIbyAQhiT42nm0cuhM7uN/W0ayrlZjGF1cbR 8NCJUL2Nwj0ywKIavC99Ipkb8AsFwpVT6U6effs6 =xybZ ------END PGP SIGNATURE----- -`, commitFromReader.Signature.Signature) - assert.EqualValues(t, `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e +-----END PGP SIGNATURE-----`, commitFromReader.Signature.Signature) + assert.Equal(t, `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e parent 26e9ccc29fad747e9c5d9f4c9ddeb7eff61cc45ef6a8dc258cbeb181afc055e8 author Adam Majer <amajer@suse.de> 1698676906 +0100 committer Adam Majer <amajer@suse.de> 1698676906 +0100 signed commit`, commitFromReader.Signature.Payload) - assert.EqualValues(t, "Adam Majer <amajer@suse.de>", commitFromReader.Author.String()) + assert.Equal(t, "Adam Majer <amajer@suse.de>", commitFromReader.Author.String()) commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n")) assert.NoError(t, err) commitFromReader.CommitMessage += "\n\n" commitFromReader.Signature.Payload += "\n\n" - assert.EqualValues(t, commitFromReader, commitFromReader2) + assert.Equal(t, commitFromReader, commitFromReader2) } func TestHasPreviousCommitSha256(t *testing.T) { diff --git a/modules/git/commit_submodule_file.go b/modules/git/commit_submodule_file.go index 2ac744fbf6..729401f752 100644 --- a/modules/git/commit_submodule_file.go +++ b/modules/git/commit_submodule_file.go @@ -46,9 +46,9 @@ func (sf *CommitSubmoduleFile) SubmoduleWebLink(ctx context.Context, optCommitID if len(optCommitID) == 2 { commitLink = sf.repoLink + "/compare/" + optCommitID[0] + "..." + optCommitID[1] } else if len(optCommitID) == 1 { - commitLink = sf.repoLink + "/commit/" + optCommitID[0] + commitLink = sf.repoLink + "/tree/" + optCommitID[0] } else { - commitLink = sf.repoLink + "/commit/" + sf.refID + commitLink = sf.repoLink + "/tree/" + sf.refID } return &SubmoduleWebLink{RepoWebLink: sf.repoLink, CommitWebLink: commitLink} } diff --git a/modules/git/commit_submodule_file_test.go b/modules/git/commit_submodule_file_test.go index 4b5b767612..6581fa8712 100644 --- a/modules/git/commit_submodule_file_test.go +++ b/modules/git/commit_submodule_file_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "testing" "github.com/stretchr/testify/assert" @@ -13,18 +12,18 @@ import ( func TestCommitSubmoduleLink(t *testing.T) { sf := NewCommitSubmoduleFile("git@github.com:user/repo.git", "aaaa") - wl := sf.SubmoduleWebLink(context.Background()) + wl := sf.SubmoduleWebLink(t.Context()) assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) - assert.Equal(t, "https://github.com/user/repo/commit/aaaa", wl.CommitWebLink) + assert.Equal(t, "https://github.com/user/repo/tree/aaaa", wl.CommitWebLink) - wl = sf.SubmoduleWebLink(context.Background(), "1111") + wl = sf.SubmoduleWebLink(t.Context(), "1111") assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) - assert.Equal(t, "https://github.com/user/repo/commit/1111", wl.CommitWebLink) + assert.Equal(t, "https://github.com/user/repo/tree/1111", wl.CommitWebLink) - wl = sf.SubmoduleWebLink(context.Background(), "1111", "2222") + wl = sf.SubmoduleWebLink(t.Context(), "1111", "2222") assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) assert.Equal(t, "https://github.com/user/repo/compare/1111...2222", wl.CommitWebLink) - wl = (*CommitSubmoduleFile)(nil).SubmoduleWebLink(context.Background()) + wl = (*CommitSubmoduleFile)(nil).SubmoduleWebLink(t.Context()) assert.Nil(t, wl) } diff --git a/modules/git/commit_test.go b/modules/git/commit_test.go index 9560c2cd94..81fb91dfc6 100644 --- a/modules/git/commit_test.go +++ b/modules/git/commit_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "os" "path/filepath" "strings" @@ -60,8 +59,7 @@ func TestGetFullCommitIDError(t *testing.T) { } func TestCommitFromReader(t *testing.T) { - commitString := `feaf4ba6bc635fec442f46ddd4512416ec43c2c2 commit 1074 -tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 + commitString := `tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 parent 37991dec2c8e592043f47155ce4808d4580f9123 author silverwind <me@silverwind.io> 1563741793 +0200 committer silverwind <me@silverwind.io> 1563741793 +0200 @@ -94,7 +92,7 @@ empty commit` assert.NoError(t, err) require.NotNil(t, commitFromReader) assert.EqualValues(t, sha, commitFromReader.ID) - assert.EqualValues(t, `-----BEGIN PGP SIGNATURE----- + assert.Equal(t, `-----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEEWPb2jX6FS2mqyJRQLmK0HJOGlEMFAl00zmEACgkQLmK0HJOG lEMDFBAAhQKKqLD1VICygJMEB8t1gBmNLgvziOLfpX4KPWdPtBk3v/QJ7OrfMrVK @@ -109,26 +107,24 @@ sD53z/f0J+We4VZjY+pidvA9BGZPFVdR3wd3xGs8/oH6UWaLJAMGkLG6dDb3qDLm mfeFhT57UbE4qukTDIQ0Y0WM40UYRTakRaDY7ubhXgLgx09Cnp9XTVMsHgT6j9/i 1pxsB104XLWjQHTjr1JtiaBQEwFh9r2OKTcpvaLcbNtYpo7CzOs= =FRsO ------END PGP SIGNATURE----- -`, commitFromReader.Signature.Signature) - assert.EqualValues(t, `tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 +-----END PGP SIGNATURE-----`, commitFromReader.Signature.Signature) + assert.Equal(t, `tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 parent 37991dec2c8e592043f47155ce4808d4580f9123 author silverwind <me@silverwind.io> 1563741793 +0200 committer silverwind <me@silverwind.io> 1563741793 +0200 empty commit`, commitFromReader.Signature.Payload) - assert.EqualValues(t, "silverwind <me@silverwind.io>", commitFromReader.Author.String()) + assert.Equal(t, "silverwind <me@silverwind.io>", commitFromReader.Author.String()) commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n")) assert.NoError(t, err) commitFromReader.CommitMessage += "\n\n" commitFromReader.Signature.Payload += "\n\n" - assert.EqualValues(t, commitFromReader, commitFromReader2) + assert.Equal(t, commitFromReader, commitFromReader2) } func TestCommitWithEncodingFromReader(t *testing.T) { - commitString := `feaf4ba6bc635fec442f46ddd4512416ec43c2c2 commit 1074 -tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 + commitString := `tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 parent 47b24e7ab977ed31c5a39989d570847d6d0052af author KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 committer KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 @@ -160,7 +156,7 @@ ISO-8859-1` assert.NoError(t, err) require.NotNil(t, commitFromReader) assert.EqualValues(t, sha, commitFromReader.ID) - assert.EqualValues(t, `-----BEGIN PGP SIGNATURE----- + assert.Equal(t, `-----BEGIN PGP SIGNATURE----- iQGzBAABCgAdFiEE9HRrbqvYxPT8PXbefPSEkrowAa8FAmYGg7IACgkQfPSEkrow Aa9olwv+P0HhtCM6CRvlUmPaqswRsDPNR4i66xyXGiSxdI9V5oJL7HLiQIM7KrFR @@ -173,22 +169,21 @@ SONRzusmu5n3DgV956REL7x62h7JuqmBz/12HZkr0z0zgXkcZ04q08pSJATX5N1F yN+tWxTsWg+zhDk96d5Esdo9JMjcFvPv0eioo30GAERaz1hoD7zCMT4jgUFTQwgz jw4YcO5u =r3UU ------END PGP SIGNATURE----- -`, commitFromReader.Signature.Signature) - assert.EqualValues(t, `tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 +-----END PGP SIGNATURE-----`, commitFromReader.Signature.Signature) + assert.Equal(t, `tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 parent 47b24e7ab977ed31c5a39989d570847d6d0052af author KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 committer KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 encoding ISO-8859-1 ISO-8859-1`, commitFromReader.Signature.Payload) - assert.EqualValues(t, "KN4CK3R <admin@oldschoolhack.me>", commitFromReader.Author.String()) + assert.Equal(t, "KN4CK3R <admin@oldschoolhack.me>", commitFromReader.Author.String()) commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n")) assert.NoError(t, err) commitFromReader.CommitMessage += "\n\n" commitFromReader.Signature.Payload += "\n\n" - assert.EqualValues(t, commitFromReader, commitFromReader2) + assert.Equal(t, commitFromReader, commitFromReader2) } func TestHasPreviousCommit(t *testing.T) { @@ -347,15 +342,15 @@ func TestGetCommitFileStatusMerges(t *testing.T) { func Test_GetCommitBranchStart(t *testing.T) { bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") - repo, err := OpenRepository(context.Background(), bareRepo1Path) + repo, err := OpenRepository(t.Context(), bareRepo1Path) assert.NoError(t, err) defer repo.Close() commit, err := repo.GetBranchCommit("branch1") assert.NoError(t, err) - assert.EqualValues(t, "2839944139e0de9737a044f78b0e4b40d989a9e3", commit.ID.String()) + assert.Equal(t, "2839944139e0de9737a044f78b0e4b40d989a9e3", commit.ID.String()) startCommitID, err := repo.GetCommitBranchStart(os.Environ(), "branch1", commit.ID.String()) assert.NoError(t, err) assert.NotEmpty(t, startCommitID) - assert.EqualValues(t, "9c9aef8dd84e02bc7ec12641deb4c930a7c30185", startCommitID) + assert.Equal(t, "95bb4d39648ee7e325106df01a621c530863a653", startCommitID) } diff --git a/modules/git/config.go b/modules/git/config.go index 9c36cf1654..234be7b955 100644 --- a/modules/git/config.go +++ b/modules/git/config.go @@ -116,7 +116,7 @@ func syncGitConfig() (err error) { } func configSet(key, value string) error { - stdout, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil) + stdout, _, err := NewCommand("config", "--global", "--get").AddDynamicArguments(key).RunStdString(DefaultContext, nil) if err != nil && !IsErrorExitCode(err, 1) { return fmt.Errorf("failed to get git config %s, err: %w", key, err) } @@ -126,7 +126,7 @@ func configSet(key, value string) error { return nil } - _, _, err = NewCommand(DefaultContext, "config", "--global").AddDynamicArguments(key, value).RunStdString(nil) + _, _, err = NewCommand("config", "--global").AddDynamicArguments(key, value).RunStdString(DefaultContext, nil) if err != nil { return fmt.Errorf("failed to set git global config %s, err: %w", key, err) } @@ -135,14 +135,14 @@ func configSet(key, value string) error { } func configSetNonExist(key, value string) error { - _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil) + _, _, err := NewCommand("config", "--global", "--get").AddDynamicArguments(key).RunStdString(DefaultContext, nil) if err == nil { // already exist return nil } if IsErrorExitCode(err, 1) { // not exist, set new config - _, _, err = NewCommand(DefaultContext, "config", "--global").AddDynamicArguments(key, value).RunStdString(nil) + _, _, err = NewCommand("config", "--global").AddDynamicArguments(key, value).RunStdString(DefaultContext, nil) if err != nil { return fmt.Errorf("failed to set git global config %s, err: %w", key, err) } @@ -153,14 +153,14 @@ func configSetNonExist(key, value string) error { } func configAddNonExist(key, value string) error { - _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil) + _, _, err := NewCommand("config", "--global", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(DefaultContext, nil) if err == nil { // already exist return nil } if IsErrorExitCode(err, 1) { // not exist, add new config - _, _, err = NewCommand(DefaultContext, "config", "--global", "--add").AddDynamicArguments(key, value).RunStdString(nil) + _, _, err = NewCommand("config", "--global", "--add").AddDynamicArguments(key, value).RunStdString(DefaultContext, nil) if err != nil { return fmt.Errorf("failed to add git global config %s, err: %w", key, err) } @@ -170,10 +170,10 @@ func configAddNonExist(key, value string) error { } func configUnsetAll(key, value string) error { - _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil) + _, _, err := NewCommand("config", "--global", "--get").AddDynamicArguments(key).RunStdString(DefaultContext, nil) if err == nil { // exist, need to remove - _, _, err = NewCommand(DefaultContext, "config", "--global", "--unset-all").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil) + _, _, err = NewCommand("config", "--global", "--unset-all").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(DefaultContext, nil) if err != nil { return fmt.Errorf("failed to unset git global config %s, err: %w", key, err) } diff --git a/modules/git/diff.go b/modules/git/diff.go index da0a2f26ba..c4df6b8063 100644 --- a/modules/git/diff.go +++ b/modules/git/diff.go @@ -34,8 +34,8 @@ func GetRawDiff(repo *Repository, commitID string, diffType RawDiffType, writer // GetReverseRawDiff dumps the reverse diff results of repository in given commit ID to io.Writer. func GetReverseRawDiff(ctx context.Context, repoPath, commitID string, writer io.Writer) error { stderr := new(bytes.Buffer) - cmd := NewCommand(ctx, "show", "--pretty=format:revert %H%n", "-R").AddDynamicArguments(commitID) - if err := cmd.Run(&RunOpts{ + cmd := NewCommand("show", "--pretty=format:revert %H%n", "-R").AddDynamicArguments(commitID) + if err := cmd.Run(ctx, &RunOpts{ Dir: repoPath, Stdout: writer, Stderr: stderr, @@ -56,7 +56,7 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff files = append(files, file) } - cmd := NewCommand(repo.Ctx) + cmd := NewCommand() switch diffType { case RawDiffNormal: if len(startCommit) != 0 { @@ -89,7 +89,7 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff } stderr := new(bytes.Buffer) - if err = cmd.Run(&RunOpts{ + if err = cmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: writer, Stderr: stderr, @@ -301,8 +301,8 @@ func GetAffectedFiles(repo *Repository, branchName, oldCommitID, newCommitID str affectedFiles := make([]string, 0, 32) // Run `git diff --name-only` to get the names of the changed files - err = NewCommand(repo.Ctx, "diff", "--name-only").AddDynamicArguments(oldCommitID, newCommitID). - Run(&RunOpts{ + err = NewCommand("diff", "--name-only").AddDynamicArguments(oldCommitID, newCommitID). + Run(repo.Ctx, &RunOpts{ Env: env, Dir: repo.Path, Stdout: stdoutWriter, diff --git a/modules/git/diff_test.go b/modules/git/diff_test.go index 0f865c52a8..7671fffcc1 100644 --- a/modules/git/diff_test.go +++ b/modules/git/diff_test.go @@ -154,7 +154,7 @@ func TestCutDiffAroundLine(t *testing.T) { } func BenchmarkCutDiffAroundLine(b *testing.B) { - for n := 0; n < b.N; n++ { + for b.Loop() { CutDiffAroundLine(strings.NewReader(exampleDiff), 3, true, 3) } } @@ -177,8 +177,8 @@ func ExampleCutDiffAroundLine() { func TestParseDiffHunkString(t *testing.T) { leftLine, leftHunk, rightLine, rightHunk := ParseDiffHunkString("@@ -19,3 +19,5 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER") - assert.EqualValues(t, 19, leftLine) - assert.EqualValues(t, 3, leftHunk) - assert.EqualValues(t, 19, rightLine) - assert.EqualValues(t, 5, rightHunk) + assert.Equal(t, 19, leftLine) + assert.Equal(t, 3, leftHunk) + assert.Equal(t, 19, rightLine) + assert.Equal(t, 5, rightHunk) } diff --git a/modules/git/error.go b/modules/git/error.go index 10fb37be07..7d131345d0 100644 --- a/modules/git/error.go +++ b/modules/git/error.go @@ -32,22 +32,6 @@ func (err ErrNotExist) Unwrap() error { return util.ErrNotExist } -// ErrBadLink entry.FollowLink error -type ErrBadLink struct { - Name string - Message string -} - -func (err ErrBadLink) Error() string { - return fmt.Sprintf("%s: %s", err.Name, err.Message) -} - -// IsErrBadLink if some error is ErrBadLink -func IsErrBadLink(err error) bool { - _, ok := err.(ErrBadLink) - return ok -} - // ErrBranchNotExist represents a "BranchNotExist" kind of error. type ErrBranchNotExist struct { Name string diff --git a/modules/git/foreachref/format.go b/modules/git/foreachref/format.go index 97e8ee4724..d9573a55d6 100644 --- a/modules/git/foreachref/format.go +++ b/modules/git/foreachref/format.go @@ -76,7 +76,7 @@ func (f Format) Parser(r io.Reader) *Parser { // would turn into "%0a%00". func (f Format) hexEscaped(delim []byte) string { escaped := "" - for i := 0; i < len(delim); i++ { + for i := range delim { escaped += "%" + hex.EncodeToString([]byte{delim[i]}) } return escaped diff --git a/modules/git/fsck.go b/modules/git/fsck.go index cec27f165b..a52684c84f 100644 --- a/modules/git/fsck.go +++ b/modules/git/fsck.go @@ -10,5 +10,5 @@ import ( // Fsck verifies the connectivity and validity of the objects in the database func Fsck(ctx context.Context, repoPath string, timeout time.Duration, args TrustedCmdArgs) error { - return NewCommand(ctx, "fsck").AddArguments(args...).Run(&RunOpts{Timeout: timeout, Dir: repoPath}) + return NewCommand("fsck").AddArguments(args...).Run(ctx, &RunOpts{Timeout: timeout, Dir: repoPath}) } diff --git a/modules/git/git.go b/modules/git/git.go index e3e5b83274..a2ffd6d289 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -30,6 +30,7 @@ type Features struct { SupportProcReceive bool // >= 2.29 SupportHashSha256 bool // >= 2.42, SHA-256 repositories no longer an ‘experimental curiosity’ SupportedObjectFormats []ObjectFormat // sha1, sha256 + SupportCheckAttrOnBare bool // >= 2.40 } var ( @@ -60,7 +61,7 @@ func DefaultFeatures() *Features { } func loadGitVersionFeatures() (*Features, error) { - stdout, _, runErr := NewCommand(DefaultContext, "version").RunStdString(nil) + stdout, _, runErr := NewCommand("version").RunStdString(DefaultContext, nil) if runErr != nil { return nil, runErr } @@ -77,6 +78,7 @@ func loadGitVersionFeatures() (*Features, error) { if features.SupportHashSha256 { features.SupportedObjectFormats = append(features.SupportedObjectFormats, Sha256ObjectFormat) } + features.SupportCheckAttrOnBare = features.CheckVersionAtLeast("2.40") return features, nil } diff --git a/modules/git/git_test.go b/modules/git/git_test.go index 5472842b76..58ba01cabc 100644 --- a/modules/git/git_test.go +++ b/modules/git/git_test.go @@ -10,18 +10,19 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/tempdir" "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" ) func testRun(m *testing.M) error { - gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + gitHomePath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("git-home") if err != nil { return fmt.Errorf("unable to create temp dir: %w", err) } - defer util.RemoveAll(gitHomePath) + defer cleanup() + setting.Git.HomePath = gitHomePath if err = InitFull(context.Background()); err != nil { diff --git a/modules/git/grep.go b/modules/git/grep.go index bf6b41a886..66711650c9 100644 --- a/modules/git/grep.go +++ b/modules/git/grep.go @@ -23,11 +23,19 @@ type GrepResult struct { LineCodes []string } +type GrepModeType string + +const ( + GrepModeExact GrepModeType = "exact" + GrepModeWords GrepModeType = "words" + GrepModeRegexp GrepModeType = "regexp" +) + type GrepOptions struct { RefName string MaxResultLimit int ContextLineNumber int - IsFuzzy bool + GrepMode GrepModeType MaxLineLength int // the maximum length of a line to parse, exceeding chars will be truncated PathspecList []string } @@ -52,21 +60,30 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO 2^@repo: go-gitea/gitea */ var results []*GrepResult - cmd := NewCommand(ctx, "grep", "--null", "--break", "--heading", "--fixed-strings", "--line-number", "--ignore-case", "--full-name") - cmd.AddOptionValues("--context", fmt.Sprint(opts.ContextLineNumber)) - if opts.IsFuzzy { + cmd := NewCommand("grep", "--null", "--break", "--heading", "--line-number", "--full-name") + cmd.AddOptionValues("--context", strconv.Itoa(opts.ContextLineNumber)) + switch opts.GrepMode { + case GrepModeExact: + cmd.AddArguments("--fixed-strings") + cmd.AddOptionValues("-e", strings.TrimLeft(search, "-")) + case GrepModeRegexp: + cmd.AddArguments("--perl-regexp") + cmd.AddOptionValues("-e", strings.TrimLeft(search, "-")) + default: /* words */ words := strings.Fields(search) - for _, word := range words { + cmd.AddArguments("--fixed-strings", "--ignore-case") + for i, word := range words { cmd.AddOptionValues("-e", strings.TrimLeft(word, "-")) + if i < len(words)-1 { + cmd.AddOptionValues("--and") + } } - } else { - cmd.AddOptionValues("-e", strings.TrimLeft(search, "-")) } cmd.AddDynamicArguments(util.IfZero(opts.RefName, "HEAD")) cmd.AddDashesAndList(opts.PathspecList...) opts.MaxResultLimit = util.IfZero(opts.MaxResultLimit, 50) stderr := bytes.Buffer{} - err = cmd.Run(&RunOpts{ + err = cmd.Run(ctx, &RunOpts{ Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr, diff --git a/modules/git/grep_test.go b/modules/git/grep_test.go index 005d539726..0dce464b7c 100644 --- a/modules/git/grep_test.go +++ b/modules/git/grep_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "path/filepath" "testing" @@ -16,7 +15,7 @@ func TestGrepSearch(t *testing.T) { assert.NoError(t, err) defer repo.Close() - res, err := GrepSearch(context.Background(), repo, "void", GrepOptions{}) + res, err := GrepSearch(t.Context(), repo, "void", GrepOptions{}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -31,7 +30,7 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "void", GrepOptions{PathspecList: []string{":(glob)java-hello/*"}}) + res, err = GrepSearch(t.Context(), repo, "void", GrepOptions{PathspecList: []string{":(glob)java-hello/*"}}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -41,7 +40,7 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "void", GrepOptions{PathspecList: []string{":(glob,exclude)java-hello/*"}}) + res, err = GrepSearch(t.Context(), repo, "void", GrepOptions{PathspecList: []string{":(glob,exclude)java-hello/*"}}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -51,7 +50,7 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "void", GrepOptions{MaxResultLimit: 1}) + res, err = GrepSearch(t.Context(), repo, "void", GrepOptions{MaxResultLimit: 1}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -61,7 +60,7 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "void", GrepOptions{MaxResultLimit: 1, MaxLineLength: 39}) + res, err = GrepSearch(t.Context(), repo, "void", GrepOptions{MaxResultLimit: 1, MaxLineLength: 39}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -71,11 +70,11 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "no-such-content", GrepOptions{}) + res, err = GrepSearch(t.Context(), repo, "no-such-content", GrepOptions{}) assert.NoError(t, err) assert.Empty(t, res) - res, err = GrepSearch(context.Background(), &Repository{Path: "no-such-git-repo"}, "no-such-content", GrepOptions{}) + res, err = GrepSearch(t.Context(), &Repository{Path: "no-such-git-repo"}, "no-such-content", GrepOptions{}) assert.Error(t, err) assert.Empty(t, res) } diff --git a/modules/git/hook.go b/modules/git/hook.go index 46f93ce13e..548a59971d 100644 --- a/modules/git/hook.go +++ b/modules/git/hook.go @@ -7,11 +7,10 @@ package git import ( "errors" "os" - "path" "path/filepath" + "slices" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -27,12 +26,7 @@ var ErrNotValidHook = errors.New("not a valid Git hook") // IsValidHookName returns true if given name is a valid Git hook. func IsValidHookName(name string) bool { - for _, hn := range hookNames { - if hn == name { - return true - } - } - return false + return slices.Contains(hookNames, name) } // Hook represents a Git hook. @@ -51,17 +45,28 @@ func GetHook(repoPath, name string) (*Hook, error) { } h := &Hook{ name: name, - path: path.Join(repoPath, "hooks", name+".d", name), + path: filepath.Join(repoPath, "hooks", name+".d", name), } - samplePath := filepath.Join(repoPath, "hooks", name+".sample") - if isFile(h.path) { + isFile, err := util.IsFile(h.path) + if err != nil { + return nil, err + } + if isFile { data, err := os.ReadFile(h.path) if err != nil { return nil, err } h.IsActive = true h.Content = string(data) - } else if isFile(samplePath) { + return h, nil + } + + samplePath := filepath.Join(repoPath, "hooks", name+".sample") + isFile, err = util.IsFile(samplePath) + if err != nil { + return nil, err + } + if isFile { data, err := os.ReadFile(samplePath) if err != nil { return nil, err @@ -79,7 +84,11 @@ func (h *Hook) Name() string { // Update updates hook settings. func (h *Hook) Update() error { if len(strings.TrimSpace(h.Content)) == 0 { - if isExist(h.path) { + exist, err := util.IsExist(h.path) + if err != nil { + return err + } + if exist { err := util.Remove(h.path) if err != nil { return err @@ -103,7 +112,10 @@ func (h *Hook) Update() error { // ListHooks returns a list of Git hooks of given repository. func ListHooks(repoPath string) (_ []*Hook, err error) { - if !isDir(path.Join(repoPath, "hooks")) { + exist, err := util.IsDir(filepath.Join(repoPath, "hooks")) + if err != nil { + return nil, err + } else if !exist { return nil, errors.New("hooks path does not exist") } @@ -116,28 +128,3 @@ func ListHooks(repoPath string) (_ []*Hook, err error) { } return hooks, nil } - -const ( - // HookPathUpdate hook update path - HookPathUpdate = "hooks/update" -) - -// SetUpdateHook writes given content to update hook of the repository. -func SetUpdateHook(repoPath, content string) (err error) { - log.Debug("Setting update hook: %s", repoPath) - hookPath := path.Join(repoPath, HookPathUpdate) - isExist, err := util.IsExist(hookPath) - if err != nil { - log.Debug("Unable to check if %s exists. Error: %v", hookPath, err) - return err - } - if isExist { - err = util.Remove(hookPath) - } else { - err = os.MkdirAll(path.Dir(hookPath), os.ModePerm) - } - if err != nil { - return err - } - return os.WriteFile(hookPath, []byte(content), 0o777) -} diff --git a/modules/git/key.go b/modules/git/key.go new file mode 100644 index 0000000000..2513c048b7 --- /dev/null +++ b/modules/git/key.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +// Based on https://git-scm.com/docs/git-config#Documentation/git-config.txt-gpgformat +const ( + SigningKeyFormatOpenPGP = "openpgp" // for GPG keys, the expected default of git cli + SigningKeyFormatSSH = "ssh" +) + +type SigningKey struct { + KeyID string + Format string +} diff --git a/modules/git/repo_language_stats.go b/modules/git/languagestats/language_stats.go index 8551ea9d24..a71284c3e4 100644 --- a/modules/git/repo_language_stats.go +++ b/modules/git/languagestats/language_stats.go @@ -1,13 +1,15 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package git +package languagestats import ( + "context" "strings" "unicode" - "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" ) const ( @@ -49,19 +51,15 @@ func mergeLanguageStats(stats map[string]int64) map[string]int64 { return res } -func TryReadLanguageAttribute(attrs map[string]string) optional.Option[string] { - language := AttributeToString(attrs, AttributeLinguistLanguage) - if language.Value() == "" { - language = AttributeToString(attrs, AttributeGitlabLanguage) - if language.Has() { - raw := language.Value() - // gitlab-language may have additional parameters after the language - // ignore them and just use the main language - // https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type - if idx := strings.IndexByte(raw, '?'); idx >= 0 { - language = optional.Some(raw[:idx]) - } - } +// GetFileLanguage tries to get the (linguist) language of the file content +func GetFileLanguage(ctx context.Context, gitRepo *git.Repository, treeish, treePath string) (string, error) { + attributesMap, err := attribute.CheckAttributes(ctx, gitRepo, treeish, attribute.CheckAttributeOpts{ + Attributes: []string{attribute.LinguistLanguage, attribute.GitlabLanguage}, + Filenames: []string{treePath}, + }) + if err != nil { + return "", err } - return language + + return attributesMap[treePath].GetLanguage().Value(), nil } diff --git a/modules/git/repo_language_stats_gogit.go b/modules/git/languagestats/language_stats_gogit.go index a34c03c781..418c05b157 100644 --- a/modules/git/repo_language_stats_gogit.go +++ b/modules/git/languagestats/language_stats_gogit.go @@ -3,13 +3,15 @@ //go:build gogit -package git +package languagestats import ( "bytes" "io" "code.gitea.io/gitea/modules/analyze" + git_module "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/optional" "github.com/go-enry/go-enry/v2" @@ -19,7 +21,7 @@ import ( ) // GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { +func GetLanguageStats(repo *git_module.Repository, commitID string) (map[string]int64, error) { r, err := git.PlainOpen(repo.Path) if err != nil { return nil, err @@ -40,8 +42,11 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } - checker, deferable := repo.CheckAttributeReader(commitID) - defer deferable() + checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes) + if err != nil { + return nil, err + } + defer checker.Close() // sizes contains the current calculated size of all files by language sizes := make(map[string]int64) @@ -62,43 +67,41 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err isDocumentation := optional.None[bool]() isDetectable := optional.None[bool]() - if checker != nil { - attrs, err := checker.CheckPath(f.Name) - if err == nil { - isVendored = AttributeToBool(attrs, AttributeLinguistVendored) - if isVendored.ValueOrDefault(false) { - return nil - } - - isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) - if isGenerated.ValueOrDefault(false) { - return nil - } + attrs, err := checker.CheckPath(f.Name) + if err == nil { + isVendored = attrs.GetVendored() + if isVendored.ValueOrDefault(false) { + return nil + } - isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) - if isDocumentation.ValueOrDefault(false) { - return nil - } + isGenerated = attrs.GetGenerated() + if isGenerated.ValueOrDefault(false) { + return nil + } - isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) - if !isDetectable.ValueOrDefault(true) { - return nil - } + isDocumentation = attrs.GetDocumentation() + if isDocumentation.ValueOrDefault(false) { + return nil + } - hasLanguage := TryReadLanguageAttribute(attrs) - if hasLanguage.Value() != "" { - language := hasLanguage.Value() + isDetectable = attrs.GetDetectable() + if !isDetectable.ValueOrDefault(true) { + return nil + } - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } + hasLanguage := attrs.GetLanguage() + if hasLanguage.Value() != "" { + language := hasLanguage.Value() - // this language will always be added to the size - sizes[language] += f.Size - return nil + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if len(group) != 0 { + language = group } + + // this language will always be added to the size + sizes[language] += f.Size + return nil } } diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/languagestats/language_stats_nogogit.go index de7707bd6c..94cf9fff8c 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/languagestats/language_stats_nogogit.go @@ -3,13 +3,15 @@ //go:build !gogit -package git +package languagestats import ( "bytes" "io" "code.gitea.io/gitea/modules/analyze" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" @@ -17,7 +19,7 @@ import ( ) // GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { +func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64, error) { // We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. // so let's create a batch stdin and stdout batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx) @@ -34,19 +36,19 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if err := writeID(commitID); err != nil { return nil, err } - shaBytes, typ, size, err := ReadBatchLine(batchReader) + shaBytes, typ, size, err := git.ReadBatchLine(batchReader) if typ != "commit" { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} + return nil, git.ErrNotExist{ID: commitID} } - sha, err := NewIDFromString(string(shaBytes)) + sha, err := git.NewIDFromString(string(shaBytes)) if err != nil { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} + return nil, git.ErrNotExist{ID: commitID} } - commit, err := CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) + commit, err := git.CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) if err != nil { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) return nil, err @@ -62,8 +64,11 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } - checker, deferable := repo.CheckAttributeReader(commitID) - defer deferable() + checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes) + if err != nil { + return nil, err + } + defer checker.Close() contentBuf := bytes.Buffer{} var content []byte @@ -92,47 +97,40 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err } isVendored := optional.None[bool]() - isGenerated := optional.None[bool]() isDocumentation := optional.None[bool]() isDetectable := optional.None[bool]() - if checker != nil { - attrs, err := checker.CheckPath(f.Name()) - if err == nil { - isVendored = AttributeToBool(attrs, AttributeLinguistVendored) - if isVendored.ValueOrDefault(false) { - continue - } - - isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) - if isGenerated.ValueOrDefault(false) { - continue - } + attrs, err := checker.CheckPath(f.Name()) + attrLinguistGenerated := optional.None[bool]() + if err == nil { + if isVendored = attrs.GetVendored(); isVendored.ValueOrDefault(false) { + continue + } - isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) - if isDocumentation.ValueOrDefault(false) { - continue - } + if attrLinguistGenerated = attrs.GetGenerated(); attrLinguistGenerated.ValueOrDefault(false) { + continue + } - isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) - if !isDetectable.ValueOrDefault(true) { - continue - } + if isDocumentation = attrs.GetDocumentation(); isDocumentation.ValueOrDefault(false) { + continue + } - hasLanguage := TryReadLanguageAttribute(attrs) - if hasLanguage.Value() != "" { - language := hasLanguage.Value() + if isDetectable = attrs.GetDetectable(); !isDetectable.ValueOrDefault(true) { + continue + } - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } + if hasLanguage := attrs.GetLanguage(); hasLanguage.Value() != "" { + language := hasLanguage.Value() - // this language will always be added to the size - sizes[language] += f.Size() - continue + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if len(group) != 0 { + language = group } + + // this language will always be added to the size + sizes[language] += f.Size() + continue } } @@ -149,7 +147,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if err := writeID(f.ID.String()); err != nil { return nil, err } - _, _, size, err := ReadBatchLine(batchReader) + _, _, size, err := git.ReadBatchLine(batchReader) if err != nil { log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err) return nil, err @@ -167,11 +165,19 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } content = contentBuf.Bytes() - if err := DiscardFull(batchReader, discard); err != nil { + if err := git.DiscardFull(batchReader, discard); err != nil { return nil, err } } - if !isGenerated.Has() && enry.IsGenerated(f.Name(), content) { + + // if "generated" attribute is set, use it, otherwise use enry.IsGenerated to guess + var isGenerated bool + if attrLinguistGenerated.Has() { + isGenerated = attrLinguistGenerated.Value() + } else { + isGenerated = enry.IsGenerated(f.Name(), content) + } + if isGenerated { continue } diff --git a/modules/git/repo_language_stats_test.go b/modules/git/languagestats/language_stats_test.go index 1ee5f4c3af..b908ae6413 100644 --- a/modules/git/repo_language_stats_test.go +++ b/modules/git/languagestats/language_stats_test.go @@ -3,34 +3,36 @@ //go:build !gogit -package git +package languagestats import ( - "path/filepath" "testing" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRepository_GetLanguageStats(t *testing.T) { - repoPath := filepath.Join(testReposDir, "language_stats_repo") - gitRepo, err := openRepositoryWithDefaultContext(repoPath) + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) require.NoError(t, err) - defer gitRepo.Close() - stats, err := gitRepo.GetLanguageStats("8fee858da5796dfb37704761701bb8e800ad9ef3") + stats, err := GetLanguageStats(gitRepo, "8fee858da5796dfb37704761701bb8e800ad9ef3") require.NoError(t, err) - assert.EqualValues(t, map[string]int64{ + assert.Equal(t, map[string]int64{ "Python": 134, "Java": 112, }, stats) } func TestMergeLanguageStats(t *testing.T) { - assert.EqualValues(t, map[string]int64{ + assert.Equal(t, map[string]int64{ "PHP": 1, "python": 10, "JAVA": 700, diff --git a/modules/git/languagestats/main_test.go b/modules/git/languagestats/main_test.go new file mode 100644 index 0000000000..707d268c81 --- /dev/null +++ b/modules/git/languagestats/main_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package languagestats + +import ( + "context" + "fmt" + "os" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func testRun(m *testing.M) error { + gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + if err != nil { + return fmt.Errorf("unable to create temp dir: %w", err) + } + defer util.RemoveAll(gitHomePath) + setting.Git.HomePath = gitHomePath + + if err = git.InitFull(context.Background()); err != nil { + return fmt.Errorf("failed to call Init: %w", err) + } + + exitCode := m.Run() + if exitCode != 0 { + return fmt.Errorf("run test failed, ExitCode=%d", exitCode) + } + return nil +} + +func TestMain(m *testing.M) { + if err := testRun(m); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Test failed: %v", err) + os.Exit(1) + } +} diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index cf9c10d7b4..cff2556083 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -13,7 +13,7 @@ import ( ) func getCacheKey(repoPath, commitID, entryPath string) string { - hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, commitID, entryPath))) + hashBytes := sha256.Sum256(fmt.Appendf(nil, "%s:%s:%s", repoPath, commitID, entryPath)) return fmt.Sprintf("last_commit:%x", hashBytes) } diff --git a/modules/git/log_name_status.go b/modules/git/log_name_status.go index 1fd58abfcd..dfdef38ef9 100644 --- a/modules/git/log_name_status.go +++ b/modules/git/log_name_status.go @@ -34,7 +34,7 @@ func LogNameStatusRepo(ctx context.Context, repository, head, treepath string, p _ = stdoutWriter.Close() } - cmd := NewCommand(ctx) + cmd := NewCommand() cmd.AddArguments("log", "--name-status", "-c", "--format=commit%x00%H %P%x00", "--parents", "--no-renames", "-t", "-z").AddDynamicArguments(head) var files []string @@ -64,7 +64,7 @@ func LogNameStatusRepo(ctx context.Context, repository, head, treepath string, p go func() { stderr := strings.Builder{} - err := cmd.Run(&RunOpts{ + err := cmd.Run(ctx, &RunOpts{ Dir: repository, Stdout: stdoutWriter, Stderr: &stderr, @@ -118,11 +118,12 @@ func (g *LogNameStatusRepoParser) Next(treepath string, paths2ids map[string]int g.buffull = false g.next, err = g.rd.ReadSlice('\x00') if err != nil { - if err == bufio.ErrBufferFull { + switch err { + case bufio.ErrBufferFull: g.buffull = true - } else if err == io.EOF { + case io.EOF: return nil, nil - } else { + default: return nil, err } } @@ -132,11 +133,12 @@ func (g *LogNameStatusRepoParser) Next(treepath string, paths2ids map[string]int if bytes.Equal(g.next, []byte("commit\000")) { g.next, err = g.rd.ReadSlice('\x00') if err != nil { - if err == bufio.ErrBufferFull { + switch err { + case bufio.ErrBufferFull: g.buffull = true - } else if err == io.EOF { + case io.EOF: return nil, nil - } else { + default: return nil, err } } @@ -214,11 +216,12 @@ diffloop: } g.next, err = g.rd.ReadSlice('\x00') if err != nil { - if err == bufio.ErrBufferFull { + switch err { + case bufio.ErrBufferFull: g.buffull = true - } else if err == io.EOF { + case io.EOF: return &ret, nil - } else { + default: return nil, err } } @@ -343,10 +346,7 @@ func WalkGitLog(ctx context.Context, repo *Repository, head *Commit, treepath st results := make([]string, len(paths)) remaining := len(paths) - nextRestart := (len(paths) * 3) / 4 - if nextRestart > 70 { - nextRestart = 70 - } + nextRestart := min((len(paths)*3)/4, 70) lastEmptyParent := head.ID.String() commitSinceLastEmptyParent := uint64(0) commitSinceNextRestart := uint64(0) diff --git a/modules/git/notes_test.go b/modules/git/notes_test.go index 267671d8fa..ca05a9e525 100644 --- a/modules/git/notes_test.go +++ b/modules/git/notes_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "path/filepath" "testing" @@ -18,7 +17,7 @@ func TestGetNotes(t *testing.T) { defer bareRepo1.Close() note := Note{} - err = GetNote(context.Background(), bareRepo1, "95bb4d39648ee7e325106df01a621c530863a653", ¬e) + err = GetNote(t.Context(), bareRepo1, "95bb4d39648ee7e325106df01a621c530863a653", ¬e) assert.NoError(t, err) assert.Equal(t, []byte("Note contents\n"), note.Message) assert.Equal(t, "Vladimir Panteleev", note.Commit.Author.Name) @@ -31,10 +30,10 @@ func TestGetNestedNotes(t *testing.T) { defer repo.Close() note := Note{} - err = GetNote(context.Background(), repo, "3e668dbfac39cbc80a9ff9c61eb565d944453ba4", ¬e) + err = GetNote(t.Context(), repo, "3e668dbfac39cbc80a9ff9c61eb565d944453ba4", ¬e) assert.NoError(t, err) assert.Equal(t, []byte("Note 2"), note.Message) - err = GetNote(context.Background(), repo, "ba0a96fa63532d6c5087ecef070b0250ed72fa47", ¬e) + err = GetNote(t.Context(), repo, "ba0a96fa63532d6c5087ecef070b0250ed72fa47", ¬e) assert.NoError(t, err) assert.Equal(t, []byte("Note 1"), note.Message) } @@ -46,7 +45,7 @@ func TestGetNonExistentNotes(t *testing.T) { defer bareRepo1.Close() note := Note{} - err = GetNote(context.Background(), bareRepo1, "non_existent_sha", ¬e) + err = GetNote(t.Context(), bareRepo1, "non_existent_sha", ¬e) assert.Error(t, err) assert.IsType(t, ErrNotExist{}, err) } diff --git a/modules/git/object_id.go b/modules/git/object_id.go index 82d30184df..25dfef3ec5 100644 --- a/modules/git/object_id.go +++ b/modules/git/object_id.go @@ -99,5 +99,5 @@ type ErrInvalidSHA struct { } func (err ErrInvalidSHA) Error() string { - return fmt.Sprintf("invalid sha: %s", err.SHA) + return "invalid sha: " + err.SHA } diff --git a/modules/git/parse.go b/modules/git/parse.go index eb26632cc0..a7f5c58e89 100644 --- a/modules/git/parse.go +++ b/modules/git/parse.go @@ -46,19 +46,9 @@ func parseLsTreeLine(line []byte) (*LsTreeEntry, error) { entry.Size = optional.Some(size) } - switch string(entryMode) { - case "100644": - entry.EntryMode = EntryModeBlob - case "100755": - entry.EntryMode = EntryModeExec - case "120000": - entry.EntryMode = EntryModeSymlink - case "160000": - entry.EntryMode = EntryModeCommit - case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons - entry.EntryMode = EntryModeTree - default: - return nil, fmt.Errorf("unknown type: %v", string(entryMode)) + entry.EntryMode, err = ParseEntryMode(string(entryMode)) + if err != nil || entry.EntryMode == EntryModeNoEntry { + return nil, fmt.Errorf("invalid ls-tree output (invalid mode): %q, err: %w", line, err) } entry.ID, err = NewIDFromString(string(entryObjectID)) diff --git a/modules/git/parse_nogogit.go b/modules/git/parse_nogogit.go index 676bb3c76c..78a0162889 100644 --- a/modules/git/parse_nogogit.go +++ b/modules/git/parse_nogogit.go @@ -19,7 +19,7 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) { return parseTreeEntries(data, nil) } -// parseTreeEntries FIXME this function's design is not right, it should make the caller read all data into memory +// parseTreeEntries FIXME this function's design is not right, it should not make the caller read all data into memory func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1) for pos := 0; pos < len(data); { diff --git a/modules/git/parse_nogogit_test.go b/modules/git/parse_nogogit_test.go index a4436ce499..6594c84269 100644 --- a/modules/git/parse_nogogit_test.go +++ b/modules/git/parse_nogogit_test.go @@ -58,7 +58,7 @@ func TestParseTreeEntriesLong(t *testing.T) { assert.NoError(t, err) assert.Len(t, entries, len(testCase.Expected)) for i, entry := range entries { - assert.EqualValues(t, testCase.Expected[i], entry) + assert.Equal(t, testCase.Expected[i], entry) } } } @@ -91,7 +91,7 @@ func TestParseTreeEntriesShort(t *testing.T) { assert.NoError(t, err) assert.Len(t, entries, len(testCase.Expected)) for i, entry := range entries { - assert.EqualValues(t, testCase.Expected[i], entry) + assert.Equal(t, testCase.Expected[i], entry) } } } diff --git a/modules/git/pipeline/catfile.go b/modules/git/pipeline/catfile.go index 4677218150..5ddc36cc01 100644 --- a/modules/git/pipeline/catfile.go +++ b/modules/git/pipeline/catfile.go @@ -25,8 +25,8 @@ func CatFileBatchCheck(ctx context.Context, shasToCheckReader *io.PipeReader, ca stderr := new(bytes.Buffer) var errbuf strings.Builder - cmd := git.NewCommand(ctx, "cat-file", "--batch-check") - if err := cmd.Run(&git.RunOpts{ + cmd := git.NewCommand("cat-file", "--batch-check") + if err := cmd.Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdin: shasToCheckReader, Stdout: catFileCheckWriter, @@ -43,8 +43,8 @@ func CatFileBatchCheckAllObjects(ctx context.Context, catFileCheckWriter *io.Pip stderr := new(bytes.Buffer) var errbuf strings.Builder - cmd := git.NewCommand(ctx, "cat-file", "--batch-check", "--batch-all-objects") - if err := cmd.Run(&git.RunOpts{ + cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects") + if err := cmd.Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdout: catFileCheckWriter, Stderr: stderr, @@ -64,7 +64,7 @@ func CatFileBatch(ctx context.Context, shasToBatchReader *io.PipeReader, catFile stderr := new(bytes.Buffer) var errbuf strings.Builder - if err := git.NewCommand(ctx, "cat-file", "--batch").Run(&git.RunOpts{ + if err := git.NewCommand("cat-file", "--batch").Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdout: catFileBatchWriter, Stdin: shasToBatchReader, diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index 92e35c5a10..c5eed73701 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -32,7 +32,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err go func() { stderr := strings.Builder{} - err := git.NewCommand(repo.Ctx, "rev-list", "--all").Run(&git.RunOpts{ + err := git.NewCommand("rev-list", "--all").Run(repo.Ctx, &git.RunOpts{ Dir: repo.Path, Stdout: revListWriter, Stderr: &stderr, diff --git a/modules/git/pipeline/namerev.go b/modules/git/pipeline/namerev.go index ad583a7479..06731c5051 100644 --- a/modules/git/pipeline/namerev.go +++ b/modules/git/pipeline/namerev.go @@ -22,7 +22,7 @@ func NameRevStdin(ctx context.Context, shasToNameReader *io.PipeReader, nameRevS stderr := new(bytes.Buffer) var errbuf strings.Builder - if err := git.NewCommand(ctx, "name-rev", "--stdin", "--name-only", "--always").Run(&git.RunOpts{ + if err := git.NewCommand("name-rev", "--stdin", "--name-only", "--always").Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdout: nameRevStdinWriter, Stdin: shasToNameReader, diff --git a/modules/git/pipeline/revlist.go b/modules/git/pipeline/revlist.go index d88ebe78ef..31627a0f3a 100644 --- a/modules/git/pipeline/revlist.go +++ b/modules/git/pipeline/revlist.go @@ -23,8 +23,8 @@ func RevListAllObjects(ctx context.Context, revListWriter *io.PipeWriter, wg *sy stderr := new(bytes.Buffer) var errbuf strings.Builder - cmd := git.NewCommand(ctx, "rev-list", "--objects", "--all") - if err := cmd.Run(&git.RunOpts{ + cmd := git.NewCommand("rev-list", "--objects", "--all") + if err := cmd.Run(ctx, &git.RunOpts{ Dir: basePath, Stdout: revListWriter, Stderr: stderr, @@ -42,11 +42,11 @@ func RevListObjects(ctx context.Context, revListWriter *io.PipeWriter, wg *sync. defer revListWriter.Close() stderr := new(bytes.Buffer) var errbuf strings.Builder - cmd := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(headSHA) + cmd := git.NewCommand("rev-list", "--objects").AddDynamicArguments(headSHA) if baseSHA != "" { cmd = cmd.AddArguments("--not").AddDynamicArguments(baseSHA) } - if err := cmd.Run(&git.RunOpts{ + if err := cmd.Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdout: revListWriter, Stderr: stderr, diff --git a/modules/git/ref.go b/modules/git/ref.go index f20a175e42..56b2db858a 100644 --- a/modules/git/ref.go +++ b/modules/git/ref.go @@ -109,8 +109,8 @@ func (ref RefName) IsFor() bool { } func (ref RefName) nameWithoutPrefix(prefix string) string { - if strings.HasPrefix(string(ref), prefix) { - return strings.TrimPrefix(string(ref), prefix) + if after, ok := strings.CutPrefix(string(ref), prefix); ok { + return after } return "" } diff --git a/modules/git/remote.go b/modules/git/remote.go index ff8c040eb1..876c3d6acb 100644 --- a/modules/git/remote.go +++ b/modules/git/remote.go @@ -17,12 +17,12 @@ import ( func GetRemoteAddress(ctx context.Context, repoPath, remoteName string) (string, error) { var cmd *Command if DefaultFeatures().CheckVersionAtLeast("2.7") { - cmd = NewCommand(ctx, "remote", "get-url").AddDynamicArguments(remoteName) + cmd = NewCommand("remote", "get-url").AddDynamicArguments(remoteName) } else { - cmd = NewCommand(ctx, "config", "--get").AddDynamicArguments("remote." + remoteName + ".url") + cmd = NewCommand("config", "--get").AddDynamicArguments("remote." + remoteName + ".url") } - result, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + result, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return "", err } diff --git a/modules/git/repo.go b/modules/git/repo.go index 0993a4ac37..f1f6902773 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -18,6 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" ) // GPGSettings represents the default GPG settings for this repository @@ -27,6 +28,7 @@ type GPGSettings struct { Email string Name string PublicKeyContent string + Format string } const prettyLogFormat = `--pretty=format:%H` @@ -42,9 +44,9 @@ func (repo *Repository) parsePrettyFormatLogToList(logs []byte) ([]*Commit, erro return commits, nil } - parts := bytes.Split(logs, []byte{'\n'}) + parts := bytes.SplitSeq(logs, []byte{'\n'}) - for _, commitID := range parts { + for commitID := range parts { commit, err := repo.GetCommit(string(commitID)) if err != nil { return nil, err @@ -57,7 +59,7 @@ func (repo *Repository) parsePrettyFormatLogToList(logs []byte) ([]*Commit, erro // IsRepoURLAccessible checks if given repository URL is accessible. func IsRepoURLAccessible(ctx context.Context, url string) bool { - _, _, err := NewCommand(ctx, "ls-remote", "-q", "-h").AddDynamicArguments(url, "HEAD").RunStdString(nil) + _, _, err := NewCommand("ls-remote", "-q", "-h").AddDynamicArguments(url, "HEAD").RunStdString(ctx, nil) return err == nil } @@ -68,7 +70,7 @@ func InitRepository(ctx context.Context, repoPath string, bare bool, objectForma return err } - cmd := NewCommand(ctx, "init") + cmd := NewCommand("init") if !IsValidObjectFormat(objectFormatName) { return fmt.Errorf("invalid object format: %s", objectFormatName) @@ -80,15 +82,15 @@ func InitRepository(ctx context.Context, repoPath string, bare bool, objectForma if bare { cmd.AddArguments("--bare") } - _, _, err = cmd.RunStdString(&RunOpts{Dir: repoPath}) + _, _, err = cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) return err } // IsEmpty Check if repository is empty. func (repo *Repository) IsEmpty() (bool, error) { var errbuf, output strings.Builder - if err := NewCommand(repo.Ctx).AddOptionFormat("--git-dir=%s", repo.Path).AddArguments("rev-list", "-n", "1", "--all"). - Run(&RunOpts{ + if err := NewCommand().AddOptionFormat("--git-dir=%s", repo.Path).AddArguments("rev-list", "-n", "1", "--all"). + Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: &output, Stderr: &errbuf, @@ -129,7 +131,7 @@ func CloneWithArgs(ctx context.Context, args TrustedCmdArgs, from, to string, op return err } - cmd := NewCommandContextNoGlobals(ctx, args...).AddArguments("clone") + cmd := NewCommandNoGlobals(args...).AddArguments("clone") if opts.SkipTLSVerify { cmd.AddArguments("-c", "http.sslVerify=false") } @@ -170,7 +172,7 @@ func CloneWithArgs(ctx context.Context, args TrustedCmdArgs, from, to string, op } stderr := new(bytes.Buffer) - if err = cmd.Run(&RunOpts{ + if err = cmd.Run(ctx, &RunOpts{ Timeout: opts.Timeout, Env: envs, Stdout: io.Discard, @@ -193,7 +195,7 @@ type PushOptions struct { // Push pushs local commits to given remote branch. func Push(ctx context.Context, repoPath string, opts PushOptions) error { - cmd := NewCommand(ctx, "push") + cmd := NewCommand("push") if opts.Force { cmd.AddArguments("-f") } @@ -206,7 +208,7 @@ func Push(ctx context.Context, repoPath string, opts PushOptions) error { } cmd.AddDashesAndList(remoteBranchArgs...) - stdout, stderr, err := cmd.RunStdString(&RunOpts{Env: opts.Env, Timeout: opts.Timeout, Dir: repoPath}) + stdout, stderr, err := cmd.RunStdString(ctx, &RunOpts{Env: opts.Env, Timeout: opts.Timeout, Dir: repoPath}) if err != nil { if strings.Contains(stderr, "non-fast-forward") { return &ErrPushOutOfDate{StdOut: stdout, StdErr: stderr, Err: err} @@ -225,8 +227,8 @@ func Push(ctx context.Context, repoPath string, opts PushOptions) error { // GetLatestCommitTime returns time for latest commit in repository (across all branches) func GetLatestCommitTime(ctx context.Context, repoPath string) (time.Time, error) { - cmd := NewCommand(ctx, "for-each-ref", "--sort=-committerdate", BranchPrefix, "--count", "1", "--format=%(committerdate)") - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + cmd := NewCommand("for-each-ref", "--sort=-committerdate", BranchPrefix, "--count", "1", "--format=%(committerdate)") + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return time.Time{}, err } @@ -242,9 +244,9 @@ type DivergeObject struct { // GetDivergingCommits returns the number of commits a targetBranch is ahead or behind a baseBranch func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch string) (do DivergeObject, err error) { - cmd := NewCommand(ctx, "rev-list", "--count", "--left-right"). + cmd := NewCommand("rev-list", "--count", "--left-right"). AddDynamicArguments(baseBranch + "..." + targetBranch).AddArguments("--") - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return do, err } @@ -266,30 +268,30 @@ func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch // CreateBundle create bundle content to the target path func (repo *Repository) CreateBundle(ctx context.Context, commit string, out io.Writer) error { - tmp, err := os.MkdirTemp(os.TempDir(), "gitea-bundle") + tmp, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-bundle") if err != nil { return err } - defer os.RemoveAll(tmp) + defer cleanup() env := append(os.Environ(), "GIT_OBJECT_DIRECTORY="+filepath.Join(repo.Path, "objects")) - _, _, err = NewCommand(ctx, "init", "--bare").RunStdString(&RunOpts{Dir: tmp, Env: env}) + _, _, err = NewCommand("init", "--bare").RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) if err != nil { return err } - _, _, err = NewCommand(ctx, "reset", "--soft").AddDynamicArguments(commit).RunStdString(&RunOpts{Dir: tmp, Env: env}) + _, _, err = NewCommand("reset", "--soft").AddDynamicArguments(commit).RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) if err != nil { return err } - _, _, err = NewCommand(ctx, "branch", "-m", "bundle").RunStdString(&RunOpts{Dir: tmp, Env: env}) + _, _, err = NewCommand("branch", "-m", "bundle").RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) if err != nil { return err } tmpFile := filepath.Join(tmp, "bundle") - _, _, err = NewCommand(ctx, "bundle", "create").AddDynamicArguments(tmpFile, "bundle", "HEAD").RunStdString(&RunOpts{Dir: tmp, Env: env}) + _, _, err = NewCommand("bundle", "create").AddDynamicArguments(tmpFile, "bundle", "HEAD").RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) if err != nil { return err } diff --git a/modules/git/repo_archive.go b/modules/git/repo_archive.go index 92f3e88f7c..0b2f6f2a45 100644 --- a/modules/git/repo_archive.go +++ b/modules/git/repo_archive.go @@ -53,7 +53,7 @@ func (repo *Repository) CreateArchive(ctx context.Context, format ArchiveType, t return fmt.Errorf("unknown format: %v", format) } - cmd := NewCommand(ctx, "archive") + cmd := NewCommand("archive") if usePrefix { cmd.AddOptionFormat("--prefix=%s", filepath.Base(strings.TrimSuffix(repo.Path, ".git"))+"/") } @@ -61,7 +61,7 @@ func (repo *Repository) CreateArchive(ctx context.Context, format ArchiveType, t cmd.AddDynamicArguments(commitID) var stderr strings.Builder - err := cmd.Run(&RunOpts{ + err := cmd.Run(ctx, &RunOpts{ Dir: repo.Path, Stdout: target, Stderr: &stderr, diff --git a/modules/git/repo_attribute.go b/modules/git/repo_attribute.go deleted file mode 100644 index 90eb783fe8..0000000000 --- a/modules/git/repo_attribute.go +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - - "code.gitea.io/gitea/modules/log" -) - -// CheckAttributeOpts represents the possible options to CheckAttribute -type CheckAttributeOpts struct { - CachedOnly bool - AllAttributes bool - Attributes []string - Filenames []string - IndexFile string - WorkTree string -} - -// CheckAttribute return the Blame object of file -func (repo *Repository) CheckAttribute(opts CheckAttributeOpts) (map[string]map[string]string, error) { - env := []string{} - - if len(opts.IndexFile) > 0 { - env = append(env, "GIT_INDEX_FILE="+opts.IndexFile) - } - if len(opts.WorkTree) > 0 { - env = append(env, "GIT_WORK_TREE="+opts.WorkTree) - } - - if len(env) > 0 { - env = append(os.Environ(), env...) - } - - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - - cmd := NewCommand(repo.Ctx, "check-attr", "-z") - - if opts.AllAttributes { - cmd.AddArguments("-a") - } else { - for _, attribute := range opts.Attributes { - if attribute != "" { - cmd.AddDynamicArguments(attribute) - } - } - } - - if opts.CachedOnly { - cmd.AddArguments("--cached") - } - - cmd.AddDashesAndList(opts.Filenames...) - - if err := cmd.Run(&RunOpts{ - Env: env, - Dir: repo.Path, - Stdout: stdOut, - Stderr: stdErr, - }); err != nil { - return nil, fmt.Errorf("failed to run check-attr: %w\n%s\n%s", err, stdOut.String(), stdErr.String()) - } - - // FIXME: This is incorrect on versions < 1.8.5 - fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) - - if len(fields)%3 != 1 { - return nil, fmt.Errorf("wrong number of fields in return from check-attr") - } - - name2attribute2info := make(map[string]map[string]string) - - for i := 0; i < (len(fields) / 3); i++ { - filename := string(fields[3*i]) - attribute := string(fields[3*i+1]) - info := string(fields[3*i+2]) - attribute2info := name2attribute2info[filename] - if attribute2info == nil { - attribute2info = make(map[string]string) - } - attribute2info[attribute] = info - name2attribute2info[filename] = attribute2info - } - - return name2attribute2info, nil -} - -// CheckAttributeReader provides a reader for check-attribute content that can be long running -type CheckAttributeReader struct { - // params - Attributes []string - Repo *Repository - IndexFile string - WorkTree string - - stdinReader io.ReadCloser - stdinWriter *os.File - stdOut attributeWriter - cmd *Command - env []string - ctx context.Context - cancel context.CancelFunc -} - -// Init initializes the CheckAttributeReader -func (c *CheckAttributeReader) Init(ctx context.Context) error { - if len(c.Attributes) == 0 { - lw := new(nulSeparatedAttributeWriter) - lw.attributes = make(chan attributeTriple) - lw.closed = make(chan struct{}) - - c.stdOut = lw - c.stdOut.Close() - return fmt.Errorf("no provided Attributes to check") - } - - c.ctx, c.cancel = context.WithCancel(ctx) - c.cmd = NewCommand(c.ctx, "check-attr", "--stdin", "-z") - - if len(c.IndexFile) > 0 { - c.cmd.AddArguments("--cached") - c.env = append(c.env, "GIT_INDEX_FILE="+c.IndexFile) - } - - if len(c.WorkTree) > 0 { - c.env = append(c.env, "GIT_WORK_TREE="+c.WorkTree) - } - - c.env = append(c.env, "GIT_FLUSH=1") - - c.cmd.AddDynamicArguments(c.Attributes...) - - var err error - - c.stdinReader, c.stdinWriter, err = os.Pipe() - if err != nil { - c.cancel() - return err - } - - lw := new(nulSeparatedAttributeWriter) - lw.attributes = make(chan attributeTriple, 5) - lw.closed = make(chan struct{}) - c.stdOut = lw - return nil -} - -// Run run cmd -func (c *CheckAttributeReader) Run() error { - defer func() { - _ = c.stdinReader.Close() - _ = c.stdOut.Close() - }() - stdErr := new(bytes.Buffer) - err := c.cmd.Run(&RunOpts{ - Env: c.env, - Dir: c.Repo.Path, - Stdin: c.stdinReader, - Stdout: c.stdOut, - Stderr: stdErr, - }) - if err != nil && !IsErrCanceledOrKilled(err) { - return fmt.Errorf("failed to run attr-check. Error: %w\nStderr: %s", err, stdErr.String()) - } - return nil -} - -// CheckPath check attr for given path -func (c *CheckAttributeReader) CheckPath(path string) (rs map[string]string, err error) { - defer func() { - if err != nil && err != c.ctx.Err() { - log.Error("Unexpected error when checking path %s in %s. Error: %v", path, c.Repo.Path, err) - } - }() - - select { - case <-c.ctx.Done(): - return nil, c.ctx.Err() - default: - } - - if _, err = c.stdinWriter.Write([]byte(path + "\x00")); err != nil { - defer c.Close() - return nil, err - } - - rs = make(map[string]string) - for range c.Attributes { - select { - case attr, ok := <-c.stdOut.ReadAttribute(): - if !ok { - return nil, c.ctx.Err() - } - rs[attr.Attribute] = attr.Value - case <-c.ctx.Done(): - return nil, c.ctx.Err() - } - } - return rs, nil -} - -// Close close pip after use -func (c *CheckAttributeReader) Close() error { - c.cancel() - err := c.stdinWriter.Close() - return err -} - -type attributeWriter interface { - io.WriteCloser - ReadAttribute() <-chan attributeTriple -} - -type attributeTriple struct { - Filename string - Attribute string - Value string -} - -type nulSeparatedAttributeWriter struct { - tmp []byte - attributes chan attributeTriple - closed chan struct{} - working attributeTriple - pos int -} - -func (wr *nulSeparatedAttributeWriter) Write(p []byte) (n int, err error) { - l, read := len(p), 0 - - nulIdx := bytes.IndexByte(p, '\x00') - for nulIdx >= 0 { - wr.tmp = append(wr.tmp, p[:nulIdx]...) - switch wr.pos { - case 0: - wr.working = attributeTriple{ - Filename: string(wr.tmp), - } - case 1: - wr.working.Attribute = string(wr.tmp) - case 2: - wr.working.Value = string(wr.tmp) - } - wr.tmp = wr.tmp[:0] - wr.pos++ - if wr.pos > 2 { - wr.attributes <- wr.working - wr.pos = 0 - } - read += nulIdx + 1 - if l > read { - p = p[nulIdx+1:] - nulIdx = bytes.IndexByte(p, '\x00') - } else { - return l, nil - } - } - wr.tmp = append(wr.tmp, p...) - return len(p), nil -} - -func (wr *nulSeparatedAttributeWriter) ReadAttribute() <-chan attributeTriple { - return wr.attributes -} - -func (wr *nulSeparatedAttributeWriter) Close() error { - select { - case <-wr.closed: - return nil - default: - } - close(wr.attributes) - close(wr.closed) - return nil -} - -// Create a check attribute reader for the current repository and provided commit ID -func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeReader, context.CancelFunc) { - indexFilename, worktree, deleteTemporaryFile, err := repo.ReadTreeToTemporaryIndex(commitID) - if err != nil { - return nil, func() {} - } - - checker := &CheckAttributeReader{ - Attributes: []string{ - AttributeLinguistVendored, - AttributeLinguistGenerated, - AttributeLinguistDocumentation, - AttributeLinguistDetectable, - AttributeLinguistLanguage, - AttributeGitlabLanguage, - }, - Repo: repo, - IndexFile: indexFilename, - WorkTree: worktree, - } - ctx, cancel := context.WithCancel(repo.Ctx) - if err := checker.Init(ctx); err != nil { - log.Error("Unable to open checker for %s. Error: %v", commitID, err) - } else { - go func() { - err := checker.Run() - if err != nil && err != ctx.Err() { - log.Error("Unable to open checker for %s. Error: %v", commitID, err) - } - cancel() - }() - } - deferable := func() { - _ = checker.Close() - cancel() - deleteTemporaryFile() - } - - return checker, deferable -} diff --git a/modules/git/repo_attribute_test.go b/modules/git/repo_attribute_test.go deleted file mode 100644 index 0fcd94b4c7..0000000000 --- a/modules/git/repo_attribute_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { - wr := &nulSeparatedAttributeWriter{ - attributes: make(chan attributeTriple, 5), - } - - testStr := ".gitignore\"\n\x00linguist-vendored\x00unspecified\x00" - - n, err := wr.Write([]byte(testStr)) - - assert.Len(t, testStr, n) - assert.NoError(t, err) - select { - case attr := <-wr.ReadAttribute(): - assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, AttributeLinguistVendored, attr.Attribute) - assert.Equal(t, "unspecified", attr.Value) - case <-time.After(100 * time.Millisecond): - assert.FailNow(t, "took too long to read an attribute from the list") - } - // Write a second attribute again - n, err = wr.Write([]byte(testStr)) - - assert.Len(t, testStr, n) - assert.NoError(t, err) - - select { - case attr := <-wr.ReadAttribute(): - assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, AttributeLinguistVendored, attr.Attribute) - assert.Equal(t, "unspecified", attr.Value) - case <-time.After(100 * time.Millisecond): - assert.FailNow(t, "took too long to read an attribute from the list") - } - - // Write a partial attribute - _, err = wr.Write([]byte("incomplete-file")) - assert.NoError(t, err) - _, err = wr.Write([]byte("name\x00")) - assert.NoError(t, err) - - select { - case <-wr.ReadAttribute(): - assert.FailNow(t, "There should not be an attribute ready to read") - case <-time.After(100 * time.Millisecond): - } - _, err = wr.Write([]byte("attribute\x00")) - assert.NoError(t, err) - select { - case <-wr.ReadAttribute(): - assert.FailNow(t, "There should not be an attribute ready to read") - case <-time.After(100 * time.Millisecond): - } - - _, err = wr.Write([]byte("value\x00")) - assert.NoError(t, err) - - attr := <-wr.ReadAttribute() - assert.Equal(t, "incomplete-filename", attr.Filename) - assert.Equal(t, "attribute", attr.Attribute) - assert.Equal(t, "value", attr.Value) - - _, err = wr.Write([]byte("shouldbe.vendor\x00linguist-vendored\x00set\x00shouldbe.vendor\x00linguist-generated\x00unspecified\x00shouldbe.vendor\x00linguist-language\x00unspecified\x00")) - assert.NoError(t, err) - attr = <-wr.ReadAttribute() - assert.NoError(t, err) - assert.EqualValues(t, attributeTriple{ - Filename: "shouldbe.vendor", - Attribute: AttributeLinguistVendored, - Value: "set", - }, attr) - attr = <-wr.ReadAttribute() - assert.NoError(t, err) - assert.EqualValues(t, attributeTriple{ - Filename: "shouldbe.vendor", - Attribute: AttributeLinguistGenerated, - Value: "unspecified", - }, attr) - attr = <-wr.ReadAttribute() - assert.NoError(t, err) - assert.EqualValues(t, attributeTriple{ - Filename: "shouldbe.vendor", - Attribute: AttributeLinguistLanguage, - Value: "unspecified", - }, attr) -} diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go index 0ca1ea79c2..293aca159c 100644 --- a/modules/git/repo_base_gogit.go +++ b/modules/git/repo_base_gogit.go @@ -49,7 +49,12 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { repoPath, err := filepath.Abs(repoPath) if err != nil { return nil, err - } else if !isDir(repoPath) { + } + exist, err := util.IsDir(repoPath) + if err != nil { + return nil, err + } + if !exist { return nil, util.NewNotExistErrorf("no such file or directory") } diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go index 477e3b8742..6f9bfd4b43 100644 --- a/modules/git/repo_base_nogogit.go +++ b/modules/git/repo_base_nogogit.go @@ -47,7 +47,12 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { repoPath, err := filepath.Abs(repoPath) if err != nil { return nil, err - } else if !isDir(repoPath) { + } + exist, err := util.IsDir(repoPath) + if err != nil { + return nil, err + } + if !exist { return nil, util.NewNotExistErrorf("no such file or directory") } @@ -62,7 +67,7 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { if repo.batch == nil { var err error - repo.batch, err = repo.NewBatch(ctx) + repo.batch, err = NewBatch(ctx, repo.Path) if err != nil { return nil, nil, nil, err } @@ -76,7 +81,7 @@ func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bu } log.Debug("Opening temporary cat file batch for: %s", repo.Path) - tempBatch, err := repo.NewBatch(ctx) + tempBatch, err := NewBatch(ctx, repo.Path) if err != nil { return nil, nil, nil, err } @@ -87,7 +92,7 @@ func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bu func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { if repo.check == nil { var err error - repo.check, err = repo.NewBatchCheck(ctx) + repo.check, err = NewBatchCheck(ctx, repo.Path) if err != nil { return nil, nil, nil, err } @@ -101,7 +106,7 @@ func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError } log.Debug("Opening temporary cat file batch-check for: %s", repo.Path) - tempBatchCheck, err := repo.NewBatchCheck(ctx) + tempBatchCheck, err := NewBatchCheck(ctx, repo.Path) if err != nil { return nil, nil, nil, err } diff --git a/modules/git/repo_blame.go b/modules/git/repo_blame.go index 139cdd7be9..6941a76c42 100644 --- a/modules/git/repo_blame.go +++ b/modules/git/repo_blame.go @@ -9,10 +9,10 @@ import ( // LineBlame returns the latest commit at the given line func (repo *Repository) LineBlame(revision, path, file string, line uint) (*Commit, error) { - res, _, err := NewCommand(repo.Ctx, "blame"). + res, _, err := NewCommand("blame"). AddOptionFormat("-L %d,%d", line, line). AddOptionValues("-p", revision). - AddDashesAndList(file).RunStdString(&RunOpts{Dir: path}) + AddDashesAndList(file).RunStdString(repo.Ctx, &RunOpts{Dir: path}) if err != nil { return nil, err } diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go index 552ae2bb8c..e7ecf53f51 100644 --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -7,7 +7,6 @@ package git import ( "context" "errors" - "fmt" "strings" ) @@ -16,7 +15,7 @@ const BranchPrefix = "refs/heads/" // IsReferenceExist returns true if given reference exists in the repository. func IsReferenceExist(ctx context.Context, repoPath, name string) bool { - _, _, err := NewCommand(ctx, "show-ref", "--verify").AddDashesAndList(name).RunStdString(&RunOpts{Dir: repoPath}) + _, _, err := NewCommand("show-ref", "--verify").AddDashesAndList(name).RunStdString(ctx, &RunOpts{Dir: repoPath}) return err == nil } @@ -25,38 +24,8 @@ func IsBranchExist(ctx context.Context, repoPath, name string) bool { return IsReferenceExist(ctx, repoPath, BranchPrefix+name) } -// Branch represents a Git branch. -type Branch struct { - Name string - Path string - - gitRepo *Repository -} - -// GetHEADBranch returns corresponding branch of HEAD. -func (repo *Repository) GetHEADBranch() (*Branch, error) { - if repo == nil { - return nil, fmt.Errorf("nil repo") - } - stdout, _, err := NewCommand(repo.Ctx, "symbolic-ref", "HEAD").RunStdString(&RunOpts{Dir: repo.Path}) - if err != nil { - return nil, err - } - stdout = strings.TrimSpace(stdout) - - if !strings.HasPrefix(stdout, BranchPrefix) { - return nil, fmt.Errorf("invalid HEAD branch: %v", stdout) - } - - return &Branch{ - Name: stdout[len(BranchPrefix):], - Path: stdout, - gitRepo: repo, - }, nil -} - func GetDefaultBranch(ctx context.Context, repoPath string) (string, error) { - stdout, _, err := NewCommand(ctx, "symbolic-ref", "HEAD").RunStdString(&RunOpts{Dir: repoPath}) + stdout, _, err := NewCommand("symbolic-ref", "HEAD").RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return "", err } @@ -67,37 +36,6 @@ func GetDefaultBranch(ctx context.Context, repoPath string) (string, error) { return strings.TrimPrefix(stdout, BranchPrefix), nil } -// GetBranch returns a branch by it's name -func (repo *Repository) GetBranch(branch string) (*Branch, error) { - if !repo.IsBranchExist(branch) { - return nil, ErrBranchNotExist{branch} - } - return &Branch{ - Path: repo.Path, - Name: branch, - gitRepo: repo, - }, nil -} - -// GetBranches returns a slice of *git.Branch -func (repo *Repository) GetBranches(skip, limit int) ([]*Branch, int, error) { - brs, countAll, err := repo.GetBranchNames(skip, limit) - if err != nil { - return nil, 0, err - } - - branches := make([]*Branch, len(brs)) - for i := range brs { - branches[i] = &Branch{ - Path: repo.Path, - Name: brs[i], - gitRepo: repo, - } - } - - return branches, countAll, nil -} - // DeleteBranchOptions Option(s) for delete branch type DeleteBranchOptions struct { Force bool @@ -105,7 +43,7 @@ type DeleteBranchOptions struct { // DeleteBranch delete a branch by name on repository. func (repo *Repository) DeleteBranch(name string, opts DeleteBranchOptions) error { - cmd := NewCommand(repo.Ctx, "branch") + cmd := NewCommand("branch") if opts.Force { cmd.AddArguments("-D") @@ -114,46 +52,41 @@ func (repo *Repository) DeleteBranch(name string, opts DeleteBranchOptions) erro } cmd.AddDashesAndList(name) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := cmd.RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // CreateBranch create a new branch func (repo *Repository) CreateBranch(branch, oldbranchOrCommit string) error { - cmd := NewCommand(repo.Ctx, "branch") + cmd := NewCommand("branch") cmd.AddDashesAndList(branch, oldbranchOrCommit) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := cmd.RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // AddRemote adds a new remote to repository. func (repo *Repository) AddRemote(name, url string, fetch bool) error { - cmd := NewCommand(repo.Ctx, "remote", "add") + cmd := NewCommand("remote", "add") if fetch { cmd.AddArguments("-f") } cmd.AddDynamicArguments(name, url) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := cmd.RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // RemoveRemote removes a remote from repository. func (repo *Repository) RemoveRemote(name string) error { - _, _, err := NewCommand(repo.Ctx, "remote", "rm").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("remote", "rm").AddDynamicArguments(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } -// GetCommit returns the head commit of a branch -func (branch *Branch) GetCommit() (*Commit, error) { - return branch.gitRepo.GetBranchCommit(branch.Name) -} - // RenameBranch rename a branch func (repo *Repository) RenameBranch(from, to string) error { - _, _, err := NewCommand(repo.Ctx, "branch", "-m").AddDynamicArguments(from, to).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("branch", "-m").AddDynamicArguments(from, to).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go index dbc4a5fedc..77aecb21eb 100644 --- a/modules/git/repo_branch_gogit.go +++ b/modules/git/repo_branch_gogit.go @@ -57,7 +57,7 @@ func (repo *Repository) IsBranchExist(name string) bool { // GetBranches returns branches from the repository, skipping "skip" initial branches and // returning at most "limit" branches, or all branches if "limit" is 0. -// Branches are returned with sort of `-commiterdate` as the nogogit +// Branches are returned with sort of `-committerdate` as the nogogit // implementation. This requires full fetch, sort and then the // skip/limit applies later as gogit returns in undefined order. func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index 0d2efd4a6b..0d11198523 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -109,7 +109,7 @@ func WalkShowRef(ctx context.Context, repoPath string, extraArgs TrustedCmdArgs, stderrBuilder := &strings.Builder{} args := TrustedCmdArgs{"for-each-ref", "--format=%(objectname) %(refname)"} args = append(args, extraArgs...) - err := NewCommand(ctx, args...).Run(&RunOpts{ + err := NewCommand(args...).Run(ctx, &RunOpts{ Dir: repoPath, Stdout: stdoutWriter, Stderr: stderrBuilder, diff --git a/modules/git/repo_branch_test.go b/modules/git/repo_branch_test.go index 5d3b8abb3a..8e8ea16fcd 100644 --- a/modules/git/repo_branch_test.go +++ b/modules/git/repo_branch_test.go @@ -21,21 +21,21 @@ func TestRepository_GetBranches(t *testing.T) { assert.NoError(t, err) assert.Len(t, branches, 2) - assert.EqualValues(t, 3, countAll) + assert.Equal(t, 3, countAll) assert.ElementsMatch(t, []string{"master", "branch2"}, branches) branches, countAll, err = bareRepo1.GetBranchNames(0, 0) assert.NoError(t, err) assert.Len(t, branches, 3) - assert.EqualValues(t, 3, countAll) + assert.Equal(t, 3, countAll) assert.ElementsMatch(t, []string{"master", "branch2", "branch1"}, branches) branches, countAll, err = bareRepo1.GetBranchNames(5, 1) assert.NoError(t, err) assert.Empty(t, branches) - assert.EqualValues(t, 3, countAll) + assert.Equal(t, 3, countAll) assert.ElementsMatch(t, []string{}, branches) } @@ -47,7 +47,7 @@ func BenchmarkRepository_GetBranches(b *testing.B) { } defer bareRepo1.Close() - for i := 0; i < b.N; i++ { + for b.Loop() { _, _, err := bareRepo1.GetBranchNames(0, 0) if err != nil { b.Fatal(err) @@ -71,15 +71,15 @@ func TestGetRefsBySha(t *testing.T) { // refs/pull/1/head branches, err = bareRepo5.GetRefsBySha("c83380d7056593c51a699d12b9c00627bd5743e9", PullPrefix) assert.NoError(t, err) - assert.EqualValues(t, []string{"refs/pull/1/head"}, branches) + assert.Equal(t, []string{"refs/pull/1/head"}, branches) branches, err = bareRepo5.GetRefsBySha("d8e0bbb45f200e67d9a784ce55bd90821af45ebd", BranchPrefix) assert.NoError(t, err) - assert.EqualValues(t, []string{"refs/heads/master", "refs/heads/master-clone"}, branches) + assert.Equal(t, []string{"refs/heads/master", "refs/heads/master-clone"}, branches) branches, err = bareRepo5.GetRefsBySha("58a4bcc53ac13e7ff76127e0fb518b5262bf09af", BranchPrefix) assert.NoError(t, err) - assert.EqualValues(t, []string{"refs/heads/test-patch-1"}, branches) + assert.Equal(t, []string{"refs/heads/test-patch-1"}, branches) } func BenchmarkGetRefsBySha(b *testing.B) { diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index 647894bb21..4066a1ca7b 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -59,7 +59,7 @@ func (repo *Repository) getCommitByPathWithID(id ObjectID, relpath string) (*Com relpath = `\` + relpath } - stdout, _, runErr := NewCommand(repo.Ctx, "log", "-1", prettyLogFormat).AddDynamicArguments(id.String()).AddDashesAndList(relpath).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand("log", "-1", prettyLogFormat).AddDynamicArguments(id.String()).AddDashesAndList(relpath).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -74,7 +74,7 @@ func (repo *Repository) getCommitByPathWithID(id ObjectID, relpath string) (*Com // GetCommitByPath returns the last commit of relative path. func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) { - stdout, _, runErr := NewCommand(repo.Ctx, "log", "-1", prettyLogFormat).AddDashesAndList(relpath).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand("log", "-1", prettyLogFormat).AddDashesAndList(relpath).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -89,8 +89,9 @@ func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) { return commits[0], nil } -func (repo *Repository) commitsByRange(id ObjectID, page, pageSize int, not string) ([]*Commit, error) { - cmd := NewCommand(repo.Ctx, "log"). +// commitsByRangeWithTime returns the specific page commits before current revision, with not, since, until support +func (repo *Repository) commitsByRangeWithTime(id ObjectID, page, pageSize int, not, since, until string) ([]*Commit, error) { + cmd := NewCommand("log"). AddOptionFormat("--skip=%d", (page-1)*pageSize). AddOptionFormat("--max-count=%d", pageSize). AddArguments(prettyLogFormat). @@ -99,8 +100,14 @@ func (repo *Repository) commitsByRange(id ObjectID, page, pageSize int, not stri if not != "" { cmd.AddOptionValues("--not", not) } + if since != "" { + cmd.AddOptionFormat("--since=%s", since) + } + if until != "" { + cmd.AddOptionFormat("--until=%s", until) + } - stdout, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } @@ -134,7 +141,7 @@ func (repo *Repository) searchCommits(id ObjectID, opts SearchCommitsOptions) ([ } // create new git log command with limit of 100 commits - cmd := NewCommand(repo.Ctx, "log", "-100", prettyLogFormat).AddDynamicArguments(id.String()) + cmd := NewCommand("log", "-100", prettyLogFormat).AddDynamicArguments(id.String()) // pretend that all refs along with HEAD were listed on command line as <commis> // https://git-scm.com/docs/git-log#Documentation/git-log.txt---all @@ -154,7 +161,7 @@ func (repo *Repository) searchCommits(id ObjectID, opts SearchCommitsOptions) ([ // search for commits matching given constraints and keywords in commit msg addCommonSearchArgs(cmd) - stdout, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } @@ -168,14 +175,14 @@ func (repo *Repository) searchCommits(id ObjectID, opts SearchCommitsOptions) ([ // ignore anything not matching a valid sha pattern if id.Type().IsValid(v) { // create new git log command with 1 commit limit - hashCmd := NewCommand(repo.Ctx, "log", "-1", prettyLogFormat) + hashCmd := NewCommand("log", "-1", prettyLogFormat) // add previous arguments except for --grep and --all addCommonSearchArgs(hashCmd) // add keyword as <commit> hashCmd.AddDynamicArguments(v) // search with given constraints for commit matching sha hash of v - hashMatching, _, err := hashCmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + hashMatching, _, err := hashCmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil || bytes.Contains(stdout, hashMatching) { continue } @@ -190,7 +197,7 @@ func (repo *Repository) searchCommits(id ObjectID, opts SearchCommitsOptions) ([ // FileChangedBetweenCommits Returns true if the file changed between commit IDs id1 and id2 // You must ensure that id1 and id2 are valid commit ids. func (repo *Repository) FileChangedBetweenCommits(filename, id1, id2 string) (bool, error) { - stdout, _, err := NewCommand(repo.Ctx, "diff", "--name-only", "-z").AddDynamicArguments(id1, id2).AddDashesAndList(filename).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("diff", "--name-only", "-z").AddDynamicArguments(id1, id2).AddDashesAndList(filename).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return false, err } @@ -212,6 +219,8 @@ type CommitsByFileAndRangeOptions struct { File string Not string Page int + Since string + Until string } // CommitsByFileAndRange return the commits according revision file and the page @@ -223,7 +232,7 @@ func (repo *Repository) CommitsByFileAndRange(opts CommitsByFileAndRangeOptions) }() go func() { stderr := strings.Builder{} - gitCmd := NewCommand(repo.Ctx, "rev-list"). + gitCmd := NewCommand("rev-list"). AddOptionFormat("--max-count=%d", setting.Git.CommitsRangeSize). AddOptionFormat("--skip=%d", (opts.Page-1)*setting.Git.CommitsRangeSize) gitCmd.AddDynamicArguments(opts.Revision) @@ -231,9 +240,15 @@ func (repo *Repository) CommitsByFileAndRange(opts CommitsByFileAndRangeOptions) if opts.Not != "" { gitCmd.AddOptionValues("--not", opts.Not) } + if opts.Since != "" { + gitCmd.AddOptionFormat("--since=%s", opts.Since) + } + if opts.Until != "" { + gitCmd.AddOptionFormat("--until=%s", opts.Until) + } gitCmd.AddDashesAndList(opts.File) - err := gitCmd.Run(&RunOpts{ + err := gitCmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr, @@ -275,11 +290,11 @@ func (repo *Repository) CommitsByFileAndRange(opts CommitsByFileAndRangeOptions) // FilesCountBetween return the number of files changed between two commits func (repo *Repository) FilesCountBetween(startCommitID, endCommitID string) (int, error) { - stdout, _, err := NewCommand(repo.Ctx, "diff", "--name-only").AddDynamicArguments(startCommitID + "..." + endCommitID).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("diff", "--name-only").AddDynamicArguments(startCommitID+"..."+endCommitID).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil && strings.Contains(err.Error(), "no merge base") { // git >= 2.28 now returns an error if startCommitID and endCommitID have become unrelated. // previously it would return the results of git diff --name-only startCommitID endCommitID so let's try that... - stdout, _, err = NewCommand(repo.Ctx, "diff", "--name-only").AddDynamicArguments(startCommitID, endCommitID).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("diff", "--name-only").AddDynamicArguments(startCommitID, endCommitID).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) } if err != nil { return 0, err @@ -293,13 +308,13 @@ func (repo *Repository) CommitsBetween(last, before *Commit) ([]*Commit, error) var stdout []byte var err error if before == nil { - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } else { - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String() + ".." + last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(before.ID.String()+".."+last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil && strings.Contains(err.Error(), "no merge base") { // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. // previously it would return the results of git rev-list before last so let's try that... - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } } if err != nil { @@ -313,22 +328,22 @@ func (repo *Repository) CommitsBetweenLimit(last, before *Commit, limit, skip in var stdout []byte var err error if before == nil { - stdout, _, err = NewCommand(repo.Ctx, "rev-list"). + stdout, _, err = NewCommand("rev-list"). AddOptionValues("--max-count", strconv.Itoa(limit)). AddOptionValues("--skip", strconv.Itoa(skip)). - AddDynamicArguments(last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + AddDynamicArguments(last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } else { - stdout, _, err = NewCommand(repo.Ctx, "rev-list"). + stdout, _, err = NewCommand("rev-list"). AddOptionValues("--max-count", strconv.Itoa(limit)). AddOptionValues("--skip", strconv.Itoa(skip)). - AddDynamicArguments(before.ID.String() + ".." + last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + AddDynamicArguments(before.ID.String()+".."+last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil && strings.Contains(err.Error(), "no merge base") { // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. // previously it would return the results of git rev-list --max-count n before last so let's try that... - stdout, _, err = NewCommand(repo.Ctx, "rev-list"). + stdout, _, err = NewCommand("rev-list"). AddOptionValues("--max-count", strconv.Itoa(limit)). AddOptionValues("--skip", strconv.Itoa(skip)). - AddDynamicArguments(before.ID.String(), last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + AddDynamicArguments(before.ID.String(), last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } } if err != nil { @@ -343,13 +358,13 @@ func (repo *Repository) CommitsBetweenNotBase(last, before *Commit, baseBranch s var stdout []byte var err error if before == nil { - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } else { - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String()+".."+last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(before.ID.String()+".."+last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil && strings.Contains(err.Error(), "no merge base") { // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. // previously it would return the results of git rev-list before last so let's try that... - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } } if err != nil { @@ -395,13 +410,13 @@ func (repo *Repository) CommitsCountBetween(start, end string) (int64, error) { // commitsBefore the limit is depth, not total number of returned commits. func (repo *Repository) commitsBefore(id ObjectID, limit int) ([]*Commit, error) { - cmd := NewCommand(repo.Ctx, "log", prettyLogFormat) + cmd := NewCommand("log", prettyLogFormat) if limit > 0 { cmd.AddOptionFormat("-%d", limit) } cmd.AddDynamicArguments(id.String()) - stdout, _, runErr := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -438,10 +453,10 @@ func (repo *Repository) getCommitsBeforeLimit(id ObjectID, num int) ([]*Commit, func (repo *Repository) getBranches(env []string, commitID string, limit int) ([]string, error) { if DefaultFeatures().CheckVersionAtLeast("2.7.0") { - stdout, _, err := NewCommand(repo.Ctx, "for-each-ref", "--format=%(refname:strip=2)"). + stdout, _, err := NewCommand("for-each-ref", "--format=%(refname:strip=2)"). AddOptionFormat("--count=%d", limit). AddOptionValues("--contains", commitID, BranchPrefix). - RunStdString(&RunOpts{ + RunStdString(repo.Ctx, &RunOpts{ Dir: repo.Path, Env: env, }) @@ -453,7 +468,7 @@ func (repo *Repository) getBranches(env []string, commitID string, limit int) ([ return branches, nil } - stdout, _, err := NewCommand(repo.Ctx, "branch").AddOptionValues("--contains", commitID).RunStdString(&RunOpts{ + stdout, _, err := NewCommand("branch").AddOptionValues("--contains", commitID).RunStdString(repo.Ctx, &RunOpts{ Dir: repo.Path, Env: env, }) @@ -495,7 +510,7 @@ func (repo *Repository) GetCommitsFromIDs(commitIDs []string) []*Commit { // IsCommitInBranch check if the commit is on the branch func (repo *Repository) IsCommitInBranch(commitID, branch string) (r bool, err error) { - stdout, _, err := NewCommand(repo.Ctx, "branch", "--contains").AddDynamicArguments(commitID, branch).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("branch", "--contains").AddDynamicArguments(commitID, branch).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return false, err } @@ -519,11 +534,12 @@ func (repo *Repository) AddLastCommitCache(cacheKey, fullName, sha string) error return nil } +// GetCommitBranchStart returns the commit where the branch diverged func (repo *Repository) GetCommitBranchStart(env []string, branch, endCommitID string) (string, error) { - cmd := NewCommand(repo.Ctx, "log", prettyLogFormat) + cmd := NewCommand("log", prettyLogFormat) cmd.AddDynamicArguments(endCommitID) - stdout, _, runErr := cmd.RunStdBytes(&RunOpts{ + stdout, _, runErr := cmd.RunStdBytes(repo.Ctx, &RunOpts{ Dir: repo.Path, Env: env, }) @@ -531,21 +547,20 @@ func (repo *Repository) GetCommitBranchStart(env []string, branch, endCommitID s return "", runErr } - parts := bytes.Split(bytes.TrimSpace(stdout), []byte{'\n'}) + parts := bytes.SplitSeq(bytes.TrimSpace(stdout), []byte{'\n'}) - var startCommitID string - for _, commitID := range parts { + // check the commits one by one until we find a commit contained by another branch + // and we think this commit is the divergence point + for commitID := range parts { branches, err := repo.getBranches(env, string(commitID), 2) if err != nil { return "", err } for _, b := range branches { if b != branch { - return startCommitID, nil + return string(commitID), nil } } - - startCommitID = string(commitID) } return "", nil diff --git a/modules/git/repo_commit_gogit.go b/modules/git/repo_commit_gogit.go index 993013eef7..a88902e209 100644 --- a/modules/git/repo_commit_gogit.go +++ b/modules/git/repo_commit_gogit.go @@ -59,7 +59,7 @@ func (repo *Repository) ConvertToGitID(commitID string) (ObjectID, error) { } } - actualCommitID, _, err := NewCommand(repo.Ctx, "rev-parse", "--verify").AddDynamicArguments(commitID).RunStdString(&RunOpts{Dir: repo.Path}) + actualCommitID, _, err := NewCommand("rev-parse", "--verify").AddDynamicArguments(commitID).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) actualCommitID = strings.TrimSpace(actualCommitID) if err != nil { if strings.Contains(err.Error(), "unknown revision or path") || diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index f5ed282a45..3ead3e2216 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -16,7 +16,7 @@ import ( // ResolveReference resolves a name to a reference func (repo *Repository) ResolveReference(name string) (string, error) { - stdout, _, err := NewCommand(repo.Ctx, "show-ref", "--hash").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("show-ref", "--hash").AddDynamicArguments(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { if strings.Contains(err.Error(), "not a valid ref") { return "", ErrNotExist{name, ""} @@ -52,13 +52,13 @@ func (repo *Repository) GetRefCommitID(name string) (string, error) { // SetReference sets the commit ID string of given reference (e.g. branch or tag). func (repo *Repository) SetReference(name, commitID string) error { - _, _, err := NewCommand(repo.Ctx, "update-ref").AddDynamicArguments(name, commitID).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("update-ref").AddDynamicArguments(name, commitID).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // RemoveReference removes the given reference (e.g. branch or tag). func (repo *Repository) RemoveReference(name string) error { - _, _, err := NewCommand(repo.Ctx, "update-ref", "--no-deref", "-d").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("update-ref", "--no-deref", "-d").AddDynamicArguments(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } @@ -68,7 +68,7 @@ func (repo *Repository) IsCommitExist(name string) bool { log.Error("IsCommitExist: %v", err) return false } - _, _, err := NewCommand(repo.Ctx, "cat-file", "-e").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("cat-file", "-e").AddDynamicArguments(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err == nil } @@ -81,10 +81,10 @@ func (repo *Repository) getCommit(id ObjectID) (*Commit, error) { _, _ = wr.Write([]byte(id.String() + "\n")) - return repo.getCommitFromBatchReader(rd, id) + return repo.getCommitFromBatchReader(wr, rd, id) } -func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id ObjectID) (*Commit, error) { +func (repo *Repository) getCommitFromBatchReader(wr WriteCloserError, rd *bufio.Reader, id ObjectID) (*Commit, error) { _, typ, size, err := ReadBatchLine(rd) if err != nil { if errors.Is(err, io.EOF) || IsErrNotExist(err) { @@ -112,7 +112,11 @@ func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id ObjectID) return nil, err } - commit, err := tag.Commit(repo) + if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { + return nil, err + } + + commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) if err != nil { return nil, err } diff --git a/modules/git/repo_commitgraph.go b/modules/git/repo_commitgraph.go index 087d5bcec4..62c6378054 100644 --- a/modules/git/repo_commitgraph.go +++ b/modules/git/repo_commitgraph.go @@ -12,7 +12,7 @@ import ( // this requires git v2.18 to be installed func WriteCommitGraph(ctx context.Context, repoPath string) error { if DefaultFeatures().CheckVersionAtLeast("2.18") { - if _, _, err := NewCommand(ctx, "commit-graph", "write").RunStdString(&RunOpts{Dir: repoPath}); err != nil { + if _, _, err := NewCommand("commit-graph", "write").RunStdString(ctx, &RunOpts{Dir: repoPath}); err != nil { return fmt.Errorf("unable to write commit-graph for '%s' : %w", repoPath, err) } } diff --git a/modules/git/repo_commitgraph_gogit.go b/modules/git/repo_commitgraph_gogit.go index d3182f15c6..c0082b62c8 100644 --- a/modules/git/repo_commitgraph_gogit.go +++ b/modules/git/repo_commitgraph_gogit.go @@ -8,7 +8,7 @@ package git import ( "os" - "path" + "path/filepath" gitealog "code.gitea.io/gitea/modules/log" @@ -18,7 +18,7 @@ import ( // CommitNodeIndex returns the index for walking commit graph func (r *Repository) CommitNodeIndex() (cgobject.CommitNodeIndex, *os.File) { - indexPath := path.Join(r.Path, "objects", "info", "commit-graph") + indexPath := filepath.Join(r.Path, "objects", "info", "commit-graph") file, err := os.Open(indexPath) if err == nil { diff --git a/modules/git/repo_compare.go b/modules/git/repo_compare.go index 877a7ff3b8..ff44506e13 100644 --- a/modules/git/repo_compare.go +++ b/modules/git/repo_compare.go @@ -39,13 +39,13 @@ func (repo *Repository) GetMergeBase(tmpRemote, base, head string) (string, stri if tmpRemote != "origin" { tmpBaseName := RemotePrefix + tmpRemote + "/tmp_" + base // Fetch commit into a temporary branch in order to be able to handle commits and tags - _, _, err := NewCommand(repo.Ctx, "fetch", "--no-tags").AddDynamicArguments(tmpRemote).AddDashesAndList(base + ":" + tmpBaseName).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("fetch", "--no-tags").AddDynamicArguments(tmpRemote).AddDashesAndList(base+":"+tmpBaseName).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err == nil { base = tmpBaseName } } - stdout, _, err := NewCommand(repo.Ctx, "merge-base").AddDashesAndList(base, head).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("merge-base").AddDashesAndList(base, head).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return strings.TrimSpace(stdout), base, err } @@ -94,9 +94,9 @@ func (repo *Repository) GetCompareInfo(basePath, baseBranch, headBranch string, if !fileOnly { // avoid: ambiguous argument 'refs/a...refs/b': unknown revision or path not in the working tree. Use '--': 'git <command> [<revision>...] -- [<file>...]' var logs []byte - logs, _, err = NewCommand(repo.Ctx, "log").AddArguments(prettyLogFormat). - AddDynamicArguments(baseCommitID + separator + headBranch).AddArguments("--"). - RunStdBytes(&RunOpts{Dir: repo.Path}) + logs, _, err = NewCommand("log").AddArguments(prettyLogFormat). + AddDynamicArguments(baseCommitID+separator+headBranch).AddArguments("--"). + RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } @@ -150,8 +150,8 @@ func (repo *Repository) GetDiffNumChangedFiles(base, head string, directComparis } // avoid: ambiguous argument 'refs/a...refs/b': unknown revision or path not in the working tree. Use '--': 'git <command> [<revision>...] -- [<file>...]' - if err := NewCommand(repo.Ctx, "diff", "-z", "--name-only").AddDynamicArguments(base + separator + head).AddArguments("--"). - Run(&RunOpts{ + if err := NewCommand("diff", "-z", "--name-only").AddDynamicArguments(base+separator+head).AddArguments("--"). + Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: w, Stderr: stderr, @@ -161,7 +161,7 @@ func (repo *Repository) GetDiffNumChangedFiles(base, head string, directComparis // previously it would return the results of git diff -z --name-only base head so let's try that... w = &lineCountWriter{} stderr.Reset() - if err = NewCommand(repo.Ctx, "diff", "-z", "--name-only").AddDynamicArguments(base, head).AddArguments("--").Run(&RunOpts{ + if err = NewCommand("diff", "-z", "--name-only").AddDynamicArguments(base, head).AddArguments("--").Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: w, Stderr: stderr, @@ -174,23 +174,15 @@ func (repo *Repository) GetDiffNumChangedFiles(base, head string, directComparis return w.numLines, nil } -// GetDiffShortStat counts number of changed files, number of additions and deletions -func (repo *Repository) GetDiffShortStat(base, head string) (numFiles, totalAdditions, totalDeletions int, err error) { - numFiles, totalAdditions, totalDeletions, err = GetDiffShortStat(repo.Ctx, repo.Path, nil, base+"..."+head) - if err != nil && strings.Contains(err.Error(), "no merge base") { - return GetDiffShortStat(repo.Ctx, repo.Path, nil, base, head) - } - return numFiles, totalAdditions, totalDeletions, err -} - -// GetDiffShortStat counts number of changed files, number of additions and deletions -func GetDiffShortStat(ctx context.Context, repoPath string, trustedArgs TrustedCmdArgs, dynamicArgs ...string) (numFiles, totalAdditions, totalDeletions int, err error) { +// GetDiffShortStatByCmdArgs counts number of changed files, number of additions and deletions +// TODO: it can be merged with another "GetDiffShortStat" in the future +func GetDiffShortStatByCmdArgs(ctx context.Context, repoPath string, trustedArgs TrustedCmdArgs, dynamicArgs ...string) (numFiles, totalAdditions, totalDeletions int, err error) { // Now if we call: // $ git diff --shortstat 1ebb35b98889ff77299f24d82da426b434b0cca0...788b8b1440462d477f45b0088875 // we get: // " 9902 files changed, 2034198 insertions(+), 298800 deletions(-)\n" - cmd := NewCommand(ctx, "diff", "--shortstat").AddArguments(trustedArgs...).AddDynamicArguments(dynamicArgs...) - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + cmd := NewCommand("diff", "--shortstat").AddArguments(trustedArgs...).AddDynamicArguments(dynamicArgs...) + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return 0, 0, 0, err } @@ -236,8 +228,8 @@ func parseDiffStat(stdout string) (numFiles, totalAdditions, totalDeletions int, // GetDiff generates and returns patch data between given revisions, optimized for human readability func (repo *Repository) GetDiff(compareArg string, w io.Writer) error { stderr := new(bytes.Buffer) - return NewCommand(repo.Ctx, "diff", "-p").AddDynamicArguments(compareArg). - Run(&RunOpts{ + return NewCommand("diff", "-p").AddDynamicArguments(compareArg). + Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: w, Stderr: stderr, @@ -246,7 +238,7 @@ func (repo *Repository) GetDiff(compareArg string, w io.Writer) error { // GetDiffBinary generates and returns patch data between given revisions, including binary diffs. func (repo *Repository) GetDiffBinary(compareArg string, w io.Writer) error { - return NewCommand(repo.Ctx, "diff", "-p", "--binary", "--histogram").AddDynamicArguments(compareArg).Run(&RunOpts{ + return NewCommand("diff", "-p", "--binary", "--histogram").AddDynamicArguments(compareArg).Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: w, }) @@ -255,8 +247,8 @@ func (repo *Repository) GetDiffBinary(compareArg string, w io.Writer) error { // GetPatch generates and returns format-patch data between given revisions, able to be used with `git apply` func (repo *Repository) GetPatch(compareArg string, w io.Writer) error { stderr := new(bytes.Buffer) - return NewCommand(repo.Ctx, "format-patch", "--binary", "--stdout").AddDynamicArguments(compareArg). - Run(&RunOpts{ + return NewCommand("format-patch", "--binary", "--stdout").AddDynamicArguments(compareArg). + Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: w, Stderr: stderr, @@ -271,13 +263,13 @@ func (repo *Repository) GetFilesChangedBetween(base, head string) ([]string, err if err != nil { return nil, err } - cmd := NewCommand(repo.Ctx, "diff-tree", "--name-only", "--root", "--no-commit-id", "-r", "-z") + cmd := NewCommand("diff-tree", "--name-only", "--root", "--no-commit-id", "-r", "-z") if base == objectFormat.EmptyObjectID().String() { cmd.AddDynamicArguments(head) } else { cmd.AddDynamicArguments(base, head) } - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := cmd.RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } diff --git a/modules/git/repo_gpg.go b/modules/git/repo_gpg.go index e2b45064fd..0021a7bda7 100644 --- a/modules/git/repo_gpg.go +++ b/modules/git/repo_gpg.go @@ -6,6 +6,7 @@ package git import ( "fmt" + "os" "strings" "code.gitea.io/gitea/modules/process" @@ -13,6 +14,14 @@ import ( // LoadPublicKeyContent will load the key from gpg func (gpgSettings *GPGSettings) LoadPublicKeyContent() error { + if gpgSettings.Format == SigningKeyFormatSSH { + content, err := os.ReadFile(gpgSettings.KeyID) + if err != nil { + return fmt.Errorf("unable to read SSH public key file: %s, %w", gpgSettings.KeyID, err) + } + gpgSettings.PublicKeyContent = string(content) + return nil + } content, stderr, err := process.GetManager().Exec( "gpg -a --export", "gpg", "-a", "--export", gpgSettings.KeyID) @@ -33,7 +42,7 @@ func (repo *Repository) GetDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, Sign: true, } - value, _, _ := NewCommand(repo.Ctx, "config", "--get", "commit.gpgsign").RunStdString(&RunOpts{Dir: repo.Path}) + value, _, _ := NewCommand("config", "--get", "commit.gpgsign").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) sign, valid := ParseBool(strings.TrimSpace(value)) if !sign || !valid { gpgSettings.Sign = false @@ -41,13 +50,16 @@ func (repo *Repository) GetDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, return gpgSettings, nil } - signingKey, _, _ := NewCommand(repo.Ctx, "config", "--get", "user.signingkey").RunStdString(&RunOpts{Dir: repo.Path}) + signingKey, _, _ := NewCommand("config", "--get", "user.signingkey").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) gpgSettings.KeyID = strings.TrimSpace(signingKey) - defaultEmail, _, _ := NewCommand(repo.Ctx, "config", "--get", "user.email").RunStdString(&RunOpts{Dir: repo.Path}) + format, _, _ := NewCommand("config", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) + gpgSettings.Format = strings.TrimSpace(format) + + defaultEmail, _, _ := NewCommand("config", "--get", "user.email").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) gpgSettings.Email = strings.TrimSpace(defaultEmail) - defaultName, _, _ := NewCommand(repo.Ctx, "config", "--get", "user.name").RunStdString(&RunOpts{Dir: repo.Path}) + defaultName, _, _ := NewCommand("config", "--get", "user.name").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) gpgSettings.Name = strings.TrimSpace(defaultName) if err := gpgSettings.LoadPublicKeyContent(); err != nil { diff --git a/modules/git/repo_index.go b/modules/git/repo_index.go index f45b6e6191..4879121a41 100644 --- a/modules/git/repo_index.go +++ b/modules/git/repo_index.go @@ -10,8 +10,7 @@ import ( "path/filepath" "strings" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/setting" ) // ReadTreeToIndex reads a treeish to the index @@ -22,7 +21,7 @@ func (repo *Repository) ReadTreeToIndex(treeish string, indexFilename ...string) } if len(treeish) != objectFormat.FullLength() { - res, _, err := NewCommand(repo.Ctx, "rev-parse", "--verify").AddDynamicArguments(treeish).RunStdString(&RunOpts{Dir: repo.Path}) + res, _, err := NewCommand("rev-parse", "--verify").AddDynamicArguments(treeish).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return err } @@ -42,7 +41,7 @@ func (repo *Repository) readTreeToIndex(id ObjectID, indexFilename ...string) er if len(indexFilename) > 0 { env = append(os.Environ(), "GIT_INDEX_FILE="+indexFilename[0]) } - _, _, err := NewCommand(repo.Ctx, "read-tree").AddDynamicArguments(id.String()).RunStdString(&RunOpts{Dir: repo.Path, Env: env}) + _, _, err := NewCommand("read-tree").AddDynamicArguments(id.String()).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path, Env: env}) if err != nil { return err } @@ -59,43 +58,35 @@ func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (tmpIndexFilena } }() - removeDirFn := func(dir string) func() { // it can't use the return value "tmpDir" directly because it is empty when error occurs - return func() { - if err := util.RemoveAll(dir); err != nil { - log.Error("failed to remove tmp index dir: %v", err) - } - } - } - - tmpDir, err = os.MkdirTemp("", "index") + tmpDir, cancel, err = setting.AppDataTempDir("git-repo-content").MkdirTempRandom("index") if err != nil { return "", "", nil, err } tmpIndexFilename = filepath.Join(tmpDir, ".tmp-index") - cancel = removeDirFn(tmpDir) + err = repo.ReadTreeToIndex(treeish, tmpIndexFilename) if err != nil { return "", "", cancel, err } - return tmpIndexFilename, tmpDir, cancel, err + return tmpIndexFilename, tmpDir, cancel, nil } // EmptyIndex empties the index func (repo *Repository) EmptyIndex() error { - _, _, err := NewCommand(repo.Ctx, "read-tree", "--empty").RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("read-tree", "--empty").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // LsFiles checks if the given filenames are in the index func (repo *Repository) LsFiles(filenames ...string) ([]string, error) { - cmd := NewCommand(repo.Ctx, "ls-files", "-z").AddDashesAndList(filenames...) - res, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + cmd := NewCommand("ls-files", "-z").AddDashesAndList(filenames...) + res, _, err := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } filelist := make([]string, 0, len(filenames)) - for _, line := range bytes.Split(res, []byte{'\000'}) { + for line := range bytes.SplitSeq(res, []byte{'\000'}) { filelist = append(filelist, string(line)) } @@ -108,7 +99,7 @@ func (repo *Repository) RemoveFilesFromIndex(filenames ...string) error { if err != nil { return err } - cmd := NewCommand(repo.Ctx, "update-index", "--remove", "-z", "--index-info") + cmd := NewCommand("update-index", "--remove", "-z", "--index-info") stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) buffer := new(bytes.Buffer) @@ -118,7 +109,7 @@ func (repo *Repository) RemoveFilesFromIndex(filenames ...string) error { buffer.WriteString("0 blob " + objectFormat.EmptyObjectID().String() + "\t" + file + "\000") } } - return cmd.Run(&RunOpts{ + return cmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdin: bytes.NewReader(buffer.Bytes()), Stdout: stdout, @@ -134,7 +125,7 @@ type IndexObjectInfo struct { // AddObjectsToIndex adds the provided object hashes to the index at the provided filenames func (repo *Repository) AddObjectsToIndex(objects ...IndexObjectInfo) error { - cmd := NewCommand(repo.Ctx, "update-index", "--add", "--replace", "-z", "--index-info") + cmd := NewCommand("update-index", "--add", "--replace", "-z", "--index-info") stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) buffer := new(bytes.Buffer) @@ -142,7 +133,7 @@ func (repo *Repository) AddObjectsToIndex(objects ...IndexObjectInfo) error { // using format: mode SP type SP sha1 TAB path buffer.WriteString(object.Mode + " blob " + object.Object.String() + "\t" + object.Filename + "\000") } - return cmd.Run(&RunOpts{ + return cmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdin: bytes.NewReader(buffer.Bytes()), Stdout: stdout, @@ -157,7 +148,7 @@ func (repo *Repository) AddObjectToIndex(mode string, object ObjectID, filename // WriteTree writes the current index as a tree to the object db and returns its hash func (repo *Repository) WriteTree() (*Tree, error) { - stdout, _, runErr := NewCommand(repo.Ctx, "write-tree").RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand("write-tree").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } diff --git a/modules/git/repo_object.go b/modules/git/repo_object.go index 3d48b91c6d..08e0413311 100644 --- a/modules/git/repo_object.go +++ b/modules/git/repo_object.go @@ -68,13 +68,13 @@ func (repo *Repository) HashObject(reader io.Reader) (ObjectID, error) { func (repo *Repository) hashObject(reader io.Reader, save bool) (string, error) { var cmd *Command if save { - cmd = NewCommand(repo.Ctx, "hash-object", "-w", "--stdin") + cmd = NewCommand("hash-object", "-w", "--stdin") } else { - cmd = NewCommand(repo.Ctx, "hash-object", "--stdin") + cmd = NewCommand("hash-object", "--stdin") } stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) - err := cmd.Run(&RunOpts{ + err := cmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdin: reader, Stdout: stdout, @@ -85,17 +85,3 @@ func (repo *Repository) hashObject(reader io.Reader, save bool) (string, error) } return strings.TrimSpace(stdout.String()), nil } - -// GetRefType gets the type of the ref based on the string -func (repo *Repository) GetRefType(ref string) ObjectType { - if repo.IsTagExist(ref) { - return ObjectTag - } else if repo.IsBranchExist(ref) { - return ObjectBranch - } else if repo.IsCommitExist(ref) { - return ObjectCommit - } else if _, err := repo.GetBlob(ref); err == nil { - return ObjectBlob - } - return ObjectType("invalid") -} diff --git a/modules/git/repo_ref.go b/modules/git/repo_ref.go index 850ec65502..554f9f73e1 100644 --- a/modules/git/repo_ref.go +++ b/modules/git/repo_ref.go @@ -18,15 +18,16 @@ func (repo *Repository) GetRefs() ([]*Reference, error) { // ListOccurrences lists all refs of the given refType the given commit appears in sorted by creation date DESC // refType should only be a literal "branch" or "tag" and nothing else func (repo *Repository) ListOccurrences(ctx context.Context, refType, commitSHA string) ([]string, error) { - cmd := NewCommand(ctx) - if refType == "branch" { + cmd := NewCommand() + switch refType { + case "branch": cmd.AddArguments("branch") - } else if refType == "tag" { + case "tag": cmd.AddArguments("tag") - } else { + default: return nil, util.NewInvalidArgumentErrorf(`can only use "branch" or "tag" for refType, but got %q`, refType) } - stdout, _, err := cmd.AddArguments("--no-color", "--sort=-creatordate", "--contains").AddDynamicArguments(commitSHA).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := cmd.AddArguments("--no-color", "--sort=-creatordate", "--contains").AddDynamicArguments(commitSHA).RunStdString(ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } diff --git a/modules/git/repo_ref_nogogit.go b/modules/git/repo_ref_nogogit.go index ac53d661b5..8d34713eaf 100644 --- a/modules/git/repo_ref_nogogit.go +++ b/modules/git/repo_ref_nogogit.go @@ -21,7 +21,7 @@ func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { go func() { stderrBuilder := &strings.Builder{} - err := NewCommand(repo.Ctx, "for-each-ref").Run(&RunOpts{ + err := NewCommand("for-each-ref").Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: stdoutWriter, Stderr: stderrBuilder, diff --git a/modules/git/repo_stats.go b/modules/git/repo_stats.go index 83220104bd..8c6f31c38c 100644 --- a/modules/git/repo_stats.go +++ b/modules/git/repo_stats.go @@ -40,7 +40,9 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) since := fromTime.Format(time.RFC3339) - stdout, _, runErr := NewCommand(repo.Ctx, "rev-list", "--count", "--no-merges", "--branches=*", "--date=iso").AddOptionFormat("--since='%s'", since).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand("rev-list", "--count", "--no-merges", "--branches=*", "--date=iso"). + AddOptionFormat("--since=%s", since). + RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -60,7 +62,8 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) _ = stdoutWriter.Close() }() - gitCmd := NewCommand(repo.Ctx, "log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso").AddOptionFormat("--since='%s'", since) + gitCmd := NewCommand("log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso"). + AddOptionFormat("--since=%s", since) if len(branch) == 0 { gitCmd.AddArguments("--branches=*") } else { @@ -68,7 +71,7 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) } stderr := new(strings.Builder) - err = gitCmd.Run(&RunOpts{ + err = gitCmd.Run(repo.Ctx, &RunOpts{ Env: []string{}, Dir: repo.Path, Stdout: stdoutWriter, diff --git a/modules/git/repo_stats_test.go b/modules/git/repo_stats_test.go index 3d032385ee..85d8807a6e 100644 --- a/modules/git/repo_stats_test.go +++ b/modules/git/repo_stats_test.go @@ -30,7 +30,7 @@ func TestRepository_GetCodeActivityStats(t *testing.T) { assert.EqualValues(t, 10, code.Additions) assert.EqualValues(t, 1, code.Deletions) assert.Len(t, code.Authors, 3) - assert.EqualValues(t, "tris.git@shoddynet.org", code.Authors[1].Email) + assert.Equal(t, "tris.git@shoddynet.org", code.Authors[1].Email) assert.EqualValues(t, 3, code.Authors[1].Commits) assert.EqualValues(t, 5, code.Authors[0].Commits) } diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index 2026a4c9f5..c8d72eee02 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -5,7 +5,6 @@ package git import ( - "context" "fmt" "io" "strings" @@ -17,20 +16,15 @@ import ( // TagPrefix tags prefix path on the repository const TagPrefix = "refs/tags/" -// IsTagExist returns true if given tag exists in the repository. -func IsTagExist(ctx context.Context, repoPath, name string) bool { - return IsReferenceExist(ctx, repoPath, TagPrefix+name) -} - // CreateTag create one tag in the repository func (repo *Repository) CreateTag(name, revision string) error { - _, _, err := NewCommand(repo.Ctx, "tag").AddDashesAndList(name, revision).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("tag").AddDashesAndList(name, revision).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // CreateAnnotatedTag create one annotated tag in the repository func (repo *Repository) CreateAnnotatedTag(name, message, revision string) error { - _, _, err := NewCommand(repo.Ctx, "tag", "-a", "-m").AddDynamicArguments(message).AddDashesAndList(name, revision).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("tag", "-a", "-m").AddDynamicArguments(message).AddDashesAndList(name, revision).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } @@ -40,13 +34,13 @@ func (repo *Repository) GetTagNameBySHA(sha string) (string, error) { return "", fmt.Errorf("SHA is too short: %s", sha) } - stdout, _, err := NewCommand(repo.Ctx, "show-ref", "--tags", "-d").RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("show-ref", "--tags", "-d").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return "", err } - tagRefs := strings.Split(stdout, "\n") - for _, tagRef := range tagRefs { + tagRefs := strings.SplitSeq(stdout, "\n") + for tagRef := range tagRefs { if len(strings.TrimSpace(tagRef)) > 0 { fields := strings.Fields(tagRef) if strings.HasPrefix(fields[0], sha) && strings.HasPrefix(fields[1], TagPrefix) { @@ -63,12 +57,12 @@ func (repo *Repository) GetTagNameBySHA(sha string) (string, error) { // GetTagID returns the object ID for a tag (annotated tags have both an object SHA AND a commit SHA) func (repo *Repository) GetTagID(name string) (string, error) { - stdout, _, err := NewCommand(repo.Ctx, "show-ref", "--tags").AddDashesAndList(name).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("show-ref", "--tags").AddDashesAndList(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return "", err } // Make sure exact match is used: "v1" != "release/v1" - for _, line := range strings.Split(stdout, "\n") { + for line := range strings.SplitSeq(stdout, "\n") { fields := strings.Fields(line) if len(fields) == 2 && fields[1] == "refs/tags/"+name { return fields[0], nil @@ -123,9 +117,9 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) { rc := &RunOpts{Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr} go func() { - err := NewCommand(repo.Ctx, "for-each-ref"). + err := NewCommand("for-each-ref"). AddOptionFormat("--format=%s", forEachRefFmt.Flag()). - AddArguments("--sort", "-*creatordate", "refs/tags").Run(rc) + AddArguments("--sort", "-*creatordate", "refs/tags").Run(repo.Ctx, rc) if err != nil { _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String())) } else { diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index e0a3104249..3d2b4f52bd 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -41,8 +41,11 @@ func (repo *Repository) GetTagType(id ObjectID) (string, error) { return "", err } _, typ, _, err := ReadBatchLine(rd) - if IsErrNotExist(err) { - return "", ErrNotExist{ID: id.String()} + if err != nil { + if IsErrNotExist(err) { + return "", ErrNotExist{ID: id.String()} + } + return "", err } return typ, nil } diff --git a/modules/git/repo_tag_test.go b/modules/git/repo_tag_test.go index f1f081680a..f1f5ff6664 100644 --- a/modules/git/repo_tag_test.go +++ b/modules/git/repo_tag_test.go @@ -27,12 +27,12 @@ func TestRepository_GetTags(t *testing.T) { } assert.Len(t, tags, 2) assert.Len(t, tags, total) - assert.EqualValues(t, "signed-tag", tags[0].Name) - assert.EqualValues(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", tags[0].ID.String()) - assert.EqualValues(t, "tag", tags[0].Type) - assert.EqualValues(t, "test", tags[1].Name) - assert.EqualValues(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[1].ID.String()) - assert.EqualValues(t, "tag", tags[1].Type) + assert.Equal(t, "signed-tag", tags[0].Name) + assert.Equal(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", tags[0].ID.String()) + assert.Equal(t, "tag", tags[0].Type) + assert.Equal(t, "test", tags[1].Name) + assert.Equal(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[1].ID.String()) + assert.Equal(t, "tag", tags[1].Type) } func TestRepository_GetTag(t *testing.T) { @@ -64,18 +64,13 @@ func TestRepository_GetTag(t *testing.T) { // and try to get the Tag for lightweight tag lTag, err := bareRepo1.GetTag(lTagName) - if err != nil { - assert.NoError(t, err) - return - } - if lTag == nil { - assert.NotNil(t, lTag) - assert.FailNow(t, "nil lTag: %s", lTagName) - } - assert.EqualValues(t, lTagName, lTag.Name) - assert.EqualValues(t, lTagCommitID, lTag.ID.String()) - assert.EqualValues(t, lTagCommitID, lTag.Object.String()) - assert.EqualValues(t, "commit", lTag.Type) + require.NoError(t, err) + require.NotNil(t, lTag, "nil lTag: %s", lTagName) + + assert.Equal(t, lTagName, lTag.Name) + assert.Equal(t, lTagCommitID, lTag.ID.String()) + assert.Equal(t, lTagCommitID, lTag.Object.String()) + assert.Equal(t, "commit", lTag.Type) // ANNOTATED TAGS aTagCommitID := "8006ff9adbf0cb94da7dad9e537e53817f9fa5c0" @@ -97,19 +92,14 @@ func TestRepository_GetTag(t *testing.T) { } aTag, err := bareRepo1.GetTag(aTagName) - if err != nil { - assert.NoError(t, err) - return - } - if aTag == nil { - assert.NotNil(t, aTag) - assert.FailNow(t, "nil aTag: %s", aTagName) - } - assert.EqualValues(t, aTagName, aTag.Name) - assert.EqualValues(t, aTagID, aTag.ID.String()) + require.NoError(t, err) + require.NotNil(t, aTag, "nil aTag: %s", aTagName) + + assert.Equal(t, aTagName, aTag.Name) + assert.Equal(t, aTagID, aTag.ID.String()) assert.NotEqual(t, aTagID, aTag.Object.String()) - assert.EqualValues(t, aTagCommitID, aTag.Object.String()) - assert.EqualValues(t, "tag", aTag.Type) + assert.Equal(t, aTagCommitID, aTag.Object.String()) + assert.Equal(t, "tag", aTag.Type) // RELEASE TAGS @@ -127,14 +117,14 @@ func TestRepository_GetTag(t *testing.T) { assert.NoError(t, err) return } - assert.EqualValues(t, rTagCommitID, rTagID) + assert.Equal(t, rTagCommitID, rTagID) oTagID, err := bareRepo1.GetTagID(lTagName) if err != nil { assert.NoError(t, err) return } - assert.EqualValues(t, lTagCommitID, oTagID) + assert.Equal(t, lTagCommitID, oTagID) } func TestRepository_GetAnnotatedTag(t *testing.T) { @@ -170,9 +160,9 @@ func TestRepository_GetAnnotatedTag(t *testing.T) { return } assert.NotNil(t, tag) - assert.EqualValues(t, aTagName, tag.Name) - assert.EqualValues(t, aTagID, tag.ID.String()) - assert.EqualValues(t, "tag", tag.Type) + assert.Equal(t, aTagName, tag.Name) + assert.Equal(t, aTagID, tag.ID.String()) + assert.Equal(t, "tag", tag.Type) // Annotated tag's Commit ID should fail tag2, err := bareRepo1.GetAnnotatedTag(aTagCommitID) diff --git a/modules/git/repo_test.go b/modules/git/repo_test.go index 9db78153a1..4638bdac1f 100644 --- a/modules/git/repo_test.go +++ b/modules/git/repo_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "path/filepath" "testing" @@ -33,21 +32,21 @@ func TestRepoIsEmpty(t *testing.T) { func TestRepoGetDivergingCommits(t *testing.T) { bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") - do, err := GetDivergingCommits(context.Background(), bareRepo1Path, "master", "branch2") + do, err := GetDivergingCommits(t.Context(), bareRepo1Path, "master", "branch2") assert.NoError(t, err) assert.Equal(t, DivergeObject{ Ahead: 1, Behind: 5, }, do) - do, err = GetDivergingCommits(context.Background(), bareRepo1Path, "master", "master") + do, err = GetDivergingCommits(t.Context(), bareRepo1Path, "master", "master") assert.NoError(t, err) assert.Equal(t, DivergeObject{ Ahead: 0, Behind: 0, }, do) - do, err = GetDivergingCommits(context.Background(), bareRepo1Path, "master", "test") + do, err = GetDivergingCommits(t.Context(), bareRepo1Path, "master", "test") assert.NoError(t, err) assert.Equal(t, DivergeObject{ Ahead: 0, diff --git a/modules/git/repo_tree.go b/modules/git/repo_tree.go index ab48d47d13..309a73d759 100644 --- a/modules/git/repo_tree.go +++ b/modules/git/repo_tree.go @@ -15,7 +15,7 @@ import ( type CommitTreeOpts struct { Parents []string Message string - KeyID string + Key *SigningKey NoGPGSign bool AlwaysSign bool } @@ -33,7 +33,7 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt "GIT_COMMITTER_EMAIL="+committer.Email, "GIT_COMMITTER_DATE="+commitTimeStr, ) - cmd := NewCommand(repo.Ctx, "commit-tree").AddDynamicArguments(tree.ID.String()) + cmd := NewCommand("commit-tree").AddDynamicArguments(tree.ID.String()) for _, parent := range opts.Parents { cmd.AddArguments("-p").AddDynamicArguments(parent) @@ -43,8 +43,13 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt _, _ = messageBytes.WriteString(opts.Message) _, _ = messageBytes.WriteString("\n") - if opts.KeyID != "" || opts.AlwaysSign { - cmd.AddOptionFormat("-S%s", opts.KeyID) + if opts.Key != nil { + if opts.Key.Format != "" { + cmd.AddConfig("gpg.format", opts.Key.Format) + } + cmd.AddOptionFormat("-S%s", opts.Key.KeyID) + } else if opts.AlwaysSign { + cmd.AddOptionFormat("-S") } if opts.NoGPGSign { @@ -53,7 +58,7 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) - err := cmd.Run(&RunOpts{ + err := cmd.Run(repo.Ctx, &RunOpts{ Env: env, Dir: repo.Path, Stdin: messageBytes, diff --git a/modules/git/repo_tree_gogit.go b/modules/git/repo_tree_gogit.go index 651794a5aa..f77cd83612 100644 --- a/modules/git/repo_tree_gogit.go +++ b/modules/git/repo_tree_gogit.go @@ -36,7 +36,7 @@ func (repo *Repository) GetTree(idStr string) (*Tree, error) { } if len(idStr) != objectFormat.FullLength() { - res, _, err := NewCommand(repo.Ctx, "rev-parse", "--verify").AddDynamicArguments(idStr).RunStdString(&RunOpts{Dir: repo.Path}) + res, _, err := NewCommand("rev-parse", "--verify").AddDynamicArguments(idStr).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go index d74769ccb2..1954f85162 100644 --- a/modules/git/repo_tree_nogogit.go +++ b/modules/git/repo_tree_nogogit.go @@ -35,7 +35,11 @@ func (repo *Repository) getTree(id ObjectID) (*Tree, error) { if err != nil { return nil, err } - commit, err := tag.Commit(repo) + + if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { + return nil, err + } + commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) if err != nil { return nil, err } diff --git a/modules/git/signature_test.go b/modules/git/signature_test.go index 92681feea9..b9b5aff61b 100644 --- a/modules/git/signature_test.go +++ b/modules/git/signature_test.go @@ -42,6 +42,6 @@ func TestParseSignatureFromCommitLine(t *testing.T) { } for _, test := range tests { got := parseSignatureFromCommitLine(test.line) - assert.EqualValues(t, test.want, got) + assert.Equal(t, test.want, got) } } diff --git a/modules/git/submodule.go b/modules/git/submodule.go index 017b644052..31a32f1a9e 100644 --- a/modules/git/submodule.go +++ b/modules/git/submodule.go @@ -45,7 +45,7 @@ func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submodul return scanner.Err() }, } - err = NewCommand(ctx, "ls-tree", "-r", "--", "HEAD").Run(opts) + err = NewCommand("ls-tree", "-r", "--", "HEAD").Run(ctx, opts) if err != nil { return nil, fmt.Errorf("GetTemplateSubmoduleCommits: error running git ls-tree: %v", err) } @@ -56,8 +56,8 @@ func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submodul // It is only for generating new repos based on existing template, requires the .gitmodules file to be already present in the work dir. func AddTemplateSubmoduleIndexes(ctx context.Context, repoPath string, submodules []TemplateSubmoduleCommit) error { for _, submodule := range submodules { - cmd := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path) - if stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}); err != nil { + cmd := NewCommand("update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path) + if stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}); err != nil { log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Path, repoPath, stdout, err) return err } diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go index d53946a27d..7893b95e3a 100644 --- a/modules/git/submodule_test.go +++ b/modules/git/submodule_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "os" "path/filepath" "testing" @@ -20,29 +19,29 @@ func TestGetTemplateSubmoduleCommits(t *testing.T) { assert.Len(t, submodules, 2) - assert.EqualValues(t, "<°)))><", submodules[0].Path) - assert.EqualValues(t, "d2932de67963f23d43e1c7ecf20173e92ee6c43c", submodules[0].Commit) + assert.Equal(t, "<°)))><", submodules[0].Path) + assert.Equal(t, "d2932de67963f23d43e1c7ecf20173e92ee6c43c", submodules[0].Commit) - assert.EqualValues(t, "libtest", submodules[1].Path) - assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[1].Commit) + assert.Equal(t, "libtest", submodules[1].Path) + assert.Equal(t, "1234567890123456789012345678901234567890", submodules[1].Commit) } func TestAddTemplateSubmoduleIndexes(t *testing.T) { - ctx := context.Background() + ctx := t.Context() tmpDir := t.TempDir() var err error - _, _, err = NewCommand(ctx, "init").RunStdString(&RunOpts{Dir: tmpDir}) + _, _, err = NewCommand("init").RunStdString(ctx, &RunOpts{Dir: tmpDir}) require.NoError(t, err) _ = os.Mkdir(filepath.Join(tmpDir, "new-dir"), 0o755) err = AddTemplateSubmoduleIndexes(ctx, tmpDir, []TemplateSubmoduleCommit{{Path: "new-dir", Commit: "1234567890123456789012345678901234567890"}}) require.NoError(t, err) - _, _, err = NewCommand(ctx, "add", "--all").RunStdString(&RunOpts{Dir: tmpDir}) + _, _, err = NewCommand("add", "--all").RunStdString(ctx, &RunOpts{Dir: tmpDir}) require.NoError(t, err) - _, _, err = NewCommand(ctx, "-c", "user.name=a", "-c", "user.email=b", "commit", "-m=test").RunStdString(&RunOpts{Dir: tmpDir}) + _, _, err = NewCommand("-c", "user.name=a", "-c", "user.email=b", "commit", "-m=test").RunStdString(ctx, &RunOpts{Dir: tmpDir}) require.NoError(t, err) submodules, err := GetTemplateSubmoduleCommits(DefaultContext, tmpDir) require.NoError(t, err) assert.Len(t, submodules, 1) - assert.EqualValues(t, "new-dir", submodules[0].Path) - assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[0].Commit) + assert.Equal(t, "new-dir", submodules[0].Path) + assert.Equal(t, "1234567890123456789012345678901234567890", submodules[0].Commit) } diff --git a/modules/git/tag.go b/modules/git/tag.go index f7666aa89b..8bf3658d62 100644 --- a/modules/git/tag.go +++ b/modules/git/tag.go @@ -21,11 +21,6 @@ type Tag struct { Signature *CommitSignature } -// Commit return the commit of the tag reference -func (tag *Tag) Commit(gitRepo *Repository) (*Commit, error) { - return gitRepo.getCommit(tag.Object) -} - func parsePayloadSignature(data []byte, messageStart int) (payload, msg, sign string) { pos := messageStart signStart, signEnd := -1, -1 diff --git a/modules/git/tests/repos/language_stats_repo/config b/modules/git/tests/repos/language_stats_repo/config index 515f483629..a4ef456cbc 100644 --- a/modules/git/tests/repos/language_stats_repo/config +++ b/modules/git/tests/repos/language_stats_repo/config @@ -1,5 +1,5 @@ [core] repositoryformatversion = 0 filemode = true - bare = false + bare = true logallrefupdates = true diff --git a/modules/git/tests/repos/repo3_notes/config b/modules/git/tests/repos/repo3_notes/config index d545cdabdb..5ed22e23d1 100644 --- a/modules/git/tests/repos/repo3_notes/config +++ b/modules/git/tests/repos/repo3_notes/config @@ -1,7 +1,7 @@ [core] repositoryformatversion = 0 filemode = false - bare = false + bare = true logallrefupdates = true symlinks = false ignorecase = true diff --git a/modules/git/tests/repos/repo4_commitsbetween/config b/modules/git/tests/repos/repo4_commitsbetween/config index d545cdabdb..5ed22e23d1 100644 --- a/modules/git/tests/repos/repo4_commitsbetween/config +++ b/modules/git/tests/repos/repo4_commitsbetween/config @@ -1,7 +1,7 @@ [core] repositoryformatversion = 0 filemode = false - bare = false + bare = true logallrefupdates = true symlinks = false ignorecase = true diff --git a/modules/git/tree.go b/modules/git/tree.go index 5a644f6c87..38fb45f3b1 100644 --- a/modules/git/tree.go +++ b/modules/git/tree.go @@ -48,15 +48,15 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) { // LsTree checks if the given filenames are in the tree func (repo *Repository) LsTree(ref string, filenames ...string) ([]string, error) { - cmd := NewCommand(repo.Ctx, "ls-tree", "-z", "--name-only"). + cmd := NewCommand("ls-tree", "-z", "--name-only"). AddDashesAndList(append([]string{ref}, filenames...)...) - res, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + res, _, err := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } filelist := make([]string, 0, len(filenames)) - for _, line := range bytes.Split(res, []byte{'\000'}) { + for line := range bytes.SplitSeq(res, []byte{'\000'}) { filelist = append(filelist, string(line)) } @@ -65,9 +65,9 @@ func (repo *Repository) LsTree(ref string, filenames ...string) ([]string, error // GetTreePathLatestCommit returns the latest commit of a tree path func (repo *Repository) GetTreePathLatestCommit(refName, treePath string) (*Commit, error) { - stdout, _, err := NewCommand(repo.Ctx, "rev-list", "-1"). + stdout, _, err := NewCommand("rev-list", "-1"). AddDynamicArguments(refName).AddDashesAndList(treePath). - RunStdString(&RunOpts{Dir: repo.Path}) + RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } diff --git a/modules/git/tree_blob_gogit.go b/modules/git/tree_blob_gogit.go index 92c25cb92c..f29e8f8b9e 100644 --- a/modules/git/tree_blob_gogit.go +++ b/modules/git/tree_blob_gogit.go @@ -21,6 +21,7 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { return &TreeEntry{ ID: t.ID, // Type: ObjectTree, + ptree: t, gogitTreeEntry: &object.TreeEntry{ Name: "", Mode: filemode.Dir, diff --git a/modules/git/tree_blob_nogogit.go b/modules/git/tree_blob_nogogit.go index b7bcf40edd..b18d0fa05e 100644 --- a/modules/git/tree_blob_nogogit.go +++ b/modules/git/tree_blob_nogogit.go @@ -11,7 +11,7 @@ import ( ) // GetTreeEntryByPath get the tree entries according the sub dir -func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { +func (t *Tree) GetTreeEntryByPath(relpath string) (_ *TreeEntry, err error) { if len(relpath) == 0 { return &TreeEntry{ ptree: t, @@ -21,27 +21,25 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { }, nil } - // FIXME: This should probably use git cat-file --batch to be a bit more efficient relpath = path.Clean(relpath) parts := strings.Split(relpath, "/") - var err error + tree := t - for i, name := range parts { - if i == len(parts)-1 { - entries, err := tree.ListEntries() - if err != nil { - return nil, err - } - for _, v := range entries { - if v.Name() == name { - return v, nil - } - } - } else { - tree, err = tree.SubTree(name) - if err != nil { - return nil, err - } + for _, name := range parts[:len(parts)-1] { + tree, err = tree.SubTree(name) + if err != nil { + return nil, err + } + } + + name := parts[len(parts)-1] + entries, err := tree.ListEntries() + if err != nil { + return nil, err + } + for _, v := range entries { + if v.Name() == name { + return v, nil } } return nil, ErrNotExist{"", relpath} diff --git a/modules/git/tree_entry.go b/modules/git/tree_entry.go index 9513121487..5099d8ee79 100644 --- a/modules/git/tree_entry.go +++ b/modules/git/tree_entry.go @@ -5,9 +5,11 @@ package git import ( - "io" + "path" "sort" "strings" + + "code.gitea.io/gitea/modules/util" ) // Type returns the type of the entry (commit, tree, blob) @@ -22,83 +24,57 @@ func (te *TreeEntry) Type() string { } } -// FollowLink returns the entry pointed to by a symlink -func (te *TreeEntry) FollowLink() (*TreeEntry, error) { +type EntryFollowResult struct { + SymlinkContent string + TargetFullPath string + TargetEntry *TreeEntry +} + +func EntryFollowLink(commit *Commit, fullPath string, te *TreeEntry) (*EntryFollowResult, error) { if !te.IsLink() { - return nil, ErrBadLink{te.Name(), "not a symlink"} + return nil, util.ErrorWrap(util.ErrUnprocessableContent, "%q is not a symlink", fullPath) } - // read the link - r, err := te.Blob().DataAsync() - if err != nil { - return nil, err + // git's filename max length is 4096, hopefully a link won't be longer than multiple of that + const maxSymlinkSize = 20 * 4096 + if te.Blob().Size() > maxSymlinkSize { + return nil, util.ErrorWrap(util.ErrUnprocessableContent, "%q content exceeds symlink limit", fullPath) } - closed := false - defer func() { - if !closed { - _ = r.Close() - } - }() - buf := make([]byte, te.Size()) - _, err = io.ReadFull(r, buf) + + link, err := te.Blob().GetBlobContent(maxSymlinkSize) if err != nil { return nil, err } - _ = r.Close() - closed = true - - lnk := string(buf) - t := te.ptree - - // traverse up directories - for ; t != nil && strings.HasPrefix(lnk, "../"); lnk = lnk[3:] { - t = t.ptree + if strings.HasPrefix(link, "/") { + // It's said that absolute path will be stored as is in Git + return &EntryFollowResult{SymlinkContent: link}, util.ErrorWrap(util.ErrUnprocessableContent, "%q is an absolute symlink", fullPath) } - if t == nil { - return nil, ErrBadLink{te.Name(), "points outside of repo"} - } - - target, err := t.GetTreeEntryByPath(lnk) + targetFullPath := path.Join(path.Dir(fullPath), link) + targetEntry, err := commit.GetTreeEntryByPath(targetFullPath) if err != nil { - if IsErrNotExist(err) { - return nil, ErrBadLink{te.Name(), "broken link"} - } - return nil, err + return &EntryFollowResult{SymlinkContent: link}, err } - return target, nil + return &EntryFollowResult{SymlinkContent: link, TargetFullPath: targetFullPath, TargetEntry: targetEntry}, nil } -// FollowLinks returns the entry ultimately pointed to by a symlink -func (te *TreeEntry) FollowLinks() (*TreeEntry, error) { - if !te.IsLink() { - return nil, ErrBadLink{te.Name(), "not a symlink"} - } - entry := te - for i := 0; i < 999; i++ { - if entry.IsLink() { - next, err := entry.FollowLink() - if err != nil { - return nil, err - } - if next.ID == entry.ID { - return nil, ErrBadLink{ - entry.Name(), - "recursive link", - } - } - entry = next - } else { +func EntryFollowLinks(commit *Commit, firstFullPath string, firstTreeEntry *TreeEntry, optLimit ...int) (res *EntryFollowResult, err error) { + limit := util.OptionalArg(optLimit, 10) + treeEntry, fullPath := firstTreeEntry, firstFullPath + for range limit { + res, err = EntryFollowLink(commit, fullPath, treeEntry) + if err != nil { + return res, err + } + treeEntry, fullPath = res.TargetEntry, res.TargetFullPath + if !treeEntry.IsLink() { break } } - if entry.IsLink() { - return nil, ErrBadLink{ - te.Name(), - "too many levels of symbolic links", - } + if treeEntry.IsLink() { + return res, util.ErrorWrap(util.ErrUnprocessableContent, "%q has too many links", firstFullPath) } - return entry, nil + return res, nil } // returns the Tree pointed to by this TreeEntry, or nil if this is not a tree diff --git a/modules/git/tree_entry_common_test.go b/modules/git/tree_entry_common_test.go new file mode 100644 index 0000000000..8b63bbb993 --- /dev/null +++ b/modules/git/tree_entry_common_test.go @@ -0,0 +1,76 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "testing" + + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFollowLink(t *testing.T) { + r, err := openRepositoryWithDefaultContext("tests/repos/repo1_bare") + require.NoError(t, err) + defer r.Close() + + commit, err := r.GetCommit("37991dec2c8e592043f47155ce4808d4580f9123") + require.NoError(t, err) + + // get the symlink + { + lnkFullPath := "foo/bar/link_to_hello" + lnk, err := commit.Tree.GetTreeEntryByPath("foo/bar/link_to_hello") + require.NoError(t, err) + assert.True(t, lnk.IsLink()) + + // should be able to dereference to target + res, err := EntryFollowLink(commit, lnkFullPath, lnk) + require.NoError(t, err) + assert.Equal(t, "hello", res.TargetEntry.Name()) + assert.Equal(t, "foo/nar/hello", res.TargetFullPath) + assert.False(t, res.TargetEntry.IsLink()) + assert.Equal(t, "b14df6442ea5a1b382985a6549b85d435376c351", res.TargetEntry.ID.String()) + } + + { + // should error when called on a normal file + entry, err := commit.Tree.GetTreeEntryByPath("file1.txt") + require.NoError(t, err) + res, err := EntryFollowLink(commit, "file1.txt", entry) + assert.ErrorIs(t, err, util.ErrUnprocessableContent) + assert.Nil(t, res) + } + + { + // should error for broken links + entry, err := commit.Tree.GetTreeEntryByPath("foo/broken_link") + require.NoError(t, err) + assert.True(t, entry.IsLink()) + res, err := EntryFollowLink(commit, "foo/broken_link", entry) + assert.ErrorIs(t, err, util.ErrNotExist) + assert.Equal(t, "nar/broken_link", res.SymlinkContent) + } + + { + // should error for external links + entry, err := commit.Tree.GetTreeEntryByPath("foo/outside_repo") + require.NoError(t, err) + assert.True(t, entry.IsLink()) + res, err := EntryFollowLink(commit, "foo/outside_repo", entry) + assert.ErrorIs(t, err, util.ErrNotExist) + assert.Equal(t, "../../outside_repo", res.SymlinkContent) + } + + { + // testing fix for short link bug + entry, err := commit.Tree.GetTreeEntryByPath("foo/link_short") + require.NoError(t, err) + res, err := EntryFollowLink(commit, "foo/link_short", entry) + assert.ErrorIs(t, err, util.ErrNotExist) + assert.Equal(t, "a", res.SymlinkContent) + } +} diff --git a/modules/git/tree_entry_gogit.go b/modules/git/tree_entry_gogit.go index eb9b012681..e6845f1c77 100644 --- a/modules/git/tree_entry_gogit.go +++ b/modules/git/tree_entry_gogit.go @@ -19,16 +19,12 @@ type TreeEntry struct { gogitTreeEntry *object.TreeEntry ptree *Tree - size int64 - sized bool - fullName string + size int64 + sized bool } // Name returns the name of the entry func (te *TreeEntry) Name() string { - if te.fullName != "" { - return te.fullName - } return te.gogitTreeEntry.Name } @@ -55,7 +51,7 @@ func (te *TreeEntry) Size() int64 { return te.size } -// IsSubModule if the entry is a sub module +// IsSubModule if the entry is a submodule func (te *TreeEntry) IsSubModule() bool { return te.gogitTreeEntry.Mode == filemode.Submodule } diff --git a/modules/git/tree_entry_mode.go b/modules/git/tree_entry_mode.go index a399118cf8..f36c07bc2a 100644 --- a/modules/git/tree_entry_mode.go +++ b/modules/git/tree_entry_mode.go @@ -3,7 +3,10 @@ package git -import "strconv" +import ( + "fmt" + "strconv" +) // EntryMode the type of the object in the git tree type EntryMode int @@ -11,16 +14,15 @@ type EntryMode int // There are only a few file modes in Git. They look like unix file modes, but they can only be // one of these. const ( - // EntryModeBlob - EntryModeBlob EntryMode = 0o100644 - // EntryModeExec - EntryModeExec EntryMode = 0o100755 - // EntryModeSymlink + // EntryModeNoEntry is possible if the file was added or removed in a commit. In the case of + // when adding the base commit doesn't have the file in its tree, a mode of 0o000000 is used. + EntryModeNoEntry EntryMode = 0o000000 + + EntryModeBlob EntryMode = 0o100644 + EntryModeExec EntryMode = 0o100755 EntryModeSymlink EntryMode = 0o120000 - // EntryModeCommit - EntryModeCommit EntryMode = 0o160000 - // EntryModeTree - EntryModeTree EntryMode = 0o040000 + EntryModeCommit EntryMode = 0o160000 + EntryModeTree EntryMode = 0o040000 ) // String converts an EntryMode to a string @@ -28,8 +30,46 @@ func (e EntryMode) String() string { return strconv.FormatInt(int64(e), 8) } -// ToEntryMode converts a string to an EntryMode -func ToEntryMode(value string) EntryMode { - v, _ := strconv.ParseInt(value, 8, 32) - return EntryMode(v) +// IsSubModule if the entry is a submodule +func (e EntryMode) IsSubModule() bool { + return e == EntryModeCommit +} + +// IsDir if the entry is a sub dir +func (e EntryMode) IsDir() bool { + return e == EntryModeTree +} + +// IsLink if the entry is a symlink +func (e EntryMode) IsLink() bool { + return e == EntryModeSymlink +} + +// IsRegular if the entry is a regular file +func (e EntryMode) IsRegular() bool { + return e == EntryModeBlob +} + +// IsExecutable if the entry is an executable file (not necessarily binary) +func (e EntryMode) IsExecutable() bool { + return e == EntryModeExec +} + +func ParseEntryMode(mode string) (EntryMode, error) { + switch mode { + case "000000": + return EntryModeNoEntry, nil + case "100644": + return EntryModeBlob, nil + case "100755": + return EntryModeExec, nil + case "120000": + return EntryModeSymlink, nil + case "160000": + return EntryModeCommit, nil + case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons + return EntryModeTree, nil + default: + return 0, fmt.Errorf("unparsable entry mode: %s", mode) + } } diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go index 81fb638d56..8fad96cdf8 100644 --- a/modules/git/tree_entry_nogogit.go +++ b/modules/git/tree_entry_nogogit.go @@ -18,7 +18,7 @@ type TreeEntry struct { sized bool } -// Name returns the name of the entry +// Name returns the name of the entry (base name) func (te *TreeEntry) Name() string { return te.name } @@ -57,29 +57,29 @@ func (te *TreeEntry) Size() int64 { return te.size } -// IsSubModule if the entry is a sub module +// IsSubModule if the entry is a submodule func (te *TreeEntry) IsSubModule() bool { - return te.entryMode == EntryModeCommit + return te.entryMode.IsSubModule() } // IsDir if the entry is a sub dir func (te *TreeEntry) IsDir() bool { - return te.entryMode == EntryModeTree + return te.entryMode.IsDir() } // IsLink if the entry is a symlink func (te *TreeEntry) IsLink() bool { - return te.entryMode == EntryModeSymlink + return te.entryMode.IsLink() } // IsRegular if the entry is a regular file func (te *TreeEntry) IsRegular() bool { - return te.entryMode == EntryModeBlob + return te.entryMode.IsRegular() } // IsExecutable if the entry is an executable file (not necessarily binary) func (te *TreeEntry) IsExecutable() bool { - return te.entryMode == EntryModeExec + return te.entryMode.IsExecutable() } // Blob returns the blob object the entry diff --git a/modules/git/tree_entry_test.go b/modules/git/tree_entry_test.go index 30eee13669..9ca82675e0 100644 --- a/modules/git/tree_entry_test.go +++ b/modules/git/tree_entry_test.go @@ -53,50 +53,3 @@ func TestEntriesCustomSort(t *testing.T) { assert.Equal(t, "bcd", entries[6].Name()) assert.Equal(t, "abc", entries[7].Name()) } - -func TestFollowLink(t *testing.T) { - r, err := openRepositoryWithDefaultContext("tests/repos/repo1_bare") - assert.NoError(t, err) - defer r.Close() - - commit, err := r.GetCommit("37991dec2c8e592043f47155ce4808d4580f9123") - assert.NoError(t, err) - - // get the symlink - lnk, err := commit.Tree.GetTreeEntryByPath("foo/bar/link_to_hello") - assert.NoError(t, err) - assert.True(t, lnk.IsLink()) - - // should be able to dereference to target - target, err := lnk.FollowLink() - assert.NoError(t, err) - assert.Equal(t, "hello", target.Name()) - assert.False(t, target.IsLink()) - assert.Equal(t, "b14df6442ea5a1b382985a6549b85d435376c351", target.ID.String()) - - // should error when called on normal file - target, err = commit.Tree.GetTreeEntryByPath("file1.txt") - assert.NoError(t, err) - _, err = target.FollowLink() - assert.EqualError(t, err, "file1.txt: not a symlink") - - // should error for broken links - target, err = commit.Tree.GetTreeEntryByPath("foo/broken_link") - assert.NoError(t, err) - assert.True(t, target.IsLink()) - _, err = target.FollowLink() - assert.EqualError(t, err, "broken_link: broken link") - - // should error for external links - target, err = commit.Tree.GetTreeEntryByPath("foo/outside_repo") - assert.NoError(t, err) - assert.True(t, target.IsLink()) - _, err = target.FollowLink() - assert.EqualError(t, err, "outside_repo: points outside of repo") - - // testing fix for short link bug - target, err = commit.Tree.GetTreeEntryByPath("foo/link_short") - assert.NoError(t, err) - _, err = target.FollowLink() - assert.EqualError(t, err, "link_short: broken link") -} diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go index 421b0ecb0f..272b018ffd 100644 --- a/modules/git/tree_gogit.go +++ b/modules/git/tree_gogit.go @@ -69,7 +69,7 @@ func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { seen := map[plumbing.Hash]bool{} walker := object.NewTreeWalker(t.gogitTree, true, seen) for { - fullName, entry, err := walker.Next() + _, entry, err := walker.Next() if err == io.EOF { break } @@ -84,7 +84,6 @@ func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { ID: ParseGogitHash(entry.Hash), gogitTreeEntry: &entry, ptree: t, - fullName: fullName, } entries = append(entries, convertedEntry) } diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 993b98edc2..f88788418e 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -70,7 +70,7 @@ func (t *Tree) ListEntries() (Entries, error) { } } - stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-l").AddDynamicArguments(t.ID.String()).RunStdBytes(&RunOpts{Dir: t.repo.Path}) + stdout, _, runErr := NewCommand("ls-tree", "-l").AddDynamicArguments(t.ID.String()).RunStdBytes(t.repo.Ctx, &RunOpts{Dir: t.repo.Path}) if runErr != nil { if strings.Contains(runErr.Error(), "fatal: Not a valid object name") || strings.Contains(runErr.Error(), "fatal: not a tree object") { return nil, ErrNotExist{ @@ -96,10 +96,10 @@ func (t *Tree) listEntriesRecursive(extraArgs TrustedCmdArgs) (Entries, error) { return t.entriesRecursive, nil } - stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r"). + stdout, _, runErr := NewCommand("ls-tree", "-t", "-r"). AddArguments(extraArgs...). AddDynamicArguments(t.ID.String()). - RunStdBytes(&RunOpts{Dir: t.repo.Path}) + RunStdBytes(t.repo.Ctx, &RunOpts{Dir: t.repo.Path}) if runErr != nil { return nil, runErr } diff --git a/modules/git/tree_test.go b/modules/git/tree_test.go index 5fee64b038..cae11c4b1b 100644 --- a/modules/git/tree_test.go +++ b/modules/git/tree_test.go @@ -19,7 +19,7 @@ func TestSubTree_Issue29101(t *testing.T) { assert.NoError(t, err) // old code could produce a different error if called multiple times - for i := 0; i < 10; i++ { + for range 10 { _, err = commit.SubTree("file1.txt") assert.Error(t, err) assert.True(t, IsErrNotExist(err)) @@ -33,10 +33,10 @@ func Test_GetTreePathLatestCommit(t *testing.T) { commitID, err := repo.GetBranchCommitID("master") assert.NoError(t, err) - assert.EqualValues(t, "544d8f7a3b15927cddf2299b4b562d6ebd71b6a7", commitID) + assert.Equal(t, "544d8f7a3b15927cddf2299b4b562d6ebd71b6a7", commitID) commit, err := repo.GetTreePathLatestCommit("master", "blame.txt") assert.NoError(t, err) assert.NotNil(t, commit) - assert.EqualValues(t, "45fb6cbc12f970b04eacd5cd4165edd11c8d7376", commit.ID.String()) + assert.Equal(t, "45fb6cbc12f970b04eacd5cd4165edd11c8d7376", commit.ID.String()) } diff --git a/modules/git/url/url.go b/modules/git/url/url.go index 1c5e8377a6..aa6fa31c5e 100644 --- a/modules/git/url/url.go +++ b/modules/git/url/url.go @@ -133,12 +133,13 @@ func ParseRepositoryURL(ctx context.Context, repoURL string) (*RepositoryURL, er } } - if parsed.URL.Scheme == "http" || parsed.URL.Scheme == "https" { + switch parsed.URL.Scheme { + case "http", "https": if !httplib.IsCurrentGiteaSiteURL(ctx, repoURL) { return ret, nil } fillPathParts(strings.TrimPrefix(parsed.URL.Path, setting.AppSubURL)) - } else if parsed.URL.Scheme == "ssh" || parsed.URL.Scheme == "git+ssh" { + case "ssh", "git+ssh": domainSSH := setting.SSH.Domain domainCur := httplib.GuessCurrentHostDomain(ctx) urlDomain, _, _ := net.SplitHostPort(parsed.URL.Host) @@ -166,9 +167,10 @@ func MakeRepositoryWebLink(repoURL *RepositoryURL) string { // now, let's guess, for example: // * git@github.com:owner/submodule.git // * https://github.com/example/submodule1.git - if repoURL.GitURL.Scheme == "http" || repoURL.GitURL.Scheme == "https" { + switch repoURL.GitURL.Scheme { + case "http", "https": return strings.TrimSuffix(repoURL.GitURL.String(), ".git") - } else if repoURL.GitURL.Scheme == "ssh" || repoURL.GitURL.Scheme == "git+ssh" { + case "ssh", "git+ssh": hostname, _, _ := net.SplitHostPort(repoURL.GitURL.Host) hostname = util.IfZero(hostname, repoURL.GitURL.Host) urlPath := strings.TrimSuffix(repoURL.GitURL.Path, ".git") diff --git a/modules/git/url/url_test.go b/modules/git/url/url_test.go index 9c020adb4d..6655c20be3 100644 --- a/modules/git/url/url_test.go +++ b/modules/git/url/url_test.go @@ -165,8 +165,8 @@ func TestParseGitURLs(t *testing.T) { t.Run(kase.kase, func(t *testing.T) { u, err := ParseGitURL(kase.kase) assert.NoError(t, err) - assert.EqualValues(t, kase.expected.extraMark, u.extraMark) - assert.EqualValues(t, *kase.expected, *u) + assert.Equal(t, kase.expected.extraMark, u.extraMark) + assert.Equal(t, *kase.expected, *u) }) } } @@ -179,7 +179,7 @@ func TestParseRepositoryURL(t *testing.T) { ctxReq := &http.Request{URL: ctxURL, Header: http.Header{}} ctxReq.Host = ctxURL.Host ctxReq.Header.Add("X-Forwarded-Proto", ctxURL.Scheme) - ctx := context.WithValue(context.Background(), httplib.RequestContextKey, ctxReq) + ctx := context.WithValue(t.Context(), httplib.RequestContextKey, ctxReq) cases := []struct { input string ownerName, repoName, remaining string @@ -249,19 +249,19 @@ func TestMakeRepositoryBaseLink(t *testing.T) { defer test.MockVariableValue(&setting.AppURL, "https://localhost:3000/subpath")() defer test.MockVariableValue(&setting.AppSubURL, "/subpath")() - u, err := ParseRepositoryURL(context.Background(), "https://localhost:3000/subpath/user/repo.git") + u, err := ParseRepositoryURL(t.Context(), "https://localhost:3000/subpath/user/repo.git") assert.NoError(t, err) assert.Equal(t, "/subpath/user/repo", MakeRepositoryWebLink(u)) - u, err = ParseRepositoryURL(context.Background(), "https://github.com/owner/repo.git") + u, err = ParseRepositoryURL(t.Context(), "https://github.com/owner/repo.git") assert.NoError(t, err) assert.Equal(t, "https://github.com/owner/repo", MakeRepositoryWebLink(u)) - u, err = ParseRepositoryURL(context.Background(), "git@github.com:owner/repo.git") + u, err = ParseRepositoryURL(t.Context(), "git@github.com:owner/repo.git") assert.NoError(t, err) assert.Equal(t, "https://github.com/owner/repo", MakeRepositoryWebLink(u)) - u, err = ParseRepositoryURL(context.Background(), "git+ssh://other:123/owner/repo.git") + u, err = ParseRepositoryURL(t.Context(), "git+ssh://other:123/owner/repo.git") assert.NoError(t, err) assert.Equal(t, "https://other/owner/repo", MakeRepositoryWebLink(u)) } diff --git a/modules/git/utils.go b/modules/git/utils.go index 56cba9087a..897306efd0 100644 --- a/modules/git/utils.go +++ b/modules/git/utils.go @@ -8,7 +8,6 @@ import ( "encoding/hex" "fmt" "io" - "os" "strconv" "strings" "sync" @@ -41,33 +40,6 @@ func (oc *ObjectCache[T]) Get(id string) (T, bool) { return obj, has } -// isDir returns true if given path is a directory, -// or returns false when it's a file or does not exist. -func isDir(dir string) bool { - f, e := os.Stat(dir) - if e != nil { - return false - } - return f.IsDir() -} - -// isFile returns true if given path is a file, -// or returns false when it's a directory or does not exist. -func isFile(filePath string) bool { - f, e := os.Stat(filePath) - if e != nil { - return false - } - return !f.IsDir() -} - -// isExist checks whether a file or directory exists. -// It returns false when the file or directory does not exist. -func isExist(path string) bool { - _, err := os.Stat(path) - return err == nil || os.IsExist(err) -} - // ConcatenateError concatenats an error with stderr string func ConcatenateError(err error, stderr string) error { if len(stderr) == 0 { diff --git a/modules/gitgraph/graph.go b/modules/gitgraph/graph.go deleted file mode 100644 index 7e12be030f..0000000000 --- a/modules/gitgraph/graph.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package gitgraph - -import ( - "bufio" - "bytes" - "context" - "os" - "strings" - - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/setting" -) - -// GetCommitGraph return a list of commit (GraphItems) from all branches -func GetCommitGraph(r *git.Repository, page, maxAllowedColors int, hidePRRefs bool, branches, files []string) (*Graph, error) { - format := "DATA:%D|%H|%ad|%h|%s" - - if page == 0 { - page = 1 - } - - graphCmd := git.NewCommand(r.Ctx, "log", "--graph", "--date-order", "--decorate=full") - - if hidePRRefs { - graphCmd.AddArguments("--exclude=" + git.PullPrefix + "*") - } - - if len(branches) == 0 { - graphCmd.AddArguments("--all") - } - - graphCmd.AddArguments("-C", "-M", "--date=iso-strict"). - AddOptionFormat("-n %d", setting.UI.GraphMaxCommitNum*page). - AddOptionFormat("--pretty=format:%s", format) - - if len(branches) > 0 { - graphCmd.AddDynamicArguments(branches...) - } - if len(files) > 0 { - graphCmd.AddDashesAndList(files...) - } - graph := NewGraph() - - stderr := new(strings.Builder) - stdoutReader, stdoutWriter, err := os.Pipe() - if err != nil { - return nil, err - } - commitsToSkip := setting.UI.GraphMaxCommitNum * (page - 1) - - scanner := bufio.NewScanner(stdoutReader) - - if err := graphCmd.Run(&git.RunOpts{ - Dir: r.Path, - Stdout: stdoutWriter, - Stderr: stderr, - PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { - _ = stdoutWriter.Close() - defer stdoutReader.Close() - parser := &Parser{} - parser.firstInUse = -1 - parser.maxAllowedColors = maxAllowedColors - if maxAllowedColors > 0 { - parser.availableColors = make([]int, maxAllowedColors) - for i := range parser.availableColors { - parser.availableColors[i] = i + 1 - } - } else { - parser.availableColors = []int{1, 2} - } - for commitsToSkip > 0 && scanner.Scan() { - line := scanner.Bytes() - dataIdx := bytes.Index(line, []byte("DATA:")) - if dataIdx < 0 { - dataIdx = len(line) - } - starIdx := bytes.IndexByte(line, '*') - if starIdx >= 0 && starIdx < dataIdx { - commitsToSkip-- - } - parser.ParseGlyphs(line[:dataIdx]) - } - - row := 0 - - // Skip initial non-commit lines - for scanner.Scan() { - line := scanner.Bytes() - if bytes.IndexByte(line, '*') >= 0 { - if err := parser.AddLineToGraph(graph, row, line); err != nil { - cancel() - return err - } - break - } - parser.ParseGlyphs(line) - } - - for scanner.Scan() { - row++ - line := scanner.Bytes() - if err := parser.AddLineToGraph(graph, row, line); err != nil { - cancel() - return err - } - } - return scanner.Err() - }, - }); err != nil { - return graph, err - } - return graph, nil -} diff --git a/modules/gitgraph/graph_models.go b/modules/gitgraph/graph_models.go deleted file mode 100644 index 191b0b3afc..0000000000 --- a/modules/gitgraph/graph_models.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package gitgraph - -import ( - "bytes" - "context" - "fmt" - "strings" - "time" - - asymkey_model "code.gitea.io/gitea/models/asymkey" - "code.gitea.io/gitea/models/db" - git_model "code.gitea.io/gitea/models/git" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" -) - -// NewGraph creates a basic graph -func NewGraph() *Graph { - graph := &Graph{} - graph.relationCommit = &Commit{ - Row: -1, - Column: -1, - } - graph.Flows = map[int64]*Flow{} - return graph -} - -// Graph represents a collection of flows -type Graph struct { - Flows map[int64]*Flow - Commits []*Commit - MinRow int - MinColumn int - MaxRow int - MaxColumn int - relationCommit *Commit -} - -// Width returns the width of the graph -func (graph *Graph) Width() int { - return graph.MaxColumn - graph.MinColumn + 1 -} - -// Height returns the height of the graph -func (graph *Graph) Height() int { - return graph.MaxRow - graph.MinRow + 1 -} - -// AddGlyph adds glyph to flows -func (graph *Graph) AddGlyph(row, column int, flowID int64, color int, glyph byte) { - flow, ok := graph.Flows[flowID] - if !ok { - flow = NewFlow(flowID, color, row, column) - graph.Flows[flowID] = flow - } - flow.AddGlyph(row, column, glyph) - - if row < graph.MinRow { - graph.MinRow = row - } - if row > graph.MaxRow { - graph.MaxRow = row - } - if column < graph.MinColumn { - graph.MinColumn = column - } - if column > graph.MaxColumn { - graph.MaxColumn = column - } -} - -// AddCommit adds a commit at row, column on flowID with the provided data -func (graph *Graph) AddCommit(row, column int, flowID int64, data []byte) error { - commit, err := NewCommit(row, column, data) - if err != nil { - return err - } - commit.Flow = flowID - graph.Commits = append(graph.Commits, commit) - - graph.Flows[flowID].Commits = append(graph.Flows[flowID].Commits, commit) - return nil -} - -// LoadAndProcessCommits will load the git.Commits for each commit in the graph, -// the associate the commit with the user author, and check the commit verification -// before finally retrieving the latest status -func (graph *Graph) LoadAndProcessCommits(ctx context.Context, repository *repo_model.Repository, gitRepo *git.Repository) error { - var err error - var ok bool - - emails := map[string]*user_model.User{} - keyMap := map[string]bool{} - - for _, c := range graph.Commits { - if len(c.Rev) == 0 { - continue - } - c.Commit, err = gitRepo.GetCommit(c.Rev) - if err != nil { - return fmt.Errorf("GetCommit: %s Error: %w", c.Rev, err) - } - - if c.Commit.Author != nil { - email := c.Commit.Author.Email - if c.User, ok = emails[email]; !ok { - c.User, _ = user_model.GetUserByEmail(ctx, email) - emails[email] = c.User - } - } - - c.Verification = asymkey_model.ParseCommitWithSignature(ctx, c.Commit) - - _ = asymkey_model.CalculateTrustStatus(c.Verification, repository.GetTrustModel(), func(user *user_model.User) (bool, error) { - return repo_model.IsOwnerMemberCollaborator(ctx, repository, user.ID) - }, &keyMap) - - statuses, _, err := git_model.GetLatestCommitStatus(ctx, repository.ID, c.Commit.ID.String(), db.ListOptions{}) - if err != nil { - log.Error("GetLatestCommitStatus: %v", err) - } else { - c.Status = git_model.CalcCommitStatus(statuses) - } - } - return nil -} - -// NewFlow creates a new flow -func NewFlow(flowID int64, color, row, column int) *Flow { - return &Flow{ - ID: flowID, - ColorNumber: color, - MinRow: row, - MinColumn: column, - MaxRow: row, - MaxColumn: column, - } -} - -// Flow represents a series of glyphs -type Flow struct { - ID int64 - ColorNumber int - Glyphs []Glyph - Commits []*Commit - MinRow int - MinColumn int - MaxRow int - MaxColumn int -} - -// Color16 wraps the color numbers around mod 16 -func (flow *Flow) Color16() int { - return flow.ColorNumber % 16 -} - -// AddGlyph adds glyph at row and column -func (flow *Flow) AddGlyph(row, column int, glyph byte) { - if row < flow.MinRow { - flow.MinRow = row - } - if row > flow.MaxRow { - flow.MaxRow = row - } - if column < flow.MinColumn { - flow.MinColumn = column - } - if column > flow.MaxColumn { - flow.MaxColumn = column - } - - flow.Glyphs = append(flow.Glyphs, Glyph{ - row, - column, - glyph, - }) -} - -// Glyph represents a co-ordinate and glyph -type Glyph struct { - Row int - Column int - Glyph byte -} - -// RelationCommit represents an empty relation commit -var RelationCommit = &Commit{ - Row: -1, -} - -func parseGitTime(timeStr string) time.Time { - t, err := time.Parse(time.RFC3339, timeStr) - if err != nil { - return time.Unix(0, 0) - } - return t -} - -// NewCommit creates a new commit from a provided line -func NewCommit(row, column int, line []byte) (*Commit, error) { - data := bytes.SplitN(line, []byte("|"), 5) - if len(data) < 5 { - return nil, fmt.Errorf("malformed data section on line %d with commit: %s", row, string(line)) - } - return &Commit{ - Row: row, - Column: column, - // 0 matches git log --pretty=format:%d => ref names, like the --decorate option of git-log(1) - Refs: newRefsFromRefNames(data[0]), - // 1 matches git log --pretty=format:%H => commit hash - Rev: string(data[1]), - // 2 matches git log --pretty=format:%ad => author date (format respects --date= option) - Date: parseGitTime(string(data[2])), - // 3 matches git log --pretty=format:%h => abbreviated commit hash - ShortRev: string(data[3]), - // 4 matches git log --pretty=format:%s => subject - Subject: string(data[4]), - }, nil -} - -func newRefsFromRefNames(refNames []byte) []git.Reference { - refBytes := bytes.Split(refNames, []byte{',', ' '}) - refs := make([]git.Reference, 0, len(refBytes)) - for _, refNameBytes := range refBytes { - if len(refNameBytes) == 0 { - continue - } - refName := string(refNameBytes) - if strings.HasPrefix(refName, "tag: ") { - refName = strings.TrimPrefix(refName, "tag: ") - } else { - refName = strings.TrimPrefix(refName, "HEAD -> ") - } - refs = append(refs, git.Reference{ - Name: refName, - }) - } - return refs -} - -// Commit represents a commit at co-ordinate X, Y with the data -type Commit struct { - Commit *git.Commit - User *user_model.User - Verification *asymkey_model.CommitVerification - Status *git_model.CommitStatus - Flow int64 - Row int - Column int - Refs []git.Reference - Rev string - Date time.Time - ShortRev string - Subject string -} - -// OnlyRelation returns whether this a relation only commit -func (c *Commit) OnlyRelation() bool { - return c.Row == -1 -} diff --git a/modules/gitgraph/graph_test.go b/modules/gitgraph/graph_test.go deleted file mode 100644 index 2f647aaf83..0000000000 --- a/modules/gitgraph/graph_test.go +++ /dev/null @@ -1,712 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package gitgraph - -import ( - "bytes" - "fmt" - "strings" - "testing" - - "code.gitea.io/gitea/modules/git" - - "github.com/stretchr/testify/assert" -) - -func BenchmarkGetCommitGraph(b *testing.B) { - currentRepo, err := git.OpenRepository(git.DefaultContext, ".") - if err != nil || currentRepo == nil { - b.Error("Could not open repository") - } - defer currentRepo.Close() - - for i := 0; i < b.N; i++ { - graph, err := GetCommitGraph(currentRepo, 1, 0, false, nil, nil) - if err != nil { - b.Error("Could get commit graph") - } - - if len(graph.Commits) < 100 { - b.Error("Should get 100 log lines.") - } - } -} - -func BenchmarkParseCommitString(b *testing.B) { - testString := "* DATA:|4e61bacab44e9b4730e44a6615d04098dd3a8eaf|2016-12-20 21:10:41 +0100|4e61bac|Add route for graph" - - parser := &Parser{} - parser.Reset() - for i := 0; i < b.N; i++ { - parser.Reset() - graph := NewGraph() - if err := parser.AddLineToGraph(graph, 0, []byte(testString)); err != nil { - b.Error("could not parse teststring") - } - if graph.Flows[1].Commits[0].Rev != "4e61bacab44e9b4730e44a6615d04098dd3a8eaf" { - b.Error("Did not get expected data") - } - } -} - -func BenchmarkParseGlyphs(b *testing.B) { - parser := &Parser{} - parser.Reset() - tgBytes := []byte(testglyphs) - var tg []byte - for i := 0; i < b.N; i++ { - parser.Reset() - tg = tgBytes - idx := bytes.Index(tg, []byte("\n")) - for idx > 0 { - parser.ParseGlyphs(tg[:idx]) - tg = tg[idx+1:] - idx = bytes.Index(tg, []byte("\n")) - } - } -} - -func TestReleaseUnusedColors(t *testing.T) { - testcases := []struct { - availableColors []int - oldColors []int - firstInUse int // these values have to be either be correct or suggest less is - firstAvailable int // available than possibly is - i.e. you cannot say 10 is available when it - }{ - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - oldColors: []int{1, 1, 1, 1, 1}, - firstAvailable: -1, - firstInUse: 1, - }, - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - oldColors: []int{1, 2, 3, 4}, - firstAvailable: 6, - firstInUse: 0, - }, - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - oldColors: []int{6, 0, 3, 5, 3, 4, 0, 0}, - firstAvailable: 6, - firstInUse: 0, - }, - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7}, - oldColors: []int{6, 1, 3, 5, 3, 4, 2, 7}, - firstAvailable: -1, - firstInUse: 0, - }, - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7}, - oldColors: []int{6, 0, 3, 5, 3, 4, 2, 7}, - firstAvailable: -1, - firstInUse: 0, - }, - } - for _, testcase := range testcases { - parser := &Parser{} - parser.Reset() - parser.availableColors = append([]int{}, testcase.availableColors...) - parser.oldColors = append(parser.oldColors, testcase.oldColors...) - parser.firstAvailable = testcase.firstAvailable - parser.firstInUse = testcase.firstInUse - parser.releaseUnusedColors() - - if parser.firstAvailable == -1 { - // All in use - for _, color := range parser.availableColors { - found := false - for _, oldColor := range parser.oldColors { - if oldColor == color { - found = true - break - } - } - if !found { - t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", - testcase.availableColors, - testcase.oldColors, - testcase.firstAvailable, - testcase.firstInUse, - parser.availableColors, - parser.oldColors, - parser.firstAvailable, - parser.firstInUse, - color) - } - } - } else if parser.firstInUse != -1 { - // Some in use - for i := parser.firstInUse; i != parser.firstAvailable; i = (i + 1) % len(parser.availableColors) { - color := parser.availableColors[i] - found := false - for _, oldColor := range parser.oldColors { - if oldColor == color { - found = true - break - } - } - if !found { - t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", - testcase.availableColors, - testcase.oldColors, - testcase.firstAvailable, - testcase.firstInUse, - parser.availableColors, - parser.oldColors, - parser.firstAvailable, - parser.firstInUse, - color) - } - } - for i := parser.firstAvailable; i != parser.firstInUse; i = (i + 1) % len(parser.availableColors) { - color := parser.availableColors[i] - found := false - for _, oldColor := range parser.oldColors { - if oldColor == color { - found = true - break - } - } - if found { - t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should not be available but is", - testcase.availableColors, - testcase.oldColors, - testcase.firstAvailable, - testcase.firstInUse, - parser.availableColors, - parser.oldColors, - parser.firstAvailable, - parser.firstInUse, - color) - } - } - } else { - // None in use - for _, color := range parser.oldColors { - if color != 0 { - t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should not be available but is", - testcase.availableColors, - testcase.oldColors, - testcase.firstAvailable, - testcase.firstInUse, - parser.availableColors, - parser.oldColors, - parser.firstAvailable, - parser.firstInUse, - color) - } - } - } - } -} - -func TestParseGlyphs(t *testing.T) { - parser := &Parser{} - parser.Reset() - tgBytes := []byte(testglyphs) - tg := tgBytes - idx := bytes.Index(tg, []byte("\n")) - row := 0 - for idx > 0 { - parser.ParseGlyphs(tg[:idx]) - tg = tg[idx+1:] - idx = bytes.Index(tg, []byte("\n")) - if parser.flows[0] != 1 { - t.Errorf("First column flow should be 1 but was %d", parser.flows[0]) - } - colorToFlow := map[int]int64{} - flowToColor := map[int64]int{} - - for i, flow := range parser.flows { - if flow == 0 { - continue - } - color := parser.colors[i] - - if fColor, in := flowToColor[flow]; in && fColor != color { - t.Errorf("Row %d column %d flow %d has color %d but should be %d", row, i, flow, color, fColor) - } - flowToColor[flow] = color - if cFlow, in := colorToFlow[color]; in && cFlow != flow { - t.Errorf("Row %d column %d flow %d has color %d but conflicts with flow %d", row, i, flow, color, cFlow) - } - colorToFlow[color] = flow - } - row++ - } - assert.Len(t, parser.availableColors, 9) -} - -func TestCommitStringParsing(t *testing.T) { - dataFirstPart := "* DATA:|4e61bacab44e9b4730e44a6615d04098dd3a8eaf|2016-12-20 21:10:41 +0100|4e61bac|" - tests := []struct { - shouldPass bool - testName string - commitMessage string - }{ - {true, "normal", "not a fancy message"}, - {true, "extra pipe", "An extra pipe: |"}, - {true, "extra 'Data:'", "DATA: might be trouble"}, - } - - for _, test := range tests { - t.Run(test.testName, func(t *testing.T) { - testString := fmt.Sprintf("%s%s", dataFirstPart, test.commitMessage) - idx := strings.Index(testString, "DATA:") - commit, err := NewCommit(0, 0, []byte(testString[idx+5:])) - if err != nil && test.shouldPass { - t.Errorf("Could not parse %s", testString) - return - } - - assert.Equal(t, test.commitMessage, commit.Subject) - }) - } -} - -var testglyphs = `* -* -* -* -* -* -* -* -|\ -* | -* | -* | -* | -* | -| * -* | -| * -| |\ -* | | -| | * -| | |\ -* | | \ -|\ \ \ \ -| * | | | -| |\| | | -* | | | | -|/ / / / -| | | * -| * | | -| * | | -| * | | -* | | | -* | | | -* | | | -* | | | -* | | | -|\ \ \ \ -| | * | | -| | |\| | -| | | * | -| | | | * -* | | | | -* | | | | -* | | | | -* | | | | -* | | | | -|\ \ \ \ \ -| * | | | | -|/| | | | | -| | |/ / / -| |/| | | -| | | | * -| * | | | -|/| | | | -| * | | | -|/| | | | -| | |/ / -| |/| | -| * | | -| * | | -| |\ \ \ -| | * | | -| |/| | | -| | | |/ -| | |/| -| * | | -| * | | -| * | | -| | * | -| | |\ \ -| | | * | -| | |/| | -| | | * | -| | | |\ \ -| | | | * | -| | | |/| | -| | * | | | -| | * | | | -| | |\ \ \ \ -| | | * | | | -| | |/| | | | -| | | | | * | -| | | | |/ / -* | | | / / -|/ / / / / -* | | | | -|\ \ \ \ \ -| * | | | | -|/| | | | | -| * | | | | -| * | | | | -| |\ \ \ \ \ -| | | * \ \ \ -| | | |\ \ \ \ -| | | | * | | | -| | | |/| | | | -| | | | | |/ / -| | | | |/| | -* | | | | | | -* | | | | | | -* | | | | | | -| | | | * | | -* | | | | | | -| | * | | | | -| |/| | | | | -* | | | | | | -| |/ / / / / -|/| | | | | -| | | | * | -| | | |/ / -| | |/| | -| * | | | -| | | | * -| | * | | -| | |\ \ \ -| | | * | | -| | |/| | | -| | | |/ / -| | | * | -| | * | | -| | |\ \ \ -| | | * | | -| | |/| | | -| | | |/ / -| | | * | -* | | | | -|\ \ \ \ \ -| * \ \ \ \ -| |\ \ \ \ \ -| | | |/ / / -| | |/| | | -| | | | * | -| | | | * | -* | | | | | -* | | | | | -|/ / / / / -| | | * | -* | | | | -* | | | | -* | | | | -* | | | | -|\ \ \ \ \ -| * | | | | -|/| | | | | -| | * | | | -| | |\ \ \ \ -| | | * | | | -| | |/| | | | -| |/| | |/ / -| | | |/| | -| | | | | * -| |_|_|_|/ -|/| | | | -| | * | | -| |/ / / -* | | | -* | | | -| | * | -* | | | -* | | | -| * | | -| | * | -| * | | -* | | | -|\ \ \ \ -| * | | | -|/| | | | -| |/ / / -| * | | -| |\ \ \ -| | * | | -| |/| | | -| | |/ / -| | * | -| | |\ \ -| | | * | -| | |/| | -* | | | | -* | | | | -|\ \ \ \ \ -| * | | | | -|/| | | | | -| | * | | | -| | * | | | -| | * | | | -| |/ / / / -| * | | | -| |\ \ \ \ -| | * | | | -| |/| | | | -* | | | | | -* | | | | | -* | | | | | -* | | | | | -* | | | | | -| | | | * | -* | | | | | -|\ \ \ \ \ \ -| * | | | | | -|/| | | | | | -| | | | | * | -| | | | |/ / -* | | | | | -|\ \ \ \ \ \ -* | | | | | | -* | | | | | | -| | | | * | | -* | | | | | | -* | | | | | | -|\ \ \ \ \ \ \ -| | |_|_|/ / / -| |/| | | | | -| | | | * | | -| | | | * | | -| | | | * | | -| | | | * | | -| | | | * | | -| | | | * | | -| | | |/ / / -| | | * | | -| | | * | | -| | | * | | -| | |/| | | -| | | * | | -| | |/| | | -| | | |/ / -| | * | | -| |/| | | -| | | * | -| | |/ / -| | * | -| * | | -| |\ \ \ -| * | | | -| | * | | -| |/| | | -| | |/ / -| | * | -| | |\ \ -| | * | | -* | | | | -|\| | | | -| * | | | -| * | | | -| * | | | -| | * | | -| * | | | -| |\| | | -| * | | | -| | * | | -| | * | | -| * | | | -| * | | | -| * | | | -| * | | | -| * | | | -| * | | | -| * | | | -| * | | | -| | * | | -| * | | | -| * | | | -| * | | | -| * | | | -| | * | | -* | | | | -|\| | | | -| | * | | -| * | | | -| |\| | | -| | * | | -| | * | | -| | * | | -| | | * | -* | | | | -|\| | | | -| | * | | -| | |/ / -| * | | -| * | | -| |\| | -* | | | -|\| | | -| | * | -| | * | -| | * | -| * | | -| | * | -| * | | -| | * | -| | * | -| | * | -| * | | -| * | | -| * | | -| * | | -| * | | -| * | | -| * | | -* | | | -|\| | | -| * | | -| |\| | -| | * | -| | |\ \ -* | | | | -|\| | | | -| * | | | -| |\| | | -| | * | | -| | | * | -| | |/ / -* | | | -* | | | -|\| | | -| * | | -| |\| | -| | * | -| | * | -| | * | -| | | * -* | | | -|\| | | -| * | | -| * | | -| | | * -| | | |\ -* | | | | -| |_|_|/ -|/| | | -| * | | -| |\| | -| | * | -| | * | -| | * | -| | * | -| | * | -| * | | -* | | | -|\| | | -| * | | -|/| | | -| |/ / -| * | -| |\ \ -| * | | -| * | | -* | | | -|\| | | -| | * | -| * | | -| * | | -| * | | -* | | | -|\| | | -| * | | -| * | | -| | * | -| | |\ \ -| | |/ / -| |/| | -| * | | -* | | | -|\| | | -| * | | -* | | | -|\| | | -| * | | -| |\ \ \ -| * | | | -| * | | | -| | | * | -| * | | | -| * | | | -| | |/ / -| |/| | -| | * | -* | | | -|\| | | -| * | | -| * | | -| * | | -| * | | -| * | | -| |\ \ \ -* | | | | -|\| | | | -| * | | | -| * | | | -* | | | | -* | | | | -|\| | | | -| | | | * -| | | | |\ -| |_|_|_|/ -|/| | | | -| * | | | -* | | | | -* | | | | -|\| | | | -| * | | | -| |\ \ \ \ -| | | |/ / -| | |/| | -| * | | | -| * | | | -| * | | | -| * | | | -| | * | | -| | | * | -| | |/ / -| |/| | -* | | | -|\| | | -| * | | -| * | | -| * | | -| * | | -| * | | -* | | | -|\| | | -| * | | -| * | | -* | | | -| * | | -| * | | -| * | | -* | | | -* | | | -* | | | -|\| | | -| * | | -* | | | -* | | | -* | | | -* | | | -| | | * -* | | | -|\| | | -| * | | -| * | | -| * | | -` diff --git a/modules/gitgraph/parser.go b/modules/gitgraph/parser.go deleted file mode 100644 index f6bf9b0b90..0000000000 --- a/modules/gitgraph/parser.go +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package gitgraph - -import ( - "bytes" - "fmt" -) - -// Parser represents a git graph parser. It is stateful containing the previous -// glyphs, detected flows and color assignments. -type Parser struct { - glyphs []byte - oldGlyphs []byte - flows []int64 - oldFlows []int64 - maxFlow int64 - colors []int - oldColors []int - availableColors []int - nextAvailable int - firstInUse int - firstAvailable int - maxAllowedColors int -} - -// Reset resets the internal parser state. -func (parser *Parser) Reset() { - parser.glyphs = parser.glyphs[0:0] - parser.oldGlyphs = parser.oldGlyphs[0:0] - parser.flows = parser.flows[0:0] - parser.oldFlows = parser.oldFlows[0:0] - parser.maxFlow = 0 - parser.colors = parser.colors[0:0] - parser.oldColors = parser.oldColors[0:0] - parser.availableColors = parser.availableColors[0:0] - parser.availableColors = append(parser.availableColors, 1, 2) - parser.nextAvailable = 0 - parser.firstInUse = -1 - parser.firstAvailable = 0 - parser.maxAllowedColors = 0 -} - -// AddLineToGraph adds the line as a row to the graph -func (parser *Parser) AddLineToGraph(graph *Graph, row int, line []byte) error { - idx := bytes.Index(line, []byte("DATA:")) - if idx < 0 { - parser.ParseGlyphs(line) - } else { - parser.ParseGlyphs(line[:idx]) - } - - var err error - commitDone := false - - for column, glyph := range parser.glyphs { - if glyph == ' ' { - continue - } - - flowID := parser.flows[column] - - graph.AddGlyph(row, column, flowID, parser.colors[column], glyph) - - if glyph == '*' { - if commitDone { - if err != nil { - err = fmt.Errorf("double commit on line %d: %s. %w", row, string(line), err) - } else { - err = fmt.Errorf("double commit on line %d: %s", row, string(line)) - } - } - commitDone = true - if idx < 0 { - if err != nil { - err = fmt.Errorf("missing data section on line %d with commit: %s. %w", row, string(line), err) - } else { - err = fmt.Errorf("missing data section on line %d with commit: %s", row, string(line)) - } - continue - } - err2 := graph.AddCommit(row, column, flowID, line[idx+5:]) - if err != nil && err2 != nil { - err = fmt.Errorf("%v %w", err2, err) - continue - } else if err2 != nil { - err = err2 - continue - } - } - } - if !commitDone { - graph.Commits = append(graph.Commits, RelationCommit) - } - return err -} - -func (parser *Parser) releaseUnusedColors() { - if parser.firstInUse > -1 { - // Here we step through the old colors, searching for them in the - // "in-use" section of availableColors (that is, the colors between - // firstInUse and firstAvailable) - // Ensure that the benchmarks are not worsened with proposed changes - stepstaken := 0 - position := parser.firstInUse - for _, color := range parser.oldColors { - if color == 0 { - continue - } - found := false - i := position - for j := stepstaken; i != parser.firstAvailable && j < len(parser.availableColors); j++ { - colorToCheck := parser.availableColors[i] - if colorToCheck == color { - found = true - break - } - i = (i + 1) % len(parser.availableColors) - } - if !found { - // Duplicate color - continue - } - // Swap them around - parser.availableColors[position], parser.availableColors[i] = parser.availableColors[i], parser.availableColors[position] - stepstaken++ - position = (parser.firstInUse + stepstaken) % len(parser.availableColors) - if position == parser.firstAvailable || stepstaken == len(parser.availableColors) { - break - } - } - if stepstaken == len(parser.availableColors) { - parser.firstAvailable = -1 - } else { - parser.firstAvailable = position - if parser.nextAvailable == -1 { - parser.nextAvailable = parser.firstAvailable - } - } - } -} - -// ParseGlyphs parses the provided glyphs and sets the internal state -func (parser *Parser) ParseGlyphs(glyphs []byte) { - // Clean state for parsing this row - parser.glyphs, parser.oldGlyphs = parser.oldGlyphs, parser.glyphs - parser.glyphs = parser.glyphs[0:0] - parser.flows, parser.oldFlows = parser.oldFlows, parser.flows - parser.flows = parser.flows[0:0] - parser.colors, parser.oldColors = parser.oldColors, parser.colors - - // Ensure we have enough flows and colors - parser.colors = parser.colors[0:0] - for range glyphs { - parser.flows = append(parser.flows, 0) - parser.colors = append(parser.colors, 0) - } - - // Copy the provided glyphs in to state.glyphs for safekeeping - parser.glyphs = append(parser.glyphs, glyphs...) - - // release unused colors - parser.releaseUnusedColors() - - for i := len(glyphs) - 1; i >= 0; i-- { - glyph := glyphs[i] - switch glyph { - case '|': - fallthrough - case '*': - parser.setUpFlow(i) - case '/': - parser.setOutFlow(i) - case '\\': - parser.setInFlow(i) - case '_': - parser.setRightFlow(i) - case '.': - fallthrough - case '-': - parser.setLeftFlow(i) - case ' ': - // no-op - default: - parser.newFlow(i) - } - } -} - -func (parser *Parser) takePreviousFlow(i, j int) { - if j < len(parser.oldFlows) && parser.oldFlows[j] > 0 { - parser.flows[i] = parser.oldFlows[j] - parser.oldFlows[j] = 0 - parser.colors[i] = parser.oldColors[j] - parser.oldColors[j] = 0 - } else { - parser.newFlow(i) - } -} - -func (parser *Parser) takeCurrentFlow(i, j int) { - if j < len(parser.flows) && parser.flows[j] > 0 { - parser.flows[i] = parser.flows[j] - parser.colors[i] = parser.colors[j] - } else { - parser.newFlow(i) - } -} - -func (parser *Parser) newFlow(i int) { - parser.maxFlow++ - parser.flows[i] = parser.maxFlow - - // Now give this flow a color - if parser.nextAvailable == -1 { - next := len(parser.availableColors) - if parser.maxAllowedColors < 1 || next < parser.maxAllowedColors { - parser.nextAvailable = next - parser.firstAvailable = next - parser.availableColors = append(parser.availableColors, next+1) - } - } - parser.colors[i] = parser.availableColors[parser.nextAvailable] - if parser.firstInUse == -1 { - parser.firstInUse = parser.nextAvailable - } - parser.availableColors[parser.firstAvailable], parser.availableColors[parser.nextAvailable] = parser.availableColors[parser.nextAvailable], parser.availableColors[parser.firstAvailable] - - parser.nextAvailable = (parser.nextAvailable + 1) % len(parser.availableColors) - parser.firstAvailable = (parser.firstAvailable + 1) % len(parser.availableColors) - - if parser.nextAvailable == parser.firstInUse { - parser.nextAvailable = parser.firstAvailable - } - if parser.nextAvailable == parser.firstInUse { - parser.nextAvailable = -1 - parser.firstAvailable = -1 - } -} - -// setUpFlow handles '|' or '*' -func (parser *Parser) setUpFlow(i int) { - // In preference order: - // - // Previous Row: '\? ' ' |' ' /' - // Current Row: ' | ' ' |' ' | ' - if i > 0 && i-1 < len(parser.oldGlyphs) && parser.oldGlyphs[i-1] == '\\' { - parser.takePreviousFlow(i, i-1) - } else if i < len(parser.oldGlyphs) && (parser.oldGlyphs[i] == '|' || parser.oldGlyphs[i] == '*') { - parser.takePreviousFlow(i, i) - } else if i+1 < len(parser.oldGlyphs) && parser.oldGlyphs[i+1] == '/' { - parser.takePreviousFlow(i, i+1) - } else { - parser.newFlow(i) - } -} - -// setOutFlow handles '/' -func (parser *Parser) setOutFlow(i int) { - // In preference order: - // - // Previous Row: ' |/' ' |_' ' |' ' /' ' _' '\' - // Current Row: '/| ' '/| ' '/ ' '/ ' '/ ' '/' - if i+2 < len(parser.oldGlyphs) && - (parser.oldGlyphs[i+1] == '|' || parser.oldGlyphs[i+1] == '*') && - (parser.oldGlyphs[i+2] == '/' || parser.oldGlyphs[i+2] == '_') && - i+1 < len(parser.glyphs) && - (parser.glyphs[i+1] == '|' || parser.glyphs[i+1] == '*') { - parser.takePreviousFlow(i, i+2) - } else if i+1 < len(parser.oldGlyphs) && - (parser.oldGlyphs[i+1] == '|' || parser.oldGlyphs[i+1] == '*' || - parser.oldGlyphs[i+1] == '/' || parser.oldGlyphs[i+1] == '_') { - parser.takePreviousFlow(i, i+1) - if parser.oldGlyphs[i+1] == '/' { - parser.glyphs[i] = '|' - } - } else if i < len(parser.oldGlyphs) && parser.oldGlyphs[i] == '\\' { - parser.takePreviousFlow(i, i) - } else { - parser.newFlow(i) - } -} - -// setInFlow handles '\' -func (parser *Parser) setInFlow(i int) { - // In preference order: - // - // Previous Row: '| ' '-. ' '| ' '\ ' '/' '---' - // Current Row: '|\' ' \' ' \' ' \' '\' ' \ ' - if i > 0 && i-1 < len(parser.oldGlyphs) && - (parser.oldGlyphs[i-1] == '|' || parser.oldGlyphs[i-1] == '*') && - (parser.glyphs[i-1] == '|' || parser.glyphs[i-1] == '*') { - parser.newFlow(i) - } else if i > 0 && i-1 < len(parser.oldGlyphs) && - (parser.oldGlyphs[i-1] == '|' || parser.oldGlyphs[i-1] == '*' || - parser.oldGlyphs[i-1] == '.' || parser.oldGlyphs[i-1] == '\\') { - parser.takePreviousFlow(i, i-1) - if parser.oldGlyphs[i-1] == '\\' { - parser.glyphs[i] = '|' - } - } else if i < len(parser.oldGlyphs) && parser.oldGlyphs[i] == '/' { - parser.takePreviousFlow(i, i) - } else { - parser.newFlow(i) - } -} - -// setRightFlow handles '_' -func (parser *Parser) setRightFlow(i int) { - // In preference order: - // - // Current Row: '__' '_/' '_|_' '_|/' - if i+1 < len(parser.glyphs) && - (parser.glyphs[i+1] == '_' || parser.glyphs[i+1] == '/') { - parser.takeCurrentFlow(i, i+1) - } else if i+2 < len(parser.glyphs) && - (parser.glyphs[i+1] == '|' || parser.glyphs[i+1] == '*') && - (parser.glyphs[i+2] == '_' || parser.glyphs[i+2] == '/') { - parser.takeCurrentFlow(i, i+2) - } else { - parser.newFlow(i) - } -} - -// setLeftFlow handles '----.' -func (parser *Parser) setLeftFlow(i int) { - if parser.glyphs[i] == '.' { - parser.newFlow(i) - } else if i+1 < len(parser.glyphs) && - (parser.glyphs[i+1] == '-' || parser.glyphs[i+1] == '.') { - parser.takeCurrentFlow(i, i+1) - } else { - parser.newFlow(i) - } -} diff --git a/modules/gitrepo/branch.go b/modules/gitrepo/branch.go index e13a4c82e1..d7857819e4 100644 --- a/modules/gitrepo/branch.go +++ b/modules/gitrepo/branch.go @@ -11,14 +11,14 @@ import ( // GetBranchesByPath returns a branch by its path // if limit = 0 it will not limit -func GetBranchesByPath(ctx context.Context, repo Repository, skip, limit int) ([]*git.Branch, int, error) { +func GetBranchesByPath(ctx context.Context, repo Repository, skip, limit int) ([]string, int, error) { gitRepo, err := OpenRepository(ctx, repo) if err != nil { return nil, 0, err } defer gitRepo.Close() - return gitRepo.GetBranches(skip, limit) + return gitRepo.GetBranchNames(skip, limit) } func GetBranchCommitID(ctx context.Context, repo Repository, branch string) (string, error) { @@ -33,9 +33,9 @@ func GetBranchCommitID(ctx context.Context, repo Repository, branch string) (str // SetDefaultBranch sets default branch of repository. func SetDefaultBranch(ctx context.Context, repo Repository, name string) error { - _, _, err := git.NewCommand(ctx, "symbolic-ref", "HEAD"). - AddDynamicArguments(git.BranchPrefix + name). - RunStdString(&git.RunOpts{Dir: repoPath(repo)}) + _, _, err := git.NewCommand("symbolic-ref", "HEAD"). + AddDynamicArguments(git.BranchPrefix+name). + RunStdString(ctx, &git.RunOpts{Dir: repoPath(repo)}) return err } @@ -44,6 +44,12 @@ func GetDefaultBranch(ctx context.Context, repo Repository) (string, error) { return git.GetDefaultBranch(ctx, repoPath(repo)) } -func GetWikiDefaultBranch(ctx context.Context, repo Repository) (string, error) { - return git.GetDefaultBranch(ctx, wikiPath(repo)) +// IsReferenceExist returns true if given reference exists in the repository. +func IsReferenceExist(ctx context.Context, repo Repository, name string) bool { + return git.IsReferenceExist(ctx, repoPath(repo), name) +} + +// IsBranchExist returns true if given branch exists in the repository. +func IsBranchExist(ctx context.Context, repo Repository, name string) bool { + return IsReferenceExist(ctx, repo, git.BranchPrefix+name) } diff --git a/modules/gitrepo/gitrepo.go b/modules/gitrepo/gitrepo.go index 540b724489..5da65e2452 100644 --- a/modules/gitrepo/gitrepo.go +++ b/modules/gitrepo/gitrepo.go @@ -5,9 +5,9 @@ package gitrepo import ( "context" + "fmt" "io" "path/filepath" - "strings" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/reqctx" @@ -15,17 +15,15 @@ import ( "code.gitea.io/gitea/modules/util" ) +// Repository represents a git repository which stored in a disk type Repository interface { - GetName() string - GetOwnerName() string + RelativePath() string // We don't assume how the directory structure of the repository is, so we only need the relative path } +// RelativePath should be an unix style path like username/reponame.git +// This method should change it according to the current OS. func repoPath(repo Repository) string { - return filepath.Join(setting.RepoRootPath, strings.ToLower(repo.GetOwnerName()), strings.ToLower(repo.GetName())+".git") -} - -func wikiPath(repo Repository) string { - return filepath.Join(setting.RepoRootPath, strings.ToLower(repo.GetOwnerName()), strings.ToLower(repo.GetName())+".wiki.git") + return filepath.Join(setting.RepoRootPath, filepath.FromSlash(repo.RelativePath())) } // OpenRepository opens the repository at the given relative path with the provided context. @@ -33,10 +31,6 @@ func OpenRepository(ctx context.Context, repo Repository) (*git.Repository, erro return git.OpenRepository(ctx, repoPath(repo)) } -func OpenWikiRepository(ctx context.Context, repo Repository) (*git.Repository, error) { - return git.OpenRepository(ctx, wikiPath(repo)) -} - // contextKey is a value for use with context.WithValue. type contextKey struct { repoPath string @@ -69,3 +63,21 @@ func RepositoryFromRequestContextOrOpen(ctx reqctx.RequestContext, repo Reposito ctx.SetContextValue(ck, gitRepo) return gitRepo, nil } + +// IsRepositoryExist returns true if the repository directory exists in the disk +func IsRepositoryExist(ctx context.Context, repo Repository) (bool, error) { + return util.IsExist(repoPath(repo)) +} + +// DeleteRepository deletes the repository directory from the disk +func DeleteRepository(ctx context.Context, repo Repository) error { + return util.RemoveAll(repoPath(repo)) +} + +// RenameRepository renames a repository's name on disk +func RenameRepository(ctx context.Context, repo, newRepo Repository) error { + if err := util.Rename(repoPath(repo), repoPath(newRepo)); err != nil { + return fmt.Errorf("rename repository directory: %w", err) + } + return nil +} diff --git a/modules/repository/hooks.go b/modules/gitrepo/hooks.go index 95849789ab..d9d4a88ff1 100644 --- a/modules/repository/hooks.go +++ b/modules/gitrepo/hooks.go @@ -1,9 +1,10 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package repository +package gitrepo import ( + "context" "fmt" "os" "path/filepath" @@ -106,9 +107,12 @@ done } // CreateDelegateHooks creates all the hooks scripts for the repo -func CreateDelegateHooks(repoPath string) (err error) { +func CreateDelegateHooks(_ context.Context, repo Repository) (err error) { + return createDelegateHooks(filepath.Join(repoPath(repo), "hooks")) +} + +func createDelegateHooks(hookDir string) (err error) { hookNames, hookTpls, giteaHookTpls := getHookTemplates() - hookDir := filepath.Join(repoPath, "hooks") for i, hookName := range hookNames { oldHookPath := filepath.Join(hookDir, hookName) @@ -170,10 +174,13 @@ func ensureExecutable(filename string) error { } // CheckDelegateHooks checks the hooks scripts for the repo -func CheckDelegateHooks(repoPath string) ([]string, error) { +func CheckDelegateHooks(_ context.Context, repo Repository) ([]string, error) { + return checkDelegateHooks(filepath.Join(repoPath(repo), "hooks")) +} + +func checkDelegateHooks(hookDir string) ([]string, error) { hookNames, hookTpls, giteaHookTpls := getHookTemplates() - hookDir := filepath.Join(repoPath, "hooks") results := make([]string, 0, 10) for i, hookName := range hookNames { diff --git a/modules/gitrepo/tag.go b/modules/gitrepo/tag.go new file mode 100644 index 0000000000..58ed204a99 --- /dev/null +++ b/modules/gitrepo/tag.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitrepo + +import ( + "context" + + "code.gitea.io/gitea/modules/git" +) + +// IsTagExist returns true if given tag exists in the repository. +func IsTagExist(ctx context.Context, repo Repository, name string) bool { + return IsReferenceExist(ctx, repo, git.TagPrefix+name) +} diff --git a/modules/globallock/globallock_test.go b/modules/globallock/globallock_test.go index f14c7d656b..8d55d9f699 100644 --- a/modules/globallock/globallock_test.go +++ b/modules/globallock/globallock_test.go @@ -66,11 +66,11 @@ func TestLockAndDo(t *testing.T) { func testLockAndDo(t *testing.T) { const concurrency = 50 - ctx := context.Background() + ctx := t.Context() count := 0 wg := sync.WaitGroup{} wg.Add(concurrency) - for i := 0; i < concurrency; i++ { + for range concurrency { go func() { defer wg.Done() err := LockAndDo(ctx, "test", func(ctx context.Context) error { diff --git a/modules/globallock/locker_test.go b/modules/globallock/locker_test.go index bee4d34b34..c9e73c25d2 100644 --- a/modules/globallock/locker_test.go +++ b/modules/globallock/locker_test.go @@ -46,14 +46,14 @@ func TestLocker(t *testing.T) { func testLocker(t *testing.T, locker Locker) { t.Run("lock", func(t *testing.T) { - parentCtx := context.Background() + parentCtx := t.Context() release, err := locker.Lock(parentCtx, "test") defer release() assert.NoError(t, err) func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() release, err := locker.Lock(ctx, "test") defer release() @@ -64,7 +64,7 @@ func testLocker(t *testing.T, locker Locker) { release() func() { - release, err := locker.Lock(context.Background(), "test") + release, err := locker.Lock(t.Context(), "test") defer release() assert.NoError(t, err) @@ -72,7 +72,7 @@ func testLocker(t *testing.T, locker Locker) { }) t.Run("try lock", func(t *testing.T) { - parentCtx := context.Background() + parentCtx := t.Context() ok, release, err := locker.TryLock(parentCtx, "test") defer release() @@ -80,7 +80,7 @@ func testLocker(t *testing.T, locker Locker) { assert.NoError(t, err) func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() ok, release, err := locker.TryLock(ctx, "test") defer release() @@ -92,7 +92,7 @@ func testLocker(t *testing.T, locker Locker) { release() func() { - ok, release, _ := locker.TryLock(context.Background(), "test") + ok, release, _ := locker.TryLock(t.Context(), "test") defer release() assert.True(t, ok) @@ -100,7 +100,7 @@ func testLocker(t *testing.T, locker Locker) { }) t.Run("wait and acquired", func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() release, err := locker.Lock(ctx, "test") require.NoError(t, err) @@ -109,7 +109,7 @@ func testLocker(t *testing.T, locker Locker) { go func() { defer wg.Done() started := time.Now() - release, err := locker.Lock(context.Background(), "test") // should be blocked for seconds + release, err := locker.Lock(t.Context(), "test") // should be blocked for seconds defer release() assert.Greater(t, time.Since(started), time.Second) assert.NoError(t, err) @@ -122,7 +122,7 @@ func testLocker(t *testing.T, locker Locker) { }) t.Run("multiple release", func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() release1, err := locker.Lock(ctx, "test") require.NoError(t, err) @@ -159,13 +159,13 @@ func testRedisLocker(t *testing.T, locker *redisLocker) { // Otherwise, it will affect other tests. t.Run("close", func(t *testing.T) { assert.NoError(t, locker.Close()) - _, err := locker.Lock(context.Background(), "test") + _, err := locker.Lock(t.Context(), "test") assert.Error(t, err) }) }() t.Run("failed extend", func(t *testing.T) { - release, err := locker.Lock(context.Background(), "test") + release, err := locker.Lock(t.Context(), "test") defer release() require.NoError(t, err) diff --git a/modules/globallock/redis_locker.go b/modules/globallock/redis_locker.go index 34ed9e389b..45dc769fd4 100644 --- a/modules/globallock/redis_locker.go +++ b/modules/globallock/redis_locker.go @@ -6,7 +6,6 @@ package globallock import ( "context" "errors" - "fmt" "sync" "sync/atomic" "time" @@ -78,7 +77,7 @@ func (l *redisLocker) Close() error { func (l *redisLocker) lock(ctx context.Context, key string, tries int) (ReleaseFunc, error) { if l.closed.Load() { - return func() {}, fmt.Errorf("locker is closed") + return func() {}, errors.New("locker is closed") } options := []redsync.Option{ diff --git a/modules/graceful/manager_windows.go b/modules/graceful/manager_windows.go index d776e0e9f9..457768d6ca 100644 --- a/modules/graceful/manager_windows.go +++ b/modules/graceful/manager_windows.go @@ -41,8 +41,7 @@ func (g *Manager) start() { // Make SVC process run := svc.Run - //lint:ignore SA1019 We use IsAnInteractiveSession because IsWindowsService has a different permissions profile - isAnInteractiveSession, err := svc.IsAnInteractiveSession() //nolint:staticcheck + isAnInteractiveSession, err := svc.IsAnInteractiveSession() //nolint:staticcheck // must use IsAnInteractiveSession because IsWindowsService has a different permissions profile if err != nil { log.Error("Unable to ascertain if running as an Windows Service: %v", err) return diff --git a/modules/graceful/releasereopen/releasereopen_test.go b/modules/graceful/releasereopen/releasereopen_test.go index 0e8b48257d..46e67c2046 100644 --- a/modules/graceful/releasereopen/releasereopen_test.go +++ b/modules/graceful/releasereopen/releasereopen_test.go @@ -30,14 +30,14 @@ func TestManager(t *testing.T) { _ = m.Register(t3) assert.NoError(t, m.ReleaseReopen()) - assert.EqualValues(t, 1, t1.count) - assert.EqualValues(t, 1, t2.count) - assert.EqualValues(t, 1, t3.count) + assert.Equal(t, 1, t1.count) + assert.Equal(t, 1, t2.count) + assert.Equal(t, 1, t3.count) c2() assert.NoError(t, m.ReleaseReopen()) - assert.EqualValues(t, 2, t1.count) - assert.EqualValues(t, 1, t2.count) - assert.EqualValues(t, 2, t3.count) + assert.Equal(t, 2, t1.count) + assert.Equal(t, 1, t2.count) + assert.Equal(t, 2, t3.count) } diff --git a/modules/gtprof/event.go b/modules/gtprof/event.go new file mode 100644 index 0000000000..da4a0faff9 --- /dev/null +++ b/modules/gtprof/event.go @@ -0,0 +1,32 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +type EventConfig struct { + attributes []*TraceAttribute +} + +type EventOption interface { + applyEvent(*EventConfig) +} + +type applyEventFunc func(*EventConfig) + +func (f applyEventFunc) applyEvent(cfg *EventConfig) { + f(cfg) +} + +func WithAttributes(attrs ...*TraceAttribute) EventOption { + return applyEventFunc(func(cfg *EventConfig) { + cfg.attributes = append(cfg.attributes, attrs...) + }) +} + +func eventConfigFromOptions(options ...EventOption) *EventConfig { + cfg := &EventConfig{} + for _, opt := range options { + opt.applyEvent(cfg) + } + return cfg +} diff --git a/modules/gtprof/trace.go b/modules/gtprof/trace.go new file mode 100644 index 0000000000..ad67c226dc --- /dev/null +++ b/modules/gtprof/trace.go @@ -0,0 +1,175 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +import ( + "context" + "fmt" + "sync" + "time" + + "code.gitea.io/gitea/modules/util" +) + +type contextKey struct { + name string +} + +var contextKeySpan = &contextKey{"span"} + +type traceStarter interface { + start(ctx context.Context, traceSpan *TraceSpan, internalSpanIdx int) (context.Context, traceSpanInternal) +} + +type traceSpanInternal interface { + addEvent(name string, cfg *EventConfig) + recordError(err error, cfg *EventConfig) + end() +} + +type TraceSpan struct { + // immutable + parent *TraceSpan + internalSpans []traceSpanInternal + internalContexts []context.Context + + // mutable, must be protected by mutex + mu sync.RWMutex + name string + statusCode uint32 + statusDesc string + startTime time.Time + endTime time.Time + attributes []*TraceAttribute + children []*TraceSpan +} + +type TraceAttribute struct { + Key string + Value TraceValue +} + +type TraceValue struct { + v any +} + +func (t *TraceValue) AsString() string { + return fmt.Sprint(t.v) +} + +func (t *TraceValue) AsInt64() int64 { + v, _ := util.ToInt64(t.v) + return v +} + +func (t *TraceValue) AsFloat64() float64 { + v, _ := util.ToFloat64(t.v) + return v +} + +var globalTraceStarters []traceStarter + +type Tracer struct { + starters []traceStarter +} + +func (s *TraceSpan) SetName(name string) { + s.mu.Lock() + defer s.mu.Unlock() + s.name = name +} + +func (s *TraceSpan) SetStatus(code uint32, desc string) { + s.mu.Lock() + defer s.mu.Unlock() + s.statusCode, s.statusDesc = code, desc +} + +func (s *TraceSpan) AddEvent(name string, options ...EventOption) { + cfg := eventConfigFromOptions(options...) + for _, tsp := range s.internalSpans { + tsp.addEvent(name, cfg) + } +} + +func (s *TraceSpan) RecordError(err error, options ...EventOption) { + cfg := eventConfigFromOptions(options...) + for _, tsp := range s.internalSpans { + tsp.recordError(err, cfg) + } +} + +func (s *TraceSpan) SetAttributeString(key, value string) *TraceSpan { + s.mu.Lock() + defer s.mu.Unlock() + + s.attributes = append(s.attributes, &TraceAttribute{Key: key, Value: TraceValue{v: value}}) + return s +} + +func (t *Tracer) Start(ctx context.Context, spanName string) (context.Context, *TraceSpan) { + starters := t.starters + if starters == nil { + starters = globalTraceStarters + } + ts := &TraceSpan{name: spanName, startTime: time.Now()} + parentSpan := GetContextSpan(ctx) + if parentSpan != nil { + parentSpan.mu.Lock() + parentSpan.children = append(parentSpan.children, ts) + parentSpan.mu.Unlock() + ts.parent = parentSpan + } + + parentCtx := ctx + for internalSpanIdx, tsp := range starters { + var internalSpan traceSpanInternal + if parentSpan != nil { + parentCtx = parentSpan.internalContexts[internalSpanIdx] + } + ctx, internalSpan = tsp.start(parentCtx, ts, internalSpanIdx) + ts.internalContexts = append(ts.internalContexts, ctx) + ts.internalSpans = append(ts.internalSpans, internalSpan) + } + ctx = context.WithValue(ctx, contextKeySpan, ts) + return ctx, ts +} + +type mutableContext interface { + context.Context + SetContextValue(key, value any) + GetContextValue(key any) any +} + +// StartInContext starts a trace span in Gitea's mutable context (usually the web request context). +// Due to the design limitation of Gitea's web framework, it can't use `context.WithValue` to bind a new span into a new context. +// So here we use our "reqctx" framework to achieve the same result: web request context could always see the latest "span". +func (t *Tracer) StartInContext(ctx mutableContext, spanName string) (*TraceSpan, func()) { + curTraceSpan := GetContextSpan(ctx) + _, newTraceSpan := GetTracer().Start(ctx, spanName) + ctx.SetContextValue(contextKeySpan, newTraceSpan) + return newTraceSpan, func() { + newTraceSpan.End() + ctx.SetContextValue(contextKeySpan, curTraceSpan) + } +} + +func (s *TraceSpan) End() { + s.mu.Lock() + s.endTime = time.Now() + s.mu.Unlock() + + for _, tsp := range s.internalSpans { + tsp.end() + } +} + +func GetTracer() *Tracer { + return &Tracer{} +} + +func GetContextSpan(ctx context.Context) *TraceSpan { + ts, _ := ctx.Value(contextKeySpan).(*TraceSpan) + return ts +} diff --git a/modules/gtprof/trace_builtin.go b/modules/gtprof/trace_builtin.go new file mode 100644 index 0000000000..2590ed3a13 --- /dev/null +++ b/modules/gtprof/trace_builtin.go @@ -0,0 +1,96 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +import ( + "context" + "fmt" + "strings" + "sync/atomic" + "time" + + "code.gitea.io/gitea/modules/tailmsg" +) + +type traceBuiltinStarter struct{} + +type traceBuiltinSpan struct { + ts *TraceSpan + + internalSpanIdx int +} + +func (t *traceBuiltinSpan) addEvent(name string, cfg *EventConfig) { + // No-op because builtin tracer doesn't need it. + // In the future we might use it to mark the time point between backend logic and network response. +} + +func (t *traceBuiltinSpan) recordError(err error, cfg *EventConfig) { + // No-op because builtin tracer doesn't need it. + // Actually Gitea doesn't handle err this way in most cases +} + +func (t *traceBuiltinSpan) toString(out *strings.Builder, indent int) { + t.ts.mu.RLock() + defer t.ts.mu.RUnlock() + + out.WriteString(strings.Repeat(" ", indent)) + out.WriteString(t.ts.name) + if t.ts.endTime.IsZero() { + out.WriteString(" duration: (not ended)") + } else { + fmt.Fprintf(out, " duration=%.4fs", t.ts.endTime.Sub(t.ts.startTime).Seconds()) + } + for _, a := range t.ts.attributes { + out.WriteString(" ") + out.WriteString(a.Key) + out.WriteString("=") + value := a.Value.AsString() + if strings.ContainsAny(value, " \t\r\n") { + quoted := false + for _, c := range "\"'`" { + if quoted = !strings.Contains(value, string(c)); quoted { + value = string(c) + value + string(c) + break + } + } + if !quoted { + value = fmt.Sprintf("%q", value) + } + } + out.WriteString(value) + } + out.WriteString("\n") + for _, c := range t.ts.children { + span := c.internalSpans[t.internalSpanIdx].(*traceBuiltinSpan) + span.toString(out, indent+2) + } +} + +func (t *traceBuiltinSpan) end() { + if t.ts.parent == nil { + // TODO: debug purpose only + // TODO: it should distinguish between http response network lag and actual processing time + threshold := time.Duration(traceBuiltinThreshold.Load()) + if threshold != 0 && t.ts.endTime.Sub(t.ts.startTime) > threshold { + sb := &strings.Builder{} + t.toString(sb, 0) + tailmsg.GetManager().GetTraceRecorder().Record(sb.String()) + } + } +} + +func (t *traceBuiltinStarter) start(ctx context.Context, traceSpan *TraceSpan, internalSpanIdx int) (context.Context, traceSpanInternal) { + return ctx, &traceBuiltinSpan{ts: traceSpan, internalSpanIdx: internalSpanIdx} +} + +func init() { + globalTraceStarters = append(globalTraceStarters, &traceBuiltinStarter{}) +} + +var traceBuiltinThreshold atomic.Int64 + +func EnableBuiltinTracer(threshold time.Duration) { + traceBuiltinThreshold.Store(int64(threshold)) +} diff --git a/modules/gtprof/trace_const.go b/modules/gtprof/trace_const.go new file mode 100644 index 0000000000..af9ce9223f --- /dev/null +++ b/modules/gtprof/trace_const.go @@ -0,0 +1,19 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +// Some interesting names could be found in https://github.com/open-telemetry/opentelemetry-go/tree/main/semconv + +const ( + TraceSpanHTTP = "http" + TraceSpanGitRun = "git-run" + TraceSpanDatabase = "database" +) + +const ( + TraceAttrFuncCaller = "func.caller" + TraceAttrDbSQL = "db.sql" + TraceAttrGitCommand = "git.command" + TraceAttrHTTPRoute = "http.route" +) diff --git a/modules/gtprof/trace_test.go b/modules/gtprof/trace_test.go new file mode 100644 index 0000000000..0f4e3facba --- /dev/null +++ b/modules/gtprof/trace_test.go @@ -0,0 +1,93 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +// "vendor span" is a simple demo for a span from a vendor library + +var vendorContextKey any = "vendorContextKey" + +type vendorSpan struct { + name string + children []*vendorSpan +} + +func vendorTraceStart(ctx context.Context, name string) (context.Context, *vendorSpan) { + span := &vendorSpan{name: name} + parentSpan, ok := ctx.Value(vendorContextKey).(*vendorSpan) + if ok { + parentSpan.children = append(parentSpan.children, span) + } + ctx = context.WithValue(ctx, vendorContextKey, span) + return ctx, span +} + +// below "testTrace*" integrate the vendor span into our trace system + +type testTraceSpan struct { + vendorSpan *vendorSpan +} + +func (t *testTraceSpan) addEvent(name string, cfg *EventConfig) {} + +func (t *testTraceSpan) recordError(err error, cfg *EventConfig) {} + +func (t *testTraceSpan) end() {} + +type testTraceStarter struct{} + +func (t *testTraceStarter) start(ctx context.Context, traceSpan *TraceSpan, internalSpanIdx int) (context.Context, traceSpanInternal) { + ctx, span := vendorTraceStart(ctx, traceSpan.name) + return ctx, &testTraceSpan{span} +} + +func TestTraceStarter(t *testing.T) { + globalTraceStarters = []traceStarter{&testTraceStarter{}} + + ctx := t.Context() + ctx, span := GetTracer().Start(ctx, "root") + defer span.End() + + func(ctx context.Context) { + ctx, span := GetTracer().Start(ctx, "span1") + defer span.End() + func(ctx context.Context) { + _, span := GetTracer().Start(ctx, "spanA") + defer span.End() + }(ctx) + func(ctx context.Context) { + _, span := GetTracer().Start(ctx, "spanB") + defer span.End() + }(ctx) + }(ctx) + + func(ctx context.Context) { + _, span := GetTracer().Start(ctx, "span2") + defer span.End() + }(ctx) + + var spanFullNames []string + var collectSpanNames func(parentFullName string, s *vendorSpan) + collectSpanNames = func(parentFullName string, s *vendorSpan) { + fullName := parentFullName + "/" + s.name + spanFullNames = append(spanFullNames, fullName) + for _, c := range s.children { + collectSpanNames(fullName, c) + } + } + collectSpanNames("", span.internalSpans[0].(*testTraceSpan).vendorSpan) + assert.Equal(t, []string{ + "/root", + "/root/span1", + "/root/span1/spanA", + "/root/span1/spanB", + "/root/span2", + }, spanFullNames) +} diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index d7ab3f7afd..77f24fa3f3 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -11,6 +11,7 @@ import ( gohtml "html" "html/template" "io" + "path" "path/filepath" "strings" "sync" @@ -83,7 +84,7 @@ func Code(fileName, language, code string) (output template.HTML, lexerName stri } if lexer == nil { - if val, ok := highlightMapping[filepath.Ext(fileName)]; ok { + if val, ok := highlightMapping[path.Ext(fileName)]; ok { // use mapped value to find lexer lexer = lexers.Get(val) } diff --git a/modules/highlight/highlight_test.go b/modules/highlight/highlight_test.go index 659688bd0f..b36de98c5c 100644 --- a/modules/highlight/highlight_test.go +++ b/modules/highlight/highlight_test.go @@ -114,7 +114,7 @@ c=2 t.Run(tt.name, func(t *testing.T) { out, lexerName, err := File(tt.name, "", []byte(tt.code)) assert.NoError(t, err) - assert.EqualValues(t, tt.want, out) + assert.Equal(t, tt.want, out) assert.Equal(t, tt.lexerName, lexerName) }) } @@ -177,7 +177,7 @@ c=2`), for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { out := PlainText([]byte(tt.code)) - assert.EqualValues(t, tt.want, out) + assert.Equal(t, tt.want, out) }) } } diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go index 1069310316..15c6371422 100644 --- a/modules/hostmatcher/hostmatcher.go +++ b/modules/hostmatcher/hostmatcher.go @@ -6,6 +6,7 @@ package hostmatcher import ( "net" "path/filepath" + "slices" "strings" ) @@ -38,7 +39,7 @@ func isBuiltin(s string) bool { // ParseHostMatchList parses the host list HostMatchList func ParseHostMatchList(settingKeyHint, hostList string) *HostMatchList { hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList} - for _, s := range strings.Split(hostList, ",") { + for s := range strings.SplitSeq(hostList, ",") { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { continue @@ -61,7 +62,7 @@ func ParseSimpleMatchList(settingKeyHint, matchList string) *HostMatchList { SettingKeyHint: settingKeyHint, SettingValue: matchList, } - for _, s := range strings.Split(matchList, ",") { + for s := range strings.SplitSeq(matchList, ",") { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { continue @@ -98,10 +99,8 @@ func (hl *HostMatchList) checkPattern(host string) bool { } func (hl *HostMatchList) checkIP(ip net.IP) bool { - for _, pattern := range hl.patterns { - if pattern == "*" { - return true - } + if slices.Contains(hl.patterns, "*") { + return true } for _, builtin := range hl.builtins { switch builtin { diff --git a/modules/htmlutil/html.go b/modules/htmlutil/html.go index 0ab0e71689..efbc174b2e 100644 --- a/modules/htmlutil/html.go +++ b/modules/htmlutil/html.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "slices" + "strings" ) // ParseSizeAndClass get size and class from string with default values @@ -31,6 +32,9 @@ func ParseSizeAndClass(defaultSize int, defaultClass string, others ...any) (int } func HTMLFormat(s template.HTML, rawArgs ...any) template.HTML { + if !strings.Contains(string(s), "%") || len(rawArgs) == 0 { + panic("HTMLFormat requires one or more arguments") + } args := slices.Clone(rawArgs) for i, v := range args { switch v := v.(type) { @@ -38,6 +42,8 @@ func HTMLFormat(s template.HTML, rawArgs ...any) template.HTML { // for most basic types (including template.HTML which is safe), just do nothing and use it case string: args[i] = template.HTMLEscapeString(v) + case template.URL: + args[i] = template.HTMLEscapeString(string(v)) case fmt.Stringer: args[i] = template.HTMLEscapeString(v.String()) default: diff --git a/modules/htmlutil/html_test.go b/modules/htmlutil/html_test.go index 5ff05d75b3..22258ce59d 100644 --- a/modules/htmlutil/html_test.go +++ b/modules/htmlutil/html_test.go @@ -10,6 +10,15 @@ import ( "github.com/stretchr/testify/assert" ) +type testStringer struct{} + +func (t testStringer) String() string { + return "&StringMethod" +} + func TestHTMLFormat(t *testing.T) { assert.Equal(t, template.HTML("<a>< < 1</a>"), HTMLFormat("<a>%s %s %d</a>", "<", template.HTML("<"), 1)) + assert.Equal(t, template.HTML("%!s(<nil>)"), HTMLFormat("%s", nil)) + assert.Equal(t, template.HTML("<>"), HTMLFormat("%s", template.URL("<>"))) + assert.Equal(t, template.HTML("&StringMethod &StringMethod"), HTMLFormat("%s %s", testStringer{}, &testStringer{})) } diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go index 2c9af94405..dd3efab7a5 100644 --- a/modules/httpcache/httpcache.go +++ b/modules/httpcache/httpcache.go @@ -4,40 +4,60 @@ package httpcache import ( - "io" + "fmt" "net/http" "strconv" "strings" "time" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) +type CacheControlOptions struct { + IsPublic bool + MaxAge time.Duration + NoTransform bool +} + // SetCacheControlInHeader sets suitable cache-control headers in the response -func SetCacheControlInHeader(h http.Header, maxAge time.Duration, additionalDirectives ...string) { - directives := make([]string, 0, 2+len(additionalDirectives)) +func SetCacheControlInHeader(h http.Header, opts *CacheControlOptions) { + directives := make([]string, 0, 4) // "max-age=0 + must-revalidate" (aka "no-cache") is preferred instead of "no-store" // because browsers may restore some input fields after navigate-back / reload a page. + publicPrivate := util.Iif(opts.IsPublic, "public", "private") if setting.IsProd { - if maxAge == 0 { + if opts.MaxAge == 0 { directives = append(directives, "max-age=0", "private", "must-revalidate") } else { - directives = append(directives, "private", "max-age="+strconv.Itoa(int(maxAge.Seconds()))) + directives = append(directives, publicPrivate, "max-age="+strconv.Itoa(int(opts.MaxAge.Seconds()))) } } else { - directives = append(directives, "max-age=0", "private", "must-revalidate") + // use dev-related controls, and remind users they are using non-prod setting. + directives = append(directives, "max-age=0", publicPrivate, "must-revalidate") + h.Set("X-Gitea-Debug", fmt.Sprintf("RUN_MODE=%v, MaxAge=%s", setting.RunMode, opts.MaxAge)) + } - // to remind users they are using non-prod setting. - h.Set("X-Gitea-Debug", "RUN_MODE="+setting.RunMode) + if opts.NoTransform { + directives = append(directives, "no-transform") } + h.Set("Cache-Control", strings.Join(directives, ", ")) +} - h.Set("Cache-Control", strings.Join(append(directives, additionalDirectives...), ", ")) +func CacheControlForPublicStatic() *CacheControlOptions { + return &CacheControlOptions{ + IsPublic: true, + MaxAge: setting.StaticCacheTime, + NoTransform: true, + } } -func ServeContentWithCacheControl(w http.ResponseWriter, req *http.Request, name string, modTime time.Time, content io.ReadSeeker) { - SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) - http.ServeContent(w, req, name, modTime, content) +func CacheControlForPrivateStatic() *CacheControlOptions { + return &CacheControlOptions{ + MaxAge: setting.StaticCacheTime, + NoTransform: true, + } } // HandleGenericETagCache handles ETag-based caching for a HTTP request. @@ -50,7 +70,8 @@ func HandleGenericETagCache(req *http.Request, w http.ResponseWriter, etag strin return true } } - SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) + // not sure whether it is a public content, so just use "private" (old behavior) + SetCacheControlInHeader(w.Header(), CacheControlForPrivateStatic()) return false } @@ -58,7 +79,7 @@ func HandleGenericETagCache(req *http.Request, w http.ResponseWriter, etag strin func checkIfNoneMatchIsValid(req *http.Request, etag string) bool { ifNoneMatch := req.Header.Get("If-None-Match") if len(ifNoneMatch) > 0 { - for _, item := range strings.Split(ifNoneMatch, ",") { + for item := range strings.SplitSeq(ifNoneMatch, ",") { item = strings.TrimPrefix(strings.TrimSpace(item), "W/") // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#directives if item == etag { return true @@ -95,6 +116,8 @@ func HandleGenericETagTimeCache(req *http.Request, w http.ResponseWriter, etag s } } } - SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) + + // not sure whether it is a public content, so just use "private" (old behavior) + SetCacheControlInHeader(w.Header(), CacheControlForPrivateStatic()) return false } diff --git a/modules/httplib/request.go b/modules/httplib/request.go index 880d7ad3cb..49ea6f4b73 100644 --- a/modules/httplib/request.go +++ b/modules/httplib/request.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "crypto/tls" + "errors" "fmt" "io" "net" @@ -99,18 +100,27 @@ func (r *Request) Param(key, value string) *Request { return r } -// Body adds request raw body. -// it supports string and []byte. +// Body adds request raw body. It supports string, []byte and io.Reader as body. func (r *Request) Body(data any) *Request { + if r == nil { + return nil + } switch t := data.(type) { + case nil: // do nothing case string: - bf := bytes.NewBufferString(t) + bf := strings.NewReader(t) r.req.Body = io.NopCloser(bf) r.req.ContentLength = int64(len(t)) case []byte: bf := bytes.NewBuffer(t) r.req.Body = io.NopCloser(bf) r.req.ContentLength = int64(len(t)) + case io.ReadCloser: + r.req.Body = t + case io.Reader: + r.req.Body = io.NopCloser(t) + default: + panic(fmt.Sprintf("unsupported request body type %T", t)) } return r } @@ -133,15 +143,15 @@ func (r *Request) getResponse() (*http.Response, error) { paramBody = paramBody[0 : len(paramBody)-1] } - if r.req.Method == "GET" && len(paramBody) > 0 { + if r.req.Method == http.MethodGet && len(paramBody) > 0 { if strings.Contains(r.url, "?") { r.url += "&" + paramBody } else { r.url = r.url + "?" + paramBody } - } else if r.req.Method == "POST" && r.req.Body == nil && len(paramBody) > 0 { + } else if r.req.Method == http.MethodPost && r.req.Body == nil && len(paramBody) > 0 { r.Header("Content-Type", "application/x-www-form-urlencoded") - r.Body(paramBody) + r.Body(paramBody) // string } var err error @@ -185,7 +195,11 @@ func (r *Request) getResponse() (*http.Response, error) { } // Response executes request client gets response manually. +// Caller MUST close the response body if no error occurs func (r *Request) Response() (*http.Response, error) { + if r == nil { + return nil, errors.New("invalid request") + } return r.getResponse() } diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go index 8fb667876e..7c1edf432d 100644 --- a/modules/httplib/serve.go +++ b/modules/httplib/serve.go @@ -33,6 +33,7 @@ type ServeHeaderOptions struct { ContentLength *int64 Disposition string // defaults to "attachment" Filename string + CacheIsPublic bool CacheDuration time.Duration // defaults to 5 minutes LastModified time.Time } @@ -72,11 +73,11 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) { header.Set("Access-Control-Expose-Headers", "Content-Disposition") } - duration := opts.CacheDuration - if duration == 0 { - duration = 5 * time.Minute - } - httpcache.SetCacheControlInHeader(header, duration) + httpcache.SetCacheControlInHeader(header, &httpcache.CacheControlOptions{ + IsPublic: opts.CacheIsPublic, + MaxAge: opts.CacheDuration, + NoTransform: true, + }) if !opts.LastModified.IsZero() { // http.TimeFormat required a UTC time, refer to https://pkg.go.dev/net/http#TimeFormat @@ -85,19 +86,15 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) { } // ServeData download file from io.Reader -func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath string, mineBuf []byte) { +func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byte, opts *ServeHeaderOptions) { // do not set "Content-Length", because the length could only be set by callers, and it needs to support range requests - opts := &ServeHeaderOptions{ - Filename: path.Base(filePath), - } - sniffedType := typesniffer.DetectContentType(mineBuf) // the "render" parameter came from year 2016: 638dd24c, it doesn't have clear meaning, so I think it could be removed later isPlain := sniffedType.IsText() || r.FormValue("render") != "" if setting.MimeTypeMap.Enabled { - fileExtension := strings.ToLower(filepath.Ext(filePath)) + fileExtension := strings.ToLower(filepath.Ext(opts.Filename)) opts.ContentType = setting.MimeTypeMap.Map[fileExtension] } @@ -114,7 +111,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri if isPlain { charset, err := charsetModule.DetectEncoding(mineBuf) if err != nil { - log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err) + log.Error("Detect raw file %s charset failed: %v, using by default utf-8", opts.Filename, err) charset = "utf-8" } opts.ContentTypeCharset = strings.ToLower(charset) @@ -142,7 +139,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri const mimeDetectionBufferLen = 1024 -func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath string, size int64, reader io.Reader) { +func ServeContentByReader(r *http.Request, w http.ResponseWriter, size int64, reader io.Reader, opts *ServeHeaderOptions) { buf := make([]byte, mimeDetectionBufferLen) n, err := util.ReadAtMost(reader, buf) if err != nil { @@ -152,7 +149,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin if n >= 0 { buf = buf[:n] } - setServeHeadersByFile(r, w, filePath, buf) + setServeHeadersByFile(r, w, buf, opts) // reset the reader to the beginning reader = io.MultiReader(bytes.NewReader(buf), reader) @@ -215,7 +212,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin _, _ = io.CopyN(w, reader, partialLength) // just like http.ServeContent, not necessary to handle the error } -func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath string, modTime *time.Time, reader io.ReadSeeker) { +func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, modTime *time.Time, reader io.ReadSeeker, opts *ServeHeaderOptions) { buf := make([]byte, mimeDetectionBufferLen) n, err := util.ReadAtMost(reader, buf) if err != nil { @@ -229,9 +226,9 @@ func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath s if n >= 0 { buf = buf[:n] } - setServeHeadersByFile(r, w, filePath, buf) + setServeHeadersByFile(r, w, buf, opts) if modTime == nil { modTime = &time.Time{} } - http.ServeContent(w, r, path.Base(filePath), *modTime, reader) + http.ServeContent(w, r, opts.Filename, *modTime, reader) } diff --git a/modules/httplib/serve_test.go b/modules/httplib/serve_test.go index e53f38b697..78b88c9b5f 100644 --- a/modules/httplib/serve_test.go +++ b/modules/httplib/serve_test.go @@ -4,11 +4,11 @@ package httplib import ( - "fmt" "net/http" "net/http/httptest" "net/url" "os" + "strconv" "strings" "testing" @@ -23,14 +23,14 @@ func TestServeContentByReader(t *testing.T) { _, rangeStr, _ := strings.Cut(t.Name(), "_range_") r := &http.Request{Header: http.Header{}, Form: url.Values{}} if rangeStr != "" { - r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + r.Header.Set("Range", "bytes="+rangeStr) } reader := strings.NewReader(data) w := httptest.NewRecorder() - ServeContentByReader(r, w, "test", int64(len(data)), reader) + ServeContentByReader(r, w, int64(len(data)), reader, &ServeHeaderOptions{}) assert.Equal(t, expectedStatusCode, w.Code) if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { - assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, strconv.Itoa(len(expectedContent)), w.Header().Get("Content-Length")) assert.Equal(t, expectedContent, w.Body.String()) } } @@ -68,7 +68,7 @@ func TestServeContentByReadSeeker(t *testing.T) { _, rangeStr, _ := strings.Cut(t.Name(), "_range_") r := &http.Request{Header: http.Header{}, Form: url.Values{}} if rangeStr != "" { - r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + r.Header.Set("Range", "bytes="+rangeStr) } seekReader, err := os.OpenFile(tmpFile, os.O_RDONLY, 0o644) @@ -76,10 +76,10 @@ func TestServeContentByReadSeeker(t *testing.T) { defer seekReader.Close() w := httptest.NewRecorder() - ServeContentByReadSeeker(r, w, "test", nil, seekReader) + ServeContentByReadSeeker(r, w, nil, seekReader, &ServeHeaderOptions{}) assert.Equal(t, expectedStatusCode, w.Code) if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { - assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, strconv.Itoa(len(expectedContent)), w.Header().Get("Content-Length")) assert.Equal(t, expectedContent, w.Body.String()) } } diff --git a/modules/httplib/url.go b/modules/httplib/url.go index f543c09190..f51506ac3b 100644 --- a/modules/httplib/url.go +++ b/modules/httplib/url.go @@ -53,28 +53,34 @@ func getRequestScheme(req *http.Request) string { return "" } -// GuessCurrentAppURL tries to guess the current full app URL (with sub-path) by http headers. It always has a '/' suffix, exactly the same as setting.AppURL +// GuessCurrentAppURL tries to guess the current full public URL (with sub-path) by http headers. It always has a '/' suffix, exactly the same as setting.AppURL +// TODO: should rename it to GuessCurrentPublicURL in the future func GuessCurrentAppURL(ctx context.Context) string { return GuessCurrentHostURL(ctx) + setting.AppSubURL + "/" } // GuessCurrentHostURL tries to guess the current full host URL (no sub-path) by http headers, there is no trailing slash. func GuessCurrentHostURL(ctx context.Context) string { - req, ok := ctx.Value(RequestContextKey).(*http.Request) - if !ok { - return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") - } - // If no scheme provided by reverse proxy, then do not guess the AppURL, use the configured one. + // Try the best guess to get the current host URL (will be used for public URL) by http headers. // At the moment, if site admin doesn't configure the proxy headers correctly, then Gitea would guess wrong. // There are some cases: // 1. The reverse proxy is configured correctly, it passes "X-Forwarded-Proto/Host" headers. Perfect, Gitea can handle it correctly. // 2. The reverse proxy is not configured correctly, doesn't pass "X-Forwarded-Proto/Host" headers, eg: only one "proxy_pass http://gitea:3000" in Nginx. // 3. There is no reverse proxy. - // Without an extra config option, Gitea is impossible to distinguish between case 2 and case 3, - // then case 2 would result in wrong guess like guessed AppURL becomes "http://gitea:3000/", which is not accessible by end users. - // So in the future maybe it should introduce a new config option, to let site admin decide how to guess the AppURL. + // Without more information, Gitea is impossible to distinguish between case 2 and case 3, then case 2 would result in + // wrong guess like guessed public URL becomes "http://gitea:3000/" behind a "https" reverse proxy, which is not accessible by end users. + // So we introduced "PUBLIC_URL_DETECTION" option, to control the guessing behavior to satisfy different use cases. + req, ok := ctx.Value(RequestContextKey).(*http.Request) + if !ok { + return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") + } reqScheme := getRequestScheme(req) if reqScheme == "" { + // if no reverse proxy header, try to use "Host" header for absolute URL + if setting.PublicURLDetection == setting.PublicURLAuto && req.Host != "" { + return util.Iif(req.TLS == nil, "http://", "https://") + req.Host + } + // fall back to default AppURL return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") } // X-Forwarded-Host has many problems: non-standard, not well-defined (X-Forwarded-Port or not), conflicts with Host header. @@ -88,8 +94,8 @@ func GuessCurrentHostDomain(ctx context.Context) string { return util.IfZero(domain, host) } -// MakeAbsoluteURL tries to make a link to an absolute URL: -// * If link is empty, it returns the current app URL. +// MakeAbsoluteURL tries to make a link to an absolute public URL: +// * If link is empty, it returns the current public URL. // * If link is absolute, it returns the link. // * Otherwise, it returns the current host URL + link, the link itself should have correct sub-path (AppSubURL) if needed. func MakeAbsoluteURL(ctx context.Context, link string) string { @@ -102,25 +108,77 @@ func MakeAbsoluteURL(ctx context.Context, link string) string { return GuessCurrentHostURL(ctx) + "/" + strings.TrimPrefix(link, "/") } -func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool { +type urlType int + +const ( + urlTypeGiteaAbsolute urlType = iota + 1 // "http://gitea/subpath" + urlTypeGiteaPageRelative // "/subpath" + urlTypeGiteaSiteRelative // "?key=val" + urlTypeUnknown // "http://other" +) + +func detectURLRoutePath(ctx context.Context, s string) (routePath string, ut urlType) { u, err := url.Parse(s) if err != nil { - return false + return "", urlTypeUnknown } + cleanedPath := "" if u.Path != "" { - cleanedPath := util.PathJoinRelX(u.Path) - if cleanedPath == "" || cleanedPath == "." { - u.Path = "/" - } else { - u.Path = "/" + cleanedPath + "/" - } + cleanedPath = util.PathJoinRelX(u.Path) + cleanedPath = util.Iif(cleanedPath == ".", "", "/"+cleanedPath) } if urlIsRelative(s, u) { - return u.Path == "" || strings.HasPrefix(strings.ToLower(u.Path), strings.ToLower(setting.AppSubURL+"/")) - } - if u.Path == "" { - u.Path = "/" + if u.Path == "" { + return "", urlTypeGiteaPageRelative + } + if strings.HasPrefix(strings.ToLower(cleanedPath+"/"), strings.ToLower(setting.AppSubURL+"/")) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaSiteRelative + } + return "", urlTypeUnknown } + u.Path = cleanedPath + "/" urlLower := strings.ToLower(u.String()) - return strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) || strings.HasPrefix(urlLower, strings.ToLower(GuessCurrentAppURL(ctx))) + if strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute + } + guessedCurURL := GuessCurrentAppURL(ctx) + if strings.HasPrefix(urlLower, strings.ToLower(guessedCurURL)) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute + } + return "", urlTypeUnknown +} + +func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool { + _, ut := detectURLRoutePath(ctx, s) + return ut != urlTypeUnknown +} + +type GiteaSiteURL struct { + RoutePath string + OwnerName string + RepoName string + RepoSubPath string +} + +func ParseGiteaSiteURL(ctx context.Context, s string) *GiteaSiteURL { + routePath, ut := detectURLRoutePath(ctx, s) + if ut == urlTypeUnknown || ut == urlTypeGiteaPageRelative { + return nil + } + ret := &GiteaSiteURL{RoutePath: routePath} + fields := strings.SplitN(strings.TrimPrefix(ret.RoutePath, "/"), "/", 3) + + // TODO: now it only does a quick check for some known reserved paths, should do more strict checks in the future + if fields[0] == "attachments" { + return ret + } + if len(fields) < 2 { + return ret + } + ret.OwnerName = fields[0] + ret.RepoName = fields[1] + if len(fields) == 3 { + ret.RepoSubPath = "/" + fields[2] + } + return ret } diff --git a/modules/httplib/url_test.go b/modules/httplib/url_test.go index fc6c91cd3a..0ffb0cac05 100644 --- a/modules/httplib/url_test.go +++ b/modules/httplib/url_test.go @@ -5,6 +5,7 @@ package httplib import ( "context" + "crypto/tls" "net/http" "testing" @@ -39,12 +40,48 @@ func TestIsRelativeURL(t *testing.T) { } } +func TestGuessCurrentHostURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + headersWithProto := http.Header{"X-Forwarded-Proto": {"https"}} + + t.Run("Legacy", func(t *testing.T) { + defer test.MockVariableValue(&setting.PublicURLDetection, setting.PublicURLLegacy)() + + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(t.Context())) + + // legacy: "Host" is not used when there is no "X-Forwarded-Proto" header + ctx := context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000"}) + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(ctx)) + + // if "X-Forwarded-Proto" exists, then use it and "Host" header + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000", Header: headersWithProto}) + assert.Equal(t, "https://req-host:3000", GuessCurrentHostURL(ctx)) + }) + + t.Run("Auto", func(t *testing.T) { + defer test.MockVariableValue(&setting.PublicURLDetection, setting.PublicURLAuto)() + + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(t.Context())) + + // auto: always use "Host" header, the scheme is determined by "X-Forwarded-Proto" header, or TLS config if no "X-Forwarded-Proto" header + ctx := context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000"}) + assert.Equal(t, "http://req-host:3000", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host", TLS: &tls.ConnectionState{}}) + assert.Equal(t, "https://req-host", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000", Header: headersWithProto}) + assert.Equal(t, "https://req-host:3000", GuessCurrentHostURL(ctx)) + }) +} + func TestMakeAbsoluteURL(t *testing.T) { defer test.MockVariableValue(&setting.Protocol, "http")() defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() defer test.MockVariableValue(&setting.AppSubURL, "/sub")() - ctx := context.Background() + ctx := t.Context() assert.Equal(t, "http://cfg-host/sub/", MakeAbsoluteURL(ctx, "")) assert.Equal(t, "http://cfg-host/foo", MakeAbsoluteURL(ctx, "foo")) assert.Equal(t, "http://cfg-host/foo", MakeAbsoluteURL(ctx, "/foo")) @@ -76,7 +113,7 @@ func TestMakeAbsoluteURL(t *testing.T) { func TestIsCurrentGiteaSiteURL(t *testing.T) { defer test.MockVariableValue(&setting.AppURL, "http://localhost:3000/sub/")() defer test.MockVariableValue(&setting.AppSubURL, "/sub")() - ctx := context.Background() + ctx := t.Context() good := []string{ "?key=val", "/sub", @@ -122,3 +159,26 @@ func TestIsCurrentGiteaSiteURL(t *testing.T) { assert.True(t, IsCurrentGiteaSiteURL(ctx, "https://user-host")) assert.False(t, IsCurrentGiteaSiteURL(ctx, "https://forwarded-host")) } + +func TestParseGiteaSiteURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://localhost:3000/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + ctx := t.Context() + tests := []struct { + url string + exp *GiteaSiteURL + }{ + {"http://localhost:3000/sub?k=v", &GiteaSiteURL{RoutePath: ""}}, + {"http://localhost:3000/sub/", &GiteaSiteURL{RoutePath: ""}}, + {"http://localhost:3000/sub/foo", &GiteaSiteURL{RoutePath: "/foo"}}, + {"http://localhost:3000/sub/foo/bar", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}}, + {"http://localhost:3000/sub/foo/bar/", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}}, + {"http://localhost:3000/sub/attachments/bar", &GiteaSiteURL{RoutePath: "/attachments/bar"}}, + {"http://localhost:3000/other", nil}, + {"http://other/", nil}, + } + for _, test := range tests { + su := ParseGiteaSiteURL(ctx, test.url) + assert.Equal(t, test.exp, su, "URL = %s", test.url) + } +} diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index 772317fa59..70f0995a01 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -16,7 +16,7 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/indexer" path_filter "code.gitea.io/gitea/modules/indexer/code/bleve/token/path" "code.gitea.io/gitea/modules/indexer/code/internal" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" @@ -24,11 +24,11 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" "github.com/blevesearch/bleve/v2" analyzer_custom "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - "github.com/blevesearch/bleve/v2/analysis/token/camelcase" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter" @@ -70,7 +70,7 @@ const ( filenameIndexerAnalyzer = "filenameIndexerAnalyzer" filenameIndexerTokenizer = "filenameIndexerTokenizer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 8 + repoIndexerLatestVersion = 9 ) // generateBleveIndexMapping generates a bleve index mapping for the repo indexer @@ -107,7 +107,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { "type": analyzer_custom.Name, "char_filters": []string{}, "tokenizer": letter.Name, - "token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name}, + "token_filters": []string{unicodeNormalizeName, lowercase.Name}, }); err != nil { return nil, err } @@ -136,6 +136,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new bleve local indexer func NewIndexer(indexDir string) *Indexer { inner := inner_bleve.NewIndexer(indexDir, repoIndexerLatestVersion, generateBleveIndexMapping) @@ -158,7 +162,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro var err error if !update.Sized { var stdout string - stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, err = git.NewCommand("cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil { return err } @@ -185,7 +189,8 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro return err } else if !typesniffer.DetectContentType(fileContents).IsText() { // FIXME: UTF-16 files will probably fail here - return nil + // Even if the file is not recognized as a "text file", we could still put its name into the indexers to make the filename become searchable, while leave the content to empty. + fileContents = nil } if _, err = batchReader.Discard(1); err != nil { @@ -211,12 +216,7 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize) if len(changes.Updates) > 0 { - r, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - return err - } - defer r.Close() - gitBatch, err := r.NewBatch(ctx) + gitBatch, err := git.NewBatch(ctx, repo.RepoPath()) if err != nil { return err } @@ -260,17 +260,31 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int var ( indexerQuery query.Query keywordQuery query.Query + contentQuery query.Query ) pathQuery := bleve.NewPrefixQuery(strings.ToLower(opts.Keyword)) pathQuery.FieldVal = "Filename" pathQuery.SetBoost(10) - contentQuery := bleve.NewMatchQuery(opts.Keyword) - contentQuery.FieldVal = "Content" - - if opts.IsKeywordFuzzy { - contentQuery.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword) + searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeExact { + // 1.21 used NewPrefixQuery, but it seems not working well, and later releases changed to NewMatchPhraseQuery + q := bleve.NewMatchPhraseQuery(opts.Keyword) + q.Analyzer = repoIndexerAnalyzer + q.FieldVal = "Content" + contentQuery = q + } else /* words */ { + q := bleve.NewMatchQuery(opts.Keyword) + q.FieldVal = "Content" + q.Analyzer = repoIndexerAnalyzer + if searchMode == indexer.SearchModeFuzzy { + // this logic doesn't seem right, it is only used to pass the test-case `Keyword: "dESCRIPTION"`, which doesn't seem to be a real-life use-case. + q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword) + } else { + q.Operator = query.MatchQueryOperatorAnd + } + contentQuery = q } keywordQuery = bleve.NewDisjunctionQuery(contentQuery, pathQuery) diff --git a/modules/indexer/code/bleve/token/path/path.go b/modules/indexer/code/bleve/token/path/path.go index 107e0da109..6dfc12f146 100644 --- a/modules/indexer/code/bleve/token/path/path.go +++ b/modules/indexer/code/bleve/token/path/path.go @@ -51,13 +51,13 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke slices.Reverse(input) } - for i := 0; i < len(input); i++ { + for i := range input { var sb strings.Builder - sb.WriteString(string(input[0].Term)) + sb.Write(input[0].Term) for j := 1; j < i; j++ { sb.WriteString("/") - sb.WriteString(string(input[j].Term)) + sb.Write(input[j].Term) } term := sb.String() @@ -97,5 +97,9 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke } func init() { - registry.RegisterTokenFilter(Name, TokenFilterConstructor) + // FIXME: move it to the bleve's init function, but do not call it in global init + err := registry.RegisterTokenFilter(Name, TokenFilterConstructor) + if err != nil { + panic(err) + } } diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index 1c4dd39eff..f925ce396a 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -15,7 +15,7 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/code/internal" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" "github.com/go-enry/go-enry/v2" "github.com/olivere/elastic/v7" @@ -45,6 +46,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new elasticsearch indexer func NewIndexer(url, indexerName string) *Indexer { inner := inner_elasticsearch.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping) @@ -142,7 +147,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro var err error if !update.Sized { var stdout string - stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, err = git.NewCommand("cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil { return nil, err } @@ -203,12 +208,7 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elasti func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { reqs := make([]elastic.BulkableRequest, 0) if len(changes.Updates) > 0 { - r, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - return err - } - defer r.Close() - batch, err := r.NewBatch(ctx) + batch, err := git.NewBatch(ctx, repo.RepoPath()) if err != nil { return err } @@ -359,13 +359,16 @@ func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLan // Search searches for codes and language stats by given conditions. func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { - searchType := esMultiMatchTypePhrasePrefix - if opts.IsKeywordFuzzy { - searchType = esMultiMatchTypeBestFields + var contentQuery elastic.Query + searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeExact { + // 1.21 used NewMultiMatchQuery().Type(esMultiMatchTypePhrasePrefix), but later releases changed to NewMatchPhraseQuery + contentQuery = elastic.NewMatchPhraseQuery("content", opts.Keyword) + } else /* words */ { + contentQuery = elastic.NewMultiMatchQuery("content", opts.Keyword).Type(esMultiMatchTypeBestFields).Operator("and") } - kwQuery := elastic.NewBoolQuery().Should( - elastic.NewMultiMatchQuery(opts.Keyword, "content").Type(searchType), + contentQuery, elastic.NewMultiMatchQuery(opts.Keyword, "filename^10").Type(esMultiMatchTypePhrasePrefix), ) query := elastic.NewBoolQuery() diff --git a/modules/indexer/code/elasticsearch/elasticsearch_test.go b/modules/indexer/code/elasticsearch/elasticsearch_test.go index a6d2af92b2..e8f1f202ce 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/code/elasticsearch/elasticsearch_test.go @@ -11,6 +11,6 @@ import ( func TestIndexPos(t *testing.T) { startIdx, endIdx := contentMatchIndexPos("test index start and end", "start", "end") - assert.EqualValues(t, 11, startIdx) - assert.EqualValues(t, 15, endIdx) + assert.Equal(t, 11, startIdx) + assert.Equal(t, 15, endIdx) } diff --git a/modules/indexer/code/git.go b/modules/indexer/code/git.go index df9783288b..41bc74e6ec 100644 --- a/modules/indexer/code/git.go +++ b/modules/indexer/code/git.go @@ -16,7 +16,7 @@ import ( ) func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (string, error) { - stdout, _, err := git.NewCommand(ctx, "show-ref", "-s").AddDynamicArguments(git.BranchPrefix + repo.DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, err := git.NewCommand("show-ref", "-s").AddDynamicArguments(git.BranchPrefix+repo.DefaultBranch).RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil { return "", err } @@ -32,8 +32,8 @@ func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision s needGenesis := len(status.CommitSha) == 0 if !needGenesis { - hasAncestorCmd := git.NewCommand(ctx, "merge-base").AddDynamicArguments(status.CommitSha, revision) - stdout, _, _ := hasAncestorCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + hasAncestorCmd := git.NewCommand("merge-base").AddDynamicArguments(status.CommitSha, revision) + stdout, _, _ := hasAncestorCmd.RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) needGenesis = len(stdout) == 0 } @@ -86,7 +86,7 @@ func parseGitLsTreeOutput(stdout []byte) ([]internal.FileUpdate, error) { // genesisChanges get changes to add repo to the indexer for the first time func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) { var changes internal.RepoChanges - stdout, _, runErr := git.NewCommand(ctx, "ls-tree", "--full-tree", "-l", "-r").AddDynamicArguments(revision).RunStdBytes(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, runErr := git.NewCommand("ls-tree", "--full-tree", "-l", "-r").AddDynamicArguments(revision).RunStdBytes(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if runErr != nil { return nil, runErr } @@ -98,8 +98,8 @@ func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision s // nonGenesisChanges get changes since the previous indexer update func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) { - diffCmd := git.NewCommand(ctx, "diff", "--name-status").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision) - stdout, _, runErr := diffCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + diffCmd := git.NewCommand("diff", "--name-status").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision) + stdout, _, runErr := diffCmd.RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if runErr != nil { // previous commit sha may have been removed by a force push, so // try rebuilding from scratch @@ -115,9 +115,9 @@ func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revisio updatedFilenames := make([]string, 0, 10) updateChanges := func() error { - cmd := git.NewCommand(ctx, "ls-tree", "--full-tree", "-l").AddDynamicArguments(revision). + cmd := git.NewCommand("ls-tree", "--full-tree", "-l").AddDynamicArguments(revision). AddDashesAndList(updatedFilenames...) - lsTreeStdout, _, err := cmd.RunStdBytes(&git.RunOpts{Dir: repo.RepoPath()}) + lsTreeStdout, _, err := cmd.RunStdBytes(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil { return err } @@ -129,8 +129,8 @@ func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revisio changes.Updates = append(changes.Updates, updates...) return nil } - lines := strings.Split(stdout, "\n") - for _, line := range lines { + lines := strings.SplitSeq(stdout, "\n") + for line := range lines { line = strings.TrimSpace(line) if len(line) == 0 { continue diff --git a/modules/indexer/code/gitgrep/gitgrep.go b/modules/indexer/code/gitgrep/gitgrep.go new file mode 100644 index 0000000000..6f6e0b47b9 --- /dev/null +++ b/modules/indexer/code/gitgrep/gitgrep.go @@ -0,0 +1,66 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgrep + +import ( + "context" + "fmt" + "strings" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/indexer" + code_indexer "code.gitea.io/gitea/modules/indexer/code" + "code.gitea.io/gitea/modules/setting" +) + +func indexSettingToGitGrepPathspecList() (list []string) { + for _, expr := range setting.Indexer.IncludePatterns { + list = append(list, ":(glob)"+expr.PatternString()) + } + for _, expr := range setting.Indexer.ExcludePatterns { + list = append(list, ":(glob,exclude)"+expr.PatternString()) + } + return list +} + +func PerformSearch(ctx context.Context, page int, repoID int64, gitRepo *git.Repository, ref git.RefName, keyword string, searchMode indexer.SearchModeType) (searchResults []*code_indexer.Result, total int, err error) { + grepMode := git.GrepModeWords + switch searchMode { + case indexer.SearchModeExact: + grepMode = git.GrepModeExact + case indexer.SearchModeRegexp: + grepMode = git.GrepModeRegexp + } + res, err := git.GrepSearch(ctx, gitRepo, keyword, git.GrepOptions{ + ContextLineNumber: 1, + GrepMode: grepMode, + RefName: ref.String(), + PathspecList: indexSettingToGitGrepPathspecList(), + }) + if err != nil { + // TODO: if no branch exists, it reports: exit status 128, fatal: this operation must be run in a work tree. + return nil, 0, fmt.Errorf("git.GrepSearch: %w", err) + } + commitID, err := gitRepo.GetRefCommitID(ref.String()) + if err != nil { + return nil, 0, fmt.Errorf("gitRepo.GetRefCommitID: %w", err) + } + + total = len(res) + pageStart := min((page-1)*setting.UI.RepoSearchPagingNum, len(res)) + pageEnd := min(page*setting.UI.RepoSearchPagingNum, len(res)) + res = res[pageStart:pageEnd] + for _, r := range res { + searchResults = append(searchResults, &code_indexer.Result{ + RepoID: repoID, + Filename: r.Filename, + CommitID: commitID, + // UpdatedUnix: not supported yet + // Language: not supported yet + // Color: not supported yet + Lines: code_indexer.HighlightSearchResultCode(r.Filename, "", r.LineNumbers, strings.Join(r.LineCodes, "\n")), + }) + } + return searchResults, total, nil +} diff --git a/modules/indexer/code/gitgrep/gitgrep_test.go b/modules/indexer/code/gitgrep/gitgrep_test.go new file mode 100644 index 0000000000..97dda9d966 --- /dev/null +++ b/modules/indexer/code/gitgrep/gitgrep_test.go @@ -0,0 +1,19 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgrep + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" +) + +func TestIndexSettingToGitGrepPathspecList(t *testing.T) { + defer test.MockVariableValue(&setting.Indexer.IncludePatterns, setting.IndexerGlobFromString("a"))() + defer test.MockVariableValue(&setting.Indexer.ExcludePatterns, setting.IndexerGlobFromString("b"))() + assert.Equal(t, []string{":(glob)a", ":(glob,exclude)b"}, indexSettingToGitGrepPathspecList()) +} diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index 728b37fab6..6035ddfe95 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/code/bleve" "code.gitea.io/gitea/modules/indexer/code/elasticsearch" "code.gitea.io/gitea/modules/indexer/code/internal" @@ -29,13 +30,11 @@ var ( // When the real indexer is not ready, it will be a dummy indexer which will return error to explain it's not ready. // So it's always safe use it as *globalIndexer.Load() and call its methods. globalIndexer atomic.Pointer[internal.Indexer] - dummyIndexer *internal.Indexer ) func init() { - i := internal.NewDummyIndexer() - dummyIndexer = &i - globalIndexer.Store(dummyIndexer) + dummyIndexer := internal.NewDummyIndexer() + globalIndexer.Store(&dummyIndexer) } func index(ctx context.Context, indexer internal.Indexer, repoID int64) error { @@ -304,3 +303,11 @@ func populateRepoIndexer(ctx context.Context) { } log.Info("Done (re)populating the repo indexer with existing repositories") } + +func SupportedSearchModes() []indexer.SearchMode { + gi := globalIndexer.Load() + if gi == nil { + return nil + } + return (*gi).SupportedSearchModes() +} diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index f358bbe785..78fea22f10 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -11,12 +11,13 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/git" + indexer_module "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/code/bleve" "code.gitea.io/gitea/modules/indexer/code/elasticsearch" "code.gitea.io/gitea/modules/indexer/code/internal" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/util" _ "code.gitea.io/gitea/models" _ "code.gitea.io/gitea/models/actions" @@ -37,13 +38,14 @@ func TestMain(m *testing.M) { func testIndexer(name string, t *testing.T, indexer internal.Indexer) { t.Run(name, func(t *testing.T) { - assert.NoError(t, setupRepositoryIndexes(git.DefaultContext, indexer)) + assert.NoError(t, setupRepositoryIndexes(t.Context(), indexer)) keywords := []struct { - RepoIDs []int64 - Keyword string - Langs int - Results []codeSearchResult + RepoIDs []int64 + Keyword string + Langs int + SearchMode indexer_module.SearchModeType + Results []codeSearchResult }{ // Search for an exact match on the contents of a file // This scenario yields a single result (the file README.md on the repo '1') @@ -184,9 +186,10 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, // Search for matches on the contents of files regardless of case. { - RepoIDs: nil, - Keyword: "dESCRIPTION", - Langs: 1, + RepoIDs: nil, + Keyword: "dESCRIPTION", + Langs: 1, + SearchMode: indexer_module.SearchModeFuzzy, Results: []codeSearchResult{ { Filename: "README.md", @@ -194,7 +197,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, }, }, - // Search for an exact match on the filename within the repo '62' (case insenstive). + // Search for an exact match on the filename within the repo '62' (case-insensitive). // This scenario yields a single result (the file avocado.md on the repo '62') { RepoIDs: []int64{62}, @@ -207,7 +210,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, }, }, - // Search for matches on the contents of files when the criteria is a expression. + // Search for matches on the contents of files when the criteria are an expression. { RepoIDs: []int64{62}, Keyword: "console.log", @@ -219,7 +222,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, }, }, - // Search for matches on the contents of files when the criteria is part of a expression. + // Search for matches on the contents of files when the criteria are parts of an expression. { RepoIDs: []int64{62}, Keyword: "log", @@ -235,17 +238,17 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { for _, kw := range keywords { t.Run(kw.Keyword, func(t *testing.T) { - total, res, langs, err := indexer.Search(context.TODO(), &internal.SearchOptions{ - RepoIDs: kw.RepoIDs, - Keyword: kw.Keyword, + total, res, langs, err := indexer.Search(t.Context(), &internal.SearchOptions{ + RepoIDs: kw.RepoIDs, + Keyword: kw.Keyword, + SearchMode: util.IfZero(kw.SearchMode, indexer_module.SearchModeWords), Paginator: &db.ListOptions{ Page: 1, PageSize: 10, }, - IsKeywordFuzzy: true, }) - assert.NoError(t, err) - assert.Len(t, langs, kw.Langs) + require.NoError(t, err) + require.Len(t, langs, kw.Langs) hits := make([]codeSearchResult, 0, len(res)) @@ -275,7 +278,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }) } - assert.NoError(t, tearDownRepositoryIndexes(indexer)) + assert.NoError(t, tearDownRepositoryIndexes(t.Context(), indexer)) }) } @@ -287,10 +290,10 @@ func TestBleveIndexAndSearch(t *testing.T) { idx := bleve.NewIndexer(dir) defer idx.Close() - _, err := idx.Init(context.Background()) + _, err := idx.Init(t.Context()) require.NoError(t, err) - testIndexer("beleve", t, idx) + testIndexer("bleve", t, idx) } func TestESIndexAndSearch(t *testing.T) { @@ -303,11 +306,11 @@ func TestESIndexAndSearch(t *testing.T) { } indexer := elasticsearch.NewIndexer(u, "gitea_codes") - if _, err := indexer.Init(context.Background()); err != nil { + if _, err := indexer.Init(t.Context()); err != nil { if indexer != nil { indexer.Close() } - assert.FailNow(t, "Unable to init ES indexer Error: %v", err) + require.NoError(t, err, "Unable to init ES indexer") } defer indexer.Close() @@ -324,9 +327,9 @@ func setupRepositoryIndexes(ctx context.Context, indexer internal.Indexer) error return nil } -func tearDownRepositoryIndexes(indexer internal.Indexer) error { +func tearDownRepositoryIndexes(ctx context.Context, indexer internal.Indexer) error { for _, repoID := range repositoriesToSearch() { - if err := indexer.Delete(context.Background(), repoID); err != nil { + if err := indexer.Delete(ctx, repoID); err != nil { return err } } diff --git a/modules/indexer/code/internal/indexer.go b/modules/indexer/code/internal/indexer.go index c259fcd26e..d58b028124 100644 --- a/modules/indexer/code/internal/indexer.go +++ b/modules/indexer/code/internal/indexer.go @@ -5,10 +5,11 @@ package internal import ( "context" - "fmt" + "errors" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/internal" ) @@ -18,6 +19,7 @@ type Indexer interface { Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error Delete(ctx context.Context, repoID int64) error Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) + SupportedSearchModes() []indexer.SearchMode } type SearchOptions struct { @@ -25,7 +27,7 @@ type SearchOptions struct { Keyword string Language string - IsKeywordFuzzy bool + SearchMode indexer.SearchModeType db.Paginator } @@ -41,14 +43,18 @@ type dummyIndexer struct { internal.Indexer } +func (d *dummyIndexer) SupportedSearchModes() []indexer.SearchMode { + return nil +} + func (d *dummyIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Delete(ctx context.Context, repoID int64) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) { - return 0, nil, nil, fmt.Errorf("indexer is not ready") + return 0, nil, nil, errors.New("indexer is not ready") } diff --git a/modules/indexer/code/internal/util.go b/modules/indexer/code/internal/util.go index 5b95783d9f..fa958be473 100644 --- a/modules/indexer/code/internal/util.go +++ b/modules/indexer/code/internal/util.go @@ -10,9 +10,7 @@ import ( "code.gitea.io/gitea/modules/log" ) -const ( - filenameMatchNumberOfLines = 7 // Copied from github search -) +const filenameMatchNumberOfLines = 7 // Copied from GitHub search func FilenameIndexerID(repoID int64, filename string) string { return internal.Base36(repoID) + "_" + filename @@ -35,7 +33,7 @@ func FilenameOfIndexerID(indexerID string) string { return indexerID[index+1:] } -// Given the contents of file, returns the boundaries of its first seven lines. +// FilenameMatchIndexPos returns the boundaries of its first seven lines. func FilenameMatchIndexPos(content string) (int, int) { count := 1 for i, c := range content { diff --git a/modules/indexer/code/search.go b/modules/indexer/code/search.go index 74c957dde6..a7a5d7d2e3 100644 --- a/modules/indexer/code/search.go +++ b/modules/indexer/code/search.go @@ -77,7 +77,7 @@ func HighlightSearchResultCode(filename, language string, lineNums []int, code s // The lineNums outputted by highlight.Code might not match the original lineNums, because "highlight" removes the last `\n` lines := make([]*ResultLine, min(len(highlightedLines), len(lineNums))) - for i := 0; i < len(lines); i++ { + for i := range lines { lines[i] = &ResultLine{ Num: lineNums[i], FormattedContent: template.HTML(highlightedLines[i]), @@ -129,7 +129,6 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res } // PerformSearch perform a search on a repository -// if isFuzzy is true set the Damerau-Levenshtein distance from 0 to 2 func PerformSearch(ctx context.Context, opts *SearchOptions) (int, []*Result, []*SearchResultLanguages, error) { if opts == nil || len(opts.Keyword) == 0 { return 0, nil, nil, nil diff --git a/modules/indexer/indexer.go b/modules/indexer/indexer.go new file mode 100644 index 0000000000..1e0f81de89 --- /dev/null +++ b/modules/indexer/indexer.go @@ -0,0 +1,54 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package indexer + +type SearchModeType string + +const ( + SearchModeExact SearchModeType = "exact" + SearchModeWords SearchModeType = "words" + SearchModeFuzzy SearchModeType = "fuzzy" + SearchModeRegexp SearchModeType = "regexp" +) + +type SearchMode struct { + ModeValue SearchModeType + TooltipTrKey string + TitleTrKey string +} + +func SearchModesExactWords() []SearchMode { + return []SearchMode{ + { + ModeValue: SearchModeExact, + TooltipTrKey: "search.exact_tooltip", + TitleTrKey: "search.exact", + }, + { + ModeValue: SearchModeWords, + TooltipTrKey: "search.words_tooltip", + TitleTrKey: "search.words", + }, + } +} + +func SearchModesExactWordsFuzzy() []SearchMode { + return append(SearchModesExactWords(), []SearchMode{ + { + ModeValue: SearchModeFuzzy, + TooltipTrKey: "search.fuzzy_tooltip", + TitleTrKey: "search.fuzzy", + }, + }...) +} + +func GitGrepSupportedSearchModes() []SearchMode { + return append(SearchModesExactWords(), []SearchMode{ + { + ModeValue: SearchModeRegexp, + TooltipTrKey: "search.regexp_tooltip", + TitleTrKey: "search.regexp", + }, + }...) +} diff --git a/modules/indexer/internal/bleve/indexer.go b/modules/indexer/internal/bleve/indexer.go index 01e53ca636..9d1e24a874 100644 --- a/modules/indexer/internal/bleve/indexer.go +++ b/modules/indexer/internal/bleve/indexer.go @@ -5,7 +5,7 @@ package bleve import ( "context" - "fmt" + "errors" "code.gitea.io/gitea/modules/indexer/internal" "code.gitea.io/gitea/modules/log" @@ -39,11 +39,11 @@ func NewIndexer(indexDir string, version int, mappingGetter func() (mapping.Inde // Init initializes the indexer func (i *Indexer) Init(_ context.Context) (bool, error) { if i == nil { - return false, fmt.Errorf("cannot init nil indexer") + return false, errors.New("cannot init nil indexer") } if i.Indexer != nil { - return false, fmt.Errorf("indexer is already initialized") + return false, errors.New("indexer is already initialized") } indexer, version, err := openIndexer(i.indexDir, i.version) @@ -83,10 +83,10 @@ func (i *Indexer) Init(_ context.Context) (bool, error) { // Ping checks if the indexer is available func (i *Indexer) Ping(_ context.Context) error { if i == nil { - return fmt.Errorf("cannot ping nil indexer") + return errors.New("cannot ping nil indexer") } if i.Indexer == nil { - return fmt.Errorf("indexer is not initialized") + return errors.New("indexer is not initialized") } return nil } diff --git a/modules/indexer/internal/bleve/query.go b/modules/indexer/internal/bleve/query.go index 1b18ca1a77..8895ae2c64 100644 --- a/modules/indexer/internal/bleve/query.go +++ b/modules/indexer/internal/bleve/query.go @@ -28,6 +28,16 @@ func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query return q } +// MatchAndQuery generates a match query for the given phrase, field and analyzer +func MatchAndQuery(matchPhrase, field, analyzer string, fuzziness int) *query.MatchQuery { + q := bleve.NewMatchQuery(matchPhrase) + q.FieldVal = field + q.Analyzer = analyzer + q.Fuzziness = fuzziness + q.Operator = query.MatchQueryOperatorAnd + return q +} + // BoolFieldQuery generates a bool field query for the given value and field func BoolFieldQuery(value bool, field string) *query.BoolFieldQuery { q := bleve.NewBoolFieldQuery(value) diff --git a/modules/indexer/internal/elasticsearch/indexer.go b/modules/indexer/internal/elasticsearch/indexer.go index 395eea3bce..265ce26585 100644 --- a/modules/indexer/internal/elasticsearch/indexer.go +++ b/modules/indexer/internal/elasticsearch/indexer.go @@ -5,6 +5,7 @@ package elasticsearch import ( "context" + "errors" "fmt" "code.gitea.io/gitea/modules/indexer/internal" @@ -36,10 +37,10 @@ func NewIndexer(url, indexName string, version int, mapping string) *Indexer { // Init initializes the indexer func (i *Indexer) Init(ctx context.Context) (bool, error) { if i == nil { - return false, fmt.Errorf("cannot init nil indexer") + return false, errors.New("cannot init nil indexer") } if i.Client != nil { - return false, fmt.Errorf("indexer is already initialized") + return false, errors.New("indexer is already initialized") } client, err := i.initClient() @@ -66,10 +67,10 @@ func (i *Indexer) Init(ctx context.Context) (bool, error) { // Ping checks if the indexer is available func (i *Indexer) Ping(ctx context.Context) error { if i == nil { - return fmt.Errorf("cannot ping nil indexer") + return errors.New("cannot ping nil indexer") } if i.Client == nil { - return fmt.Errorf("indexer is not initialized") + return errors.New("indexer is not initialized") } resp, err := i.Client.ClusterHealth().Do(ctx) diff --git a/modules/indexer/internal/indexer.go b/modules/indexer/internal/indexer.go index c7f356da1e..3442bbaff2 100644 --- a/modules/indexer/internal/indexer.go +++ b/modules/indexer/internal/indexer.go @@ -5,7 +5,7 @@ package internal import ( "context" - "fmt" + "errors" ) // Indexer defines an basic indexer interface @@ -27,11 +27,11 @@ func NewDummyIndexer() Indexer { type dummyIndexer struct{} func (d *dummyIndexer) Init(ctx context.Context) (bool, error) { - return false, fmt.Errorf("indexer is not ready") + return false, errors.New("indexer is not ready") } func (d *dummyIndexer) Ping(ctx context.Context) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Close() {} diff --git a/modules/indexer/internal/meilisearch/indexer.go b/modules/indexer/internal/meilisearch/indexer.go index 01bb49bbfc..65db75bb55 100644 --- a/modules/indexer/internal/meilisearch/indexer.go +++ b/modules/indexer/internal/meilisearch/indexer.go @@ -5,6 +5,7 @@ package meilisearch import ( "context" + "errors" "fmt" "github.com/meilisearch/meilisearch-go" @@ -33,11 +34,11 @@ func NewIndexer(url, apiKey, indexName string, version int, settings *meilisearc // Init initializes the indexer func (i *Indexer) Init(_ context.Context) (bool, error) { if i == nil { - return false, fmt.Errorf("cannot init nil indexer") + return false, errors.New("cannot init nil indexer") } if i.Client != nil { - return false, fmt.Errorf("indexer is already initialized") + return false, errors.New("indexer is already initialized") } i.Client = meilisearch.New(i.url, meilisearch.WithAPIKey(i.apiKey)) @@ -62,10 +63,10 @@ func (i *Indexer) Init(_ context.Context) (bool, error) { // Ping checks if the indexer is available func (i *Indexer) Ping(ctx context.Context) error { if i == nil { - return fmt.Errorf("cannot ping nil indexer") + return errors.New("cannot ping nil indexer") } if i.Client == nil { - return fmt.Errorf("indexer is not initialized") + return errors.New("indexer is not initialized") } resp, err := i.Client.Health() if err != nil { diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index bf51bd6c14..39d96cab98 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -5,10 +5,14 @@ package bleve import ( "context" + "strconv" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/util" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" @@ -120,6 +124,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWordsFuzzy() +} + // NewIndexer creates a new bleve local indexer func NewIndexer(indexDir string) *Indexer { inner := inner_bleve.NewIndexer(indexDir, issueIndexerLatestVersion, generateIssueIndexMapping) @@ -157,16 +165,24 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( var queries []query.Query if options.Keyword != "" { - fuzziness := 0 - if options.IsFuzzyKeyword { - fuzziness = inner_bleve.GuessFuzzinessByKeyword(options.Keyword) + searchMode := util.IfZero(options.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeWords || searchMode == indexer.SearchModeFuzzy { + fuzziness := 0 + if searchMode == indexer.SearchModeFuzzy { + fuzziness = inner_bleve.GuessFuzzinessByKeyword(options.Keyword) + } + queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ + inner_bleve.MatchAndQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness), + inner_bleve.MatchAndQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness), + inner_bleve.MatchAndQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness), + }...)) + } else /* exact */ { + queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ + inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, 0), + inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, 0), + inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, 0), + }...)) } - - queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ - inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness), - inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness), - inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness), - }...)) } if len(options.RepoIDs) > 0 || options.AllPublic { @@ -232,12 +248,20 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectColumnID.Value(), "project_board_id")) } - if options.PosterID.Has() { - queries = append(queries, inner_bleve.NumericEqualityQuery(options.PosterID.Value(), "poster_id")) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + queries = append(queries, inner_bleve.NumericEqualityQuery(posterIDInt64, "poster_id")) } - if options.AssigneeID.Has() { - queries = append(queries, inner_bleve.NumericEqualityQuery(options.AssigneeID.Value(), "assignee_id")) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + queries = append(queries, inner_bleve.NumericRangeInclusiveQuery(optional.Some[int64](1), optional.None[int64](), "assignee_id")) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + queries = append(queries, inner_bleve.NumericEqualityQuery(assigneeIDInt64, "assignee_id")) + } } if options.MentionID.Has() { diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 6c9cfcf670..50951f9c88 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -5,29 +5,35 @@ package db import ( "context" + "strings" + "sync" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_db "code.gitea.io/gitea/modules/indexer/internal/db" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/util" "xorm.io/builder" ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = (*Indexer)(nil) // Indexer implements Indexer interface to use database's like search type Indexer struct { indexer_internal.Indexer } -func NewIndexer() *Indexer { - return &Indexer{ - Indexer: &inner_db.Indexer{}, - } +func (i *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() } +var GetIndexer = sync.OnceValue(func() *Indexer { + return &Indexer{Indexer: &inner_db.Indexer{}} +}) + // Index dummy function func (i *Indexer) Index(_ context.Context, _ ...*internal.IndexerData) error { return nil @@ -38,6 +44,26 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error { return nil } +func buildMatchQuery(mode indexer.SearchModeType, colName, keyword string) builder.Cond { + if mode == indexer.SearchModeExact { + return db.BuildCaseInsensitiveLike(colName, keyword) + } + + // match words + cond := builder.NewCond() + fields := strings.Fields(keyword) + if len(fields) == 0 { + return builder.Expr("1=1") + } + for _, field := range fields { + if field == "" { + continue + } + cond = cond.And(db.BuildCaseInsensitiveLike(colName, field)) + } + return cond +} + // Search searches for issues func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { // FIXME: I tried to avoid importing models here, but it seems to be impossible. @@ -58,16 +84,16 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( repoCond = builder.Eq{"repo_id": options.RepoIDs[0]} } subQuery := builder.Select("id").From("issue").Where(repoCond) - + searchMode := util.IfZero(options.SearchMode, i.SupportedSearchModes()[0].ModeValue) cond = builder.Or( - db.BuildCaseInsensitiveLike("issue.name", options.Keyword), - db.BuildCaseInsensitiveLike("issue.content", options.Keyword), + buildMatchQuery(searchMode, "issue.name", options.Keyword), + buildMatchQuery(searchMode, "issue.content", options.Keyword), builder.In("issue.id", builder.Select("issue_id"). From("comment"). Where(builder.And( builder.Eq{"type": issue_model.CommentTypeComment}, builder.In("issue_id", subQuery), - db.BuildCaseInsensitiveLike("content", options.Keyword), + buildMatchQuery(searchMode, "content", options.Keyword), )), ), ) @@ -95,7 +121,11 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( }, nil } - ids, total, err := issue_model.IssueIDs(ctx, opt, cond) + return i.FindWithIssueOptions(ctx, opt, cond) +} + +func (i *Indexer) FindWithIssueOptions(ctx context.Context, opt *issue_model.IssuesOptions, otherConds ...builder.Cond) (*internal.SearchResult, error) { + ids, total, err := issue_model.IssueIDs(ctx, opt, otherConds...) if err != nil { return nil, err } diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index 42834f6e88..380a25dc23 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -6,6 +6,7 @@ package db import ( "context" "fmt" + "strings" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" @@ -34,7 +35,11 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m case internal.SortByDeadlineAsc: sortType = "nearduedate" default: - sortType = "newest" + if strings.HasPrefix(string(options.SortBy), issue_model.ScopeSortPrefix) { + sortType = string(options.SortBy) + } else { + sortType = "newest" + } } // See the comment of issues_model.SearchOptions for the reason why we need to convert @@ -54,7 +59,7 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m RepoIDs: options.RepoIDs, AllPublic: options.AllPublic, RepoCond: nil, - AssigneeID: optional.Some(convertID(options.AssigneeID)), + AssigneeID: options.AssigneeID, PosterID: options.PosterID, MentionedID: convertID(options.MentionID), ReviewRequestedID: convertID(options.ReviewRequestedID), @@ -68,14 +73,13 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m ExcludedLabelNames: nil, IncludeMilestones: nil, SortType: sortType, - IssueIDs: nil, UpdatedAfterUnix: options.UpdatedAfterUnix.Value(), UpdatedBeforeUnix: options.UpdatedBeforeUnix.Value(), PriorityRepoID: 0, IsArchived: options.IsArchived, - Org: nil, + Owner: nil, Team: nil, - User: nil, + Doer: nil, } if len(options.MilestoneIDs) == 1 && options.MilestoneIDs[0] == 0 { diff --git a/modules/indexer/issues/dboptions.go b/modules/indexer/issues/dboptions.go index 4f6ad96d22..f17724664d 100644 --- a/modules/indexer/issues/dboptions.go +++ b/modules/indexer/issues/dboptions.go @@ -4,12 +4,19 @@ package issues import ( + "strings" + "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/indexer/issues/internal" "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/setting" ) func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOptions { + if opts.IssueIDs != nil { + setting.PanicInDevOrTesting("Indexer SearchOptions doesn't support IssueIDs") + } searchOpt := &SearchOptions{ Keyword: keyword, RepoIDs: opts.RepoIDs, @@ -45,11 +52,7 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp searchOpt.ProjectID = optional.Some[int64](0) // Those issues with no project(projectid==0) } - if opts.AssigneeID.Value() == db.NoConditionID { - searchOpt.AssigneeID = optional.Some[int64](0) // FIXME: this is inconsistent from other places, 0 means "no assignee" - } else if opts.AssigneeID.Value() != 0 { - searchOpt.AssigneeID = opts.AssigneeID - } + searchOpt.AssigneeID = opts.AssigneeID // See the comment of issues_model.SearchOptions for the reason why we need to convert convertID := func(id int64) optional.Option[int64] { @@ -99,7 +102,11 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp // Unsupported sort type for search fallthrough default: - searchOpt.SortBy = SortByUpdatedDesc + if strings.HasPrefix(opts.SortType, issues_model.ScopeSortPrefix) { + searchOpt.SortBy = internal.SortBy(opts.SortType) + } else { + searchOpt.SortBy = SortByUpdatedDesc + } } return searchOpt diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 4c293f3f2a..9d627466ef 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -5,14 +5,15 @@ package elasticsearch import ( "context" - "fmt" "strconv" "strings" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/util" "github.com/olivere/elastic/v7" ) @@ -33,6 +34,11 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + // TODO: es supports fuzzy search, but our code doesn't at the moment, and actually the default fuzziness is already "AUTO" + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new elasticsearch indexer func NewIndexer(url, indexerName string) *Indexer { inner := inner_elasticsearch.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping) @@ -89,7 +95,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er issue := issues[0] _, err := b.inner.Client.Index(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", issue.ID)). + Id(strconv.FormatInt(issue.ID, 10)). BodyJson(issue). Do(ctx) return err @@ -100,7 +106,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er reqs = append(reqs, elastic.NewBulkIndexRequest(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", issue.ID)). + Id(strconv.FormatInt(issue.ID, 10)). Doc(issue), ) } @@ -119,7 +125,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { } else if len(ids) == 1 { _, err := b.inner.Client.Delete(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", ids[0])). + Id(strconv.FormatInt(ids[0], 10)). Do(ctx) return err } @@ -129,7 +135,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { reqs = append(reqs, elastic.NewBulkDeleteRequest(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", id)), + Id(strconv.FormatInt(id, 10)), ) } @@ -146,12 +152,12 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query := elastic.NewBoolQuery() if options.Keyword != "" { - searchType := esMultiMatchTypePhrasePrefix - if options.IsFuzzyKeyword { - searchType = esMultiMatchTypeBestFields + searchMode := util.IfZero(options.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeExact { + query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypePhrasePrefix)) + } else /* words */ { + query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypeBestFields).Operator("and")) } - - query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(searchType)) } if len(options.RepoIDs) > 0 { @@ -205,12 +211,22 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query.Must(elastic.NewTermQuery("project_board_id", options.ProjectColumnID.Value())) } - if options.PosterID.Has() { - query.Must(elastic.NewTermQuery("poster_id", options.PosterID.Value())) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + query.Must(elastic.NewTermQuery("poster_id", posterIDInt64)) } - if options.AssigneeID.Has() { - query.Must(elastic.NewTermQuery("assignee_id", options.AssigneeID.Value())) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + q := elastic.NewRangeQuery("assignee_id") + q.Gte(1) + query.Must(q) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + query.Must(elastic.NewTermQuery("assignee_id", assigneeIDInt64)) + } } if options.MentionID.Has() { diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index c82dc0867e..8f25c84b76 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -14,6 +14,7 @@ import ( db_model "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/issues/bleve" "code.gitea.io/gitea/modules/indexer/issues/db" "code.gitea.io/gitea/modules/indexer/issues/elasticsearch" @@ -102,7 +103,7 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err) } case "db": - issueIndexer = db.NewIndexer() + issueIndexer = db.GetIndexer() case "meilisearch": issueIndexer = meilisearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName) existed, err = issueIndexer.Init(ctx) @@ -216,7 +217,7 @@ func PopulateIssueIndexer(ctx context.Context) error { return fmt.Errorf("shutdown before completion: %w", ctx.Err()) default: } - repos, _, err := repo_model.SearchRepositoryByName(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{ ListOptions: db_model.ListOptions{Page: page, PageSize: repo_model.RepositoryListDefaultPageSize}, OrderBy: db_model.SearchOrderByID, Private: true, @@ -281,7 +282,7 @@ const ( // SearchIssues search issues by options. func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, error) { - indexer := *globalIndexer.Load() + ix := *globalIndexer.Load() if opts.Keyword == "" || opts.IsKeywordNumeric() { // This is a conservative shortcut. @@ -290,20 +291,22 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, err // So if the user creates an issue and list issues immediately, the issue may not be listed because the indexer needs time to index the issue. // Even worse, the external indexer like elastic search may not be available for a while, // and the user may not be able to list issues completely until it is available again. - indexer = db.NewIndexer() + ix = db.GetIndexer() } - result, err := indexer.Search(ctx, opts) + result, err := ix.Search(ctx, opts) if err != nil { return nil, 0, err } + return SearchResultToIDSlice(result), result.Total, nil +} +func SearchResultToIDSlice(result *internal.SearchResult) []int64 { ret := make([]int64, 0, len(result.Hits)) for _, hit := range result.Hits { ret = append(ret, hit.ID) } - - return ret, result.Total, nil + return ret } // CountIssues counts issues by options. It is a shortcut of SearchIssues(ctx, opts) but only returns the total count. @@ -313,3 +316,11 @@ func CountIssues(ctx context.Context, opts *SearchOptions) (int64, error) { _, total, err := SearchIssues(ctx, opts) return total, err } + +func SupportedSearchModes() []indexer.SearchMode { + gi := globalIndexer.Load() + if gi == nil { + return nil + } + return (*gi).SupportedSearchModes() +} diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index 8043d33eeb..3e38ac49b7 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -4,7 +4,6 @@ package issues import ( - "context" "testing" "code.gitea.io/gitea/models/db" @@ -45,6 +44,7 @@ func TestDBSearchIssues(t *testing.T) { t.Run("search issues with order", searchIssueWithOrder) t.Run("search issues in project", searchIssueInProject) t.Run("search issues with paginator", searchIssueWithPaginator) + t.Run("search issues with any assignee", searchIssueWithAnyAssignee) } func searchIssueWithKeyword(t *testing.T) { @@ -83,9 +83,11 @@ func searchIssueWithKeyword(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - require.NoError(t, err) - assert.Equal(t, test.expectedIDs, issueIDs) + t.Run(test.opts.Keyword, func(t *testing.T) { + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) + assert.Equal(t, test.expectedIDs, issueIDs) + }) } } @@ -118,7 +120,7 @@ func searchIssueByIndex(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -162,7 +164,7 @@ func searchIssueInRepo(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -175,19 +177,19 @@ func searchIssueByID(t *testing.T) { }{ { opts: SearchOptions{ - PosterID: optional.Some(int64(1)), + PosterID: "1", }, expectedIDs: []int64{11, 6, 3, 2, 1}, }, { opts: SearchOptions{ - AssigneeID: optional.Some(int64(1)), + AssigneeID: "1", }, expectedIDs: []int64{6, 1}, }, { - // NOTE: This tests no assignees filtering and also ToSearchOptions() to ensure it will set AssigneeID to 0 when it is passed as -1. - opts: *ToSearchOptions("", &issues.IssuesOptions{AssigneeID: optional.Some(db.NoConditionID)}), + // NOTE: This tests no assignees filtering and also ToSearchOptions() to ensure it handles the filter correctly + opts: *ToSearchOptions("", &issues.IssuesOptions{AssigneeID: "(none)"}), expectedIDs: []int64{22, 21, 16, 15, 14, 13, 12, 11, 20, 5, 19, 18, 10, 7, 4, 9, 8, 3, 2}, }, { @@ -232,7 +234,7 @@ func searchIssueByID(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -257,7 +259,7 @@ func searchIssueIsPull(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -282,7 +284,7 @@ func searchIssueIsClosed(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -307,7 +309,7 @@ func searchIssueIsArchived(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -332,7 +334,7 @@ func searchIssueByMilestoneID(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -363,7 +365,7 @@ func searchIssueByLabelID(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -382,7 +384,7 @@ func searchIssueByTime(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -401,7 +403,7 @@ func searchIssueWithOrder(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -432,7 +434,7 @@ func searchIssueInProject(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -455,7 +457,29 @@ func searchIssueWithPaginator(t *testing.T) { }, } for _, test := range tests { - issueIDs, total, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, total, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) + assert.Equal(t, test.expectedIDs, issueIDs) + assert.Equal(t, test.expectedTotal, total) + } +} + +func searchIssueWithAnyAssignee(t *testing.T) { + tests := []struct { + opts SearchOptions + expectedIDs []int64 + expectedTotal int64 + }{ + { + SearchOptions{ + AssigneeID: "(any)", + }, + []int64{17, 6, 1}, + 3, + }, + } + for _, test := range tests { + issueIDs, total, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) assert.Equal(t, test.expectedTotal, total) diff --git a/modules/indexer/issues/internal/indexer.go b/modules/indexer/issues/internal/indexer.go index 95740bc598..59c6f48485 100644 --- a/modules/indexer/issues/internal/indexer.go +++ b/modules/indexer/issues/internal/indexer.go @@ -5,8 +5,9 @@ package internal import ( "context" - "fmt" + "errors" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/internal" ) @@ -16,6 +17,7 @@ type Indexer interface { Index(ctx context.Context, issue ...*IndexerData) error Delete(ctx context.Context, ids ...int64) error Search(ctx context.Context, options *SearchOptions) (*SearchResult, error) + SupportedSearchModes() []indexer.SearchMode } // NewDummyIndexer returns a dummy indexer @@ -29,14 +31,18 @@ type dummyIndexer struct { internal.Indexer } +func (d *dummyIndexer) SupportedSearchModes() []indexer.SearchMode { + return nil +} + func (d *dummyIndexer) Index(_ context.Context, _ ...*IndexerData) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Delete(_ context.Context, _ ...int64) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Search(_ context.Context, _ *SearchOptions) (*SearchResult, error) { - return nil, fmt.Errorf("indexer is not ready") + return nil, errors.New("indexer is not ready") } diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 09dcbf4804..0d4f0f727d 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -7,6 +7,7 @@ import ( "strconv" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/timeutil" ) @@ -77,7 +78,7 @@ type SearchResult struct { type SearchOptions struct { Keyword string // keyword to search - IsFuzzyKeyword bool // if false the levenshtein distance is 0 + SearchMode indexer.SearchModeType RepoIDs []int64 // repository IDs which the issues belong to AllPublic bool // if include all public repositories @@ -96,9 +97,8 @@ type SearchOptions struct { ProjectID optional.Option[int64] // project the issues belong to ProjectColumnID optional.Option[int64] // project column the issues belong to - PosterID optional.Option[int64] // poster of the issues - - AssigneeID optional.Option[int64] // assignee of the issues, zero means no assignee + PosterID string // poster of the issues, "(none)" or "(any)" or a user ID + AssigneeID string // assignee of the issues, "(none)" or "(any)" or a user ID MentionID optional.Option[int64] // mentioned user of the issues diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 94ce8520bf..7aebbbcd58 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -8,7 +8,6 @@ package tests import ( - "context" "fmt" "slices" "testing" @@ -24,10 +23,10 @@ import ( ) func TestIndexer(t *testing.T, indexer internal.Indexer) { - _, err := indexer.Init(context.Background()) + _, err := indexer.Init(t.Context()) require.NoError(t, err) - require.NoError(t, indexer.Ping(context.Background())) + require.NoError(t, indexer.Ping(t.Context())) var ( ids []int64 @@ -39,32 +38,32 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { ids = append(ids, v.ID) data[v.ID] = v } - require.NoError(t, indexer.Index(context.Background(), d...)) - require.NoError(t, waitData(indexer, int64(len(data)))) + require.NoError(t, indexer.Index(t.Context(), d...)) + waitData(t, indexer, int64(len(data))) } defer func() { - require.NoError(t, indexer.Delete(context.Background(), ids...)) + require.NoError(t, indexer.Delete(t.Context(), ids...)) }() for _, c := range cases { t.Run(c.Name, func(t *testing.T) { if len(c.ExtraData) > 0 { - require.NoError(t, indexer.Index(context.Background(), c.ExtraData...)) + require.NoError(t, indexer.Index(t.Context(), c.ExtraData...)) for _, v := range c.ExtraData { data[v.ID] = v } - require.NoError(t, waitData(indexer, int64(len(data)))) + waitData(t, indexer, int64(len(data))) defer func() { for _, v := range c.ExtraData { - require.NoError(t, indexer.Delete(context.Background(), v.ID)) + require.NoError(t, indexer.Delete(t.Context(), v.ID)) delete(data, v.ID) } - require.NoError(t, waitData(indexer, int64(len(data)))) + waitData(t, indexer, int64(len(data))) }() } - result, err := indexer.Search(context.Background(), c.SearchOptions) + result, err := indexer.Search(t.Context(), c.SearchOptions) require.NoError(t, err) if c.Expected != nil { @@ -80,7 +79,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { // test counting c.SearchOptions.Paginator = &db.ListOptions{PageSize: 0} - countResult, err := indexer.Search(context.Background(), c.SearchOptions) + countResult, err := indexer.Search(t.Context(), c.SearchOptions) require.NoError(t, err) assert.Empty(t, countResult.Hits) assert.Equal(t, result.Total, countResult.Total) @@ -93,7 +92,7 @@ var cases = []*testIndexerCase{ Name: "default", SearchOptions: &internal.SearchOptions{}, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) }, }, @@ -379,7 +378,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - PosterID: optional.Some(int64(1)), + PosterID: "1", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -397,7 +396,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - AssigneeID: optional.Some(int64(1)), + AssigneeID: "1", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -415,7 +414,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - AssigneeID: optional.Some(int64(0)), + AssigneeID: "(none)", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -526,7 +525,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCreatedDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -542,7 +541,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByUpdatedDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -558,7 +557,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCommentsDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -574,7 +573,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByDeadlineDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -590,7 +589,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCreatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -606,7 +605,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByUpdatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -622,7 +621,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCommentsAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -638,7 +637,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByDeadlineAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -647,6 +646,21 @@ var cases = []*testIndexerCase{ } }, }, + { + Name: "SearchAnyAssignee", + SearchOptions: &internal.SearchOptions{ + AssigneeID: "(any)", + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Len(t, result.Hits, 180) + for _, v := range result.Hits { + assert.GreaterOrEqual(t, data[v.ID].AssigneeID, int64(1)) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.AssigneeID >= 1 + }), result.Total) + }, + }, } type testIndexerCase struct { @@ -736,22 +750,10 @@ func countIndexerData(data map[int64]*internal.IndexerData, f func(v *internal.I // waitData waits for the indexer to index all data. // Some engines like Elasticsearch index data asynchronously, so we need to wait for a while. -func waitData(indexer internal.Indexer, total int64) error { - var actual int64 - for i := 0; i < 100; i++ { - result, err := indexer.Search(context.Background(), &internal.SearchOptions{ - Paginator: &db.ListOptions{ - PageSize: 0, - }, - }) - if err != nil { - return err - } - actual = result.Total - if actual == total { - return nil - } - time.Sleep(100 * time.Millisecond) - } - return fmt.Errorf("waitData: expected %d, actual %d", total, actual) +func waitData(t *testing.T, indexer internal.Indexer, total int64) { + assert.Eventually(t, func() bool { + result, err := indexer.Search(t.Context(), &internal.SearchOptions{Paginator: &db.ListOptions{}}) + require.NoError(t, err) + return result.Total == total + }, 10*time.Second, 100*time.Millisecond, "expected total=%d", total) } diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 1066e96272..759a98473f 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_meilisearch "code.gitea.io/gitea/modules/indexer/internal/meilisearch" "code.gitea.io/gitea/modules/indexer/issues/internal" @@ -35,6 +36,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_meilisearch.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new meilisearch indexer func NewIndexer(url, apiKey, indexerName string) *Indexer { settings := &meilisearch.Settings{ @@ -182,12 +187,20 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query.And(inner_meilisearch.NewFilterEq("project_board_id", options.ProjectColumnID.Value())) } - if options.PosterID.Has() { - query.And(inner_meilisearch.NewFilterEq("poster_id", options.PosterID.Value())) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + query.And(inner_meilisearch.NewFilterEq("poster_id", posterIDInt64)) } - if options.AssigneeID.Has() { - query.And(inner_meilisearch.NewFilterEq("assignee_id", options.AssigneeID.Value())) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + query.And(inner_meilisearch.NewFilterGte("assignee_id", 1)) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + query.And(inner_meilisearch.NewFilterEq("assignee_id", assigneeIDInt64)) + } } if options.MentionID.Has() { @@ -230,9 +243,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( limit = 1 } - keyword := options.Keyword - if !options.IsFuzzyKeyword { - // to make it non fuzzy ("typo tolerance" in meilisearch terms), we have to quote the keyword(s) + keyword := options.Keyword // default to match "words" + if options.SearchMode == indexer.SearchModeExact { // https://www.meilisearch.com/docs/reference/api/search#phrase-search keyword = doubleQuoteKeyword(keyword) } diff --git a/modules/indexer/issues/meilisearch/meilisearch_test.go b/modules/indexer/issues/meilisearch/meilisearch_test.go index a3a332554a..2fea4004cb 100644 --- a/modules/indexer/issues/meilisearch/meilisearch_test.go +++ b/modules/indexer/issues/meilisearch/meilisearch_test.go @@ -74,13 +74,13 @@ func TestConvertHits(t *testing.T) { } hits, err := convertHits(validResponse) assert.NoError(t, err) - assert.EqualValues(t, []internal.Match{{ID: 11}, {ID: 22}, {ID: 33}}, hits) + assert.Equal(t, []internal.Match{{ID: 11}, {ID: 22}, {ID: 33}}, hits) } func TestDoubleQuoteKeyword(t *testing.T) { - assert.EqualValues(t, "", doubleQuoteKeyword("")) - assert.EqualValues(t, `"a" "b" "c"`, doubleQuoteKeyword("a b c")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword(`a "" "d" """g`)) + assert.Empty(t, doubleQuoteKeyword("")) + assert.Equal(t, `"a" "b" "c"`, doubleQuoteKeyword("a b c")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword(`a "" "d" """g`)) } diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index deb19adc49..19d835a1d8 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -92,6 +92,11 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD projectID = issue.Project.ID } + projectColumnID, err := issue.ProjectColumnID(ctx) + if err != nil { + return nil, false, err + } + return &internal.IndexerData{ ID: issue.ID, RepoID: issue.RepoID, @@ -106,7 +111,7 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD NoLabel: len(labels) == 0, MilestoneID: issue.MilestoneID, ProjectID: projectID, - ProjectColumnID: issue.ProjectColumnID(ctx), + ProjectColumnID: projectColumnID, PosterID: issue.PosterID, AssigneeID: issue.AssigneeID, MentionIDs: mentionIDs, diff --git a/modules/indexer/stats/db.go b/modules/indexer/stats/db.go index 98a977c700..199d493e97 100644 --- a/modules/indexer/stats/db.go +++ b/modules/indexer/stats/db.go @@ -8,6 +8,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/languagestats" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" @@ -49,10 +50,10 @@ func (db *DBIndexer) Index(id int64) error { commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch) if err != nil { if git.IsErrBranchNotExist(err) || git.IsErrNotExist(err) || setting.IsInTesting { - log.Debug("Unable to get commit ID for default branch %s in %s ... skipping this repository", repo.DefaultBranch, repo.RepoPath()) + log.Debug("Unable to get commit ID for default branch %s in %s ... skipping this repository", repo.DefaultBranch, repo.FullName()) return nil } - log.Error("Unable to get commit ID for default branch %s in %s. Error: %v", repo.DefaultBranch, repo.RepoPath(), err) + log.Error("Unable to get commit ID for default branch %s in %s. Error: %v", repo.DefaultBranch, repo.FullName(), err) return err } @@ -62,20 +63,20 @@ func (db *DBIndexer) Index(id int64) error { } // Calculate and save language statistics to database - stats, err := gitRepo.GetLanguageStats(commitID) + stats, err := languagestats.GetLanguageStats(gitRepo, commitID) if err != nil { if !setting.IsInTesting { - log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err) + log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err) } return err } err = repo_model.UpdateLanguageStats(ctx, repo, commitID, stats) if err != nil { - log.Error("Unable to update language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err) + log.Error("Unable to update language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err) return err } - log.Debug("DBIndexer completed language stats for ID %s for default branch %s in %s. stats count: %d", commitID, repo.DefaultBranch, repo.RepoPath(), len(stats)) + log.Debug("DBIndexer completed language stats for ID %s for default branch %s in %s. stats count: %d", commitID, repo.DefaultBranch, repo.FullName(), len(stats)) return nil } diff --git a/modules/indexer/stats/indexer_test.go b/modules/indexer/stats/indexer_test.go index 5be45d7a3b..d32a8bf151 100644 --- a/modules/indexer/stats/indexer_test.go +++ b/modules/indexer/stats/indexer_test.go @@ -4,7 +4,6 @@ package stats import ( - "context" "testing" "time" @@ -40,7 +39,7 @@ func TestRepoStatsIndex(t *testing.T) { err = UpdateRepoIndexer(repo) assert.NoError(t, err) - assert.NoError(t, queue.GetManager().FlushAll(context.Background(), 5*time.Second)) + assert.NoError(t, queue.GetManager().FlushAll(t.Context(), 5*time.Second)) status, err := repo_model.GetIndexerStatus(db.DefaultContext, repo, repo_model.RepoIndexerTypeStats) assert.NoError(t, err) diff --git a/modules/indexer/stats/queue.go b/modules/indexer/stats/queue.go index d002bd57cf..69cde321d8 100644 --- a/modules/indexer/stats/queue.go +++ b/modules/indexer/stats/queue.go @@ -4,7 +4,7 @@ package stats import ( - "fmt" + "errors" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" @@ -31,7 +31,7 @@ func handler(items ...int64) []int64 { func initStatsQueue() error { statsQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_stats_update", handler) if statsQueue == nil { - return fmt.Errorf("unable to create repo_stats_update queue") + return errors.New("unable to create repo_stats_update queue") } go graceful.GetManager().RunWithCancel(statsQueue) return nil diff --git a/modules/issue/template/template.go b/modules/issue/template/template.go index a2b9d6b33e..192aaf8e01 100644 --- a/modules/issue/template/template.go +++ b/modules/issue/template/template.go @@ -4,9 +4,11 @@ package template import ( + "errors" "fmt" "net/url" "regexp" + "slices" "strconv" "strings" @@ -31,17 +33,17 @@ func Validate(template *api.IssueTemplate) error { func validateMetadata(template *api.IssueTemplate) error { if strings.TrimSpace(template.Name) == "" { - return fmt.Errorf("'name' is required") + return errors.New("'name' is required") } if strings.TrimSpace(template.About) == "" { - return fmt.Errorf("'about' is required") + return errors.New("'about' is required") } return nil } func validateYaml(template *api.IssueTemplate) error { if len(template.Fields) == 0 { - return fmt.Errorf("'body' is required") + return errors.New("'body' is required") } ids := make(container.Set[string]) for idx, field := range template.Fields { @@ -401,7 +403,7 @@ func (f *valuedField) Render() string { } func (f *valuedField) Value() string { - return strings.TrimSpace(f.Get(fmt.Sprintf("form-field-%s", f.ID))) + return strings.TrimSpace(f.Get("form-field-" + f.ID)) } func (f *valuedField) Options() []*valuedOption { @@ -444,14 +446,9 @@ func (o *valuedOption) Label() string { func (o *valuedOption) IsChecked() bool { switch o.field.Type { case api.IssueFormFieldTypeDropdown: - checks := strings.Split(o.field.Get(fmt.Sprintf("form-field-%s", o.field.ID)), ",") + checks := strings.Split(o.field.Get("form-field-"+o.field.ID), ",") idx := strconv.Itoa(o.index) - for _, v := range checks { - if v == idx { - return true - } - } - return false + return slices.Contains(checks, idx) case api.IssueFormFieldTypeCheckboxes: return o.field.Get(fmt.Sprintf("form-field-%s-%d", o.field.ID, o.index)) == "on" } diff --git a/modules/issue/template/template_test.go b/modules/issue/template/template_test.go index 575e23def9..7fec9431b6 100644 --- a/modules/issue/template/template_test.go +++ b/modules/issue/template/template_test.go @@ -904,7 +904,7 @@ Option 1 of dropdown, Option 2 of dropdown t.Fatal(err) } if got := RenderToMarkdown(template, tt.args.values); got != tt.want { - assert.EqualValues(t, tt.want, got) + assert.Equal(t, tt.want, got) } }) } diff --git a/modules/json/json.go b/modules/json/json.go index 34568c75c6..444dc8526a 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -3,11 +3,10 @@ package json -// Allow "encoding/json" import. import ( "bytes" "encoding/binary" - "encoding/json" //nolint:depguard + "encoding/json" //nolint:depguard // this package wraps it "io" jsoniter "github.com/json-iterator/go" @@ -145,6 +144,12 @@ func Valid(data []byte) bool { // UnmarshalHandleDoubleEncode - due to a bug in xorm (see https://gitea.com/xorm/xorm/pulls/1957) - it's // possible that a Blob may be double encoded or gain an unwanted prefix of 0xff 0xfe. func UnmarshalHandleDoubleEncode(bs []byte, v any) error { + if len(bs) == 0 { + // json.Unmarshal will report errors if input is empty (nil or zero-length) + // It seems that XORM ignores the nil but still passes zero-length string into this function + // To be consistent, we should treat all empty inputs as success + return nil + } err := json.Unmarshal(bs, v) if err != nil { ok := true diff --git a/modules/json/json_test.go b/modules/json/json_test.go new file mode 100644 index 0000000000..ace7167913 --- /dev/null +++ b/modules/json/json_test.go @@ -0,0 +1,18 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package json + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGiteaDBJSONUnmarshal(t *testing.T) { + var m map[any]any + err := UnmarshalHandleDoubleEncode(nil, &m) + assert.NoError(t, err) + err = UnmarshalHandleDoubleEncode([]byte(""), &m) + assert.NoError(t, err) +} diff --git a/modules/label/label.go b/modules/label/label.go index d3ef0e1dc9..3e68c4d26e 100644 --- a/modules/label/label.go +++ b/modules/label/label.go @@ -7,19 +7,24 @@ import ( "fmt" "regexp" "strings" -) + "sync" -// colorPattern is a regexp which can validate label color -var colorPattern = regexp.MustCompile("^#?(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{3})$") + "code.gitea.io/gitea/modules/util" +) // Label represents label information loaded from template type Label struct { - Name string `yaml:"name"` - Color string `yaml:"color"` - Description string `yaml:"description,omitempty"` - Exclusive bool `yaml:"exclusive,omitempty"` + Name string `yaml:"name"` + Color string `yaml:"color"` + Description string `yaml:"description,omitempty"` + Exclusive bool `yaml:"exclusive,omitempty"` + ExclusiveOrder int `yaml:"exclusive_order,omitempty"` } +var colorPattern = sync.OnceValue(func() *regexp.Regexp { + return regexp.MustCompile(`^#([\da-fA-F]{3}|[\da-fA-F]{6})$`) +}) + // NormalizeColor normalizes a color string to a 6-character hex code func NormalizeColor(color string) (string, error) { // normalize case @@ -30,8 +35,8 @@ func NormalizeColor(color string) (string, error) { color = "#" + color } - if !colorPattern.MatchString(color) { - return "", fmt.Errorf("bad color code: %s", color) + if !colorPattern().MatchString(color) { + return "", util.NewInvalidArgumentErrorf("invalid color: %s", color) } // convert 3-character shorthand into 6-character version diff --git a/modules/label/parser.go b/modules/label/parser.go index 511bac823f..2a10152062 100644 --- a/modules/label/parser.go +++ b/modules/label/parser.go @@ -72,7 +72,7 @@ func parseYamlFormat(fileName string, data []byte) ([]*Label, error) { func parseLegacyFormat(fileName string, data []byte) ([]*Label, error) { lines := strings.Split(string(data), "\n") list := make([]*Label, 0, len(lines)) - for i := 0; i < len(lines); i++ { + for i := range lines { line := strings.TrimSpace(lines[i]) if len(line) == 0 { continue @@ -108,7 +108,7 @@ func LoadTemplateDescription(fileName string) (string, error) { return "", err } - for i := 0; i < len(list); i++ { + for i := range list { if i > 0 { buf.WriteString(", ") } diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 3acd23b8f7..4b51193846 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -70,12 +70,16 @@ func (c *HTTPClient) transferNames() []string { func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { log.Trace("BATCH operation with objects: %v", objects) - url := fmt.Sprintf("%s/objects/batch", c.endpoint) + url := c.endpoint + "/objects/batch" + // Original: In some lfs server implementations, they require the ref attribute. #32838 // `ref` is an "optional object describing the server ref that the objects belong to" - // but some (incorrect) lfs servers require it, so maybe adding an empty ref here doesn't break the correct ones. + // but some (incorrect) lfs servers like aliyun require it, so maybe adding an empty ref here doesn't break the correct ones. // https://github.com/git-lfs/git-lfs/blob/a32a02b44bf8a511aa14f047627c49e1a7fd5021/docs/api/batch.md?plain=1#L37 - request := &BatchRequest{operation, c.transferNames(), &Reference{}, objects} + // + // UPDATE: it can't use "empty ref" here because it breaks others like https://github.com/go-gitea/gitea/issues/33453 + request := &BatchRequest{operation, c.transferNames(), nil, objects} + payload := new(bytes.Buffer) err := json.NewEncoder(payload).Encode(request) if err != nil { diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index aa7e3c45c4..179bcdb29a 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -31,7 +31,7 @@ func (a *DummyTransferAdapter) Name() string { } func (a *DummyTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) { - return io.NopCloser(bytes.NewBufferString("dummy")), nil + return io.NopCloser(strings.NewReader("dummy")), nil } func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error { @@ -49,7 +49,7 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { if strings.Contains(url, "status-not-ok") { return &http.Response{StatusCode: http.StatusBadRequest} } else if strings.Contains(url, "invalid-json-response") { - return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBufferString("invalid json"))} + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("invalid json"))} } else if strings.Contains(url, "valid-batch-request-download") { batchResponse = &BatchResponse{ Transfer: "dummy", @@ -248,7 +248,7 @@ func TestHTTPClientDownload(t *testing.T) { }, } - err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { + err := client.Download(t.Context(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { if objectError != nil { return objectError } @@ -348,7 +348,7 @@ func TestHTTPClientUpload(t *testing.T) { }, } - err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) { + err := client.Upload(t.Context(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) { return io.NopCloser(new(bytes.Buffer)), objectError }) if c.expectedError != "" { diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index ebde20f826..9c95613057 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -15,15 +15,13 @@ import ( "strings" ) +// spec: https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md const ( - blobSizeCutoff = 1024 + MetaFileMaxSize = 1024 // spec says the maximum size of a pointer file must be smaller than 1024 - // MetaFileIdentifier is the string appearing at the first line of LFS pointer files. - // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" + MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" // the first line of a pointer file - // MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. - MetaFileOidPrefix = "oid sha256:" + MetaFileOidPrefix = "oid sha256:" // spec says the only supported hash is sha256 at the moment ) var ( @@ -39,7 +37,7 @@ var ( // ReadPointer tries to read LFS pointer data from the reader func ReadPointer(reader io.Reader) (Pointer, error) { - buf := make([]byte, blobSizeCutoff) + buf := make([]byte, MetaFileMaxSize) n, err := io.ReadFull(reader, buf) if err != nil && err != io.ErrUnexpectedEOF { return Pointer{}, err @@ -65,6 +63,7 @@ func ReadPointerFromBuffer(buf []byte) (Pointer, error) { return p, ErrInvalidStructure } + // spec says "key/value pairs MUST be sorted alphabetically in ascending order (version is exception and must be the first)" oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix) if len(oid) != 64 || !oidPattern.MatchString(oid) { return p, ErrInvalidOIDFormat diff --git a/modules/lfs/pointer_scanner_gogit.go b/modules/lfs/pointer_scanner_gogit.go index f4302c23bc..e153b8e24e 100644 --- a/modules/lfs/pointer_scanner_gogit.go +++ b/modules/lfs/pointer_scanner_gogit.go @@ -31,7 +31,7 @@ func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan c default: } - if blob.Size > blobSizeCutoff { + if blob.Size > MetaFileMaxSize { return nil } diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index a430b71a5f..ef72d76db4 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -5,7 +5,6 @@ package lfs import ( "bytes" - "context" "io" "net/http" "strings" @@ -33,7 +32,7 @@ func TestBasicTransferAdapter(t *testing.T) { if strings.Contains(url, "download-request") { assert.Equal(t, "GET", req.Method) - return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBufferString("dummy"))} + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("dummy"))} } else if strings.Contains(url, "upload-request") { assert.Equal(t, "PUT", req.Method) assert.Equal(t, "application/octet-stream", req.Header.Get("Content-Type")) @@ -94,7 +93,7 @@ func TestBasicTransferAdapter(t *testing.T) { } for n, c := range cases { - _, err := a.Download(context.Background(), c.link) + _, err := a.Download(t.Context(), c.link) if len(c.expectederror) > 0 { assert.Contains(t, err.Error(), c.expectederror, "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { @@ -127,7 +126,7 @@ func TestBasicTransferAdapter(t *testing.T) { } for n, c := range cases { - err := a.Upload(context.Background(), c.link, p, bytes.NewBufferString("dummy")) + err := a.Upload(t.Context(), c.link, p, strings.NewReader("dummy")) if len(c.expectederror) > 0 { assert.Contains(t, err.Error(), c.expectederror, "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { @@ -160,7 +159,7 @@ func TestBasicTransferAdapter(t *testing.T) { } for n, c := range cases { - err := a.Verify(context.Background(), c.link, p) + err := a.Verify(t.Context(), c.link, p) if len(c.expectederror) > 0 { assert.Contains(t, err.Error(), c.expectederror, "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { diff --git a/modules/lfstransfer/backend/backend.go b/modules/lfstransfer/backend/backend.go index 2b1fe49fda..dd4108ea56 100644 --- a/modules/lfstransfer/backend/backend.go +++ b/modules/lfstransfer/backend/backend.go @@ -4,7 +4,6 @@ package backend import ( - "bytes" "context" "encoding/base64" "fmt" @@ -29,7 +28,7 @@ var Capabilities = []string{ "locking", } -var _ transfer.Backend = &GiteaBackend{} +var _ transfer.Backend = (*GiteaBackend)(nil) // GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API type GiteaBackend struct { @@ -48,7 +47,7 @@ func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (t return nil, err } server = server.JoinPath("api/internal/repo", repo, "info/lfs") - return &GiteaBackend{ctx: ctx, server: server, op: op, authToken: token, internalAuth: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil + return &GiteaBackend{ctx: ctx, server: server, op: op, authToken: token, internalAuth: "Bearer " + setting.InternalToken, logger: logger}, nil } // Batch implements transfer.Backend @@ -71,24 +70,23 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans g.logger.Log("json marshal error", err) return nil, err } - url := g.server.JoinPath("objects/batch").String() headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + req := newInternalRequestLFS(g.ctx, g.server.JoinPath("objects/batch").String(), http.MethodPost, headers, bodyBytes) resp, err := req.Response() if err != nil { g.logger.Log("http request error", err) return nil, err } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode)) return nil, statusCodeToErr(resp.StatusCode) } - defer resp.Body.Close() respBytes, err := io.ReadAll(resp.Body) if err != nil { g.logger.Log("http read error", err) @@ -158,8 +156,7 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans return pointers, nil } -// Download implements transfer.Backend. The returned reader must be closed by the -// caller. +// Download implements transfer.Backend. The returned reader must be closed by the caller. func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, int64, error) { idMapStr, exists := args[argID] if !exists { @@ -181,31 +178,30 @@ func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, g.logger.Log("argument id incorrect") return nil, 0, transfer.ErrCorruptData } - url := action.Href headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeOctetStream, } - req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil) + req := newInternalRequestLFS(g.ctx, toInternalLFSURL(action.Href), http.MethodGet, headers, nil) resp, err := req.Response() if err != nil { - return nil, 0, err + return nil, 0, fmt.Errorf("failed to get response: %w", err) } + // no need to close the body here by "defer resp.Body.Close()", see below if resp.StatusCode != http.StatusOK { return nil, 0, statusCodeToErr(resp.StatusCode) } - defer resp.Body.Close() - respBytes, err := io.ReadAll(resp.Body) + + respSize, err := strconv.ParseInt(resp.Header.Get("X-Gitea-LFS-Content-Length"), 10, 64) if err != nil { - return nil, 0, err + return nil, 0, fmt.Errorf("failed to parse content length: %w", err) } - respSize := int64(len(respBytes)) - respBuf := io.NopCloser(bytes.NewBuffer(respBytes)) - return respBuf, respSize, nil + // transfer.Backend will check io.Closer interface and close this Body reader + return resp.Body, respSize, nil } -// StartUpload implements transfer.Backend. +// Upload implements transfer.Backend. func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer.Args) error { idMapStr, exists := args[argID] if !exists { @@ -227,22 +223,20 @@ func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer g.logger.Log("argument id incorrect") return transfer.ErrCorruptData } - url := action.Href headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerContentType: mimeOctetStream, headerContentLength: strconv.FormatInt(size, 10), } - reqBytes, err := io.ReadAll(r) - if err != nil { - return err - } - req := newInternalRequest(g.ctx, url, http.MethodPut, headers, reqBytes) + + req := newInternalRequestLFS(g.ctx, toInternalLFSURL(action.Href), http.MethodPut, headers, nil) + req.Body(r) resp, err := req.Response() if err != nil { return err } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return statusCodeToErr(resp.StatusCode) } @@ -277,18 +271,18 @@ func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (trans // the server sent no verify action return transfer.SuccessStatus(), nil } - url := action.Href headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + req := newInternalRequestLFS(g.ctx, toInternalLFSURL(action.Href), http.MethodPost, headers, bodyBytes) resp, err := req.Response() if err != nil { return transfer.NewStatus(transfer.StatusInternalServerError), err } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return transfer.NewStatus(uint32(resp.StatusCode), http.StatusText(resp.StatusCode)), statusCodeToErr(resp.StatusCode) } diff --git a/modules/lfstransfer/backend/lock.go b/modules/lfstransfer/backend/lock.go index f094cce1db..2c3c16a9bb 100644 --- a/modules/lfstransfer/backend/lock.go +++ b/modules/lfstransfer/backend/lock.go @@ -5,6 +5,7 @@ package backend import ( "context" + "errors" "fmt" "io" "net/http" @@ -43,14 +44,13 @@ func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) { g.logger.Log("json marshal error", err) return nil, err } - url := g.server.String() headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + req := newInternalRequestLFS(g.ctx, g.server.String(), http.MethodPost, headers, bodyBytes) resp, err := req.Response() if err != nil { g.logger.Log("http request error", err) @@ -75,7 +75,7 @@ func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) { if respBody.Lock == nil { g.logger.Log("api returned nil lock") - return nil, fmt.Errorf("api returned nil lock") + return nil, errors.New("api returned nil lock") } respLock := respBody.Lock owner := userUnknown @@ -95,14 +95,13 @@ func (g *giteaLockBackend) Unlock(lock transfer.Lock) error { g.logger.Log("json marshal error", err) return err } - url := g.server.JoinPath(lock.ID(), "unlock").String() headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + req := newInternalRequestLFS(g.ctx, g.server.JoinPath(lock.ID(), "unlock").String(), http.MethodPost, headers, bodyBytes) resp, err := req.Response() if err != nil { g.logger.Log("http request error", err) @@ -176,16 +175,15 @@ func (g *giteaLockBackend) Range(cursor string, limit int, iter func(transfer.Lo } func (g *giteaLockBackend) queryLocks(v url.Values) ([]transfer.Lock, string, error) { - urlq := g.server.JoinPath() // get a copy - urlq.RawQuery = v.Encode() - url := urlq.String() + serverURLWithQuery := g.server.JoinPath() // get a copy + serverURLWithQuery.RawQuery = v.Encode() headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil) + req := newInternalRequestLFS(g.ctx, serverURLWithQuery.String(), http.MethodGet, headers, nil) resp, err := req.Response() if err != nil { g.logger.Log("http request error", err) @@ -266,7 +264,7 @@ func (g *giteaLock) CurrentUser() (string, error) { // AsLockSpec implements transfer.Lock func (g *giteaLock) AsLockSpec(ownerID bool) ([]string, error) { msgs := []string{ - fmt.Sprintf("lock %s", g.ID()), + "lock " + g.ID(), fmt.Sprintf("path %s %s", g.ID(), g.Path()), fmt.Sprintf("locked-at %s %s", g.ID(), g.FormattedTimestamp()), fmt.Sprintf("ownername %s %s", g.ID(), g.OwnerName()), @@ -288,9 +286,9 @@ func (g *giteaLock) AsLockSpec(ownerID bool) ([]string, error) { // AsArguments implements transfer.Lock func (g *giteaLock) AsArguments() []string { return []string{ - fmt.Sprintf("id=%s", g.ID()), - fmt.Sprintf("path=%s", g.Path()), - fmt.Sprintf("locked-at=%s", g.FormattedTimestamp()), - fmt.Sprintf("ownername=%s", g.OwnerName()), + "id=" + g.ID(), + "path=" + g.Path(), + "locked-at=" + g.FormattedTimestamp(), + "ownername=" + g.OwnerName(), } } diff --git a/modules/lfstransfer/backend/util.go b/modules/lfstransfer/backend/util.go index cffefef375..afe02f799c 100644 --- a/modules/lfstransfer/backend/util.go +++ b/modules/lfstransfer/backend/util.go @@ -5,15 +5,16 @@ package backend import ( "context" - "crypto/tls" "fmt" - "net" + "io" "net/http" - "time" + "net/url" + "strings" "code.gitea.io/gitea/modules/httplib" - "code.gitea.io/gitea/modules/proxyprotocol" + "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "github.com/charmbracelet/git-lfs-transfer/transfer" ) @@ -60,8 +61,7 @@ const ( // Operations enum const ( - opNone = iota - opDownload + opDownload = iota + 1 opUpload ) @@ -89,53 +89,61 @@ func statusCodeToErr(code int) error { } } -func newInternalRequest(ctx context.Context, url, method string, headers map[string]string, body []byte) *httplib.Request { - req := httplib.NewRequest(url, method). - SetContext(ctx). - SetTimeout(10*time.Second, 60*time.Second). - SetTLSClientConfig(&tls.Config{ - InsecureSkipVerify: true, - }) - - if setting.Protocol == setting.HTTPUnix { - req.SetTransport(&http.Transport{ - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - var d net.Dialer - conn, err := d.DialContext(ctx, "unix", setting.HTTPAddr) - if err != nil { - return conn, err - } - if setting.LocalUseProxyProtocol { - if err = proxyprotocol.WriteLocalHeader(conn); err != nil { - _ = conn.Close() - return nil, err - } - } - return conn, err - }, - }) - } else if setting.LocalUseProxyProtocol { - req.SetTransport(&http.Transport{ - DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - var d net.Dialer - conn, err := d.DialContext(ctx, network, address) - if err != nil { - return conn, err - } - if err = proxyprotocol.WriteLocalHeader(conn); err != nil { - _ = conn.Close() - return nil, err - } - return conn, err - }, - }) +func toInternalLFSURL(s string) string { + pos1 := strings.Index(s, "://") + if pos1 == -1 { + return "" } + appSubURLWithSlash := setting.AppSubURL + "/" + pos2 := strings.Index(s[pos1+3:], appSubURLWithSlash) + if pos2 == -1 { + return "" + } + routePath := s[pos1+3+pos2+len(appSubURLWithSlash):] + fields := strings.SplitN(routePath, "/", 3) + if len(fields) < 3 || !strings.HasPrefix(fields[2], "info/lfs") { + return "" + } + return setting.LocalURL + "api/internal/repo/" + routePath +} + +func isInternalLFSURL(s string) bool { + if !strings.HasPrefix(s, setting.LocalURL) { + return false + } + u, err := url.Parse(s) + if err != nil { + return false + } + routePath := util.PathJoinRelX(u.Path) + subRoutePath, cut := strings.CutPrefix(routePath, "api/internal/repo/") + if !cut { + return false + } + fields := strings.SplitN(subRoutePath, "/", 3) + if len(fields) < 3 || !strings.HasPrefix(fields[2], "info/lfs") { + return false + } + return true +} +func newInternalRequestLFS(ctx context.Context, internalURL, method string, headers map[string]string, body any) *httplib.Request { + if !isInternalLFSURL(internalURL) { + return nil + } + req := private.NewInternalRequest(ctx, internalURL, method) + req.SetReadWriteTimeout(0) for k, v := range headers { req.Header(k, v) } - - req.Body(body) - + switch body := body.(type) { + case nil: // do nothing + case []byte: + req.Body(body) // []byte + case io.Reader: + req.Body(body) // io.Reader or io.ReadCloser + default: + panic(fmt.Sprintf("unsupported request body type %T", body)) + } return req } diff --git a/modules/lfstransfer/backend/util_test.go b/modules/lfstransfer/backend/util_test.go new file mode 100644 index 0000000000..408b53c369 --- /dev/null +++ b/modules/lfstransfer/backend/util_test.go @@ -0,0 +1,53 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package backend + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" +) + +func TestToInternalLFSURL(t *testing.T) { + defer test.MockVariableValue(&setting.LocalURL, "http://localurl/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + cases := []struct { + url string + expected string + }{ + {"http://appurl/any", ""}, + {"http://appurl/sub/any", ""}, + {"http://appurl/sub/owner/repo/any", ""}, + {"http://appurl/sub/owner/repo/info/any", ""}, + {"http://appurl/sub/owner/repo/info/lfs/any", "http://localurl/api/internal/repo/owner/repo/info/lfs/any"}, + } + for _, c := range cases { + assert.Equal(t, c.expected, toInternalLFSURL(c.url), c.url) + } +} + +func TestIsInternalLFSURL(t *testing.T) { + defer test.MockVariableValue(&setting.LocalURL, "http://localurl/")() + defer test.MockVariableValue(&setting.InternalToken, "mock-token")() + cases := []struct { + url string + expected bool + }{ + {"", false}, + {"http://otherurl/api/internal/repo/owner/repo/info/lfs/any", false}, + {"http://localurl/api/internal/repo/owner/repo/info/lfs/any", true}, + {"http://localurl/api/internal/repo/owner/repo/info", false}, + {"http://localurl/api/internal/misc/owner/repo/info/lfs/any", false}, + {"http://localurl/api/internal/owner/repo/info/lfs/any", false}, + {"http://localurl/api/internal/foo/bar", false}, + } + for _, c := range cases { + req := newInternalRequestLFS(t.Context(), c.url, "GET", nil, nil) + assert.Equal(t, c.expected, req != nil, c.url) + assert.Equal(t, c.expected, isInternalLFSURL(c.url), c.url) + } +} diff --git a/modules/log/event_format.go b/modules/log/event_format.go index 8fda0a4980..4cf471d223 100644 --- a/modules/log/event_format.go +++ b/modules/log/event_format.go @@ -125,15 +125,19 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms buf = append(buf, resetBytes...) } } - if flags&(Lshortfile|Llongfile) != 0 { + if flags&(Lshortfile|Llongfile) != 0 && event.Filename != "" { if mode.Colorize { buf = append(buf, fgGreenBytes...) } file := event.Filename if flags&Lmedfile == Lmedfile { - startIndex := len(file) - 20 - if startIndex > 0 { - file = "..." + file[startIndex:] + fileLen := len(file) + const softLimit = 20 + if fileLen > softLimit { + slashIndex := strings.LastIndexByte(file[:fileLen-softLimit], '/') + if slashIndex != -1 { + file = ".../" + file[slashIndex+1:] + } } } else if flags&Lshortfile != 0 { startIndex := strings.LastIndexByte(file, '/') @@ -157,14 +161,22 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms if mode.Colorize { buf = append(buf, fgGreenBytes...) } - funcname := event.Caller + funcName := event.Caller + shortFuncName := funcName if flags&Lshortfuncname != 0 { - lastIndex := strings.LastIndexByte(funcname, '.') - if lastIndex > 0 && len(funcname) > lastIndex+1 { - funcname = funcname[lastIndex+1:] + // funcName = "code.gitea.io/gitea/modules/foo/bar.MyFunc.func1.2()" + slashPos := strings.LastIndexByte(funcName, '/') + dotPos := strings.IndexByte(funcName[slashPos+1:], '.') + if dotPos > 0 { + // shortFuncName = "MyFunc.func1.2()" + shortFuncName = funcName[slashPos+1+dotPos+1:] + if strings.Contains(shortFuncName, ".") { + shortFuncName = strings.ReplaceAll(shortFuncName, ".func", ".") + } } + funcName = shortFuncName } - buf = append(buf, funcname...) + buf = append(buf, funcName...) if mode.Colorize { buf = append(buf, resetBytes...) } @@ -200,7 +212,7 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms } } if hasColorValue { - msg = []byte(fmt.Sprintf(msgFormat, msgArgs...)) + msg = fmt.Appendf(nil, msgFormat, msgArgs...) } } // try to re-use the pre-formatted simple text message @@ -231,8 +243,8 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms buf = append(buf, msg...) if event.Stacktrace != "" && mode.StacktraceLevel <= event.Level { - lines := bytes.Split([]byte(event.Stacktrace), []byte("\n")) - for _, line := range lines { + lines := bytes.SplitSeq([]byte(event.Stacktrace), []byte("\n")) + for line := range lines { buf = append(buf, "\n\t"...) buf = append(buf, line...) } diff --git a/modules/log/event_writer_base.go b/modules/log/event_writer_base.go index c327c48ca2..9189ca4e90 100644 --- a/modules/log/event_writer_base.go +++ b/modules/log/event_writer_base.go @@ -105,7 +105,7 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) { case io.WriterTo: _, err = msg.WriteTo(b.OutputWriteCloser) default: - _, err = b.OutputWriteCloser.Write([]byte(fmt.Sprint(msg))) + _, err = fmt.Fprint(b.OutputWriteCloser, msg) } if err != nil { FallbackErrorf("unable to write log message of %q (%v): %v", b.Name, err, event.Msg) diff --git a/modules/log/event_writer_conn_test.go b/modules/log/event_writer_conn_test.go index 69e87aa8c4..2aff37812d 100644 --- a/modules/log/event_writer_conn_test.go +++ b/modules/log/event_writer_conn_test.go @@ -4,7 +4,6 @@ package log import ( - "context" "fmt" "io" "net" @@ -40,7 +39,7 @@ func TestConnLogger(t *testing.T) { level := INFO flags := LstdFlags | LUTC | Lfuncname - logger := NewLoggerWithWriters(context.Background(), "test", NewEventWriterConn("test-conn", WriterMode{ + logger := NewLoggerWithWriters(t.Context(), "test", NewEventWriterConn("test-conn", WriterMode{ Level: level, Prefix: prefix, Flags: FlagsFromBits(flags), diff --git a/modules/log/flags.go b/modules/log/flags.go index 8064c91745..f409261150 100644 --- a/modules/log/flags.go +++ b/modules/log/flags.go @@ -123,7 +123,7 @@ func FlagsFromString(from string, def ...uint32) Flags { return Flags{defined: true, flags: def[0]} } flags := uint32(0) - for _, flag := range strings.Split(strings.ToLower(from), ",") { + for flag := range strings.SplitSeq(strings.ToLower(from), ",") { flags |= flagFromString[strings.TrimSpace(flag)] } return Flags{defined: true, flags: flags} diff --git a/modules/log/flags_test.go b/modules/log/flags_test.go index 03972a9fb0..6eb65d8114 100644 --- a/modules/log/flags_test.go +++ b/modules/log/flags_test.go @@ -12,19 +12,19 @@ import ( ) func TestFlags(t *testing.T) { - assert.EqualValues(t, Ldefault, Flags{}.Bits()) + assert.Equal(t, Ldefault, Flags{}.Bits()) assert.EqualValues(t, 0, FlagsFromString("").Bits()) - assert.EqualValues(t, Lgopid, FlagsFromString("", Lgopid).Bits()) + assert.Equal(t, Lgopid, FlagsFromString("", Lgopid).Bits()) assert.EqualValues(t, 0, FlagsFromString("none", Lgopid).Bits()) - assert.EqualValues(t, Ldate|Ltime, FlagsFromString("date,time", Lgopid).Bits()) + assert.Equal(t, Ldate|Ltime, FlagsFromString("date,time", Lgopid).Bits()) - assert.EqualValues(t, "stdflags", FlagsFromString("stdflags").String()) - assert.EqualValues(t, "medfile", FlagsFromString("medfile").String()) + assert.Equal(t, "stdflags", FlagsFromString("stdflags").String()) + assert.Equal(t, "medfile", FlagsFromString("medfile").String()) bs, err := json.Marshal(FlagsFromString("utc,level")) assert.NoError(t, err) - assert.EqualValues(t, `"level,utc"`, string(bs)) + assert.Equal(t, `"level,utc"`, string(bs)) var flags Flags assert.NoError(t, json.Unmarshal(bs, &flags)) - assert.EqualValues(t, LUTC|Llevel, flags.Bits()) + assert.Equal(t, LUTC|Llevel, flags.Bits()) } diff --git a/modules/log/level_test.go b/modules/log/level_test.go index cd18a807d8..0e59af6cb7 100644 --- a/modules/log/level_test.go +++ b/modules/log/level_test.go @@ -32,11 +32,11 @@ func TestLevelMarshalUnmarshalJSON(t *testing.T) { assert.NoError(t, err) assert.Equal(t, INFO, testLevel.Level) - err = json.Unmarshal([]byte(fmt.Sprintf(`{"level":%d}`, 2)), &testLevel) + err = json.Unmarshal(fmt.Appendf(nil, `{"level":%d}`, 2), &testLevel) assert.NoError(t, err) assert.Equal(t, INFO, testLevel.Level) - err = json.Unmarshal([]byte(fmt.Sprintf(`{"level":%d}`, 10012)), &testLevel) + err = json.Unmarshal(fmt.Appendf(nil, `{"level":%d}`, 10012), &testLevel) assert.NoError(t, err) assert.Equal(t, INFO, testLevel.Level) @@ -51,5 +51,5 @@ func TestLevelMarshalUnmarshalJSON(t *testing.T) { } func makeTestLevelBytes(level string) []byte { - return []byte(fmt.Sprintf(`{"level":"%s"}`, level)) + return fmt.Appendf(nil, `{"level":"%s"}`, level) } diff --git a/modules/log/logger.go b/modules/log/logger.go index a833b6ef0f..8b89e0eb5a 100644 --- a/modules/log/logger.go +++ b/modules/log/logger.go @@ -24,7 +24,7 @@ package log // BaseLogger provides the basic logging functions type BaseLogger interface { - Log(skip int, level Level, format string, v ...any) + Log(skip int, event *Event, format string, v ...any) GetLevel() Level } @@ -45,6 +45,6 @@ type Logger interface { LevelLogger } -type LogStringer interface { //nolint:revive +type LogStringer interface { //nolint:revive // export stutter LogString() string } diff --git a/modules/log/logger_global.go b/modules/log/logger_global.go index 6ce8b70fed..07c25cd62f 100644 --- a/modules/log/logger_global.go +++ b/modules/log/logger_global.go @@ -18,7 +18,7 @@ func GetLevel() Level { } func Log(skip int, level Level, format string, v ...any) { - GetLogger(DEFAULT).Log(skip+1, level, format, v...) + GetLogger(DEFAULT).Log(skip+1, &Event{Level: level}, format, v...) } func Trace(format string, v ...any) { diff --git a/modules/log/logger_impl.go b/modules/log/logger_impl.go index 76dd5f43fb..551c1454aa 100644 --- a/modules/log/logger_impl.go +++ b/modules/log/logger_impl.go @@ -5,6 +5,7 @@ package log import ( "context" + "reflect" "runtime" "strings" "sync" @@ -175,29 +176,42 @@ func (l *LoggerImpl) IsEnabled() bool { return l.level.Load() < int32(FATAL) && len(l.eventWriters) > 0 } +func asLogStringer(v any) LogStringer { + if s, ok := v.(LogStringer); ok { + return s + } else if a := reflect.ValueOf(v); a.Kind() == reflect.Struct { + // in case the receiver is a pointer, but the value is a struct + vp := reflect.New(a.Type()) + vp.Elem().Set(a) + if s, ok := vp.Interface().(LogStringer); ok { + return s + } + } + return nil +} + // Log prepares the log event, if the level matches, the event will be sent to the writers -func (l *LoggerImpl) Log(skip int, level Level, format string, logArgs ...any) { - if Level(l.level.Load()) > level { +func (l *LoggerImpl) Log(skip int, event *Event, format string, logArgs ...any) { + if Level(l.level.Load()) > event.Level { return } - event := &Event{ - Time: time.Now(), - Level: level, - Caller: "?()", + if event.Time.IsZero() { + event.Time = time.Now() } - - pc, filename, line, ok := runtime.Caller(skip + 1) - if ok { - fn := runtime.FuncForPC(pc) - if fn != nil { - event.Caller = fn.Name() + "()" + if event.Caller == "" { + pc, filename, line, ok := runtime.Caller(skip + 1) + if ok { + fn := runtime.FuncForPC(pc) + if fn != nil { + fnName := fn.Name() + event.Caller = strings.ReplaceAll(fnName, "[...]", "") + "()" // generic function names are "foo[...]" + } + } + event.Filename, event.Line = strings.TrimPrefix(filename, projectPackagePrefix), line + if l.stacktraceLevel.Load() <= int32(event.Level) { + event.Stacktrace = Stack(skip + 1) } - } - event.Filename, event.Line = strings.TrimPrefix(filename, projectPackagePrefix), line - - if l.stacktraceLevel.Load() <= int32(level) { - event.Stacktrace = Stack(skip + 1) } // get a simple text message without color @@ -207,11 +221,11 @@ func (l *LoggerImpl) Log(skip int, level Level, format string, logArgs ...any) { // handle LogStringer values for i, v := range msgArgs { if cv, ok := v.(*ColoredValue); ok { - if s, ok := cv.v.(LogStringer); ok { - cv.v = logStringFormatter{v: s} + if ls := asLogStringer(cv.v); ls != nil { + cv.v = logStringFormatter{v: ls} } - } else if s, ok := v.(LogStringer); ok { - msgArgs[i] = logStringFormatter{v: s} + } else if ls := asLogStringer(v); ls != nil { + msgArgs[i] = logStringFormatter{v: ls} } } diff --git a/modules/log/logger_test.go b/modules/log/logger_test.go index 0de14eb411..a74139dc51 100644 --- a/modules/log/logger_test.go +++ b/modules/log/logger_test.go @@ -4,7 +4,7 @@ package log import ( - "context" + "regexp" "sync" "testing" "time" @@ -35,11 +35,11 @@ func (d *dummyWriter) Close() error { return nil } -func (d *dummyWriter) GetLogs() []string { +func (d *dummyWriter) FetchLogs() []string { d.mu.Lock() defer d.mu.Unlock() - logs := make([]string, len(d.logs)) - copy(logs, d.logs) + logs := d.logs + d.logs = nil return logs } @@ -53,20 +53,20 @@ func newDummyWriter(name string, level Level, delay time.Duration) *dummyWriter } func TestLogger(t *testing.T) { - logger := NewLoggerWithWriters(context.Background(), "test") + logger := NewLoggerWithWriters(t.Context(), "test") dump := logger.DumpWriters() assert.Empty(t, dump) - assert.EqualValues(t, NONE, logger.GetLevel()) + assert.Equal(t, NONE, logger.GetLevel()) assert.False(t, logger.IsEnabled()) w1 := newDummyWriter("dummy-1", DEBUG, 0) logger.AddWriters(w1) - assert.EqualValues(t, DEBUG, logger.GetLevel()) + assert.Equal(t, DEBUG, logger.GetLevel()) w2 := newDummyWriter("dummy-2", WARN, 200*time.Millisecond) logger.AddWriters(w2) - assert.EqualValues(t, DEBUG, logger.GetLevel()) + assert.Equal(t, DEBUG, logger.GetLevel()) dump = logger.DumpWriters() assert.Len(t, dump, 2) @@ -77,18 +77,18 @@ func TestLogger(t *testing.T) { // w2 is slow, so only w1 has logs time.Sleep(100 * time.Millisecond) - assert.Equal(t, []string{"debug-level\n", "error-level\n"}, w1.GetLogs()) - assert.Equal(t, []string{}, w2.GetLogs()) + assert.Equal(t, []string{"debug-level\n", "error-level\n"}, w1.FetchLogs()) + assert.Empty(t, w2.FetchLogs()) logger.Close() // after Close, all logs are flushed - assert.Equal(t, []string{"debug-level\n", "error-level\n"}, w1.GetLogs()) - assert.Equal(t, []string{"error-level\n"}, w2.GetLogs()) + assert.Empty(t, w1.FetchLogs()) + assert.Equal(t, []string{"error-level\n"}, w2.FetchLogs()) } func TestLoggerPause(t *testing.T) { - logger := NewLoggerWithWriters(context.Background(), "test") + logger := NewLoggerWithWriters(t.Context(), "test") w1 := newDummyWriter("dummy-1", DEBUG, 0) logger.AddWriters(w1) @@ -98,12 +98,12 @@ func TestLoggerPause(t *testing.T) { logger.Info("info-level") time.Sleep(100 * time.Millisecond) - assert.Equal(t, []string{}, w1.GetLogs()) + assert.Empty(t, w1.FetchLogs()) GetManager().ResumeAll() time.Sleep(100 * time.Millisecond) - assert.Equal(t, []string{"info-level\n"}, w1.GetLogs()) + assert.Equal(t, []string{"info-level\n"}, w1.FetchLogs()) logger.Close() } @@ -116,21 +116,54 @@ func (t testLogString) LogString() string { return "log-string" } -func TestLoggerLogString(t *testing.T) { - logger := NewLoggerWithWriters(context.Background(), "test") +type testLogStringPtrReceiver struct { + Field string +} - w1 := newDummyWriter("dummy-1", DEBUG, 0) - w1.Mode.Colorize = true - logger.AddWriters(w1) +func (t *testLogStringPtrReceiver) LogString() string { + return "log-string-ptr-receiver" +} - logger.Info("%s %s %#v %v", testLogString{}, &testLogString{}, testLogString{Field: "detail"}, NewColoredValue(testLogString{}, FgRed)) - logger.Close() +func genericFunc[T any](logger Logger, v T) { + logger.Info("from genericFunc: %v", v) +} - assert.Equal(t, []string{"log-string log-string log.testLogString{Field:\"detail\"} \x1b[31mlog-string\x1b[0m\n"}, w1.GetLogs()) +func TestLoggerOutput(t *testing.T) { + t.Run("LogString", func(t *testing.T) { + logger := NewLoggerWithWriters(t.Context(), "test") + w1 := newDummyWriter("dummy-1", DEBUG, 0) + w1.Mode.Colorize = true + logger.AddWriters(w1) + logger.Info("%s %s %#v %v", testLogString{}, &testLogString{}, testLogString{Field: "detail"}, NewColoredValue(testLogString{}, FgRed)) + logger.Info("%s %s %#v %v", testLogStringPtrReceiver{}, &testLogStringPtrReceiver{}, testLogStringPtrReceiver{Field: "detail"}, NewColoredValue(testLogStringPtrReceiver{}, FgRed)) + logger.Close() + + assert.Equal(t, []string{ + "log-string log-string log.testLogString{Field:\"detail\"} \x1b[31mlog-string\x1b[0m\n", + "log-string-ptr-receiver log-string-ptr-receiver &log.testLogStringPtrReceiver{Field:\"detail\"} \x1b[31mlog-string-ptr-receiver\x1b[0m\n", + }, w1.FetchLogs()) + }) + + t.Run("Caller", func(t *testing.T) { + logger := NewLoggerWithWriters(t.Context(), "test") + w1 := newDummyWriter("dummy-1", DEBUG, 0) + w1.EventWriterBaseImpl.Mode.Flags.flags = Lmedfile | Lshortfuncname + logger.AddWriters(w1) + anonymousFunc := func(logger Logger) { + logger.Info("from anonymousFunc") + } + genericFunc(logger, "123") + anonymousFunc(logger) + logger.Close() + logs := w1.FetchLogs() + assert.Len(t, logs, 2) + assert.Regexp(t, `modules/log/logger_test.go:\w+:`+regexp.QuoteMeta(`genericFunc() from genericFunc: 123`), logs[0]) + assert.Regexp(t, `modules/log/logger_test.go:\w+:`+regexp.QuoteMeta(`TestLoggerOutput.2.1() from anonymousFunc`), logs[1]) + }) } func TestLoggerExpressionFilter(t *testing.T) { - logger := NewLoggerWithWriters(context.Background(), "test") + logger := NewLoggerWithWriters(t.Context(), "test") w1 := newDummyWriter("dummy-1", DEBUG, 0) w1.Mode.Expression = "foo.*" @@ -142,5 +175,5 @@ func TestLoggerExpressionFilter(t *testing.T) { logger.SendLogEvent(&Event{Level: INFO, Filename: "foo.go", MsgSimpleText: "by filename"}) logger.Close() - assert.Equal(t, []string{"foo\n", "foo bar\n", "by filename\n"}, w1.GetLogs()) + assert.Equal(t, []string{"foo\n", "foo bar\n", "by filename\n"}, w1.FetchLogs()) } diff --git a/modules/log/manager_test.go b/modules/log/manager_test.go index b8fbf84613..beddbccb73 100644 --- a/modules/log/manager_test.go +++ b/modules/log/manager_test.go @@ -37,6 +37,6 @@ func TestSharedWorker(t *testing.T) { m.Close() - logs := w.(*dummyWriter).GetLogs() + logs := w.(*dummyWriter).FetchLogs() assert.Equal(t, []string{"msg-1\n", "msg-2\n", "msg-3\n"}, logs) } diff --git a/modules/log/misc.go b/modules/log/misc.go index ae4ce04cf3..c9d230e4ac 100644 --- a/modules/log/misc.go +++ b/modules/log/misc.go @@ -19,8 +19,8 @@ func BaseLoggerToGeneralLogger(b BaseLogger) Logger { var _ Logger = (*baseToLogger)(nil) -func (s *baseToLogger) Log(skip int, level Level, format string, v ...any) { - s.base.Log(skip+1, level, format, v...) +func (s *baseToLogger) Log(skip int, event *Event, format string, v ...any) { + s.base.Log(skip+1, event, format, v...) } func (s *baseToLogger) GetLevel() Level { @@ -32,27 +32,27 @@ func (s *baseToLogger) LevelEnabled(level Level) bool { } func (s *baseToLogger) Trace(format string, v ...any) { - s.base.Log(1, TRACE, format, v...) + s.base.Log(1, &Event{Level: TRACE}, format, v...) } func (s *baseToLogger) Debug(format string, v ...any) { - s.base.Log(1, DEBUG, format, v...) + s.base.Log(1, &Event{Level: DEBUG}, format, v...) } func (s *baseToLogger) Info(format string, v ...any) { - s.base.Log(1, INFO, format, v...) + s.base.Log(1, &Event{Level: INFO}, format, v...) } func (s *baseToLogger) Warn(format string, v ...any) { - s.base.Log(1, WARN, format, v...) + s.base.Log(1, &Event{Level: WARN}, format, v...) } func (s *baseToLogger) Error(format string, v ...any) { - s.base.Log(1, ERROR, format, v...) + s.base.Log(1, &Event{Level: ERROR}, format, v...) } func (s *baseToLogger) Critical(format string, v ...any) { - s.base.Log(1, CRITICAL, format, v...) + s.base.Log(1, &Event{Level: CRITICAL}, format, v...) } type PrintfLogger struct { diff --git a/modules/markup/common/footnote.go b/modules/markup/common/footnote.go index 4406803694..1ece436c66 100644 --- a/modules/markup/common/footnote.go +++ b/modules/markup/common/footnote.go @@ -53,7 +53,7 @@ type FootnoteLink struct { // Dump implements Node.Dump. func (n *FootnoteLink) Dump(source []byte, level int) { m := map[string]string{} - m["Index"] = fmt.Sprintf("%v", n.Index) + m["Index"] = strconv.Itoa(n.Index) m["Name"] = fmt.Sprintf("%v", n.Name) ast.DumpHelper(n, source, level, m, nil) } @@ -85,7 +85,7 @@ type FootnoteBackLink struct { // Dump implements Node.Dump. func (n *FootnoteBackLink) Dump(source []byte, level int) { m := map[string]string{} - m["Index"] = fmt.Sprintf("%v", n.Index) + m["Index"] = strconv.Itoa(n.Index) m["Name"] = fmt.Sprintf("%v", n.Name) ast.DumpHelper(n, source, level, m, nil) } @@ -151,7 +151,7 @@ type FootnoteList struct { // Dump implements Node.Dump. func (n *FootnoteList) Dump(source []byte, level int) { m := map[string]string{} - m["Count"] = fmt.Sprintf("%v", n.Count) + m["Count"] = strconv.Itoa(n.Count) ast.DumpHelper(n, source, level, m, nil) } @@ -197,7 +197,7 @@ func (b *footnoteBlockParser) Open(parent ast.Node, reader text.Reader, pc parse return nil, parser.NoChildren } open := pos + 1 - closure := util.FindClosure(line[pos+1:], '[', ']', false, false) //nolint + closure := util.FindClosure(line[pos+1:], '[', ']', false, false) //nolint:staticcheck // deprecated function closes := pos + 1 + closure next := closes + 1 if closure > -1 { @@ -287,7 +287,7 @@ func (s *footnoteParser) Parse(parent ast.Node, block text.Reader, pc parser.Con return nil } open := pos - closure := util.FindClosure(line[pos:], '[', ']', false, false) //nolint + closure := util.FindClosure(line[pos:], '[', ']', false, false) //nolint:staticcheck // deprecated function if closure < 0 { return nil } @@ -409,9 +409,9 @@ func (r *FootnoteHTMLRenderer) renderFootnoteLink(w util.BufWriter, source []byt _, _ = w.Write(n.Name) _, _ = w.WriteString(`"><a href="#fn:`) _, _ = w.Write(n.Name) - _, _ = w.WriteString(`" class="footnote-ref" role="doc-noteref">`) + _, _ = w.WriteString(`" class="footnote-ref" role="doc-noteref">`) // FIXME: here and below, need to keep the classes _, _ = w.WriteString(is) - _, _ = w.WriteString(`</a></sup>`) + _, _ = w.WriteString(` </a></sup>`) // the style doesn't work at the moment, so add a space to separate the names } return ast.WalkContinue, nil } diff --git a/modules/markup/common/linkify.go b/modules/markup/common/linkify.go index 52888958fa..3eecb97eac 100644 --- a/modules/markup/common/linkify.go +++ b/modules/markup/common/linkify.go @@ -85,9 +85,10 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont } else if lastChar == ')' { closing := 0 for i := m[1] - 1; i >= m[0]; i-- { - if line[i] == ')' { + switch line[i] { + case ')': closing++ - } else if line[i] == '(' { + case '(': closing-- } } diff --git a/modules/markup/console/console.go b/modules/markup/console/console.go index 06f3acfa68..492579b0a5 100644 --- a/modules/markup/console/console.go +++ b/modules/markup/console/console.go @@ -6,13 +6,14 @@ package console import ( "bytes" "io" - "path" + "unicode/utf8" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" trend "github.com/buildkite/terminal-to-html/v3" - "github.com/go-enry/go-enry/v2" ) func init() { @@ -22,6 +23,8 @@ func init() { // Renderer implements markup.Renderer type Renderer struct{} +var _ markup.RendererContentDetector = (*Renderer)(nil) + // Name implements markup.Renderer func (Renderer) Name() string { return "console" @@ -40,15 +43,36 @@ func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { } // CanRender implements markup.RendererContentDetector -func (Renderer) CanRender(filename string, input io.Reader) bool { - buf, err := io.ReadAll(input) - if err != nil { +func (Renderer) CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool { + if !sniffedType.IsTextPlain() { return false } - if enry.GetLanguage(path.Base(filename), buf) != enry.OtherLanguage { + + s := util.UnsafeBytesToString(prefetchBuf) + rs := []rune(s) + cnt := 0 + firstErrPos := -1 + isCtrlSep := func(p int) bool { + return p < len(rs) && (rs[p] == ';' || rs[p] == 'm') + } + for i, c := range rs { + if c == 0 { + return false + } + if c == '\x1b' { + match := i+1 < len(rs) && rs[i+1] == '[' + if match && (isCtrlSep(i+2) || isCtrlSep(i+3) || isCtrlSep(i+4) || isCtrlSep(i+5)) { + cnt++ + } + } + if c == utf8.RuneError && firstErrPos == -1 { + firstErrPos = i + } + } + if firstErrPos != -1 && firstErrPos != len(rs)-1 { return false } - return bytes.ContainsRune(buf, '\x1b') + return cnt >= 2 // only render it as console output if there are at least two escape sequences } // Render renders terminal colors to HTML with all specific handling stuff. diff --git a/modules/markup/console/console_test.go b/modules/markup/console/console_test.go index e1f0da1f01..d1192bebc2 100644 --- a/modules/markup/console/console_test.go +++ b/modules/markup/console/console_test.go @@ -4,28 +4,43 @@ package console import ( - "context" "strings" "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/typesniffer" "github.com/stretchr/testify/assert" ) func TestRenderConsole(t *testing.T) { - var render Renderer - kases := map[string]string{ - "\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok": "<span class=\"term-fg37 term-bg40\">npm</span> <span class=\"term-fg32\">info</span> <span class=\"term-fg35\">it worked if it ends with</span> ok", + cases := []struct { + input string + expected string + }{ + {"\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok", `<span class="term-fg37 term-bg40">npm</span> <span class="term-fg32">info</span> <span class="term-fg35">it worked if it ends with</span> ok`}, + {"\x1b[1;2m \x1b[123m 啊", `<span class="term-fg2"> 啊</span>`}, + {"\x1b[1;2m \x1b[123m \xef", `<span class="term-fg2"> �</span>`}, + {"\x1b[1;2m \x1b[123m \xef \xef", ``}, + {"\x1b[12", ``}, + {"\x1b[1", ``}, + {"\x1b[FOO\x1b[", ``}, + {"\x1b[mFOO\x1b[m", `FOO`}, } - for k, v := range kases { + var render Renderer + for i, c := range cases { var buf strings.Builder - canRender := render.CanRender("test", strings.NewReader(k)) - assert.True(t, canRender) + st := typesniffer.DetectContentType([]byte(c.input)) + canRender := render.CanRender("test", st, []byte(c.input)) + if c.expected == "" { + assert.False(t, canRender, "case %d: expected not to render", i) + continue + } - err := render.Render(markup.NewRenderContext(context.Background()), strings.NewReader(k), &buf) + assert.True(t, canRender) + err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(c.input), &buf) assert.NoError(t, err) - assert.EqualValues(t, v, buf.String()) + assert.Equal(t, c.expected, buf.String()) } } diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index 4c47170c30..fff7f0baca 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -4,7 +4,6 @@ package markup import ( - "context" "strings" "testing" @@ -24,8 +23,8 @@ func TestRenderCSV(t *testing.T) { for k, v := range kases { var buf strings.Builder - err := render.Render(markup.NewRenderContext(context.Background()), strings.NewReader(k), &buf) + err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(k), &buf) assert.NoError(t, err) - assert.EqualValues(t, v, buf.String()) + assert.Equal(t, v, buf.String()) } } diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 03242e569e..39861ade12 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -12,11 +12,9 @@ import ( "runtime" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) // RegisterRenderers registers all supported third part renderers according settings @@ -77,27 +75,22 @@ func envMark(envName string) string { // Render renders the data of the document to HTML via the external tool. func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { - var ( - command = strings.NewReplacer( - envMark("GITEA_PREFIX_SRC"), ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault), - envMark("GITEA_PREFIX_RAW"), ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw), - ).Replace(p.Command) - commands = strings.Fields(command) - args = commands[1:] - ) + baseLinkSrc := ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault) + baseLinkRaw := ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw) + command := strings.NewReplacer( + envMark("GITEA_PREFIX_SRC"), baseLinkSrc, + envMark("GITEA_PREFIX_RAW"), baseLinkRaw, + ).Replace(p.Command) + commands := strings.Fields(command) + args := commands[1:] if p.IsInputFile { // write to temp file - f, err := os.CreateTemp("", "gitea_input") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("gitea_input") if err != nil { return fmt.Errorf("%s create temp file when rendering %s failed: %w", p.Name(), p.Command, err) } - tmpPath := f.Name() - defer func() { - if err := util.Remove(tmpPath); err != nil { - log.Warn("Unable to remove temporary file: %s: Error: %v", tmpPath, err) - } - }() + defer cleanup() _, err = io.Copy(f, input) if err != nil { @@ -112,14 +105,14 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. args = append(args, f.Name()) } - processCtx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Render [%s] for %s", commands[0], ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault))) + processCtx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Render [%s] for %s", commands[0], baseLinkSrc)) defer finished() cmd := exec.CommandContext(processCtx, commands[0], args...) cmd.Env = append( os.Environ(), - "GITEA_PREFIX_SRC="+ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault), - "GITEA_PREFIX_RAW="+ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw), + "GITEA_PREFIX_SRC="+baseLinkSrc, + "GITEA_PREFIX_RAW="+baseLinkRaw, ) if !p.IsInputFile { cmd.Stdin = input diff --git a/modules/markup/html.go b/modules/markup/html.go index bb12febf27..51afd4be00 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "regexp" + "slices" "strings" "sync" @@ -32,7 +33,6 @@ type globalVarsType struct { comparePattern *regexp.Regexp fullURLPattern *regexp.Regexp emailRegex *regexp.Regexp - blackfridayExtRegex *regexp.Regexp emojiShortCodeRegex *regexp.Regexp issueFullPattern *regexp.Regexp filesChangedFullPattern *regexp.Regexp @@ -47,7 +47,7 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // NOTE: All below regex matching do not perform any extra validation. // Thus a link is produced even if the linked entity does not exist. // While fast, this is also incorrect and lead to false positives. - // TODO: fix invalid linking issue + // TODO: fix invalid linking issue (update: stale TODO, what issues? maybe no TODO anymore) // valid chars in encoded path and parameter: [-+~_%.a-zA-Z0-9/] @@ -72,10 +72,8 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // it is still accepted by the CommonMark specification, as well as the HTML5 spec: // http://spec.commonmark.org/0.28/#email-address // https://html.spec.whatwg.org/multipage/input.html#e-mail-state-(type%3Demail) - v.emailRegex = regexp.MustCompile("(?:\\s|^|\\(|\\[)([a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)(?:\\s|$|\\)|\\]|;|,|\\?|!|\\.(\\s|$))") - - // blackfridayExtRegex is for blackfriday extensions create IDs like fn:user-content-footnote - v.blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`) + // At the moment, we use stricter rule for rendering purpose: only allow the "name" part starting after the word boundary + v.emailRegex = regexp.MustCompile(`\b([-\w.!#$%&'*+/=?^{|}~]*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)\b`) // emojiShortCodeRegex find emoji by alias like :smile: v.emojiShortCodeRegex = regexp.MustCompile(`:[-+\w]+:`) @@ -89,22 +87,18 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // codePreviewPattern matches "http://domain/.../{owner}/{repo}/src/commit/{commit}/{filepath}#L10-L20" v.codePreviewPattern = regexp.MustCompile(`https?://\S+/([^\s/]+)/([^\s/]+)/src/commit/([0-9a-f]{7,64})(/\S+)#(L\d+(-L\d+)?)`) - v.tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`) + // cleans: "<foo/bar", "<any words/", ("<html", "<head", "<script", "<style", "<?", "<%") + v.tagCleaner = regexp.MustCompile(`(?i)<(/?\w+/\w+|/[\w ]+/|/?(html|head|script|style|%|\?)\b)`) v.nulCleaner = strings.NewReplacer("\000", "") return v }) -// IsFullURLBytes reports whether link fits valid format. -func IsFullURLBytes(link []byte) bool { - return globalVars().fullURLPattern.Match(link) -} - func IsFullURLString(link string) bool { return globalVars().fullURLPattern.MatchString(link) } func IsNonEmptyRelativePath(link string) bool { - return link != "" && !IsFullURLString(link) && link[0] != '/' && link[0] != '?' && link[0] != '#' + return link != "" && !IsFullURLString(link) && link[0] != '?' && link[0] != '#' } // CustomLinkURLSchemes allows for additional schemes to be detected when parsing links within text @@ -116,13 +110,7 @@ func CustomLinkURLSchemes(schemes []string) { if !validScheme.MatchString(s) { continue } - without := false - for _, sna := range xurls.SchemesNoAuthority { - if s == sna { - without = true - break - } - } + without := slices.Contains(xurls.SchemesNoAuthority, s) if without { s += ":" } else { @@ -260,7 +248,7 @@ func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output node, err := html.Parse(io.MultiReader( // prepend "<html><body>" strings.NewReader("<html><body>"), - // Strip out nuls - they're always invalid + // strip out NULLs (they're always invalid), and escape known tags bytes.NewReader(globalVars().tagCleaner.ReplaceAll([]byte(globalVars().nulCleaner.Replace(string(rawHTML))), []byte("<$1"))), // close the tags strings.NewReader("</body></html>"), @@ -316,44 +304,39 @@ func isEmojiNode(node *html.Node) bool { } func visitNode(ctx *RenderContext, procs []processor, node *html.Node) *html.Node { - // Add user-content- to IDs and "#" links if they don't already have them - for idx, attr := range node.Attr { - val := strings.TrimPrefix(attr.Val, "#") - notHasPrefix := !(strings.HasPrefix(val, "user-content-") || globalVars().blackfridayExtRegex.MatchString(val)) - - if attr.Key == "id" && notHasPrefix { - node.Attr[idx].Val = "user-content-" + attr.Val - } - - if attr.Key == "href" && strings.HasPrefix(attr.Val, "#") && notHasPrefix { - node.Attr[idx].Val = "#user-content-" + val - } - } - - switch node.Type { - case html.TextNode: + if node.Type == html.TextNode { for _, proc := range procs { proc(ctx, node) // it might add siblings } + return node.NextSibling + } + if node.Type != html.ElementNode { + return node.NextSibling + } - case html.ElementNode: - if isEmojiNode(node) { - // TextNode emoji will be converted to `<span class="emoji">`, then the next iteration will visit the "span" - // if we don't stop it, it will go into the TextNode again and create an infinite recursion - return node.NextSibling - } else if node.Data == "code" || node.Data == "pre" { - return node.NextSibling // ignore code and pre nodes - } else if node.Data == "img" { - return visitNodeImg(ctx, node) - } else if node.Data == "video" { - return visitNodeVideo(ctx, node) - } else if node.Data == "a" { - procs = emojiProcessors // Restrict text in links to emojis - } - for n := node.FirstChild; n != nil; { - n = visitNode(ctx, procs, n) - } - default: + processNodeAttrID(node) + processFootnoteNode(ctx, node) // FIXME: the footnote processing should be done in the "footnote.go" renderer directly + + if isEmojiNode(node) { + // TextNode emoji will be converted to `<span class="emoji">`, then the next iteration will visit the "span" + // if we don't stop it, it will go into the TextNode again and create an infinite recursion + return node.NextSibling + } else if node.Data == "code" || node.Data == "pre" { + return node.NextSibling // ignore code and pre nodes + } else if node.Data == "img" { + return visitNodeImg(ctx, node) + } else if node.Data == "video" { + return visitNodeVideo(ctx, node) + } + + if node.Data == "a" { + processNodeA(ctx, node) + // only use emoji processors for the content in the "A" tag, + // because the content there is not processable, for example: the content is a commit id or a full URL. + procs = emojiProcessors + } + for n := node.FirstChild; n != nil; { + n = visitNode(ctx, procs, n) } return node.NextSibling } diff --git a/modules/markup/html_commit.go b/modules/markup/html_commit.go index aa1b7d034a..fe7a034967 100644 --- a/modules/markup/html_commit.go +++ b/modules/markup/html_commit.go @@ -43,7 +43,6 @@ func createCodeLink(href, content, class string) *html.Node { code := &html.Node{ Type: html.ElementNode, Data: atom.Code.String(), - Attr: []html.Attribute{{Key: "class", Val: "nohighlight"}}, } code.AppendChild(text) @@ -63,7 +62,7 @@ func anyHashPatternExtract(s string) (ret anyHashPatternResult, ok bool) { // if url ends in '.', it's very likely that it is not part of the actual url but used to finish a sentence. ret.PosEnd-- ret.FullURL = ret.FullURL[:len(ret.FullURL)-1] - for i := 0; i < len(m); i++ { + for i := range m { m[i] = min(m[i], ret.PosEnd) } } @@ -189,7 +188,7 @@ func hashCurrentPatternProcessor(ctx *RenderContext, node *html.Node) { continue } - link := ctx.RenderHelper.ResolveLink(util.URLJoin(ctx.RenderOptions.Metas["user"], ctx.RenderOptions.Metas["repo"], "commit", hash), LinkTypeApp) + link := "/:root/" + util.URLJoin(ctx.RenderOptions.Metas["user"], ctx.RenderOptions.Metas["repo"], "commit", hash) replaceContent(node, m[2], m[3], createCodeLink(link, base.ShortSha(hash), "commit")) start = 0 node = node.NextSibling.NextSibling @@ -205,9 +204,9 @@ func commitCrossReferencePatternProcessor(ctx *RenderContext, node *html.Node) { return } - reftext := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) - linkHref := ctx.RenderHelper.ResolveLink(util.URLJoin(ref.Owner, ref.Name, "commit", ref.CommitSha), LinkTypeApp) - link := createLink(ctx, linkHref, reftext, "commit") + refText := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) + linkHref := "/:root/" + util.URLJoin(ref.Owner, ref.Name, "commit", ref.CommitSha) + link := createLink(ctx, linkHref, refText, "commit") replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link) node = node.NextSibling.NextSibling diff --git a/modules/markup/html_email.go b/modules/markup/html_email.go index cbfae8b829..cf18e99d98 100644 --- a/modules/markup/html_email.go +++ b/modules/markup/html_email.go @@ -3,7 +3,11 @@ package markup -import "golang.org/x/net/html" +import ( + "strings" + + "golang.org/x/net/html" +) // emailAddressProcessor replaces raw email addresses with a mailto: link. func emailAddressProcessor(ctx *RenderContext, node *html.Node) { @@ -14,6 +18,14 @@ func emailAddressProcessor(ctx *RenderContext, node *html.Node) { return } + var nextByte byte + if len(node.Data) > m[3] { + nextByte = node.Data[m[3]] + } + if strings.IndexByte(":/", nextByte) != -1 { + // for cases: "git@gitea.com:owner/repo.git", "https://git@gitea.com/owner/repo.git" + return + } mail := node.Data[m[2]:m[3]] replaceContent(node, m[2], m[3], createLink(ctx, "mailto:"+mail, mail, "" /*mailto*/)) node = node.NextSibling.NextSibling diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index 159d712955..467cc509d0 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -107,7 +107,7 @@ func TestRender_IssueIndexPattern2(t *testing.T) { isExternal := false if marker == "!" { path = "pulls" - prefix = "http://localhost:3000/someUser/someRepo/pulls/" + prefix = "/someUser/someRepo/pulls/" } else { path = "issues" prefix = "https://someurl.com/someUser/someRepo/" @@ -116,7 +116,7 @@ func TestRender_IssueIndexPattern2(t *testing.T) { links := make([]any, len(indices)) for i, index := range indices { - links[i] = numericIssueLink(util.URLJoin(TestRepoURL, path), "ref-issue", index, marker) + links[i] = numericIssueLink(util.URLJoin("/test-owner/test-repo", path), "ref-issue", index, marker) } expectedNil := fmt.Sprintf(expectedFmt, links...) testRenderIssueIndexPattern(t, s, expectedNil, NewTestRenderContext(TestAppURL, localMetas)) @@ -293,13 +293,13 @@ func TestRender_AutoLink(t *testing.T) { // render valid commit URLs tmp := util.URLJoin(TestRepoURL, "commit", "d8a994ef243349f321568f9e36d5c3f444b99cae") - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24</code></a>") tmp += "#diff-2" - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24 (diff-2)</code></a>") // render other commit URLs tmp = "https://external-link.gitea.io/go-gitea/gitea/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2" - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24 (diff-2)</code></a>") } func TestRender_FullIssueURLs(t *testing.T) { @@ -405,10 +405,10 @@ func TestRegExp_anySHA1Pattern(t *testing.T) { if v.CommitID == "" { assert.False(t, ok) } else { - assert.EqualValues(t, strings.TrimSuffix(k, "."), ret.FullURL) - assert.EqualValues(t, v.CommitID, ret.CommitID) - assert.EqualValues(t, v.SubPath, ret.SubPath) - assert.EqualValues(t, v.QueryHash, ret.QueryHash) + assert.Equal(t, strings.TrimSuffix(k, "."), ret.FullURL) + assert.Equal(t, v.CommitID, ret.CommitID) + assert.Equal(t, v.SubPath, ret.SubPath) + assert.Equal(t, v.QueryHash, ret.QueryHash) } } } diff --git a/modules/markup/html_issue.go b/modules/markup/html_issue.go index 7a6f33011a..85bec5db20 100644 --- a/modules/markup/html_issue.go +++ b/modules/markup/html_issue.go @@ -82,7 +82,7 @@ func createIssueLinkContentWithSummary(ctx *RenderContext, linkHref string, ref h, err := DefaultRenderHelperFuncs.RenderRepoIssueIconTitle(ctx, RenderIssueIconTitleOptions{ OwnerName: ref.Owner, RepoName: ref.Name, - LinkHref: linkHref, + LinkHref: ctx.RenderHelper.ResolveLink(linkHref, LinkTypeDefault), IssueIndex: issueIndex, }) if err != nil { @@ -162,7 +162,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { issueOwner := util.Iif(ref.Owner == "", ctx.RenderOptions.Metas["user"], ref.Owner) issueRepo := util.Iif(ref.Owner == "", ctx.RenderOptions.Metas["repo"], ref.Name) issuePath := util.Iif(ref.IsPull, "pulls", "issues") - linkHref := ctx.RenderHelper.ResolveLink(util.URLJoin(issueOwner, issueRepo, issuePath, ref.Issue), LinkTypeApp) + linkHref := "/:root/" + util.URLJoin(issueOwner, issueRepo, issuePath, ref.Issue) // at the moment, only render the issue index in a full line (or simple line) as icon+title // otherwise it would be too noisy for "take #1 as an example" in a sentence diff --git a/modules/markup/html_issue_test.go b/modules/markup/html_issue_test.go index 8d189fbdf6..39cd9dcf6a 100644 --- a/modules/markup/html_issue_test.go +++ b/modules/markup/html_issue_test.go @@ -30,6 +30,7 @@ func TestRender_IssueList(t *testing.T) { rctx := markup.NewTestRenderContext(markup.TestAppURL, map[string]string{ "user": "test-user", "repo": "test-repo", "markupAllowShortIssuePattern": "true", + "footnoteContextId": "12345", }) out, err := markdown.RenderString(rctx, input) require.NoError(t, err) @@ -39,7 +40,7 @@ func TestRender_IssueList(t *testing.T) { t.Run("NormalIssueRef", func(t *testing.T) { test( "#12345", - `<p><a href="http://localhost:3000/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a></p>`, + `<p><a href="/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a></p>`, ) }) @@ -56,7 +57,7 @@ func TestRender_IssueList(t *testing.T) { test( "* foo #12345 bar", `<ul> -<li>foo <a href="http://localhost:3000/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a> bar</li> +<li>foo <a href="/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a> bar</li> </ul>`, ) }) @@ -69,4 +70,22 @@ func TestRender_IssueList(t *testing.T) { </ul>`, ) }) + + t.Run("IssueFootnote", func(t *testing.T) { + test( + "foo[^1][^2]\n\n[^1]: bar\n[^2]: baz", + `<p>foo<sup id="fnref:user-content-1-12345"><a href="#fn:user-content-1-12345" rel="nofollow">1 </a></sup><sup id="fnref:user-content-2-12345"><a href="#fn:user-content-2-12345" rel="nofollow">2 </a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-1-12345"> +<p>bar <a href="#fnref:user-content-1-12345" rel="nofollow">↩︎</a></p> +</li> +<li id="fn:user-content-2-12345"> +<p>baz <a href="#fnref:user-content-2-12345" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div>`, + ) + }) } diff --git a/modules/markup/html_link.go b/modules/markup/html_link.go index 0e7a988d36..43faef1681 100644 --- a/modules/markup/html_link.go +++ b/modules/markup/html_link.go @@ -31,8 +31,8 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { // It makes page handling terrible, but we prefer GitHub syntax // And fall back to MediaWiki only when it is obvious from the look // Of text and link contents - sl := strings.Split(content, "|") - for _, v := range sl { + sl := strings.SplitSeq(content, "|") + for v := range sl { if equalPos := strings.IndexByte(v, '='); equalPos == -1 { // There is no equal in this argument; this is a mandatory arg if props["name"] == "" { @@ -125,7 +125,6 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { } } if image { - link = ctx.RenderHelper.ResolveLink(link, LinkTypeMedia) title := props["title"] if title == "" { title = props["alt"] @@ -151,7 +150,6 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { childNode.Attr = childNode.Attr[:2] } } else { - link = ctx.RenderHelper.ResolveLink(link, LinkTypeDefault) childNode.Type = html.TextNode childNode.Data = name } diff --git a/modules/markup/html_mention.go b/modules/markup/html_mention.go index fffa12e7b7..f97c034cf3 100644 --- a/modules/markup/html_mention.go +++ b/modules/markup/html_mention.go @@ -33,7 +33,7 @@ func mentionProcessor(ctx *RenderContext, node *html.Node) { if ok && strings.Contains(mention, "/") { mentionOrgAndTeam := strings.Split(mention, "/") if mentionOrgAndTeam[0][1:] == ctx.RenderOptions.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") { - link := ctx.RenderHelper.ResolveLink(util.URLJoin("org", ctx.RenderOptions.Metas["org"], "teams", mentionOrgAndTeam[1]), LinkTypeApp) + link := "/:root/" + util.URLJoin("org", ctx.RenderOptions.Metas["org"], "teams", mentionOrgAndTeam[1]) replaceContent(node, loc.Start, loc.End, createLink(ctx, link, mention, "" /*mention*/)) node = node.NextSibling.NextSibling start = 0 @@ -45,7 +45,7 @@ func mentionProcessor(ctx *RenderContext, node *html.Node) { mentionedUsername := mention[1:] if DefaultRenderHelperFuncs != nil && DefaultRenderHelperFuncs.IsUsernameMentionable(ctx, mentionedUsername) { - link := ctx.RenderHelper.ResolveLink(mentionedUsername, LinkTypeApp) + link := "/:root/" + mentionedUsername replaceContent(node, loc.Start, loc.End, createLink(ctx, link, mention, "" /*mention*/)) node = node.NextSibling.NextSibling start = 0 diff --git a/modules/markup/html_node.go b/modules/markup/html_node.go index 6e8ca67900..4eb78fdd2b 100644 --- a/modules/markup/html_node.go +++ b/modules/markup/html_node.go @@ -4,42 +4,105 @@ package markup import ( + "strings" + "golang.org/x/net/html" ) +func isAnchorIDUserContent(s string) bool { + // blackfridayExtRegex is for blackfriday extensions create IDs like fn:user-content-footnote + // old logic: blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`) + return strings.HasPrefix(s, "user-content-") || strings.Contains(s, ":user-content-") +} + +func isAnchorIDFootnote(s string) bool { + return strings.HasPrefix(s, "fnref:user-content-") || strings.HasPrefix(s, "fn:user-content-") +} + +func isAnchorHrefFootnote(s string) bool { + return strings.HasPrefix(s, "#fnref:user-content-") || strings.HasPrefix(s, "#fn:user-content-") +} + +func processNodeAttrID(node *html.Node) { + // Add user-content- to IDs and "#" links if they don't already have them, + // and convert the link href to a relative link to the host root + for idx, attr := range node.Attr { + if attr.Key == "id" { + if !isAnchorIDUserContent(attr.Val) { + node.Attr[idx].Val = "user-content-" + attr.Val + } + } + } +} + +func processFootnoteNode(ctx *RenderContext, node *html.Node) { + for idx, attr := range node.Attr { + if (attr.Key == "id" && isAnchorIDFootnote(attr.Val)) || + (attr.Key == "href" && isAnchorHrefFootnote(attr.Val)) { + if footnoteContextID := ctx.RenderOptions.Metas["footnoteContextId"]; footnoteContextID != "" { + node.Attr[idx].Val = attr.Val + "-" + footnoteContextID + } + continue + } + } +} + +func processNodeA(ctx *RenderContext, node *html.Node) { + for idx, attr := range node.Attr { + if attr.Key == "href" { + if anchorID, ok := strings.CutPrefix(attr.Val, "#"); ok { + if !isAnchorIDUserContent(attr.Val) { + node.Attr[idx].Val = "#user-content-" + anchorID + } + } else { + node.Attr[idx].Val = ctx.RenderHelper.ResolveLink(attr.Val, LinkTypeDefault) + } + } + } +} + func visitNodeImg(ctx *RenderContext, img *html.Node) (next *html.Node) { next = img.NextSibling - for i, attr := range img.Attr { - if attr.Key != "src" { + attrSrc, hasLazy := "", false + for i, imgAttr := range img.Attr { + hasLazy = hasLazy || imgAttr.Key == "loading" && imgAttr.Val == "lazy" + if imgAttr.Key != "src" { + attrSrc = imgAttr.Val continue } - if IsNonEmptyRelativePath(attr.Val) { - attr.Val = ctx.RenderHelper.ResolveLink(attr.Val, LinkTypeMedia) + imgSrcOrigin := imgAttr.Val + isLinkable := imgSrcOrigin != "" && !strings.HasPrefix(imgSrcOrigin, "data:") - // By default, the "<img>" tag should also be clickable, - // because frontend use `<img>` to paste the re-scaled image into the markdown, - // so it must match the default markdown image behavior. - hasParentAnchor := false - for p := img.Parent; p != nil; p = p.Parent { - if hasParentAnchor = p.Type == html.ElementNode && p.Data == "a"; hasParentAnchor { - break - } - } - if !hasParentAnchor { - imgA := &html.Node{Type: html.ElementNode, Data: "a", Attr: []html.Attribute{ - {Key: "href", Val: attr.Val}, - {Key: "target", Val: "_blank"}, - }} - parent := img.Parent - imgNext := img.NextSibling - parent.RemoveChild(img) - parent.InsertBefore(imgA, imgNext) - imgA.AppendChild(img) + // By default, the "<img>" tag should also be clickable, + // because frontend uses `<img>` to paste the re-scaled image into the Markdown, + // so it must match the default Markdown image behavior. + cnt := 0 + for p := img.Parent; isLinkable && p != nil && cnt < 2; p = p.Parent { + if hasParentAnchor := p.Type == html.ElementNode && p.Data == "a"; hasParentAnchor { + isLinkable = false + break } + cnt++ } - attr.Val = camoHandleLink(attr.Val) - img.Attr[i] = attr + if isLinkable { + wrapper := &html.Node{Type: html.ElementNode, Data: "a", Attr: []html.Attribute{ + {Key: "href", Val: ctx.RenderHelper.ResolveLink(imgSrcOrigin, LinkTypeDefault)}, + {Key: "target", Val: "_blank"}, + }} + parent := img.Parent + imgNext := img.NextSibling + parent.RemoveChild(img) + parent.InsertBefore(wrapper, imgNext) + wrapper.AppendChild(img) + } + + imgAttr.Val = ctx.RenderHelper.ResolveLink(imgSrcOrigin, LinkTypeMedia) + imgAttr.Val = camoHandleLink(imgAttr.Val) + img.Attr[i] = imgAttr + } + if !RenderBehaviorForTesting.DisableAdditionalAttributes && !hasLazy && !strings.HasPrefix(attrSrc, "data:") { + img.Attr = append(img.Attr, html.Attribute{Key: "loading", Val: "lazy"}) } return next } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 6d8f24184b..5fdbf43f7c 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -35,6 +35,7 @@ func TestRender_Commits(t *testing.T) { sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" repo := markup.TestAppURL + testRepoOwnerName + "/" + testRepoName + "/" commit := util.URLJoin(repo, "commit", sha) + commitPath := "/user13/repo11/commit/" + sha tree := util.URLJoin(repo, "tree", sha, "src") file := util.URLJoin(repo, "commit", sha, "example.txt") @@ -44,9 +45,9 @@ func TestRender_Commits(t *testing.T) { commitCompare := util.URLJoin(repo, "compare", sha+"..."+sha) commitCompareWithHash := commitCompare + "#L2" - test(sha, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) - test(sha[:7], `<p><a href="`+commit[:len(commit)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`) - test(sha[:39], `<p><a href="`+commit[:len(commit)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(sha, `<p><a href="`+commitPath+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(sha[:7], `<p><a href="`+commitPath[:len(commitPath)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`) + test(sha[:39], `<p><a href="`+commitPath[:len(commitPath)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test(commit, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test(tree, `<p><a href="`+tree+`" rel="nofollow"><code>65f1bf27bc/src</code></a></p>`) @@ -57,13 +58,13 @@ func TestRender_Commits(t *testing.T) { test(commitCompare, `<p><a href="`+commitCompare+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc</code></a></p>`) test(commitCompareWithHash, `<p><a href="`+commitCompareWithHash+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc (L2)</code></a></p>`) - test("commit "+sha, `<p>commit <a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test("commit "+sha, `<p>commit <a href="`+commitPath+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test("/home/gitea/"+sha, "<p>/home/gitea/"+sha+"</p>") test("deadbeef", `<p>deadbeef</p>`) test("d27ace93", `<p>d27ace93</p>`) test(sha[:14]+".x", `<p>`+sha[:14]+`.x</p>`) - expected14 := `<a href="` + commit[:len(commit)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>` + expected14 := `<a href="` + commitPath[:len(commitPath)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>` test(sha[:14]+".", `<p>`+expected14+`.</p>`) test(sha[:14]+",", `<p>`+expected14+`,</p>`) test("["+sha[:14]+"]", `<p>[`+expected14+`]</p>`) @@ -80,10 +81,10 @@ func TestRender_CrossReferences(t *testing.T) { test( "test-owner/test-repo#12345", - `<p><a href="`+util.URLJoin(markup.TestAppURL, "test-owner", "test-repo", "issues", "12345")+`" class="ref-issue" rel="nofollow">test-owner/test-repo#12345</a></p>`) + `<p><a href="/test-owner/test-repo/issues/12345" class="ref-issue" rel="nofollow">test-owner/test-repo#12345</a></p>`) test( "go-gitea/gitea#12345", - `<p><a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) + `<p><a href="/go-gitea/gitea/issues/12345" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) test( "/home/gitea/go-gitea/gitea#12345", `<p>/home/gitea/go-gitea/gitea#12345</p>`) @@ -224,10 +225,10 @@ func TestRender_email(t *testing.T) { test := func(input, expected string) { res, err := markup.RenderString(markup.NewTestRenderContext().WithRelativePath("a.md"), input) assert.NoError(t, err) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res)) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res), "input: %s", input) } - // Text that should be turned into email link + // Text that should be turned into email link test( "info@gitea.com", `<p><a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a></p>`) @@ -259,28 +260,48 @@ func TestRender_email(t *testing.T) { <a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>? <a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>!</p>`) + // match GitHub behavior + test("email@domain@domain.com", `<p>email@<a href="mailto:domain@domain.com" rel="nofollow">domain@domain.com</a></p>`) + + // match GitHub behavior + test(`"info@gitea.com"`, `<p>"<a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a>"</p>`) + // Test that should *not* be turned into email links test( - "\"info@gitea.com\"", - `<p>"info@gitea.com"</p>`) - test( "/home/gitea/mailstore/info@gitea/com", `<p>/home/gitea/mailstore/info@gitea/com</p>`) test( "git@try.gitea.io:go-gitea/gitea.git", `<p>git@try.gitea.io:go-gitea/gitea.git</p>`) test( + "https://foo:bar@gitea.io", + `<p><a href="https://foo:bar@gitea.io" rel="nofollow">https://foo:bar@gitea.io</a></p>`) + test( "gitea@3", `<p>gitea@3</p>`) test( "gitea@gmail.c", `<p>gitea@gmail.c</p>`) test( - "email@domain@domain.com", - `<p>email@domain@domain.com</p>`) - test( "email@domain..com", `<p>email@domain..com</p>`) + + cases := []struct { + input, expected string + }{ + // match GitHub behavior + {"?a@d.zz", `<p>?<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + {"*a@d.zz", `<p>*<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + {"~a@d.zz", `<p>~<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + + // the following cases don't match GitHub behavior, but they are valid email addresses ... + // maybe we should reduce the candidate characters for the "name" part in the future + {"a*a@d.zz", `<p><a href="mailto:a*a@d.zz" rel="nofollow">a*a@d.zz</a></p>`}, + {"a~a@d.zz", `<p><a href="mailto:a~a@d.zz" rel="nofollow">a~a@d.zz</a></p>`}, + } + for _, c := range cases { + test(c.input, c.expected) + } } func TestRender_emoji(t *testing.T) { @@ -468,7 +489,7 @@ func Test_ParseClusterFuzz(t *testing.T) { assert.NotContains(t, res.String(), "<html") } -func TestPostProcess_RenderDocument(t *testing.T) { +func TestPostProcess(t *testing.T) { setting.StaticURLPrefix = markup.TestAppURL // can't run standalone defer testModule.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() @@ -479,7 +500,7 @@ func TestPostProcess_RenderDocument(t *testing.T) { assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res.String())) } - // Issue index shouldn't be post processing in a document. + // Issue index shouldn't be post-processing in a document. test( "#1", "#1") @@ -487,9 +508,9 @@ func TestPostProcess_RenderDocument(t *testing.T) { // But cross-referenced issue index should work. test( "go-gitea/gitea#12345", - `<a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue">go-gitea/gitea#12345</a>`) + `<a href="/go-gitea/gitea/issues/12345" class="ref-issue">go-gitea/gitea#12345</a>`) - // Test that other post processing still works. + // Test that other post-processing still works. test( ":gitea:", `<span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span>`) @@ -498,6 +519,16 @@ func TestPostProcess_RenderDocument(t *testing.T) { `Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle`) test("http://localhost:3000/person/repo/issues/4#issuecomment-1234", `<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234" class="ref-issue">person/repo#4 (comment)</a>`) + + // special tags, GitHub's behavior, and for unclosed tags, output as text content as much as possible + test("<script>a", `<script>a`) + test("<script>a</script>", `<script>a</script>`) + test("<STYLE>a", `<STYLE>a`) + test("<style>a</STYLE>", `<style>a</STYLE>`) + + // other special tags, our special behavior + test("<?php\nfoo", "<?php\nfoo") + test("<%asp\nfoo", "<%asp\nfoo") } func TestIssue16020(t *testing.T) { @@ -522,7 +553,7 @@ func BenchmarkEmojiPostprocess(b *testing.B) { data += data } b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { var res strings.Builder err := markup.PostProcessDefault(markup.NewTestRenderContext(localMetas), strings.NewReader(data), &res) assert.NoError(b, err) @@ -543,7 +574,7 @@ func TestIssue18471(t *testing.T) { err := markup.PostProcessDefault(markup.NewTestRenderContext(localMetas), strings.NewReader(data), &res) assert.NoError(t, err) - assert.Equal(t, `<a href="http://domain/org/repo/compare/783b039...da951ce" class="compare"><code class="nohighlight">783b039...da951ce</code></a>`, res.String()) + assert.Equal(t, `<a href="http://domain/org/repo/compare/783b039...da951ce" class="compare"><code>783b039...da951ce</code></a>`, res.String()) } func TestIsFullURL(t *testing.T) { diff --git a/modules/markup/internal/internal_test.go b/modules/markup/internal/internal_test.go index 98ff3bc079..590bcbb67f 100644 --- a/modules/markup/internal/internal_test.go +++ b/modules/markup/internal/internal_test.go @@ -35,7 +35,7 @@ func TestRenderInternal(t *testing.T) { assert.EqualValues(t, c.protected, protected) _, _ = io.WriteString(in, string(protected)) _ = in.Close() - assert.EqualValues(t, c.recovered, out.String()) + assert.Equal(t, c.recovered, out.String()) } var r1, r2 RenderInternal @@ -44,11 +44,11 @@ func TestRenderInternal(t *testing.T) { _ = r1.init("sec", nil) protected = r1.ProtectSafeAttrs(`<div class="test"></div>`) assert.EqualValues(t, `<div data-attr-class="sec:test"></div>`, protected) - assert.EqualValues(t, "data-attr-class", r1.SafeAttr("class")) - assert.EqualValues(t, "sec:val", r1.SafeValue("val")) + assert.Equal(t, "data-attr-class", r1.SafeAttr("class")) + assert.Equal(t, "sec:val", r1.SafeValue("val")) recovered, ok := r1.RecoverProtectedValue("sec:val") assert.True(t, ok) - assert.EqualValues(t, "val", recovered) + assert.Equal(t, "val", recovered) recovered, ok = r1.RecoverProtectedValue("other:val") assert.False(t, ok) assert.Empty(t, recovered) @@ -57,5 +57,5 @@ func TestRenderInternal(t *testing.T) { in2 := r2.init("sec-other", out2) _, _ = io.WriteString(in2, string(protected)) _ = in2.Close() - assert.EqualValues(t, `<div data-attr-class="sec:test"></div>`, out2.String(), "different secureID should not recover the value") + assert.Equal(t, `<div data-attr-class="sec:test"></div>`, out2.String(), "different secureID should not recover the value") } diff --git a/modules/markup/markdown/ast.go b/modules/markup/markdown/ast.go index ca165b1ba0..f29f883734 100644 --- a/modules/markup/markdown/ast.go +++ b/modules/markup/markdown/ast.go @@ -4,6 +4,7 @@ package markdown import ( + "html/template" "strconv" "github.com/yuin/goldmark/ast" @@ -29,9 +30,7 @@ func (n *Details) Kind() ast.NodeKind { // NewDetails returns a new Paragraph node. func NewDetails() *Details { - return &Details{ - BaseBlock: ast.BaseBlock{}, - } + return &Details{} } // Summary is a block that contains the summary of details block @@ -54,9 +53,7 @@ func (n *Summary) Kind() ast.NodeKind { // NewSummary returns a new Summary node. func NewSummary() *Summary { - return &Summary{ - BaseBlock: ast.BaseBlock{}, - } + return &Summary{} } // TaskCheckBoxListItem is a block that represents a list item of a markdown block with a checkbox @@ -95,29 +92,6 @@ type Icon struct { Name []byte } -// Dump implements Node.Dump . -func (n *Icon) Dump(source []byte, level int) { - m := map[string]string{} - m["Name"] = string(n.Name) - ast.DumpHelper(n, source, level, m, nil) -} - -// KindIcon is the NodeKind for Icon -var KindIcon = ast.NewNodeKind("Icon") - -// Kind implements Node.Kind. -func (n *Icon) Kind() ast.NodeKind { - return KindIcon -} - -// NewIcon returns a new Paragraph node. -func NewIcon(name string) *Icon { - return &Icon{ - BaseInline: ast.BaseInline{}, - Name: []byte(name), - } -} - // ColorPreview is an inline for a color preview type ColorPreview struct { ast.BaseInline @@ -175,3 +149,24 @@ func NewAttention(attentionType string) *Attention { AttentionType: attentionType, } } + +var KindRawHTML = ast.NewNodeKind("RawHTML") + +type RawHTML struct { + ast.BaseBlock + rawHTML template.HTML +} + +func (n *RawHTML) Dump(source []byte, level int) { + m := map[string]string{} + m["RawHTML"] = string(n.rawHTML) + ast.DumpHelper(n, source, level, m, nil) +} + +func (n *RawHTML) Kind() ast.NodeKind { + return KindRawHTML +} + +func NewRawHTML(rawHTML template.HTML) *RawHTML { + return &RawHTML{rawHTML: rawHTML} +} diff --git a/modules/markup/markdown/convertyaml.go b/modules/markup/markdown/convertyaml.go index 1675b68be2..04664a9c1d 100644 --- a/modules/markup/markdown/convertyaml.go +++ b/modules/markup/markdown/convertyaml.go @@ -4,23 +4,22 @@ package markdown import ( + "strings" + + "code.gitea.io/gitea/modules/htmlutil" + "code.gitea.io/gitea/modules/svg" + "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" "gopkg.in/yaml.v3" ) func nodeToTable(meta *yaml.Node) ast.Node { - for { - if meta == nil { - return nil - } - switch meta.Kind { - case yaml.DocumentNode: - meta = meta.Content[0] - continue - default: - } - break + for meta != nil && meta.Kind == yaml.DocumentNode { + meta = meta.Content[0] + } + if meta == nil { + return nil } switch meta.Kind { case yaml.MappingNode: @@ -72,12 +71,28 @@ func sequenceNodeToTable(meta *yaml.Node) ast.Node { return table } -func nodeToDetails(meta *yaml.Node, icon string) ast.Node { +func nodeToDetails(g *ASTTransformer, meta *yaml.Node) ast.Node { + for meta != nil && meta.Kind == yaml.DocumentNode { + meta = meta.Content[0] + } + if meta == nil { + return nil + } + if meta.Kind != yaml.MappingNode { + return nil + } + var keys []string + for i := 0; i < len(meta.Content); i += 2 { + if meta.Content[i].Kind == yaml.ScalarNode { + keys = append(keys, meta.Content[i].Value) + } + } details := NewDetails() + details.SetAttributeString(g.renderInternal.SafeAttr("class"), g.renderInternal.SafeValue("frontmatter-content")) summary := NewSummary() - summary.AppendChild(summary, NewIcon(icon)) + summaryInnerHTML := htmlutil.HTMLFormat("%s %s", svg.RenderHTML("octicon-table", 12), strings.Join(keys, ", ")) + summary.AppendChild(summary, NewRawHTML(summaryInnerHTML)) details.AppendChild(details, summary) details.AppendChild(details, nodeToTable(meta)) - return details } diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 69c2a96ff1..b28fa9824e 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -5,14 +5,10 @@ package markdown import ( "fmt" - "regexp" - "strings" - "sync" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/internal" - "code.gitea.io/gitea/modules/setting" "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" @@ -51,7 +47,7 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa tocList := make([]Header, 0, 20) if rc.yamlNode != nil { - metaNode := rc.toMetaNode() + metaNode := rc.toMetaNode(g) if metaNode != nil { node.InsertBefore(node, firstChild, metaNode) } @@ -68,23 +64,12 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa g.transformHeading(ctx, v, reader, &tocList) case *ast.Paragraph: g.applyElementDir(v) - case *ast.Image: - g.transformImage(ctx, v) - case *ast.Link: - g.transformLink(ctx, v) case *ast.List: g.transformList(ctx, v, rc) case *ast.Text: if v.SoftLineBreak() && !v.HardLineBreak() { - // TODO: this was a quite unclear part, old code: `if metas["mode"] != "document" { use comment link break setting }` - // many places render non-comment contents with no mode=document, then these contents also use comment's hard line break setting - // especially in many tests. - markdownLineBreakStyle := ctx.RenderOptions.Metas["markdownLineBreakStyle"] - if markdownLineBreakStyle == "comment" { - v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInComments) - } else if markdownLineBreakStyle == "document" { - v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInDocuments) - } + newLineHardBreak := ctx.RenderOptions.Metas["markdownNewLineHardBreak"] == "true" + v.SetHardLineBreak(newLineHardBreak) } case *ast.CodeSpan: g.transformCodeSpan(ctx, v, reader) @@ -111,11 +96,6 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa } } -// it is copied from old code, which is quite doubtful whether it is correct -var reValidIconName = sync.OnceValue(func() *regexp.Regexp { - return regexp.MustCompile(`^[-\w]+$`) // old: regexp.MustCompile("^[a-z ]+$") -}) - // NewHTMLRenderer creates a HTMLRenderer to render in the gitea form. func NewHTMLRenderer(renderInternal *internal.RenderInternal, opts ...html.Option) renderer.NodeRenderer { r := &HTMLRenderer{ @@ -140,11 +120,11 @@ func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { reg.Register(ast.KindDocument, r.renderDocument) reg.Register(KindDetails, r.renderDetails) reg.Register(KindSummary, r.renderSummary) - reg.Register(KindIcon, r.renderIcon) reg.Register(ast.KindCodeSpan, r.renderCodeSpan) reg.Register(KindAttention, r.renderAttention) reg.Register(KindTaskCheckBoxListItem, r.renderTaskCheckBoxListItem) reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox) + reg.Register(KindRawHTML, r.renderRawHTML) } func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { @@ -155,7 +135,7 @@ func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast. if entering { _, err = w.WriteString("<div") if err == nil { - _, err = w.WriteString(fmt.Sprintf(` lang=%q`, val)) + _, err = fmt.Fprintf(w, ` lang=%q`, val) } if err == nil { _, err = w.WriteRune('>') @@ -206,30 +186,14 @@ func (r *HTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.N return ast.WalkContinue, nil } -func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *HTMLRenderer) renderRawHTML(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } - - n := node.(*Icon) - - name := strings.TrimSpace(strings.ToLower(string(n.Name))) - - if len(name) == 0 { - // skip this - return ast.WalkContinue, nil - } - - if !reValidIconName().MatchString(name) { - // skip this - return ast.WalkContinue, nil - } - - // FIXME: the "icon xxx" is from Fomantic UI, it's really questionable whether it still works correctly - err := r.renderInternal.FormatWithSafeAttrs(w, `<i class="icon %s"></i>`, name) + n := node.(*RawHTML) + _, err := w.WriteString(string(r.renderInternal.ProtectSafeAttrs(n.rawHTML))) if err != nil { return ast.WalkStop, err } - return ast.WalkContinue, nil } diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index b5fffccdb9..3b788432ba 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -5,7 +5,7 @@ package markdown import ( - "fmt" + "errors" "html/template" "io" "strings" @@ -48,7 +48,7 @@ func (l *limitWriter) Write(data []byte) (int, error) { if err != nil { return n, err } - return n, fmt.Errorf("rendered content too large - truncating render") + return n, errors.New("rendered content too large - truncating render") } n, err := l.w.Write(data) l.sum += int64(n) @@ -86,20 +86,15 @@ func (r *GlodmarkRender) highlightingRenderer(w util.BufWriter, c highlighting.C preClasses += " is-loading" } - err := r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<pre class="%s">`, preClasses) - if err != nil { - return - } - // include language-x class as part of commonmark spec, "chroma" class is used to highlight the code // the "display" class is used by "js/markup/math.ts" to render the code element as a block // the "math.ts" strictly depends on the structure: <pre class="code-block is-loading"><code class="language-math display">...</code></pre> - err = r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<code class="chroma language-%s display">`, languageStr) + err := r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<div class="code-block-container code-overflow-scroll"><pre class="%s"><code class="chroma language-%s display">`, preClasses, languageStr) if err != nil { return } } else { - _, err := w.WriteString("</code></pre>") + _, err := w.WriteString("</code></pre></div>") if err != nil { return } @@ -126,11 +121,11 @@ func SpecializedMarkdown(ctx *markup.RenderContext) *GlodmarkRender { highlighting.WithWrapperRenderer(r.highlightingRenderer), ), math.NewExtension(&ctx.RenderInternal, math.Options{ - Enabled: setting.Markdown.EnableMath, - ParseDollarInline: true, - ParseDollarBlock: true, - ParseSquareBlock: true, // TODO: this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping, it should be deprecated in the future (by some config options) - // ParseBracketInline: true, // TODO: this is also a bad syntax "\( ... \)", it also conflicts, it should be deprecated in the future + Enabled: setting.Markdown.EnableMath, + ParseInlineDollar: setting.Markdown.MathCodeBlockOptions.ParseInlineDollar, + ParseInlineParentheses: setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses, // this is a bad syntax "\( ... \)", it conflicts with normal markdown escaping + ParseBlockDollar: setting.Markdown.MathCodeBlockOptions.ParseBlockDollar, + ParseBlockSquareBrackets: setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets, // this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping }), meta.Meta, ), @@ -159,6 +154,14 @@ func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error limit: setting.UI.MaxDisplayFileSize * 3, } + // FIXME: Don't read all to memory, but goldmark doesn't support + buf, err := io.ReadAll(input) + if err != nil { + log.Error("Unable to ReadAll: %v", err) + return err + } + buf = giteautil.NormalizeEOL(buf) + // FIXME: should we include a timeout to abort the renderer if it takes too long? defer func() { err := recover() @@ -166,35 +169,20 @@ func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error return } - log.Warn("Unable to render markdown due to panic in goldmark: %v", err) - if (!setting.IsProd && !setting.IsInTesting) || log.IsDebug() { - log.Error("Panic in markdown: %v\n%s", err, log.Stack(2)) - } + log.Error("Panic in markdown: %v\n%s", err, log.Stack(2)) + escapedHTML := template.HTMLEscapeString(giteautil.UnsafeBytesToString(buf)) + _, _ = output.Write(giteautil.UnsafeStringToBytes(escapedHTML)) }() - // FIXME: Don't read all to memory, but goldmark doesn't support pc := newParserContext(ctx) - buf, err := io.ReadAll(input) - if err != nil { - log.Error("Unable to ReadAll: %v", err) - return err - } - buf = giteautil.NormalizeEOL(buf) // Preserve original length. bufWithMetadataLength := len(buf) - rc := &RenderConfig{ - Meta: markup.RenderMetaAsDetails, - Icon: "table", - Lang: "", - } + rc := &RenderConfig{Meta: markup.RenderMetaAsDetails} buf, _ = ExtractMetadataBytes(buf, rc) - metaLength := bufWithMetadataLength - len(buf) - if metaLength < 0 { - metaLength = 0 - } + metaLength := max(bufWithMetadataLength-len(buf), 0) rc.metaLength = metaLength pc.Set(renderConfigKey, rc) diff --git a/modules/markup/markdown/markdown_attention_test.go b/modules/markup/markdown/markdown_attention_test.go index f6ec775b2c..7b54653ec0 100644 --- a/modules/markup/markdown/markdown_attention_test.go +++ b/modules/markup/markdown/markdown_attention_test.go @@ -23,6 +23,11 @@ func TestAttention(t *testing.T) { defer svg.MockIcon("octicon-alert")() defer svg.MockIcon("octicon-stop")() + test := func(input, expected string) { + result, err := markdown.RenderString(markup.NewTestRenderContext(), input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(result))) + } renderAttention := func(attention, icon string) string { tmpl := `<blockquote class="attention-header attention-{attention}"><p><svg class="attention-icon attention-{attention} svg {icon}" width="16" height="16"></svg><strong class="attention-{attention}">{Attention}</strong></p>` tmpl = strings.ReplaceAll(tmpl, "{attention}", attention) @@ -31,12 +36,6 @@ func TestAttention(t *testing.T) { return tmpl } - test := func(input, expected string) { - result, err := markdown.RenderString(markup.NewTestRenderContext(), input) - assert.NoError(t, err) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(result))) - } - test(` > [!NOTE] > text @@ -53,4 +52,7 @@ func TestAttention(t *testing.T) { // legacy GitHub style test(`> **warning**`, renderAttention("warning", "octicon-alert")+"\n</blockquote>") + + // edge case (it used to cause panic) + test(">\ntext", "<blockquote>\n</blockquote>\n<p>text</p>") } diff --git a/modules/markup/markdown/markdown_benchmark_test.go b/modules/markup/markdown/markdown_benchmark_test.go index 0f7e3eea6f..e08612f064 100644 --- a/modules/markup/markdown/markdown_benchmark_test.go +++ b/modules/markup/markdown/markdown_benchmark_test.go @@ -12,14 +12,14 @@ import ( func BenchmarkSpecializedMarkdown(b *testing.B) { // 240856 4719 ns/op - for i := 0; i < b.N; i++ { + for b.Loop() { markdown.SpecializedMarkdown(&markup.RenderContext{}) } } func BenchmarkMarkdownRender(b *testing.B) { // 23202 50840 ns/op - for i := 0; i < b.N; i++ { + for b.Loop() { _, _ = markdown.RenderString(markup.NewTestRenderContext(), "https://example.com\n- a\n- b\n") } } diff --git a/modules/markup/markdown/markdown_math_test.go b/modules/markup/markdown/markdown_math_test.go index 813f050965..a75f18d36a 100644 --- a/modules/markup/markdown/markdown_math_test.go +++ b/modules/markup/markdown/markdown_math_test.go @@ -8,6 +8,8 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" ) @@ -15,6 +17,7 @@ import ( const nl = "\n" func TestMathRender(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseInlineParentheses: true} testcases := []struct { testcase string expected string @@ -69,7 +72,7 @@ func TestMathRender(t *testing.T) { }, { "$$a$$", - `<code class="language-math display">a</code>` + nl, + `<p><code class="language-math">a</code></p>` + nl, }, { "$$a$$ test", @@ -111,6 +114,7 @@ func TestMathRender(t *testing.T) { } func TestMathRenderBlockIndent(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseBlockDollar: true, ParseBlockSquareBrackets: true} testcases := []struct { name string testcase string @@ -243,3 +247,64 @@ x }) } } + +func TestMathRenderOptions(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{} + defer test.MockVariableValue(&setting.Markdown.MathCodeBlockOptions) + test := func(t *testing.T, expected, input string) { + res, err := RenderString(markup.NewTestRenderContext(), input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(res)), "input: %s", input) + } + + // default (non-conflict) inline syntax + test(t, `<p><code class="language-math">a</code></p>`, "$`a`$") + + // ParseInlineDollar + test(t, `<p>$a$</p>`, `$a$`) + setting.Markdown.MathCodeBlockOptions.ParseInlineDollar = true + test(t, `<p><code class="language-math">a</code></p>`, `$a$`) + + // ParseInlineParentheses + test(t, `<p>(a)</p>`, `\(a\)`) + setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses = true + test(t, `<p><code class="language-math">a</code></p>`, `\(a\)`) + + // ParseBlockDollar + test(t, `<p>$$ +a +$$</p> +`, ` +$$ +a +$$ +`) + setting.Markdown.MathCodeBlockOptions.ParseBlockDollar = true + test(t, `<pre class="code-block is-loading"><code class="language-math display"> +a +</code></pre> +`, ` +$$ +a +$$ +`) + + // ParseBlockSquareBrackets + test(t, `<p>[ +a +]</p> +`, ` +\[ +a +\] +`) + setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets = true + test(t, `<pre class="code-block is-loading"><code class="language-math display"> +a +</code></pre> +`, ` +\[ +a +\] +`) +} diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index 7a09be8665..4eb01bcc2d 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -47,7 +47,7 @@ func TestRender_StandardLinks(t *testing.T) { func TestRender_Images(t *testing.T) { setting.AppURL = AppURL - test := func(input, expected string) { + render := func(input, expected string) { buffer, err := markdown.RenderString(markup.NewTestRenderContext(FullURL), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) @@ -59,27 +59,32 @@ func TestRender_Images(t *testing.T) { result := util.URLJoin(FullURL, url) // hint: With Markdown v2.5.2, there is a new syntax: [link](URL){:target="_blank"} , but we do not support it now - test( + render( "!["+title+"]("+url+")", `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "[["+title+"|"+url+"]]", `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) - test( + render( "[!["+title+"]("+url+")]("+href+")", `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "!["+title+"]("+url+")", `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "[["+title+"|"+url+"]]", `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) - test( + render( "[!["+title+"]("+url+")]("+href+")", `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) + + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, false)() + render( + "<a><img src='a.jpg'></a>", // by the way, empty "a" tag will be removed + `<p dir="auto"><img src="http://localhost:3000/user13/repo11/a.jpg" loading="lazy"/></p>`) } func TestTotal_RenderString(t *testing.T) { @@ -223,7 +228,7 @@ This PR has been generated by [Renovate Bot](https://github.com/renovatebot/reno <dd>This is another definition of the second term.</dd> </dl> <h3 id="user-content-footnotes">Footnotes</h3> -<p>Here is a simple footnote,<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1</a></sup> and here is a longer one.<sup id="fnref:user-content-bignote"><a href="#fn:user-content-bignote" rel="nofollow">2</a></sup></p> +<p>Here is a simple footnote,<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1 </a></sup> and here is a longer one.<sup id="fnref:user-content-bignote"><a href="#fn:user-content-bignote" rel="nofollow">2 </a></sup></p> <div> <hr/> <ol> @@ -252,7 +257,7 @@ This PR has been generated by [Renovate Bot](https://github.com/renovatebot/reno return username == "r-lyeh" }, }) - for i := 0; i < len(sameCases); i++ { + for i := range sameCases { line, err := markdown.RenderString(markup.NewTestRenderContext(localMetas), sameCases[i]) assert.NoError(t, err) assert.Equal(t, testAnswers[i], string(line)) @@ -308,12 +313,12 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) { testcase := `  ` - expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"></a> -<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"></a></p> + expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"/></a> +<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"/></a></p> ` - res, err := markdown.RenderRawString(markup.NewTestRenderContext(), testcase) + res, err := markdown.RenderString(markup.NewTestRenderContext(), testcase) assert.NoError(t, err) - assert.Equal(t, expected, res) + assert.Equal(t, expected, string(res)) } func TestRenderEmojiInLinks_Issue12331(t *testing.T) { @@ -383,18 +388,74 @@ func TestColorPreview(t *testing.T) { } } -func TestTaskList(t *testing.T) { +func TestMarkdownFrontmatter(t *testing.T) { testcases := []struct { - testcase string + name string + input string expected string }{ { + "MapInFrontmatter", + `--- +key1: val1 +key2: val2 +--- +test +`, + `<details class="frontmatter-content"><summary><span>octicon-table(12/)</span> key1, key2</summary><table> +<thead> +<tr> +<th>key1</th> +<th>key2</th> +</tr> +</thead> +<tbody> +<tr> +<td>val1</td> +<td>val2</td> +</tr> +</tbody> +</table> +</details><p>test</p> +`, + }, + + { + "ListInFrontmatter", + `--- +- item1 +- item2 +--- +test +`, + `- item1 +- item2 + +<p>test</p> +`, + }, + + { + "StringInFrontmatter", + `--- +anything +--- +test +`, + `anything + +<p>test</p> +`, + }, + + { // data-source-position should take into account YAML frontmatter. + "ListAfterFrontmatter", `--- foo: bar --- - [ ] task 1`, - `<details><summary><i class="icon table"></i></summary><table> + `<details class="frontmatter-content"><summary><span>octicon-table(12/)</span> foo</summary><table> <thead> <tr> <th>foo</th> @@ -414,9 +475,9 @@ foo: bar } for _, test := range testcases { - res, err := markdown.RenderString(markup.NewTestRenderContext(), test.testcase) - assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) - assert.Equal(t, template.HTML(test.expected), res, "Unexpected result in testcase %q", test.testcase) + res, err := markdown.RenderString(markup.NewTestRenderContext(), test.input) + assert.NoError(t, err, "Unexpected error in testcase: %q", test.name) + assert.Equal(t, test.expected, string(res), "Unexpected result in testcase %q", test.name) } } @@ -473,3 +534,16 @@ space</p> assert.NoError(t, err) assert.Equal(t, expected, string(result)) } + +func TestMarkdownLink(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + input := `<a href=foo>link1</a> +<a href='/foo'>link2</a> +<a href="#foo">link3</a>` + result, err := markdown.RenderString(markup.NewTestRenderContext("/base", localMetas), input) + assert.NoError(t, err) + assert.Equal(t, `<p><a href="/base/foo" rel="nofollow">link1</a> +<a href="/base/foo" rel="nofollow">link2</a> +<a href="#user-content-foo" rel="nofollow">link3</a></p> +`, string(result)) +} diff --git a/modules/markup/markdown/math/block_renderer.go b/modules/markup/markdown/math/block_renderer.go index 412e4d0dee..95a336a02c 100644 --- a/modules/markup/markdown/math/block_renderer.go +++ b/modules/markup/markdown/math/block_renderer.go @@ -42,7 +42,7 @@ func (r *BlockRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { func (r *BlockRenderer) writeLines(w util.BufWriter, source []byte, n gast.Node) { l := n.Lines().Len() - for i := 0; i < l; i++ { + for i := range l { line := n.Lines().At(i) _, _ = w.Write(util.EscapeHTML(line.Value(source))) } @@ -51,8 +51,8 @@ func (r *BlockRenderer) writeLines(w util.BufWriter, source []byte, n gast.Node) func (r *BlockRenderer) renderBlock(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { n := node.(*Block) if entering { - code := giteaUtil.Iif(n.Inline, "", `<pre class="code-block is-loading">`) + `<code class="language-math display">` - _ = r.renderInternal.FormatWithSafeAttrs(w, template.HTML(code)) + codeHTML := giteaUtil.Iif[template.HTML](n.Inline, "", `<pre class="code-block is-loading">`) + `<code class="language-math display">` + _, _ = w.WriteString(string(r.renderInternal.ProtectSafeAttrs(codeHTML))) r.writeLines(w, source, n) } else { _, _ = w.WriteString(`</code>` + giteaUtil.Iif(n.Inline, "", `</pre>`) + "\n") diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go index a57abe9f9b..a711d1e1cd 100644 --- a/modules/markup/markdown/math/inline_parser.go +++ b/modules/markup/markdown/math/inline_parser.go @@ -15,26 +15,26 @@ type inlineParser struct { trigger []byte endBytesSingleDollar []byte endBytesDoubleDollar []byte - endBytesBracket []byte + endBytesParentheses []byte + enableInlineDollar bool } -var defaultInlineDollarParser = &inlineParser{ - trigger: []byte{'$'}, - endBytesSingleDollar: []byte{'$'}, - endBytesDoubleDollar: []byte{'$', '$'}, -} - -func NewInlineDollarParser() parser.InlineParser { - return defaultInlineDollarParser +func NewInlineDollarParser(enableInlineDollar bool) parser.InlineParser { + return &inlineParser{ + trigger: []byte{'$'}, + endBytesSingleDollar: []byte{'$'}, + endBytesDoubleDollar: []byte{'$', '$'}, + enableInlineDollar: enableInlineDollar, + } } -var defaultInlineBracketParser = &inlineParser{ - trigger: []byte{'\\', '('}, - endBytesBracket: []byte{'\\', ')'}, +var defaultInlineParenthesesParser = &inlineParser{ + trigger: []byte{'\\', '('}, + endBytesParentheses: []byte{'\\', ')'}, } -func NewInlineBracketParser() parser.InlineParser { - return defaultInlineBracketParser +func NewInlineParenthesesParser() parser.InlineParser { + return defaultInlineParenthesesParser } // Trigger triggers this parser on $ or \ @@ -46,7 +46,7 @@ func isPunctuation(b byte) bool { return b == '.' || b == '!' || b == '?' || b == ',' || b == ';' || b == ':' } -func isBracket(b byte) bool { +func isParenthesesClose(b byte) bool { return b == ')' } @@ -70,10 +70,11 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. startMarkLen = 1 stopMark = parser.endBytesSingleDollar if len(line) > 1 { - if line[1] == '$' { + switch line[1] { + case '$': startMarkLen = 2 stopMark = parser.endBytesDoubleDollar - } else if line[1] == '`' { + case '`': pos := 1 for ; pos < len(line) && line[pos] == '`'; pos++ { } @@ -85,7 +86,11 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. } } else { startMarkLen = 2 - stopMark = parser.endBytesBracket + stopMark = parser.endBytesParentheses + } + + if line[0] == '$' && !parser.enableInlineDollar && (len(line) == 1 || line[1] != '`') { + return nil } if checkSurrounding { @@ -109,7 +114,7 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. succeedingCharacter = line[i+len(stopMark)] } // check valid ending character - isValidEndingChar := isPunctuation(succeedingCharacter) || isBracket(succeedingCharacter) || + isValidEndingChar := isPunctuation(succeedingCharacter) || isParenthesesClose(succeedingCharacter) || succeedingCharacter == ' ' || succeedingCharacter == '\n' || succeedingCharacter == 0 if checkSurrounding && !isValidEndingChar { break @@ -121,9 +126,10 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. i++ continue } - if line[i] == '{' { + switch line[i] { + case '{': depth++ - } else if line[i] == '}' { + case '}': depth-- } } diff --git a/modules/markup/markdown/math/inline_renderer.go b/modules/markup/markdown/math/inline_renderer.go index d000a7b317..eeeb60cc7e 100644 --- a/modules/markup/markdown/math/inline_renderer.go +++ b/modules/markup/markdown/math/inline_renderer.go @@ -28,7 +28,7 @@ func NewInlineRenderer(renderInternal *internal.RenderInternal) renderer.NodeRen func (r *InlineRenderer) renderInline(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { if entering { - _ = r.renderInternal.FormatWithSafeAttrs(w, `<code class="language-math">`) + _, _ = w.WriteString(string(r.renderInternal.ProtectSafeAttrs(`<code class="language-math">`))) for c := n.FirstChild(); c != nil; c = c.NextSibling() { segment := c.(*ast.Text).Segment value := util.EscapeHTML(segment.Value(source)) diff --git a/modules/markup/markdown/math/math.go b/modules/markup/markdown/math/math.go index a6ff593d62..4b74db2d76 100644 --- a/modules/markup/markdown/math/math.go +++ b/modules/markup/markdown/math/math.go @@ -14,10 +14,11 @@ import ( ) type Options struct { - Enabled bool - ParseDollarInline bool - ParseDollarBlock bool - ParseSquareBlock bool + Enabled bool + ParseInlineDollar bool // inline $$ xxx $$ text + ParseInlineParentheses bool // inline \( xxx \) text + ParseBlockDollar bool // block $$ multiple-line $$ text + ParseBlockSquareBrackets bool // block \[ multiple-line \] text } // Extension is a math extension @@ -42,16 +43,16 @@ func (e *Extension) Extend(m goldmark.Markdown) { return } - inlines := []util.PrioritizedValue{util.Prioritized(NewInlineBracketParser(), 501)} - if e.options.ParseDollarInline { - inlines = append(inlines, util.Prioritized(NewInlineDollarParser(), 502)) + var inlines []util.PrioritizedValue + if e.options.ParseInlineParentheses { + inlines = append(inlines, util.Prioritized(NewInlineParenthesesParser(), 501)) } - m.Parser().AddOptions(parser.WithInlineParsers(inlines...)) + inlines = append(inlines, util.Prioritized(NewInlineDollarParser(e.options.ParseInlineDollar), 502)) + m.Parser().AddOptions(parser.WithInlineParsers(inlines...)) m.Parser().AddOptions(parser.WithBlockParsers( - util.Prioritized(NewBlockParser(e.options.ParseDollarBlock, e.options.ParseSquareBlock), 701), + util.Prioritized(NewBlockParser(e.options.ParseBlockDollar, e.options.ParseBlockSquareBrackets), 701), )) - m.Renderer().AddOptions(renderer.WithNodeRenderers( util.Prioritized(NewBlockRenderer(e.renderInternal), 501), util.Prioritized(NewInlineRenderer(e.renderInternal), 502), diff --git a/modules/markup/markdown/meta_test.go b/modules/markup/markdown/meta_test.go index 278c33f1d2..283d289d48 100644 --- a/modules/markup/markdown/meta_test.go +++ b/modules/markup/markdown/meta_test.go @@ -51,7 +51,7 @@ func TestExtractMetadata(t *testing.T) { var meta IssueTemplate body, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) assert.NoError(t, err) - assert.Equal(t, "", body) + assert.Empty(t, body) assert.Equal(t, metaTest, meta) assert.True(t, meta.Valid()) }) @@ -60,7 +60,7 @@ func TestExtractMetadata(t *testing.T) { func TestExtractMetadataBytes(t *testing.T) { t.Run("ValidFrontAndBody", func(t *testing.T) { var meta IssueTemplate - body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta) + body, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest), &meta) assert.NoError(t, err) assert.Equal(t, bodyTest, string(body)) assert.Equal(t, metaTest, meta) @@ -69,21 +69,21 @@ func TestExtractMetadataBytes(t *testing.T) { t.Run("NoFirstSeparator", func(t *testing.T) { var meta IssueTemplate - _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", frontTest, sepTest, bodyTest)), &meta) + _, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", frontTest, sepTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoLastSeparator", func(t *testing.T) { var meta IssueTemplate - _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, bodyTest)), &meta) + _, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", sepTest, frontTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoBody", func(t *testing.T) { var meta IssueTemplate - body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta) + body, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) assert.NoError(t, err) - assert.Equal(t, "", string(body)) + assert.Empty(t, string(body)) assert.Equal(t, metaTest, meta) assert.True(t, meta.Valid()) }) diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index f4c48d1b3d..d8b1b10ce6 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -16,7 +16,6 @@ import ( // RenderConfig represents rendering configuration for this file type RenderConfig struct { Meta markup.RenderMetaMode - Icon string TOC string // "false": hide, "side"/empty: in sidebar, "main"/"true": in main view Lang string yamlNode *yaml.Node @@ -74,7 +73,7 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { type yamlRenderConfig struct { Meta *string `yaml:"meta"` - Icon *string `yaml:"details_icon"` + Icon *string `yaml:"details_icon"` // deprecated, because there is no font icon, so no custom icon TOC *string `yaml:"include_toc"` Lang *string `yaml:"lang"` } @@ -96,10 +95,6 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { rc.Meta = renderMetaModeFromString(*cfg.Gitea.Meta) } - if cfg.Gitea.Icon != nil { - rc.Icon = strings.TrimSpace(strings.ToLower(*cfg.Gitea.Icon)) - } - if cfg.Gitea.Lang != nil && *cfg.Gitea.Lang != "" { rc.Lang = *cfg.Gitea.Lang } @@ -111,7 +106,7 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { return nil } -func (rc *RenderConfig) toMetaNode() ast.Node { +func (rc *RenderConfig) toMetaNode(g *ASTTransformer) ast.Node { if rc.yamlNode == nil { return nil } @@ -119,7 +114,7 @@ func (rc *RenderConfig) toMetaNode() ast.Node { case markup.RenderMetaAsTable: return nodeToTable(rc.yamlNode) case markup.RenderMetaAsDetails: - return nodeToDetails(rc.yamlNode, rc.Icon) + return nodeToDetails(g, rc.yamlNode) default: return nil } diff --git a/modules/markup/markdown/renderconfig_test.go b/modules/markup/markdown/renderconfig_test.go index 13346570fa..53c52177a7 100644 --- a/modules/markup/markdown/renderconfig_test.go +++ b/modules/markup/markdown/renderconfig_test.go @@ -21,42 +21,36 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "empty", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", }, "", }, { "lang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "test", }, "lang: test", }, { "metatable", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", }, "gitea: table", }, { "metanone", &RenderConfig{ Meta: "none", - Icon: "table", Lang: "", }, "gitea: none", }, { "metadetails", &RenderConfig{ Meta: "details", - Icon: "table", Lang: "", }, "gitea: details", }, { "metawrong", &RenderConfig{ Meta: "details", - Icon: "table", Lang: "", }, "gitea: wrong", }, @@ -64,7 +58,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { "toc", &RenderConfig{ TOC: "true", Meta: "table", - Icon: "table", Lang: "", }, "include_toc: true", }, @@ -72,14 +65,12 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { "tocfalse", &RenderConfig{ TOC: "false", Meta: "table", - Icon: "table", Lang: "", }, "include_toc: false", }, { "toclang", &RenderConfig{ Meta: "table", - Icon: "table", TOC: "true", Lang: "testlang", }, ` @@ -90,7 +81,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` gitea: @@ -100,7 +90,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang2", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` lang: notright @@ -111,7 +100,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` gitea: @@ -123,7 +111,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Lang: "two", Meta: "table", TOC: "true", - Icon: "smiley", }, ` lang: one include_toc: true @@ -139,14 +126,12 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got := &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", } err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got) require.NoError(t, err) assert.Equal(t, tt.expected.Meta, got.Meta) - assert.Equal(t, tt.expected.Icon, got.Icon) assert.Equal(t, tt.expected.Lang, got.Lang) assert.Equal(t, tt.expected.TOC, got.TOC) }) diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go index ea1af83a3e..a11b9d0390 100644 --- a/modules/markup/markdown/toc.go +++ b/modules/markup/markdown/toc.go @@ -4,7 +4,6 @@ package markdown import ( - "fmt" "net/url" "code.gitea.io/gitea/modules/translation" @@ -50,7 +49,7 @@ func createTOCNode(toc []Header, lang string, detailsAttrs map[string]string) as } li := ast.NewListItem(currentLevel * 2) a := ast.NewLink() - a.Destination = []byte(fmt.Sprintf("#%s", url.QueryEscape(header.ID))) + a.Destination = []byte("#" + url.QueryEscape(header.ID)) a.AppendChild(a, ast.NewString([]byte(header.Text))) li.AppendChild(li, a) ul.AppendChild(ul, li) diff --git a/modules/markup/markdown/transform_blockquote.go b/modules/markup/markdown/transform_blockquote.go index 2651d44a69..bf17f01681 100644 --- a/modules/markup/markdown/transform_blockquote.go +++ b/modules/markup/markdown/transform_blockquote.go @@ -46,7 +46,7 @@ func (g *ASTTransformer) extractBlockquoteAttentionEmphasis(firstParagraph ast.N if !ok { return "", nil } - val1 := string(node1.Text(reader.Source())) //nolint:staticcheck + val1 := string(node1.Text(reader.Source())) //nolint:staticcheck // Text is deprecated attentionType := strings.ToLower(val1) if g.attentionTypes.Contains(attentionType) { return attentionType, []ast.Node{node1} @@ -115,6 +115,9 @@ func (g *ASTTransformer) transformBlockquote(v *ast.Blockquote, reader text.Read // grab these nodes and make sure we adhere to the attention blockquote structure firstParagraph := v.FirstChild() + if firstParagraph == nil { + return ast.WalkContinue, nil + } g.applyElementDir(firstParagraph) attentionType, processedNodes := g.extractBlockquoteAttentionEmphasis(firstParagraph, reader) diff --git a/modules/markup/markdown/transform_codespan.go b/modules/markup/markdown/transform_codespan.go index bccc43aad2..c2e4295bc2 100644 --- a/modules/markup/markdown/transform_codespan.go +++ b/modules/markup/markdown/transform_codespan.go @@ -68,7 +68,7 @@ func cssColorHandler(value string) bool { } func (g *ASTTransformer) transformCodeSpan(_ *markup.RenderContext, v *ast.CodeSpan, reader text.Reader) { - colorContent := v.Text(reader.Source()) //nolint:staticcheck + colorContent := v.Text(reader.Source()) //nolint:staticcheck // Text is deprecated if cssColorHandler(string(colorContent)) { v.AppendChild(v, NewColorPreview(colorContent)) } diff --git a/modules/markup/markdown/transform_heading.go b/modules/markup/markdown/transform_heading.go index 5f8a12794d..a229a7b1a4 100644 --- a/modules/markup/markdown/transform_heading.go +++ b/modules/markup/markdown/transform_heading.go @@ -16,10 +16,10 @@ import ( func (g *ASTTransformer) transformHeading(_ *markup.RenderContext, v *ast.Heading, reader text.Reader, tocList *[]Header) { for _, attr := range v.Attributes() { if _, ok := attr.Value.([]byte); !ok { - v.SetAttribute(attr.Name, []byte(fmt.Sprintf("%v", attr.Value))) + v.SetAttribute(attr.Name, fmt.Appendf(nil, "%v", attr.Value)) } } - txt := v.Text(reader.Source()) //nolint:staticcheck + txt := v.Text(reader.Source()) //nolint:staticcheck // Text is deprecated header := Header{ Text: util.UnsafeBytesToString(txt), Level: v.Level, diff --git a/modules/markup/markdown/transform_image.go b/modules/markup/markdown/transform_image.go deleted file mode 100644 index 36512e59a8..0000000000 --- a/modules/markup/markdown/transform_image.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markdown - -import ( - "code.gitea.io/gitea/modules/markup" - - "github.com/yuin/goldmark/ast" -) - -func (g *ASTTransformer) transformImage(ctx *markup.RenderContext, v *ast.Image) { - // Images need two things: - // - // 1. Their src needs to munged to be a real value - // 2. If they're not wrapped with a link they need a link wrapper - - // Check if the destination is a real link - if len(v.Destination) > 0 && !markup.IsFullURLBytes(v.Destination) { - v.Destination = []byte(ctx.RenderHelper.ResolveLink(string(v.Destination), markup.LinkTypeMedia)) - } - - parent := v.Parent() - // Create a link around image only if parent is not already a link - if _, ok := parent.(*ast.Link); !ok && parent != nil { - next := v.NextSibling() - - // Create a link wrapper - wrap := ast.NewLink() - wrap.Destination = v.Destination - wrap.Title = v.Title - wrap.SetAttributeString("target", []byte("_blank")) - - // Duplicate the current image node - image := ast.NewImage(ast.NewLink()) - image.Destination = v.Destination - image.Title = v.Title - for _, attr := range v.Attributes() { - image.SetAttribute(attr.Name, attr.Value) - } - for child := v.FirstChild(); child != nil; { - next := child.NextSibling() - image.AppendChild(image, child) - child = next - } - - // Append our duplicate image to the wrapper link - wrap.AppendChild(wrap, image) - - // Wire in the next sibling - wrap.SetNextSibling(next) - - // Replace the current node with the wrapper link - parent.ReplaceChild(parent, v, wrap) - - // But most importantly ensure the next sibling is still on the old image too - v.SetNextSibling(next) - } -} diff --git a/modules/markup/markdown/transform_link.go b/modules/markup/markdown/transform_link.go deleted file mode 100644 index 51c2c915d8..0000000000 --- a/modules/markup/markdown/transform_link.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markdown - -import ( - "code.gitea.io/gitea/modules/markup" - - "github.com/yuin/goldmark/ast" -) - -func resolveLink(ctx *markup.RenderContext, link, userContentAnchorPrefix string) (result string, resolved bool) { - isAnchorFragment := link != "" && link[0] == '#' - if !isAnchorFragment && !markup.IsFullURLString(link) { - link, resolved = ctx.RenderHelper.ResolveLink(link, markup.LinkTypeDefault), true - } - if isAnchorFragment && userContentAnchorPrefix != "" { - link, resolved = userContentAnchorPrefix+link[1:], true - } - return link, resolved -} - -func (g *ASTTransformer) transformLink(ctx *markup.RenderContext, v *ast.Link) { - if link, resolved := resolveLink(ctx, string(v.Destination), "#user-content-"); resolved { - v.Destination = []byte(link) - } -} diff --git a/modules/markup/mdstripper/mdstripper.go b/modules/markup/mdstripper/mdstripper.go index fe0eabb473..6e392444b4 100644 --- a/modules/markup/mdstripper/mdstripper.go +++ b/modules/markup/mdstripper/mdstripper.go @@ -46,7 +46,7 @@ func (r *stripRenderer) Render(w io.Writer, source []byte, doc ast.Node) error { coalesce := prevSibIsText r.processString( w, - v.Text(source), //nolint:staticcheck + v.Text(source), //nolint:staticcheck // Text is deprecated coalesce) if v.SoftLineBreak() { r.doubleSpace(w) @@ -107,11 +107,12 @@ func (r *stripRenderer) processAutoLink(w io.Writer, link []byte) { } var sep string - if parts[3] == "issues" { + switch parts[3] { + case "issues": sep = "#" - } else if parts[3] == "pulls" { + case "pulls": sep = "!" - } else { + default: // Process out of band r.links = append(r.links, linkStr) return diff --git a/modules/markup/mdstripper/mdstripper_test.go b/modules/markup/mdstripper/mdstripper_test.go index ea34df0a3b..7fb49c1e01 100644 --- a/modules/markup/mdstripper/mdstripper_test.go +++ b/modules/markup/mdstripper/mdstripper_test.go @@ -79,7 +79,7 @@ A HIDDEN ` + "`" + `GHOST` + "`" + ` IN THIS LINE. lines = append(lines, line) } } - assert.EqualValues(t, test.expectedText, lines) - assert.EqualValues(t, test.expectedLinks, links) + assert.Equal(t, test.expectedText, lines) + assert.Equal(t, test.expectedLinks, links) } } diff --git a/modules/markup/orgmode/orgmode.go b/modules/markup/orgmode/orgmode.go index 70d02c1321..93c335d244 100644 --- a/modules/markup/orgmode/orgmode.go +++ b/modules/markup/orgmode/orgmode.go @@ -1,7 +1,7 @@ // Copyright 2017 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package markup +package orgmode import ( "fmt" @@ -125,27 +125,13 @@ type orgWriter struct { var _ org.Writer = (*orgWriter)(nil) -func (r *orgWriter) resolveLink(kind, link string) string { - link = strings.TrimPrefix(link, "file:") - if !strings.HasPrefix(link, "#") && // not a URL fragment - !markup.IsFullURLString(link) { - if kind == "regular" { - // orgmode reports the link kind as "regular" for "[[ImageLink.svg][The Image Desc]]" - // so we need to try to guess the link kind again here - kind = org.RegularLink{URL: link}.Kind() - } - if kind == "image" || kind == "video" { - link = r.rctx.RenderHelper.ResolveLink(link, markup.LinkTypeMedia) - } else { - link = r.rctx.RenderHelper.ResolveLink(link, markup.LinkTypeDefault) - } - } - return link +func (r *orgWriter) resolveLink(link string) string { + return strings.TrimPrefix(link, "file:") } // WriteRegularLink renders images, links or videos func (r *orgWriter) WriteRegularLink(l org.RegularLink) { - link := r.resolveLink(l.Kind(), l.URL) + link := r.resolveLink(l.URL) printHTML := func(html template.HTML, a ...any) { _, _ = fmt.Fprint(r, htmlutil.HTMLFormat(html, a...)) @@ -156,14 +142,14 @@ func (r *orgWriter) WriteRegularLink(l org.RegularLink) { if l.Description == nil { printHTML(`<img src="%s" alt="%s">`, link, link) } else { - imageSrc := r.resolveLink(l.Kind(), org.String(l.Description...)) + imageSrc := r.resolveLink(org.String(l.Description...)) printHTML(`<a href="%s"><img src="%s" alt="%s"></a>`, link, imageSrc, imageSrc) } case "video": if l.Description == nil { printHTML(`<video src="%s">%s</video>`, link, link) } else { - videoSrc := r.resolveLink(l.Kind(), org.String(l.Description...)) + videoSrc := r.resolveLink(org.String(l.Description...)) printHTML(`<a href="%s"><video src="%s">%s</video></a>`, link, videoSrc, videoSrc) } default: diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go index de39bafebe..df4bb38ad1 100644 --- a/modules/markup/orgmode/orgmode_test.go +++ b/modules/markup/orgmode/orgmode_test.go @@ -1,7 +1,7 @@ // Copyright 2017 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package markup +package orgmode_test import ( "os" @@ -9,6 +9,7 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/orgmode" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" @@ -22,7 +23,7 @@ func TestMain(m *testing.M) { func TestRender_StandardLinks(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("/relative-path/media/branch/main/"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -30,37 +31,37 @@ func TestRender_StandardLinks(t *testing.T) { test("[[https://google.com/]]", `<p><a href="https://google.com/">https://google.com/</a></p>`) test("[[ImageLink.svg][The Image Desc]]", - `<p><a href="/relative-path/media/branch/main/ImageLink.svg">The Image Desc</a></p>`) + `<p><a href="ImageLink.svg">The Image Desc</a></p>`) } func TestRender_InternalLinks(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("/relative-path/src/branch/main"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } test("[[file:test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="test.org">Test</a></p>`) test("[[./test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="./test.org">Test</a></p>`) test("[[test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="test.org">Test</a></p>`) test("[[path/to/test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/path/to/test.org">Test</a></p>`) + `<p><a href="path/to/test.org">Test</a></p>`) } func TestRender_Media(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("./relative-path"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } test("[[file:../../.images/src/02/train.jpg]]", - `<p><img src=".images/src/02/train.jpg" alt=".images/src/02/train.jpg"></p>`) + `<p><img src="../../.images/src/02/train.jpg" alt="../../.images/src/02/train.jpg"></p>`) test("[[file:train.jpg]]", - `<p><img src="relative-path/train.jpg" alt="relative-path/train.jpg"></p>`) + `<p><img src="train.jpg" alt="train.jpg"></p>`) // With description. test("[[https://example.com][https://example.com/example.svg]]", @@ -91,7 +92,7 @@ func TestRender_Media(t *testing.T) { func TestRender_Source(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext(), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } diff --git a/modules/markup/render.go b/modules/markup/render.go index 37a2a86687..79f1f473c2 100644 --- a/modules/markup/render.go +++ b/modules/markup/render.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/url" + "strconv" "strings" "time" @@ -46,7 +47,7 @@ type RenderOptions struct { // user&repo, format&style®exp (for external issue pattern), teams&org (for mention) // RefTypeNameSubURL (for iframe&asciicast) // markupAllowShortIssuePattern - // markdownLineBreakStyle (comment, document) + // markdownNewLineHardBreak Metas map[string]string // used by external render. the router "/org/repo/render/..." will output the rendered content in a standalone page @@ -247,7 +248,8 @@ func Init(renderHelpFuncs *RenderHelperFuncs) { } func ComposeSimpleDocumentMetas() map[string]string { - return map[string]string{"markdownLineBreakStyle": "document"} + // TODO: there is no separate config option for "simple document" rendering, so temporarily use the same config as "repo file" + return map[string]string{"markdownNewLineHardBreak": strconv.FormatBool(setting.Markdown.RenderOptionsRepoFile.NewLineHardBreak)} } type TestRenderHelper struct { @@ -261,8 +263,14 @@ func (r *TestRenderHelper) IsCommitIDExisting(commitID string) bool { return strings.HasPrefix(commitID, "65f1bf2") //|| strings.HasPrefix(commitID, "88fc37a") } -func (r *TestRenderHelper) ResolveLink(link string, likeType LinkType) string { - return r.ctx.ResolveLinkRelative(r.BaseLink, "", link) +func (r *TestRenderHelper) ResolveLink(link, preferLinkType string) string { + linkType, link := ParseRenderedLink(link, preferLinkType) + switch linkType { + case LinkTypeRoot: + return r.ctx.ResolveLinkRoot(link) + default: + return r.ctx.ResolveLinkRelative(r.BaseLink, "", link) + } } var _ RenderHelper = (*TestRenderHelper)(nil) diff --git a/modules/markup/render_helper.go b/modules/markup/render_helper.go index 8ff0e7d6fb..b16f1189c5 100644 --- a/modules/markup/render_helper.go +++ b/modules/markup/render_helper.go @@ -10,13 +10,11 @@ import ( "code.gitea.io/gitea/modules/setting" ) -type LinkType string - const ( - LinkTypeApp LinkType = "app" // the link is relative to the AppSubURL - LinkTypeDefault LinkType = "default" // the link is relative to the default base (eg: repo link, or current ref tree path) - LinkTypeMedia LinkType = "media" // the link should be used to access media files (images, videos) - LinkTypeRaw LinkType = "raw" // not really useful, mainly for environment GITEA_PREFIX_RAW for external renders + LinkTypeDefault = "" + LinkTypeRoot = "/:root" // the link is relative to the AppSubURL(ROOT_URL) + LinkTypeMedia = "/:media" // the link should be used to access media files (images, videos) + LinkTypeRaw = "/:raw" // not really useful, mainly for environment GITEA_PREFIX_RAW for external renders ) type RenderHelper interface { @@ -27,7 +25,7 @@ type RenderHelper interface { // but not make processors to guess "is it rendering a comment or a wiki?" or "does it need to check commit ID?" IsCommitIDExisting(commitID string) bool - ResolveLink(link string, likeType LinkType) string + ResolveLink(link, preferLinkType string) string } // RenderHelperFuncs is used to decouple cycle-import @@ -51,7 +49,8 @@ func (r *SimpleRenderHelper) IsCommitIDExisting(commitID string) bool { return false } -func (r *SimpleRenderHelper) ResolveLink(link string, likeType LinkType) string { +func (r *SimpleRenderHelper) ResolveLink(link, preferLinkType string) string { + _, link = ParseRenderedLink(link, preferLinkType) return resolveLinkRelative(context.Background(), setting.AppSubURL+"/", "", link, false) } diff --git a/modules/markup/render_link.go b/modules/markup/render_link.go index b2e0699681..046544ce81 100644 --- a/modules/markup/render_link.go +++ b/modules/markup/render_link.go @@ -33,10 +33,24 @@ func resolveLinkRelative(ctx context.Context, base, cur, link string, absolute b return finalLink } -func (ctx *RenderContext) ResolveLinkRelative(base, cur, link string) (finalLink string) { +func (ctx *RenderContext) ResolveLinkRelative(base, cur, link string) string { + if strings.HasPrefix(link, "/:") { + setting.PanicInDevOrTesting("invalid link %q, forgot to cut?", link) + } return resolveLinkRelative(ctx, base, cur, link, ctx.RenderOptions.UseAbsoluteLink) } -func (ctx *RenderContext) ResolveLinkApp(link string) string { +func (ctx *RenderContext) ResolveLinkRoot(link string) string { return ctx.ResolveLinkRelative(setting.AppSubURL+"/", "", link) } + +func ParseRenderedLink(s, preferLinkType string) (linkType, link string) { + if strings.HasPrefix(s, "/:") { + p := strings.IndexByte(s[1:], '/') + if p == -1 { + return s, "" + } + return s[:p+1], s[p+2:] + } + return preferLinkType, s +} diff --git a/modules/markup/render_link_test.go b/modules/markup/render_link_test.go index c904ec7f18..972e15308c 100644 --- a/modules/markup/render_link_test.go +++ b/modules/markup/render_link_test.go @@ -4,7 +4,6 @@ package markup import ( - "context" "testing" "code.gitea.io/gitea/modules/setting" @@ -13,7 +12,7 @@ import ( ) func TestResolveLinkRelative(t *testing.T) { - ctx := context.Background() + ctx := t.Context() setting.AppURL = "http://localhost:3000" assert.Equal(t, "/a", resolveLinkRelative(ctx, "/a", "", "", false)) assert.Equal(t, "/a/b", resolveLinkRelative(ctx, "/a", "b", "", false)) diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 35f90eb46c..b6e9c348b7 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -4,12 +4,12 @@ package markup import ( - "bytes" "io" "path" "strings" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" ) // Renderer defines an interface for rendering markup file to HTML @@ -37,7 +37,7 @@ type ExternalRenderer interface { // RendererContentDetector detects if the content can be rendered // by specified renderer type RendererContentDetector interface { - CanRender(filename string, input io.Reader) bool + CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool } var ( @@ -60,13 +60,9 @@ func GetRendererByFileName(filename string) Renderer { } // DetectRendererType detects the markup type of the content -func DetectRendererType(filename string, input io.Reader) string { - buf, err := io.ReadAll(input) - if err != nil { - return "" - } +func DetectRendererType(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) string { for _, renderer := range renderers { - if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, bytes.NewReader(buf)) { + if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, sniffedType, prefetchBuf) { return renderer.Name() } } diff --git a/modules/markup/renderer_test.go b/modules/markup/renderer_test.go deleted file mode 100644 index 0791081f94..0000000000 --- a/modules/markup/renderer_test.go +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markup_test diff --git a/modules/markup/sanitizer_default.go b/modules/markup/sanitizer_default.go index 14161eb533..0fbf0f0b24 100644 --- a/modules/markup/sanitizer_default.go +++ b/modules/markup/sanitizer_default.go @@ -4,6 +4,7 @@ package markup import ( + "html/template" "io" "net/url" "regexp" @@ -52,6 +53,8 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy { policy.AllowAttrs("src", "autoplay", "controls").OnElements("video") + policy.AllowAttrs("loading").OnElements("img") + // Allow generally safe attributes (reference: https://github.com/jch/html-pipeline) generalSafeAttrs := []string{ "abbr", "accept", "accept-charset", @@ -90,9 +93,9 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy { return policy } -// Sanitize takes a string that contains a HTML fragment or document and applies policy whitelist. -func Sanitize(s string) string { - return GetDefaultSanitizer().defaultPolicy.Sanitize(s) +// Sanitize use default sanitizer policy to sanitize a string +func Sanitize(s string) template.HTML { + return template.HTML(GetDefaultSanitizer().defaultPolicy.Sanitize(s)) } // SanitizeReader sanitizes a Reader diff --git a/modules/markup/sanitizer_default_test.go b/modules/markup/sanitizer_default_test.go index e6fbae5056..e5ba018e1b 100644 --- a/modules/markup/sanitizer_default_test.go +++ b/modules/markup/sanitizer_default_test.go @@ -62,9 +62,13 @@ func TestSanitizer(t *testing.T) { `<a href="javascript:alert('xss')">bad</a>`, `bad`, `<a href="vbscript:no">bad</a>`, `bad`, `<a href="data:1234">bad</a>`, `bad`, + + // Some classes and attributes are used by the frontend framework and will execute JS code, so make sure they are removed + `<div class="link-action" data-attr-class="foo" data-url="xxx">txt</div>`, `<div data-attr-class="foo">txt</div>`, + `<div class="form-fetch-action" data-markdown-generated-content="bar" data-global-init="a" data-global-click="b">txt</div>`, `<div data-markdown-generated-content="bar">txt</div>`, } for i := 0; i < len(testCases); i += 2 { - assert.Equal(t, testCases[i+1], Sanitize(testCases[i])) + assert.Equal(t, testCases[i+1], string(Sanitize(testCases[i]))) } } diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 230260ff94..4d2ec287a9 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -184,7 +184,7 @@ func NewCollector() Collector { Users: prometheus.NewDesc( namespace+"users", "Number of Users", - nil, nil, + []string{"state"}, nil, ), Watches: prometheus.NewDesc( namespace+"watches", @@ -373,7 +373,14 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( c.Users, prometheus.GaugeValue, - float64(stats.Counter.User), + float64(stats.Counter.UsersActive), + "active", // state label + ) + ch <- prometheus.MustNewConstMetric( + c.Users, + prometheus.GaugeValue, + float64(stats.Counter.UsersNotActive), + "inactive", // state label ) ch <- prometheus.MustNewConstMetric( c.Watches, diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index 08dbbc29a9..669222dea2 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -12,18 +12,17 @@ import ( // Downloader downloads the site repo information type Downloader interface { - SetContext(context.Context) - GetRepoInfo() (*Repository, error) - GetTopics() ([]string, error) - GetMilestones() ([]*Milestone, error) - GetReleases() ([]*Release, error) - GetLabels() ([]*Label, error) - GetIssues(page, perPage int) ([]*Issue, bool, error) - GetComments(commentable Commentable) ([]*Comment, bool, error) - GetAllComments(page, perPage int) ([]*Comment, bool, error) + GetRepoInfo(ctx context.Context) (*Repository, error) + GetTopics(ctx context.Context) ([]string, error) + GetMilestones(ctx context.Context) ([]*Milestone, error) + GetReleases(ctx context.Context) ([]*Release, error) + GetLabels(ctx context.Context) ([]*Label, error) + GetIssues(ctx context.Context, page, perPage int) ([]*Issue, bool, error) + GetComments(ctx context.Context, commentable Commentable) ([]*Comment, bool, error) + GetAllComments(ctx context.Context, page, perPage int) ([]*Comment, bool, error) SupportGetRepoComments() bool - GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) - GetReviews(reviewable Reviewable) ([]*Review, error) + GetPullRequests(ctx context.Context, page, perPage int) ([]*PullRequest, bool, error) + GetReviews(ctx context.Context, reviewable Reviewable) ([]*Review, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) } diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index e5b69331df..e488f6914f 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -13,56 +13,53 @@ type NullDownloader struct{} var _ Downloader = &NullDownloader{} -// SetContext set context -func (n NullDownloader) SetContext(_ context.Context) {} - // GetRepoInfo returns a repository information -func (n NullDownloader) GetRepoInfo() (*Repository, error) { +func (n NullDownloader) GetRepoInfo(_ context.Context) (*Repository, error) { return nil, ErrNotSupported{Entity: "RepoInfo"} } // GetTopics return repository topics -func (n NullDownloader) GetTopics() ([]string, error) { +func (n NullDownloader) GetTopics(_ context.Context) ([]string, error) { return nil, ErrNotSupported{Entity: "Topics"} } // GetMilestones returns milestones -func (n NullDownloader) GetMilestones() ([]*Milestone, error) { +func (n NullDownloader) GetMilestones(_ context.Context) ([]*Milestone, error) { return nil, ErrNotSupported{Entity: "Milestones"} } // GetReleases returns releases -func (n NullDownloader) GetReleases() ([]*Release, error) { +func (n NullDownloader) GetReleases(_ context.Context) ([]*Release, error) { return nil, ErrNotSupported{Entity: "Releases"} } // GetLabels returns labels -func (n NullDownloader) GetLabels() ([]*Label, error) { +func (n NullDownloader) GetLabels(_ context.Context) ([]*Label, error) { return nil, ErrNotSupported{Entity: "Labels"} } // GetIssues returns issues according start and limit -func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { +func (n NullDownloader) GetIssues(_ context.Context, page, perPage int) ([]*Issue, bool, error) { return nil, false, ErrNotSupported{Entity: "Issues"} } // GetComments returns comments of an issue or PR -func (n NullDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { +func (n NullDownloader) GetComments(_ context.Context, commentable Commentable) ([]*Comment, bool, error) { return nil, false, ErrNotSupported{Entity: "Comments"} } // GetAllComments returns paginated comments -func (n NullDownloader) GetAllComments(page, perPage int) ([]*Comment, bool, error) { +func (n NullDownloader) GetAllComments(_ context.Context, page, perPage int) ([]*Comment, bool, error) { return nil, false, ErrNotSupported{Entity: "AllComments"} } // GetPullRequests returns pull requests according page and perPage -func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { +func (n NullDownloader) GetPullRequests(_ context.Context, page, perPage int) ([]*PullRequest, bool, error) { return nil, false, ErrNotSupported{Entity: "PullRequests"} } // GetReviews returns pull requests review -func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { +func (n NullDownloader) GetReviews(_ context.Context, reviewable Reviewable) ([]*Review, error) { return nil, ErrNotSupported{Entity: "Reviews"} } diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go index 1cacf5f375..69804b7767 100644 --- a/modules/migration/retry_downloader.go +++ b/modules/migration/retry_downloader.go @@ -13,57 +13,49 @@ var _ Downloader = &RetryDownloader{} // RetryDownloader retry the downloads type RetryDownloader struct { Downloader - ctx context.Context RetryTimes int // the total execute times RetryDelay int // time to delay seconds } // NewRetryDownloader creates a retry downloader -func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader { +func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *RetryDownloader { return &RetryDownloader{ Downloader: downloader, - ctx: ctx, RetryTimes: retryTimes, RetryDelay: retryDelay, } } -func (d *RetryDownloader) retry(work func() error) error { +func (d *RetryDownloader) retry(ctx context.Context, work func(context.Context) error) error { var ( times = d.RetryTimes err error ) for ; times > 0; times-- { - if err = work(); err == nil { + if err = work(ctx); err == nil { return nil } if IsErrNotSupported(err) { return err } select { - case <-d.ctx.Done(): - return d.ctx.Err() + case <-ctx.Done(): + return ctx.Err() case <-time.After(time.Second * time.Duration(d.RetryDelay)): } } return err } -// SetContext set context -func (d *RetryDownloader) SetContext(ctx context.Context) { - d.ctx = ctx - d.Downloader.SetContext(ctx) -} - // GetRepoInfo returns a repository information with retry -func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { +func (d *RetryDownloader) GetRepoInfo(ctx context.Context) (*Repository, error) { var ( repo *Repository err error ) - err = d.retry(func() error { - repo, err = d.Downloader.GetRepoInfo() + err = d.retry(ctx, func(ctx context.Context) error { + repo, err = d.Downloader.GetRepoInfo(ctx) return err }) @@ -71,14 +63,14 @@ func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { } // GetTopics returns a repository's topics with retry -func (d *RetryDownloader) GetTopics() ([]string, error) { +func (d *RetryDownloader) GetTopics(ctx context.Context) ([]string, error) { var ( topics []string err error ) - err = d.retry(func() error { - topics, err = d.Downloader.GetTopics() + err = d.retry(ctx, func(ctx context.Context) error { + topics, err = d.Downloader.GetTopics(ctx) return err }) @@ -86,14 +78,14 @@ func (d *RetryDownloader) GetTopics() ([]string, error) { } // GetMilestones returns a repository's milestones with retry -func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) { +func (d *RetryDownloader) GetMilestones(ctx context.Context) ([]*Milestone, error) { var ( milestones []*Milestone err error ) - err = d.retry(func() error { - milestones, err = d.Downloader.GetMilestones() + err = d.retry(ctx, func(ctx context.Context) error { + milestones, err = d.Downloader.GetMilestones(ctx) return err }) @@ -101,14 +93,14 @@ func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) { } // GetReleases returns a repository's releases with retry -func (d *RetryDownloader) GetReleases() ([]*Release, error) { +func (d *RetryDownloader) GetReleases(ctx context.Context) ([]*Release, error) { var ( releases []*Release err error ) - err = d.retry(func() error { - releases, err = d.Downloader.GetReleases() + err = d.retry(ctx, func(ctx context.Context) error { + releases, err = d.Downloader.GetReleases(ctx) return err }) @@ -116,14 +108,14 @@ func (d *RetryDownloader) GetReleases() ([]*Release, error) { } // GetLabels returns a repository's labels with retry -func (d *RetryDownloader) GetLabels() ([]*Label, error) { +func (d *RetryDownloader) GetLabels(ctx context.Context) ([]*Label, error) { var ( labels []*Label err error ) - err = d.retry(func() error { - labels, err = d.Downloader.GetLabels() + err = d.retry(ctx, func(ctx context.Context) error { + labels, err = d.Downloader.GetLabels(ctx) return err }) @@ -131,15 +123,15 @@ func (d *RetryDownloader) GetLabels() ([]*Label, error) { } // GetIssues returns a repository's issues with retry -func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { +func (d *RetryDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*Issue, bool, error) { var ( issues []*Issue isEnd bool err error ) - err = d.retry(func() error { - issues, isEnd, err = d.Downloader.GetIssues(page, perPage) + err = d.retry(ctx, func(ctx context.Context) error { + issues, isEnd, err = d.Downloader.GetIssues(ctx, page, perPage) return err }) @@ -147,15 +139,15 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns a repository's comments with retry -func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { +func (d *RetryDownloader) GetComments(ctx context.Context, commentable Commentable) ([]*Comment, bool, error) { var ( comments []*Comment isEnd bool err error ) - err = d.retry(func() error { - comments, isEnd, err = d.Downloader.GetComments(commentable) + err = d.retry(ctx, func(context.Context) error { + comments, isEnd, err = d.Downloader.GetComments(ctx, commentable) return err }) @@ -163,15 +155,15 @@ func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool } // GetPullRequests returns a repository's pull requests with retry -func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { +func (d *RetryDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*PullRequest, bool, error) { var ( prs []*PullRequest err error isEnd bool ) - err = d.retry(func() error { - prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage) + err = d.retry(ctx, func(ctx context.Context) error { + prs, isEnd, err = d.Downloader.GetPullRequests(ctx, page, perPage) return err }) @@ -179,14 +171,13 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo } // GetReviews returns pull requests reviews -func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { +func (d *RetryDownloader) GetReviews(ctx context.Context, reviewable Reviewable) ([]*Review, error) { var ( reviews []*Review err error ) - - err = d.retry(func() error { - reviews, err = d.Downloader.GetReviews(reviewable) + err = d.retry(ctx, func(ctx context.Context) error { + reviews, err = d.Downloader.GetReviews(ctx, reviewable) return err }) diff --git a/modules/migration/schemas_bindata.go b/modules/migration/schemas_bindata.go index c5db3b3461..695c2c1135 100644 --- a/modules/migration/schemas_bindata.go +++ b/modules/migration/schemas_bindata.go @@ -3,6 +3,28 @@ //go:build bindata +//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas bindata.dat + package migration -//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas migration bindata.go +import ( + "io" + "io/fs" + "path" + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() fs.FS { + return assetfs.NewEmbeddedFS(bindata) +}) + +func openSchema(filename string) (io.ReadCloser, error) { + return BuiltinAssets().Open(path.Base(filename)) +} diff --git a/modules/migration/schemas_static.go b/modules/migration/schemas_static.go deleted file mode 100644 index 8a0c340a65..0000000000 --- a/modules/migration/schemas_static.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package migration - -import ( - "io" - "path" -) - -func openSchema(filename string) (io.ReadCloser, error) { - return Assets.Open(path.Base(filename)) -} diff --git a/modules/migration/uploader.go b/modules/migration/uploader.go index ff642aa4fa..65752e248e 100644 --- a/modules/migration/uploader.go +++ b/modules/migration/uploader.go @@ -4,20 +4,22 @@ package migration +import "context" + // Uploader uploads all the information of one repository type Uploader interface { MaxBatchInsertSize(tp string) int - CreateRepo(repo *Repository, opts MigrateOptions) error - CreateTopics(topic ...string) error - CreateMilestones(milestones ...*Milestone) error - CreateReleases(releases ...*Release) error - SyncTags() error - CreateLabels(labels ...*Label) error - CreateIssues(issues ...*Issue) error - CreateComments(comments ...*Comment) error - CreatePullRequests(prs ...*PullRequest) error - CreateReviews(reviews ...*Review) error + CreateRepo(ctx context.Context, repo *Repository, opts MigrateOptions) error + CreateTopics(ctx context.Context, topic ...string) error + CreateMilestones(ctx context.Context, milestones ...*Milestone) error + CreateReleases(ctx context.Context, releases ...*Release) error + SyncTags(ctx context.Context) error + CreateLabels(ctx context.Context, labels ...*Label) error + CreateIssues(ctx context.Context, issues ...*Issue) error + CreateComments(ctx context.Context, comments ...*Comment) error + CreatePullRequests(ctx context.Context, prs ...*PullRequest) error + CreateReviews(ctx context.Context, reviews ...*Review) error Rollback() error - Finish() error + Finish(ctx context.Context) error Close() } diff --git a/modules/optional/option.go b/modules/optional/option.go index af9e5ac852..6075c6347e 100644 --- a/modules/optional/option.go +++ b/modules/optional/option.go @@ -3,6 +3,14 @@ package optional +import "strconv" + +// Option is a generic type that can hold a value of type T or be empty (None). +// +// It must use the slice type to work with "chi" form values binding: +// * non-existing value are represented as an empty slice (None) +// * existing value is represented as a slice with one element (Some) +// * multiple values are represented as a slice with multiple elements (Some), the Value is the first element (not well-defined in this case) type Option[T any] []T func None[T any]() Option[T] { @@ -43,3 +51,12 @@ func (o Option[T]) ValueOrDefault(v T) T { } return v } + +// ParseBool get the corresponding optional.Option[bool] of a string using strconv.ParseBool +func ParseBool(s string) Option[bool] { + v, e := strconv.ParseBool(s) + if e != nil { + return None[bool]() + } + return Some(v) +} diff --git a/modules/optional/option_test.go b/modules/optional/option_test.go index 203e9221e3..f600ff5a2c 100644 --- a/modules/optional/option_test.go +++ b/modules/optional/option_test.go @@ -57,3 +57,16 @@ func TestOption(t *testing.T) { assert.True(t, opt3.Has()) assert.Equal(t, int(1), opt3.Value()) } + +func Test_ParseBool(t *testing.T) { + assert.Equal(t, optional.None[bool](), optional.ParseBool("")) + assert.Equal(t, optional.None[bool](), optional.ParseBool("x")) + + assert.Equal(t, optional.Some(false), optional.ParseBool("0")) + assert.Equal(t, optional.Some(false), optional.ParseBool("f")) + assert.Equal(t, optional.Some(false), optional.ParseBool("False")) + + assert.Equal(t, optional.Some(true), optional.ParseBool("1")) + assert.Equal(t, optional.Some(true), optional.ParseBool("t")) + assert.Equal(t, optional.Some(true), optional.ParseBool("True")) +} diff --git a/modules/optional/serialization_test.go b/modules/optional/serialization_test.go index 09a4bddea0..cf81a94cfc 100644 --- a/modules/optional/serialization_test.go +++ b/modules/optional/serialization_test.go @@ -4,7 +4,7 @@ package optional_test import ( - std_json "encoding/json" //nolint:depguard + std_json "encoding/json" //nolint:depguard // for testing purpose "testing" "code.gitea.io/gitea/modules/json" @@ -51,11 +51,11 @@ func TestOptionalToJson(t *testing.T) { t.Run(tc.name, func(t *testing.T) { b, err := json.Marshal(tc.obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, string(b), "gitea json module returned unexpected") + assert.Equal(t, tc.want, string(b), "gitea json module returned unexpected") b, err = std_json.Marshal(tc.obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, string(b), "std json module returned unexpected") + assert.Equal(t, tc.want, string(b), "std json module returned unexpected") }) } } @@ -89,12 +89,12 @@ func TestOptionalFromJson(t *testing.T) { var obj1 testSerializationStruct err := json.Unmarshal([]byte(tc.data), &obj1) assert.NoError(t, err) - assert.EqualValues(t, tc.want, obj1, "gitea json module returned unexpected") + assert.Equal(t, tc.want, obj1, "gitea json module returned unexpected") var obj2 testSerializationStruct err = std_json.Unmarshal([]byte(tc.data), &obj2) assert.NoError(t, err) - assert.EqualValues(t, tc.want, obj2, "std json module returned unexpected") + assert.Equal(t, tc.want, obj2, "std json module returned unexpected") }) } } @@ -135,7 +135,7 @@ optional_two_string: null t.Run(tc.name, func(t *testing.T) { b, err := yaml.Marshal(tc.obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, string(b), "yaml module returned unexpected") + assert.Equal(t, tc.want, string(b), "yaml module returned unexpected") }) } } @@ -184,7 +184,7 @@ optional_twostring: null var obj testSerializationStruct err := yaml.Unmarshal([]byte(tc.data), &obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, obj, "yaml module returned unexpected") + assert.Equal(t, tc.want, obj, "yaml module returned unexpected") }) } } diff --git a/modules/options/options_bindata.go b/modules/options/options_bindata.go index 29151cb3cb..b2321d7eb5 100644 --- a/modules/options/options_bindata.go +++ b/modules/options/options_bindata.go @@ -3,6 +3,21 @@ //go:build bindata +//go:generate go run ../../build/generate-bindata.go ../../options bindata.dat + package options -//go:generate go run ../../build/generate-bindata.go ../../options options bindata.go +import ( + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", assetfs.NewEmbeddedFS(bindata)) +}) diff --git a/modules/options/dynamic.go b/modules/options/options_dynamic.go index 085492d11c..085492d11c 100644 --- a/modules/options/dynamic.go +++ b/modules/options/options_dynamic.go diff --git a/modules/options/static.go b/modules/options/static.go deleted file mode 100644 index 72b28e990e..0000000000 --- a/modules/options/static.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package options - -import ( - "code.gitea.io/gitea/modules/assetfs" -) - -func BuiltinAssets() *assetfs.Layer { - return assetfs.Bindata("builtin(bindata)", Assets) -} diff --git a/modules/packages/conda/metadata.go b/modules/packages/conda/metadata.go index 76ba95eace..5eb2890115 100644 --- a/modules/packages/conda/metadata.go +++ b/modules/packages/conda/metadata.go @@ -17,9 +17,9 @@ import ( ) var ( - ErrInvalidStructure = util.SilentWrap{Message: "package structure is invalid", Err: util.ErrInvalidArgument} - ErrInvalidName = util.SilentWrap{Message: "package name is invalid", Err: util.ErrInvalidArgument} - ErrInvalidVersion = util.SilentWrap{Message: "package version is invalid", Err: util.ErrInvalidArgument} + ErrInvalidStructure = util.NewInvalidArgumentErrorf("package structure is invalid") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") ) const ( diff --git a/modules/packages/container/const.go b/modules/packages/container/const.go new file mode 100644 index 0000000000..6c7c9b46d1 --- /dev/null +++ b/modules/packages/container/const.go @@ -0,0 +1,11 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package container + +const ( + ContentTypeDockerDistributionManifestV2 = "application/vnd.docker.distribution.manifest.v2+json" + + ManifestFilename = "manifest.json" + UploadVersion = "_upload" +) diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go index 2a41fb9105..3ef0684d13 100644 --- a/modules/packages/container/metadata.go +++ b/modules/packages/container/metadata.go @@ -71,14 +71,34 @@ type Manifest struct { Size int64 `json:"size"` } +func IsMediaTypeValid(mt string) bool { + return strings.HasPrefix(mt, "application/vnd.docker.") || strings.HasPrefix(mt, "application/vnd.oci.") +} + +func IsMediaTypeImageManifest(mt string) bool { + return strings.EqualFold(mt, oci.MediaTypeImageManifest) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.v2+json") +} + +func IsMediaTypeImageIndex(mt string) bool { + return strings.EqualFold(mt, oci.MediaTypeImageIndex) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.list.v2+json") +} + // ParseImageConfig parses the metadata of an image config -func ParseImageConfig(mt string, r io.Reader) (*Metadata, error) { - if strings.EqualFold(mt, helm.ConfigMediaType) { +func ParseImageConfig(mediaType string, r io.Reader) (*Metadata, error) { + if strings.EqualFold(mediaType, helm.ConfigMediaType) { return parseHelmConfig(r) } // fallback to OCI Image Config - return parseOCIImageConfig(r) + // FIXME: this fallback is not right, we should strictly check the media type in the future + metadata, err := parseOCIImageConfig(r) + if err != nil { + if !IsMediaTypeImageManifest(mediaType) { + return &Metadata{Platform: "unknown/unknown"}, nil + } + return nil, err + } + return metadata, nil } func parseOCIImageConfig(r io.Reader) (*Metadata, error) { diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go index 665499b2e6..0f2d702925 100644 --- a/modules/packages/container/metadata_test.go +++ b/modules/packages/container/metadata_test.go @@ -11,6 +11,7 @@ import ( oci "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestParseImageConfig(t *testing.T) { @@ -58,4 +59,8 @@ func TestParseImageConfig(t *testing.T) { assert.ElementsMatch(t, []string{author}, metadata.Authors) assert.Equal(t, projectURL, metadata.ProjectURL) assert.Equal(t, repositoryURL, metadata.RepositoryURL) + + metadata, err = ParseImageConfig("anything-unknown", strings.NewReader("")) + require.NoError(t, err) + assert.Equal(t, &Metadata{Platform: "unknown/unknown"}, metadata) } diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index 37612556d7..dadb7eaefc 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -28,8 +28,7 @@ func NewContentStore() *ContentStore { return contentStore } -// Get gets a package blob -func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { +func (s *ContentStore) OpenBlob(key BlobHash256Key) (storage.Object, error) { return s.store.Open(KeyToRelativePath(key)) } diff --git a/modules/packages/goproxy/metadata.go b/modules/packages/goproxy/metadata.go index 40f7d20508..a67b149f4d 100644 --- a/modules/packages/goproxy/metadata.go +++ b/modules/packages/goproxy/metadata.go @@ -5,7 +5,6 @@ package goproxy import ( "archive/zip" - "fmt" "io" "path" "strings" @@ -88,7 +87,7 @@ func ParsePackage(r io.ReaderAt, size int64) (*Package, error) { return nil, ErrInvalidStructure } - p.GoMod = fmt.Sprintf("module %s", p.Name) + p.GoMod = "module " + p.Name return p, nil } diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go index 4ab45edcec..0cd657cd44 100644 --- a/modules/packages/hashed_buffer.go +++ b/modules/packages/hashed_buffer.go @@ -6,6 +6,7 @@ package packages import ( "io" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util/filebuffer" ) @@ -34,11 +35,11 @@ func NewHashedBuffer() (*HashedBuffer, error) { // NewHashedBufferWithSize creates a hashed buffer with a specific memory size func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { - b, err := filebuffer.New(maxMemorySize) + tempDir, err := setting.AppDataTempDir("package-hashed-buffer").MkdirAllSub("") if err != nil { return nil, err } - + b := filebuffer.New(maxMemorySize, tempDir) hash := NewMultiHasher() combinedWriter := io.MultiWriter(b, hash) diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go index 564e782f18..5104c1fb25 100644 --- a/modules/packages/hashed_buffer_test.go +++ b/modules/packages/hashed_buffer_test.go @@ -9,10 +9,13 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestHashedBuffer(t *testing.T) { + setting.AppDataPath = t.TempDir() cases := []struct { MaxMemorySize int Data string diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 8ba4dbfba7..11b5123c27 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -58,7 +58,7 @@ type PackageMetadata struct { Time map[string]time.Time `json:"time,omitempty"` Homepage string `json:"homepage,omitempty"` Keywords []string `json:"keywords,omitempty"` - Repository Repository `json:"repository,omitempty"` + Repository Repository `json:"repository"` Author User `json:"author"` ReadmeFilename string `json:"readmeFilename,omitempty"` Users map[string]bool `json:"users,omitempty"` @@ -75,7 +75,7 @@ type PackageMetadataVersion struct { Author User `json:"author"` Homepage string `json:"homepage,omitempty"` License string `json:"license,omitempty"` - Repository Repository `json:"repository,omitempty"` + Repository Repository `json:"repository"` Keywords []string `json:"keywords,omitempty"` Dependencies map[string]string `json:"dependencies,omitempty"` BundleDependencies []string `json:"bundleDependencies,omitempty"` diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index d1d0263387..362d0470d5 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -23,5 +23,5 @@ type Metadata struct { OptionalDependencies map[string]string `json:"optional_dependencies,omitempty"` Bin map[string]string `json:"bin,omitempty"` Readme string `json:"readme,omitempty"` - Repository Repository `json:"repository,omitempty"` + Repository Repository `json:"repository"` } diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 1e98ddffde..a122590bf1 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -57,14 +57,24 @@ type Package struct { // Metadata represents the metadata of a Nuget package type Metadata struct { - Description string `json:"description,omitempty"` - ReleaseNotes string `json:"release_notes,omitempty"` - Readme string `json:"readme,omitempty"` - Authors string `json:"authors,omitempty"` - ProjectURL string `json:"project_url,omitempty"` - RepositoryURL string `json:"repository_url,omitempty"` - RequireLicenseAcceptance bool `json:"require_license_acceptance"` - Dependencies map[string][]Dependency `json:"dependencies,omitempty"` + Authors string `json:"authors,omitempty"` + Copyright string `json:"copyright,omitempty"` + Description string `json:"description,omitempty"` + DevelopmentDependency bool `json:"development_dependency,omitempty"` + IconURL string `json:"icon_url,omitempty"` + Language string `json:"language,omitempty"` + LicenseURL string `json:"license_url,omitempty"` + MinClientVersion string `json:"min_client_version,omitempty"` + Owners string `json:"owners,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Readme string `json:"readme,omitempty"` + ReleaseNotes string `json:"release_notes,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + RequireLicenseAcceptance bool `json:"require_license_acceptance"` + Tags string `json:"tags,omitempty"` + Title string `json:"title,omitempty"` + + Dependencies map[string][]Dependency `json:"dependencies,omitempty"` } // Dependency represents a dependency of a Nuget package @@ -74,24 +84,30 @@ type Dependency struct { } // https://learn.microsoft.com/en-us/nuget/reference/nuspec +// https://github.com/NuGet/NuGet.Client/blob/dev/src/NuGet.Core/NuGet.Packaging/compiler/resources/nuspec.xsd type nuspecPackage struct { Metadata struct { - ID string `xml:"id"` - Version string `xml:"version"` - Authors string `xml:"authors"` - RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + // required fields + Authors string `xml:"authors"` + Description string `xml:"description"` + ID string `xml:"id"` + Version string `xml:"version"` + + // optional fields + Copyright string `xml:"copyright"` + DevelopmentDependency bool `xml:"developmentDependency"` + IconURL string `xml:"iconUrl"` + Language string `xml:"language"` + LicenseURL string `xml:"licenseUrl"` + MinClientVersion string `xml:"minClientVersion,attr"` + Owners string `xml:"owners"` ProjectURL string `xml:"projectUrl"` - Description string `xml:"description"` - ReleaseNotes string `xml:"releaseNotes"` Readme string `xml:"readme"` - PackageTypes struct { - PackageType []struct { - Name string `xml:"name,attr"` - } `xml:"packageType"` - } `xml:"packageTypes"` - Repository struct { - URL string `xml:"url,attr"` - } `xml:"repository"` + ReleaseNotes string `xml:"releaseNotes"` + RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + Tags string `xml:"tags"` + Title string `xml:"title"` + Dependencies struct { Dependency []struct { ID string `xml:"id,attr"` @@ -107,6 +123,14 @@ type nuspecPackage struct { } `xml:"dependency"` } `xml:"group"` } `xml:"dependencies"` + PackageTypes struct { + PackageType []struct { + Name string `xml:"name,attr"` + } `xml:"packageType"` + } `xml:"packageTypes"` + Repository struct { + URL string `xml:"url,attr"` + } `xml:"repository"` } `xml:"metadata"` } @@ -167,13 +191,23 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { } m := &Metadata{ - Description: p.Metadata.Description, - ReleaseNotes: p.Metadata.ReleaseNotes, Authors: p.Metadata.Authors, + Copyright: p.Metadata.Copyright, + Description: p.Metadata.Description, + DevelopmentDependency: p.Metadata.DevelopmentDependency, + IconURL: p.Metadata.IconURL, + Language: p.Metadata.Language, + LicenseURL: p.Metadata.LicenseURL, + MinClientVersion: p.Metadata.MinClientVersion, + Owners: p.Metadata.Owners, ProjectURL: p.Metadata.ProjectURL, + ReleaseNotes: p.Metadata.ReleaseNotes, RepositoryURL: p.Metadata.Repository.URL, RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance, - Dependencies: make(map[string][]Dependency), + Tags: p.Metadata.Tags, + Title: p.Metadata.Title, + + Dependencies: make(map[string][]Dependency), } if p.Metadata.Readme != "" { @@ -227,13 +261,13 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { func toNormalizedVersion(v *version.Version) string { var buf bytes.Buffer segments := v.Segments64() - fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) + _, _ = fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) if len(segments) > 3 && segments[3] > 0 { - fmt.Fprintf(&buf, ".%d", segments[3]) + _, _ = fmt.Fprintf(&buf, ".%d", segments[3]) } pre := v.Prerelease() if pre != "" { - fmt.Fprint(&buf, "-", pre) + _, _ = fmt.Fprint(&buf, "-", pre) } return buf.String() } diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go index f466492f8a..90c3e8dfeb 100644 --- a/modules/packages/nuget/metadata_test.go +++ b/modules/packages/nuget/metadata_test.go @@ -12,44 +12,62 @@ import ( ) const ( - id = "System.Gitea" - semver = "1.0.1" - authors = "Gitea Authors" - projectURL = "https://gitea.io" - description = "Package Description" - releaseNotes = "Package Release Notes" - readme = "Readme" - repositoryURL = "https://gitea.io/gitea/gitea" - targetFramework = ".NETStandard2.1" - dependencyID = "System.Text.Json" - dependencyVersion = "5.0.0" + authors = "Gitea Authors" + copyright = "Package Copyright" + dependencyID = "System.Text.Json" + dependencyVersion = "5.0.0" + developmentDependency = true + description = "Package Description" + iconURL = "https://gitea.io/favicon.png" + id = "System.Gitea" + language = "Package Language" + licenseURL = "https://gitea.io/license" + minClientVersion = "1.0.0.0" + owners = "Package Owners" + projectURL = "https://gitea.io" + readme = "Readme" + releaseNotes = "Package Release Notes" + repositoryURL = "https://gitea.io/gitea/gitea" + requireLicenseAcceptance = true + tags = "tag_1 tag_2 tag_3" + targetFramework = ".NETStandard2.1" + title = "Package Title" + versionStr = "1.0.1" ) const nuspecContent = `<?xml version="1.0" encoding="utf-8"?> <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> - <metadata> - <id>` + id + `</id> - <version>` + semver + `</version> - <authors>` + authors + `</authors> - <requireLicenseAcceptance>true</requireLicenseAcceptance> - <projectUrl>` + projectURL + `</projectUrl> - <description>` + description + `</description> - <releaseNotes>` + releaseNotes + `</releaseNotes> - <repository url="` + repositoryURL + `" /> - <readme>README.md</readme> - <dependencies> - <group targetFramework="` + targetFramework + `"> - <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" /> - </group> - </dependencies> - </metadata> + <metadata minClientVersion="` + minClientVersion + `"> + <authors>` + authors + `</authors> + <copyright>` + copyright + `</copyright> + <description>` + description + `</description> + <developmentDependency>true</developmentDependency> + <iconUrl>` + iconURL + `</iconUrl> + <id>` + id + `</id> + <language>` + language + `</language> + <licenseUrl>` + licenseURL + `</licenseUrl> + <owners>` + owners + `</owners> + <projectUrl>` + projectURL + `</projectUrl> + <readme>README.md</readme> + <releaseNotes>` + releaseNotes + `</releaseNotes> + <repository url="` + repositoryURL + `" /> + <requireLicenseAcceptance>true</requireLicenseAcceptance> + <tags>` + tags + `</tags> + <title>` + title + `</title> + <version>` + versionStr + `</version> + <dependencies> + <group targetFramework="` + targetFramework + `"> + <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" /> + </group> + </dependencies> + </metadata> </package>` const symbolsNuspecContent = `<?xml version="1.0" encoding="utf-8"?> <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> <metadata> <id>` + id + `</id> - <version>` + semver + `</version> + <version>` + versionStr + `</version> <description>` + description + `</description> <packageTypes> <packageType name="SymbolsPackage" /> @@ -140,14 +158,26 @@ func TestParsePackageMetaData(t *testing.T) { assert.NotNil(t, np) assert.Equal(t, DependencyPackage, np.PackageType) - assert.Equal(t, id, np.ID) - assert.Equal(t, semver, np.Version) assert.Equal(t, authors, np.Metadata.Authors) - assert.Equal(t, projectURL, np.Metadata.ProjectURL) assert.Equal(t, description, np.Metadata.Description) - assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) + assert.Equal(t, id, np.ID) + assert.Equal(t, versionStr, np.Version) + + assert.Equal(t, copyright, np.Metadata.Copyright) + assert.Equal(t, developmentDependency, np.Metadata.DevelopmentDependency) + assert.Equal(t, iconURL, np.Metadata.IconURL) + assert.Equal(t, language, np.Metadata.Language) + assert.Equal(t, licenseURL, np.Metadata.LicenseURL) + assert.Equal(t, minClientVersion, np.Metadata.MinClientVersion) + assert.Equal(t, owners, np.Metadata.Owners) + assert.Equal(t, projectURL, np.Metadata.ProjectURL) assert.Equal(t, readme, np.Metadata.Readme) + assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL) + assert.Equal(t, requireLicenseAcceptance, np.Metadata.RequireLicenseAcceptance) + assert.Equal(t, tags, np.Metadata.Tags) + assert.Equal(t, title, np.Metadata.Title) + assert.Len(t, np.Metadata.Dependencies, 1) assert.Contains(t, np.Metadata.Dependencies, targetFramework) deps := np.Metadata.Dependencies[targetFramework] @@ -180,7 +210,7 @@ func TestParsePackageMetaData(t *testing.T) { assert.Equal(t, SymbolsPackage, np.PackageType) assert.Equal(t, id, np.ID) - assert.Equal(t, semver, np.Version) + assert.Equal(t, versionStr, np.Version) assert.Equal(t, description, np.Metadata.Description) assert.Empty(t, np.Metadata.Dependencies) }) diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go index 81bf0371a0..9c952e1f10 100644 --- a/modules/packages/nuget/symbol_extractor.go +++ b/modules/packages/nuget/symbol_extractor.go @@ -34,7 +34,7 @@ type PortablePdbList []*PortablePdb func (l PortablePdbList) Close() { for _, pdb := range l { - pdb.Content.Close() + _ = pdb.Content.Close() } } @@ -65,7 +65,7 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { buf, err := packages.CreateHashedBufferFromReader(f) - f.Close() + _ = f.Close() if err != nil { return err @@ -73,12 +73,12 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { id, err := ParseDebugHeaderID(buf) if err != nil { - buf.Close() + _ = buf.Close() return fmt.Errorf("Invalid PDB file: %w", err) } if _, err := buf.Seek(0, io.SeekStart); err != nil { - buf.Close() + _ = buf.Close() return err } diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go index fa1b80ee82..e841e377d9 100644 --- a/modules/packages/nuget/symbol_extractor_test.go +++ b/modules/packages/nuget/symbol_extractor_test.go @@ -9,6 +9,8 @@ import ( "encoding/base64" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) @@ -17,18 +19,19 @@ fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==` func TestExtractPortablePdb(t *testing.T) { + setting.AppDataPath = t.TempDir() createArchive := func(name string, content []byte) []byte { var buf bytes.Buffer archive := zip.NewWriter(&buf) w, _ := archive.Create(name) - w.Write(content) - archive.Close() + _, _ = w.Write(content) + _ = archive.Close() return buf.Bytes() } t.Run("MissingPdbFiles", func(t *testing.T) { var buf bytes.Buffer - zip.NewWriter(&buf).Close() + _ = zip.NewWriter(&buf).Close() pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len())) assert.ErrorIs(t, err, ErrMissingPdbFiles) diff --git a/modules/packages/rubygems/marshal.go b/modules/packages/rubygems/marshal.go index 4e6a5fc5f8..1505221acc 100644 --- a/modules/packages/rubygems/marshal.go +++ b/modules/packages/rubygems/marshal.go @@ -250,7 +250,7 @@ func (e *MarshalEncoder) marshalArray(arr reflect.Value) error { return err } - for i := 0; i < length; i++ { + for i := range length { if err := e.marshal(arr.Index(i).Interface()); err != nil { return err } diff --git a/modules/packages/swift/metadata.go b/modules/packages/swift/metadata.go index 24c4262ab7..85beb57607 100644 --- a/modules/packages/swift/metadata.go +++ b/modules/packages/swift/metadata.go @@ -47,7 +47,7 @@ type Metadata struct { Keywords []string `json:"keywords,omitempty"` RepositoryURL string `json:"repository_url,omitempty"` License string `json:"license,omitempty"` - Author Person `json:"author,omitempty"` + Author Person `json:"author"` Manifests map[string]*Manifest `json:"manifests,omitempty"` } diff --git a/modules/paginator/paginator.go b/modules/paginator/paginator.go index 8258d194c2..0f64e89d9a 100644 --- a/modules/paginator/paginator.go +++ b/modules/paginator/paginator.go @@ -4,6 +4,8 @@ package paginator +import "code.gitea.io/gitea/modules/util" + /* In template: @@ -32,25 +34,43 @@ Output: // Paginator represents a set of results of pagination calculations. type Paginator struct { - total int // total rows count + total int // total rows count, -1 means unknown + totalPages int // total pages count, -1 means unknown + current int // current page number + curRows int // current page rows count + pagingNum int // how many rows in one page - current int // current page number numPages int // how many pages to show on the UI } // New initialize a new pagination calculation and returns a Paginator as result. func New(total, pagingNum, current, numPages int) *Paginator { - if pagingNum <= 0 { - pagingNum = 1 + pagingNum = max(pagingNum, 1) + totalPages := util.Iif(total == -1, -1, (total+pagingNum-1)/pagingNum) + if total >= 0 { + current = min(current, totalPages) } - if current <= 0 { - current = 1 + current = max(current, 1) + return &Paginator{ + total: total, + totalPages: totalPages, + current: current, + pagingNum: pagingNum, + numPages: numPages, } - p := &Paginator{total, pagingNum, current, numPages} - if p.current > p.TotalPages() { - p.current = p.TotalPages() +} + +func (p *Paginator) SetCurRows(rows int) { + // For "unlimited paging", we need to know the rows of current page to determine if there is a next page. + // There is still an edge case: when curRows==pagingNum, then the "next page" will be an empty page. + // Ideally we should query one more row to determine if there is really a next page, but it's impossible in current framework. + p.curRows = rows + if p.total == -1 && p.current == 1 && !p.HasNext() { + // if there is only one page for the "unlimited paging", set total rows/pages count + // then the tmpl could decide to hide the nav bar. + p.total = rows + p.totalPages = util.Iif(p.total == 0, 0, 1) } - return p } // IsFirst returns true if current page is the first page. @@ -72,7 +92,10 @@ func (p *Paginator) Previous() int { // HasNext returns true if there is a next page relative to current page. func (p *Paginator) HasNext() bool { - return p.total > p.current*p.pagingNum + if p.total == -1 { + return p.curRows >= p.pagingNum + } + return p.current*p.pagingNum < p.total } func (p *Paginator) Next() int { @@ -84,10 +107,7 @@ func (p *Paginator) Next() int { // IsLast returns true if current page is the last page. func (p *Paginator) IsLast() bool { - if p.total == 0 { - return true - } - return p.total > (p.current-1)*p.pagingNum && !p.HasNext() + return !p.HasNext() } // Total returns number of total rows. @@ -97,10 +117,7 @@ func (p *Paginator) Total() int { // TotalPages returns number of total pages. func (p *Paginator) TotalPages() int { - if p.total == 0 { - return 1 - } - return (p.total + p.pagingNum - 1) / p.pagingNum + return p.totalPages } // Current returns current page number. @@ -135,10 +152,10 @@ func getMiddleIdx(numPages int) int { // If value is -1 means "..." that more pages are not showing. func (p *Paginator) Pages() []*Page { if p.numPages == 0 { - return []*Page{} - } else if p.numPages == 1 && p.TotalPages() == 1 { + return nil + } else if p.total == -1 || (p.numPages == 1 && p.TotalPages() == 1) { // Only show current page. - return []*Page{{1, true}} + return []*Page{{p.current, true}} } // Total page number is less or equal. diff --git a/modules/paginator/paginator_test.go b/modules/paginator/paginator_test.go index 8a56ee5121..ed46ecea94 100644 --- a/modules/paginator/paginator_test.go +++ b/modules/paginator/paginator_test.go @@ -76,9 +76,7 @@ func TestPaginator(t *testing.T) { t.Run("Only current page", func(t *testing.T) { p := New(0, 10, 1, 1) pages := p.Pages() - assert.Len(t, pages, 1) - assert.Equal(t, 1, pages[0].Num()) - assert.True(t, pages[0].IsCurrent()) + assert.Empty(t, pages) // no "total", so no pages p = New(1, 10, 1, 1) pages = p.Pages() diff --git a/modules/private/actions.go b/modules/private/actions.go index 311a283650..e68f2f85b0 100644 --- a/modules/private/actions.go +++ b/modules/private/actions.go @@ -17,7 +17,7 @@ type GenerateTokenRequest struct { func GenerateActionsRunnerToken(ctx context.Context, scope string) (*ResponseText, ResponseExtra) { reqURL := setting.LocalURL + "api/internal/actions/generate_actions_runner_token" - req := newInternalRequest(ctx, reqURL, "POST", GenerateTokenRequest{ + req := newInternalRequestAPI(ctx, reqURL, "POST", GenerateTokenRequest{ Scope: scope, }) diff --git a/modules/private/hook.go b/modules/private/hook.go index 745c200619..215996b9b9 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -7,9 +7,9 @@ import ( "context" "fmt" "net/url" - "time" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" ) @@ -82,29 +82,32 @@ type HookProcReceiveRefResult struct { HeadBranch string } +func newInternalRequestAPIForHooks(ctx context.Context, hookName, ownerName, repoName string, opts HookOptions) *httplib.Request { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/%s/%s/%s", hookName, url.PathEscape(ownerName), url.PathEscape(repoName)) + req := newInternalRequestAPI(ctx, reqURL, "POST", opts) + // This "timeout" applies to http.Client's timeout: A Timeout of zero means no timeout. + // This "timeout" was previously set to `time.Duration(60+len(opts.OldCommitIDs))` seconds, but it caused unnecessary timeout failures. + // It should be good enough to remove the client side timeout, only respect the "ctx" and server side timeout. + req.SetReadWriteTimeout(0) + return req +} + // HookPreReceive check whether the provided commits are allowed func HookPreReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) ResponseExtra { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - req := newInternalRequest(ctx, reqURL, "POST", opts) - req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second) + req := newInternalRequestAPIForHooks(ctx, "pre-receive", ownerName, repoName, opts) _, extra := requestJSONResp(req, &ResponseText{}) return extra } // HookPostReceive updates services and users func HookPostReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) (*HookPostReceiveResult, ResponseExtra) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - req := newInternalRequest(ctx, reqURL, "POST", opts) - req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second) + req := newInternalRequestAPIForHooks(ctx, "post-receive", ownerName, repoName, opts) return requestJSONResp(req, &HookPostReceiveResult{}) } // HookProcReceive proc-receive hook func HookProcReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) (*HookProcReceiveResult, ResponseExtra) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/proc-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - - req := newInternalRequest(ctx, reqURL, "POST", opts) - req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second) + req := newInternalRequestAPIForHooks(ctx, "proc-receive", ownerName, repoName, opts) return requestJSONResp(req, &HookProcReceiveResult{}) } @@ -115,7 +118,7 @@ func SetDefaultBranch(ctx context.Context, ownerName, repoName, branch string) R url.PathEscape(repoName), url.PathEscape(branch), ) - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") _, extra := requestJSONResp(req, &ResponseText{}) return extra } @@ -123,7 +126,7 @@ func SetDefaultBranch(ctx context.Context, ownerName, repoName, branch string) R // SSHLog sends ssh error log response func SSHLog(ctx context.Context, isErr bool, msg string) error { reqURL := setting.LocalURL + "api/internal/ssh/log" - req := newInternalRequest(ctx, reqURL, "POST", &SSHLogOption{IsError: isErr, Message: msg}) + req := newInternalRequestAPI(ctx, reqURL, "POST", &SSHLogOption{IsError: isErr, Message: msg}) _, extra := requestJSONResp(req, &ResponseText{}) return extra.Error } diff --git a/modules/private/internal.go b/modules/private/internal.go index c7e7773524..e599c6eb8e 100644 --- a/modules/private/internal.go +++ b/modules/private/internal.go @@ -6,7 +6,6 @@ package private import ( "context" "crypto/tls" - "fmt" "net" "net/http" "os" @@ -34,16 +33,20 @@ func getClientIP() string { return strings.Fields(sshConnEnv)[0] } -func newInternalRequest(ctx context.Context, url, method string, body ...any) *httplib.Request { +func NewInternalRequest(ctx context.Context, url, method string) *httplib.Request { if setting.InternalToken == "" { log.Fatal(`The INTERNAL_TOKEN setting is missing from the configuration file: %q. Ensure you are running in the correct environment or set the correct configuration file with -c.`, setting.CustomConf) } + if !strings.HasPrefix(url, setting.LocalURL) { + log.Fatal("Invalid internal request URL: %q", url) + } + req := httplib.NewRequest(url, method). SetContext(ctx). Header("X-Real-IP", getClientIP()). - Header("X-Gitea-Internal-Auth", fmt.Sprintf("Bearer %s", setting.InternalToken)). + Header("X-Gitea-Internal-Auth", "Bearer "+setting.InternalToken). SetTLSClientConfig(&tls.Config{ InsecureSkipVerify: true, ServerName: setting.Domain, @@ -82,13 +85,17 @@ Ensure you are running in the correct environment or set the correct configurati }, }) } + return req +} +func newInternalRequestAPI(ctx context.Context, url, method string, body ...any) *httplib.Request { + req := NewInternalRequest(ctx, url, method) if len(body) == 1 { req.Header("Content-Type", "application/json") jsonBytes, _ := json.Marshal(body[0]) req.Body(jsonBytes) } else if len(body) > 1 { - log.Fatal("Too many arguments for newInternalRequest") + log.Fatal("Too many arguments for newInternalRequestAPI") } req.SetTimeout(10*time.Second, 60*time.Second) diff --git a/modules/private/key.go b/modules/private/key.go index dcd1714856..114683b343 100644 --- a/modules/private/key.go +++ b/modules/private/key.go @@ -14,7 +14,7 @@ import ( func UpdatePublicKeyInRepo(ctx context.Context, keyID, repoID int64) error { // Ask for running deliver hook and test pull request tasks. reqURL := setting.LocalURL + fmt.Sprintf("api/internal/ssh/%d/update/%d", keyID, repoID) - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") _, extra := requestJSONResp(req, &ResponseText{}) return extra.Error } @@ -24,7 +24,7 @@ func UpdatePublicKeyInRepo(ctx context.Context, keyID, repoID int64) error { func AuthorizedPublicKeyByContent(ctx context.Context, content string) (*ResponseText, ResponseExtra) { // Ask for running deliver hook and test pull request tasks. reqURL := setting.LocalURL + "api/internal/ssh/authorized_keys" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") req.Param("content", content) return requestJSONResp(req, &ResponseText{}) } diff --git a/modules/private/mail.go b/modules/private/mail.go index 08de5b7e28..3904e37bea 100644 --- a/modules/private/mail.go +++ b/modules/private/mail.go @@ -23,7 +23,7 @@ type Email struct { func SendEmail(ctx context.Context, subject, message string, to []string) (*ResponseText, ResponseExtra) { reqURL := setting.LocalURL + "api/internal/mail/send" - req := newInternalRequest(ctx, reqURL, "POST", Email{ + req := newInternalRequestAPI(ctx, reqURL, "POST", Email{ Subject: subject, Message: message, To: to, diff --git a/modules/private/manager.go b/modules/private/manager.go index 6055e553bd..e3d5ad57e0 100644 --- a/modules/private/manager.go +++ b/modules/private/manager.go @@ -18,21 +18,21 @@ import ( // Shutdown calls the internal shutdown function func Shutdown(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/shutdown" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Shutting down") } // Restart calls the internal restart function func Restart(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/restart" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Restarting") } // ReloadTemplates calls the internal reload-templates function func ReloadTemplates(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/reload-templates" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Reloaded") } @@ -45,7 +45,7 @@ type FlushOptions struct { // FlushQueues calls the internal flush-queues function func FlushQueues(ctx context.Context, timeout time.Duration, nonBlocking bool) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/flush-queues" - req := newInternalRequest(ctx, reqURL, "POST", FlushOptions{Timeout: timeout, NonBlocking: nonBlocking}) + req := newInternalRequestAPI(ctx, reqURL, "POST", FlushOptions{Timeout: timeout, NonBlocking: nonBlocking}) if timeout > 0 { req.SetReadWriteTimeout(timeout + 10*time.Second) } @@ -55,28 +55,28 @@ func FlushQueues(ctx context.Context, timeout time.Duration, nonBlocking bool) R // PauseLogging pauses logging func PauseLogging(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/pause-logging" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Logging Paused") } // ResumeLogging resumes logging func ResumeLogging(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/resume-logging" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Logging Restarted") } // ReleaseReopenLogging releases and reopens logging files func ReleaseReopenLogging(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/release-and-reopen-logging" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Logging Restarted") } // SetLogSQL sets database logging func SetLogSQL(ctx context.Context, on bool) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/set-log-sql?on=" + strconv.FormatBool(on) - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Log SQL setting set") } @@ -91,7 +91,7 @@ type LoggerOptions struct { // AddLogger adds a logger func AddLogger(ctx context.Context, logger, writer, mode string, config map[string]any) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/add-logger" - req := newInternalRequest(ctx, reqURL, "POST", LoggerOptions{ + req := newInternalRequestAPI(ctx, reqURL, "POST", LoggerOptions{ Logger: logger, Writer: writer, Mode: mode, @@ -103,7 +103,7 @@ func AddLogger(ctx context.Context, logger, writer, mode string, config map[stri // RemoveLogger removes a logger func RemoveLogger(ctx context.Context, logger, writer string) ResponseExtra { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/remove-logger/%s/%s", url.PathEscape(logger), url.PathEscape(writer)) - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Removed") } @@ -111,7 +111,7 @@ func RemoveLogger(ctx context.Context, logger, writer string) ResponseExtra { func Processes(ctx context.Context, out io.Writer, flat, noSystem, stacktraces, json bool, cancel string) ResponseExtra { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/processes?flat=%t&no-system=%t&stacktraces=%t&json=%t&cancel-pid=%s", flat, noSystem, stacktraces, json, url.QueryEscape(cancel)) - req := newInternalRequest(ctx, reqURL, "GET") + req := newInternalRequestAPI(ctx, reqURL, "GET") callback := func(resp *http.Response, extra *ResponseExtra) { _, extra.Error = io.Copy(out, resp.Body) } diff --git a/modules/private/restore_repo.go b/modules/private/restore_repo.go index 496209d3cb..9c3a008142 100644 --- a/modules/private/restore_repo.go +++ b/modules/private/restore_repo.go @@ -24,7 +24,7 @@ type RestoreParams struct { func RestoreRepo(ctx context.Context, repoDir, ownerName, repoName string, units []string, validation bool) ResponseExtra { reqURL := setting.LocalURL + "api/internal/restore_repo" - req := newInternalRequest(ctx, reqURL, "POST", RestoreParams{ + req := newInternalRequestAPI(ctx, reqURL, "POST", RestoreParams{ RepoDir: repoDir, OwnerName: ownerName, RepoName: repoName, diff --git a/modules/private/serv.go b/modules/private/serv.go index 480a446954..b1dafbd81b 100644 --- a/modules/private/serv.go +++ b/modules/private/serv.go @@ -23,7 +23,7 @@ type KeyAndOwner struct { // ServNoCommand returns information about the provided key func ServNoCommand(ctx context.Context, keyID int64) (*asymkey_model.PublicKey, *user_model.User, error) { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/serv/none/%d", keyID) - req := newInternalRequest(ctx, reqURL, "GET") + req := newInternalRequestAPI(ctx, reqURL, "GET") keyAndOwner, extra := requestJSONResp(req, &KeyAndOwner{}) if extra.HasError() { return nil, nil, extra.Error @@ -46,18 +46,16 @@ type ServCommandResults struct { } // ServCommand preps for a serv call -func ServCommand(ctx context.Context, keyID int64, ownerName, repoName string, mode perm.AccessMode, verbs ...string) (*ServCommandResults, ResponseExtra) { +func ServCommand(ctx context.Context, keyID int64, ownerName, repoName string, mode perm.AccessMode, verb, lfsVerb string) (*ServCommandResults, ResponseExtra) { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/serv/command/%d/%s/%s?mode=%d", keyID, url.PathEscape(ownerName), url.PathEscape(repoName), mode, ) - for _, verb := range verbs { - if verb != "" { - reqURL += fmt.Sprintf("&verb=%s", url.QueryEscape(verb)) - } - } - req := newInternalRequest(ctx, reqURL, "GET") + reqURL += "&verb=" + url.QueryEscape(verb) + // reqURL += "&lfs_verb=" + url.QueryEscape(lfsVerb) // TODO: actually there is no use of this parameter. In the future, the URL construction should be more flexible + _ = lfsVerb + req := newInternalRequestAPI(ctx, reqURL, "GET") return requestJSONResp(req, &ServCommandResults{}) } diff --git a/modules/process/manager_test.go b/modules/process/manager_test.go index 36b2a912ea..0d637c8acc 100644 --- a/modules/process/manager_test.go +++ b/modules/process/manager_test.go @@ -23,7 +23,7 @@ func TestGetManager(t *testing.T) { func TestManager_AddContext(t *testing.T) { pm := Manager{processMap: make(map[IDType]*process), next: 1} - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) defer cancel() p1Ctx, _, finished := pm.AddContext(ctx, "foo") @@ -42,7 +42,7 @@ func TestManager_AddContext(t *testing.T) { func TestManager_Cancel(t *testing.T) { pm := Manager{processMap: make(map[IDType]*process), next: 1} - ctx, _, finished := pm.AddContext(context.Background(), "foo") + ctx, _, finished := pm.AddContext(t.Context(), "foo") defer finished() pm.Cancel(GetPID(ctx)) @@ -54,7 +54,7 @@ func TestManager_Cancel(t *testing.T) { } finished() - ctx, cancel, finished := pm.AddContext(context.Background(), "foo") + ctx, cancel, finished := pm.AddContext(t.Context(), "foo") defer finished() cancel() @@ -70,7 +70,7 @@ func TestManager_Cancel(t *testing.T) { func TestManager_Remove(t *testing.T) { pm := Manager{processMap: make(map[IDType]*process), next: 1} - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) defer cancel() p1Ctx, _, finished := pm.AddContext(ctx, "foo") diff --git a/modules/proxyprotocol/errors.go b/modules/proxyprotocol/errors.go index 5439a86bd8..f76c82b7f6 100644 --- a/modules/proxyprotocol/errors.go +++ b/modules/proxyprotocol/errors.go @@ -20,7 +20,7 @@ type ErrBadAddressType struct { } func (e *ErrBadAddressType) Error() string { - return fmt.Sprintf("Unexpected proxy header address type: %s", e.Address) + return "Unexpected proxy header address type: " + e.Address } // ErrBadRemote is an error demonstrating a bad proxy header with bad Remote diff --git a/modules/public/public.go b/modules/public/public.go index abc6b46158..a7eace1538 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -44,7 +44,7 @@ func FileHandlerFunc() http.HandlerFunc { func parseAcceptEncoding(val string) container.Set[string] { parts := strings.Split(val, ";") types := make(container.Set[string]) - for _, v := range strings.Split(parts[0], ",") { + for v := range strings.SplitSeq(parts[0], ",") { types.Add(strings.TrimSpace(v)) } return types @@ -86,33 +86,28 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, return } - serveContent(w, req, fi, fi.ModTime(), f) + servePublicAsset(w, req, fi, fi.ModTime(), f) } -type GzipBytesProvider interface { - GzipBytes() []byte -} - -// serveContent serve http content -func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { +// servePublicAsset serve http content +func servePublicAsset(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { setWellKnownContentType(w, fi.Name()) - + httpcache.SetCacheControlInHeader(w.Header(), httpcache.CacheControlForPublicStatic()) encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) - if encodings.Contains("gzip") { - // try to provide gzip content directly from bindata (provided by vfsgen۰CompressedFileInfo) - if compressed, ok := fi.(GzipBytesProvider); ok { - rdGzip := bytes.NewReader(compressed.GzipBytes()) + fiEmbedded, _ := fi.(assetfs.EmbeddedFileInfo) + if encodings.Contains("gzip") && fiEmbedded != nil { + // try to provide gzip content directly from bindata + if gzipBytes, ok := fiEmbedded.GetGzipContent(); ok { + rdGzip := bytes.NewReader(gzipBytes) // all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name // then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data if w.Header().Get("Content-Type") == "" { w.Header().Set("Content-Type", "application/octet-stream") } w.Header().Set("Content-Encoding", "gzip") - httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, rdGzip) + http.ServeContent(w, req, fi.Name(), modtime, rdGzip) return } } - - httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, content) - return + http.ServeContent(w, req, fi.Name(), modtime, content) } diff --git a/modules/public/public_bindata.go b/modules/public/public_bindata.go index 4878f88ad1..2dcf3e72e4 100644 --- a/modules/public/public_bindata.go +++ b/modules/public/public_bindata.go @@ -5,4 +5,19 @@ package public -//go:generate go run ../../build/generate-bindata.go ../../public public bindata.go true +//go:generate go run ../../build/generate-bindata.go ../../public bindata.dat + +import ( + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", assetfs.NewEmbeddedFS(bindata)) +}) diff --git a/modules/public/serve_dynamic.go b/modules/public/public_dynamic.go index a668b17c34..a668b17c34 100644 --- a/modules/public/serve_dynamic.go +++ b/modules/public/public_dynamic.go diff --git a/modules/public/serve_static.go b/modules/public/serve_static.go deleted file mode 100644 index e79085021e..0000000000 --- a/modules/public/serve_static.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package public - -import ( - "time" - - "code.gitea.io/gitea/modules/assetfs" - "code.gitea.io/gitea/modules/timeutil" -) - -var _ GzipBytesProvider = (*vfsgen۰CompressedFileInfo)(nil) - -// GlobalModTime provide a global mod time for embedded asset files -func GlobalModTime(filename string) time.Time { - return timeutil.GetExecutableModTime() -} - -func BuiltinAssets() *assetfs.Layer { - return assetfs.Bindata("builtin(bindata)", Assets) -} diff --git a/modules/queue/base_levelqueue_common.go b/modules/queue/base_levelqueue_common.go index 78d3b85a8a..d37093b84d 100644 --- a/modules/queue/base_levelqueue_common.go +++ b/modules/queue/base_levelqueue_common.go @@ -83,7 +83,7 @@ func prepareLevelDB(cfg *BaseConfig) (conn string, db *leveldb.DB, err error) { } conn = cfg.ConnStr } - for i := 0; i < 10; i++ { + for range 10 { if db, err = nosql.GetManager().GetLevelDB(conn); err == nil { break } diff --git a/modules/queue/base_redis.go b/modules/queue/base_redis.go index a1e234943d..bea0fd7a98 100644 --- a/modules/queue/base_redis.go +++ b/modules/queue/base_redis.go @@ -29,7 +29,7 @@ func newBaseRedisGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { client := nosql.GetManager().GetRedisClient(cfg.ConnStr) var err error - for i := 0; i < 10; i++ { + for range 10 { err = client.Ping(graceful.GetManager().ShutdownContext()).Err() if err == nil { break diff --git a/modules/queue/base_test.go b/modules/queue/base_test.go index 01b52b3c16..8e7c18d740 100644 --- a/modules/queue/base_test.go +++ b/modules/queue/base_test.go @@ -17,11 +17,11 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) q, err := newFn(cfg) assert.NoError(t, err) - ctx := context.Background() + ctx := t.Context() _ = q.RemoveAll(ctx) cnt, err := q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 0, cnt) + assert.Equal(t, 0, cnt) // push the first item err = q.PushItem(ctx, []byte("foo")) @@ -29,7 +29,7 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) cnt, err = q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 1, cnt) + assert.Equal(t, 1, cnt) // push a duplicate item err = q.PushItem(ctx, []byte("foo")) @@ -45,10 +45,10 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) has, err := q.HasItem(ctx, []byte("foo")) assert.NoError(t, err) if !isUnique { - assert.EqualValues(t, 2, cnt) + assert.Equal(t, 2, cnt) assert.False(t, has) // non-unique queues don't check for duplicates } else { - assert.EqualValues(t, 1, cnt) + assert.Equal(t, 1, cnt) assert.True(t, has) } @@ -59,18 +59,18 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) // pop the first item (and the duplicate if non-unique) it, err := q.PopItem(ctx) assert.NoError(t, err) - assert.EqualValues(t, "foo", string(it)) + assert.Equal(t, "foo", string(it)) if !isUnique { it, err = q.PopItem(ctx) assert.NoError(t, err) - assert.EqualValues(t, "foo", string(it)) + assert.Equal(t, "foo", string(it)) } // pop another item it, err = q.PopItem(ctx) assert.NoError(t, err) - assert.EqualValues(t, "bar", string(it)) + assert.Equal(t, "bar", string(it)) // pop an empty queue (timeout, cancel) ctxTimed, cancel := context.WithTimeout(ctx, 10*time.Millisecond) @@ -87,7 +87,7 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) // test blocking push if queue is full for i := 0; i < cfg.Length; i++ { - err = q.PushItem(ctx, []byte(fmt.Sprintf("item-%d", i))) + err = q.PushItem(ctx, fmt.Appendf(nil, "item-%d", i)) assert.NoError(t, err) } ctxTimed, cancel = context.WithTimeout(ctx, 10*time.Millisecond) @@ -107,13 +107,13 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) // remove all cnt, err = q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, cfg.Length, cnt) + assert.Equal(t, cfg.Length, cnt) _ = q.RemoveAll(ctx) cnt, err = q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 0, cnt) + assert.Equal(t, 0, cnt) }) } @@ -121,12 +121,12 @@ func TestBaseDummy(t *testing.T) { q, err := newBaseDummy(&BaseConfig{}, true) assert.NoError(t, err) - ctx := context.Background() + ctx := t.Context() assert.NoError(t, q.PushItem(ctx, []byte("foo"))) cnt, err := q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 0, cnt) + assert.Equal(t, 0, cnt) has, err := q.HasItem(ctx, []byte("foo")) assert.NoError(t, err) diff --git a/modules/queue/manager.go b/modules/queue/manager.go index 079e2bee7a..ae6c51872d 100644 --- a/modules/queue/manager.go +++ b/modules/queue/manager.go @@ -6,6 +6,7 @@ package queue import ( "context" "errors" + "maps" "sync" "time" @@ -70,9 +71,7 @@ func (m *Manager) ManagedQueues() map[int64]ManagedWorkerPoolQueue { defer m.mu.Unlock() queues := make(map[int64]ManagedWorkerPoolQueue, len(m.Queues)) - for k, v := range m.Queues { - queues[k] = v - } + maps.Copy(queues, m.Queues) return queues } diff --git a/modules/queue/manager_test.go b/modules/queue/manager_test.go index 15dd1b4f2f..fda498cc84 100644 --- a/modules/queue/manager_test.go +++ b/modules/queue/manager_test.go @@ -4,7 +4,6 @@ package queue import ( - "context" "path/filepath" "testing" @@ -48,7 +47,7 @@ CONN_STR = redis:// assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/common"), q.baseConfig.DataFullDir) assert.Equal(t, 100000, q.baseConfig.Length) assert.Equal(t, 20, q.batchLength) - assert.Equal(t, "", q.baseConfig.ConnStr) + assert.Empty(t, q.baseConfig.ConnStr) assert.Equal(t, "default_queue", q.baseConfig.QueueFullName) assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName) assert.NotZero(t, q.GetWorkerMaxNumber()) @@ -80,7 +79,7 @@ MAX_WORKERS = 123 assert.NoError(t, err) - q1 := createWorkerPoolQueue[string](context.Background(), "no-such", cfgProvider, nil, false) + q1 := createWorkerPoolQueue[string](t.Context(), "no-such", cfgProvider, nil, false) assert.Equal(t, "no-such", q1.GetName()) assert.Equal(t, "dummy", q1.GetType()) // no handler, so it becomes dummy assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir1"), q1.baseConfig.DataFullDir) @@ -96,13 +95,13 @@ MAX_WORKERS = 123 assert.Equal(t, "string", q1.GetItemTypeName()) qid1 := GetManager().qidCounter - q2 := createWorkerPoolQueue(context.Background(), "sub", cfgProvider, func(s ...int) (unhandled []int) { return nil }, false) + q2 := createWorkerPoolQueue(t.Context(), "sub", cfgProvider, func(s ...int) (unhandled []int) { return nil }, false) assert.Equal(t, "sub", q2.GetName()) assert.Equal(t, "level", q2.GetType()) assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir2"), q2.baseConfig.DataFullDir) assert.Equal(t, 102, q2.baseConfig.Length) assert.Equal(t, 22, q2.batchLength) - assert.Equal(t, "", q2.baseConfig.ConnStr) + assert.Empty(t, q2.baseConfig.ConnStr) assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName) assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName) assert.Equal(t, 123, q2.GetWorkerMaxNumber()) @@ -118,7 +117,7 @@ MAX_WORKERS = 123 assert.Equal(t, 120, q1.workerMaxNum) stop := runWorkerPoolQueue(q2) - assert.NoError(t, GetManager().GetManagedQueue(qid2).FlushWithContext(context.Background(), 0)) - assert.NoError(t, GetManager().FlushAll(context.Background(), 0)) + assert.NoError(t, GetManager().GetManagedQueue(qid2).FlushWithContext(t.Context(), 0)) + assert.NoError(t, GetManager().FlushAll(t.Context(), 0)) stop() } diff --git a/modules/queue/workerqueue_test.go b/modules/queue/workerqueue_test.go index c0841a1752..a6c369d5f9 100644 --- a/modules/queue/workerqueue_test.go +++ b/modules/queue/workerqueue_test.go @@ -4,7 +4,6 @@ package queue import ( - "context" "slices" "strconv" "sync" @@ -58,15 +57,15 @@ func TestWorkerPoolQueueUnhandled(t *testing.T) { testRecorder.Record("push:%v", i) assert.NoError(t, q.Push(i)) } - assert.NoError(t, q.FlushWithContext(context.Background(), 0)) + assert.NoError(t, q.FlushWithContext(t.Context(), 0)) stop() ok := true for i := 0; i < queueSetting.Length; i++ { if i%2 == 0 { - ok = ok && assert.EqualValues(t, 2, m[i], "test %s: item %d", t.Name(), i) + ok = ok && assert.Equal(t, 2, m[i], "test %s: item %d", t.Name(), i) } else { - ok = ok && assert.EqualValues(t, 1, m[i], "test %s: item %d", t.Name(), i) + ok = ok && assert.Equal(t, 1, m[i], "test %s: item %d", t.Name(), i) } } if !ok { @@ -78,17 +77,17 @@ func TestWorkerPoolQueueUnhandled(t *testing.T) { runCount := 2 // we can run these tests even hundreds times to see its stability t.Run("1/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { test(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1}) } }) t.Run("3/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { test(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1}) } }) t.Run("4/5", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { test(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5}) } }) @@ -97,17 +96,17 @@ func TestWorkerPoolQueueUnhandled(t *testing.T) { func TestWorkerPoolQueuePersistence(t *testing.T) { runCount := 2 // we can run these tests even hundreds times to see its stability t.Run("1/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1, Length: 100}) } }) t.Run("3/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1, Length: 100}) } }) t.Run("4/5", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5, Length: 100}) } }) @@ -142,7 +141,7 @@ func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSett q, _ := newWorkerPoolQueueForTest("pr_patch_checker_test", queueSetting, testHandler, true) stop := runWorkerPoolQueue(q) - for i := 0; i < testCount; i++ { + for i := range testCount { _ = q.Push("task-" + strconv.Itoa(i)) } close(startWhenAllReady) @@ -166,7 +165,7 @@ func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSett q, _ := newWorkerPoolQueueForTest("pr_patch_checker_test", queueSetting, testHandler, true) stop := runWorkerPoolQueue(q) - assert.NoError(t, q.FlushWithContext(context.Background(), 0)) + assert.NoError(t, q.FlushWithContext(t.Context(), 0)) stop() } @@ -174,7 +173,7 @@ func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSett assert.NotEmpty(t, tasksQ1) assert.NotEmpty(t, tasksQ2) - assert.EqualValues(t, testCount, len(tasksQ1)+len(tasksQ2)) + assert.Equal(t, testCount, len(tasksQ1)+len(tasksQ2)) } func TestWorkerPoolQueueActiveWorkers(t *testing.T) { @@ -187,34 +186,34 @@ func TestWorkerPoolQueueActiveWorkers(t *testing.T) { q, _ := newWorkerPoolQueueForTest("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 1, Length: 100}, handler, false) stop := runWorkerPoolQueue(q) - for i := 0; i < 5; i++ { + for i := range 5 { assert.NoError(t, q.Push(i)) } time.Sleep(50 * time.Millisecond) - assert.EqualValues(t, 1, q.GetWorkerNumber()) - assert.EqualValues(t, 1, q.GetWorkerActiveNumber()) + assert.Equal(t, 1, q.GetWorkerNumber()) + assert.Equal(t, 1, q.GetWorkerActiveNumber()) time.Sleep(500 * time.Millisecond) - assert.EqualValues(t, 1, q.GetWorkerNumber()) - assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 1, q.GetWorkerNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) time.Sleep(workerIdleDuration) - assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + assert.Equal(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working stop() q, _ = newWorkerPoolQueueForTest("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 3, Length: 100}, handler, false) stop = runWorkerPoolQueue(q) - for i := 0; i < 15; i++ { + for i := range 15 { assert.NoError(t, q.Push(i)) } time.Sleep(50 * time.Millisecond) - assert.EqualValues(t, 3, q.GetWorkerNumber()) - assert.EqualValues(t, 3, q.GetWorkerActiveNumber()) + assert.Equal(t, 3, q.GetWorkerNumber()) + assert.Equal(t, 3, q.GetWorkerActiveNumber()) time.Sleep(500 * time.Millisecond) - assert.EqualValues(t, 3, q.GetWorkerNumber()) - assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 3, q.GetWorkerNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) time.Sleep(workerIdleDuration) - assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + assert.Equal(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working stop() } @@ -241,13 +240,13 @@ func TestWorkerPoolQueueShutdown(t *testing.T) { } <-handlerCalled time.Sleep(200 * time.Millisecond) // wait for a while to make sure all workers are active - assert.EqualValues(t, 4, q.GetWorkerActiveNumber()) + assert.Equal(t, 4, q.GetWorkerActiveNumber()) stop() // stop triggers shutdown - assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) // no item was ever handled, so we still get all of them again q, _ = newWorkerPoolQueueForTest("test-workpoolqueue", qs, handler, false) - assert.EqualValues(t, 20, q.GetQueueItemNumber()) + assert.Equal(t, 20, q.GetQueueItemNumber()) } func TestWorkerPoolQueueWorkerIdleReset(t *testing.T) { @@ -275,7 +274,7 @@ func TestWorkerPoolQueueWorkerIdleReset(t *testing.T) { } q, _ = newWorkerPoolQueueForTest("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 2, Length: 100}, handler, false) stop := runWorkerPoolQueue(q) - for i := 0; i < 100; i++ { + for i := range 100 { assert.NoError(t, q.Push(i)) } time.Sleep(500 * time.Millisecond) diff --git a/modules/references/references.go b/modules/references/references.go index a5b102b7f2..592bd4cbe4 100644 --- a/modules/references/references.go +++ b/modules/references/references.go @@ -462,11 +462,12 @@ func findAllIssueReferencesBytes(content []byte, links []string) []*rawReference continue } var sep string - if parts[3] == "issues" { + switch parts[3] { + case "issues": sep = "#" - } else if parts[3] == "pulls" { + case "pulls": sep = "!" - } else { + default: continue } // Note: closing/reopening keywords not supported with URLs diff --git a/modules/references/references_test.go b/modules/references/references_test.go index 1b6a968d6a..a15ae99f79 100644 --- a/modules/references/references_test.go +++ b/modules/references/references_test.go @@ -46,7 +46,7 @@ owner/repo!123456789 contentBytes := []byte(test) convertFullHTMLReferencesToShortRefs(re, &contentBytes) result := string(contentBytes) - assert.EqualValues(t, expect, result) + assert.Equal(t, expect, result) } func TestFindAllIssueReferences(t *testing.T) { @@ -283,9 +283,9 @@ func testFixtures(t *testing.T, fixtures []testFixture, context string) { } expref := rawToIssueReferenceList(expraw) refs := FindAllIssueReferencesMarkdown(fixture.input) - assert.EqualValues(t, expref, refs, "[%s] Failed to parse: {%s}", context, fixture.input) + assert.Equal(t, expref, refs, "[%s] Failed to parse: {%s}", context, fixture.input) rawrefs := findAllIssueReferencesMarkdown(fixture.input) - assert.EqualValues(t, expraw, rawrefs, "[%s] Failed to parse: {%s}", context, fixture.input) + assert.Equal(t, expraw, rawrefs, "[%s] Failed to parse: {%s}", context, fixture.input) } // Restore for other tests that may rely on the original value @@ -294,7 +294,7 @@ func testFixtures(t *testing.T, fixtures []testFixture, context string) { func TestFindAllMentions(t *testing.T) { res := FindAllMentionsBytes([]byte("@tasha, @mike; @lucy: @john")) - assert.EqualValues(t, []RefSpan{ + assert.Equal(t, []RefSpan{ {Start: 0, End: 6}, {Start: 8, End: 13}, {Start: 15, End: 20}, @@ -554,7 +554,7 @@ func TestParseCloseKeywords(t *testing.T) { res := pat.FindAllStringSubmatch(test.match, -1) assert.Len(t, res, 1) assert.Len(t, res[0], 2) - assert.EqualValues(t, test.expected, res[0][1]) + assert.Equal(t, test.expected, res[0][1]) } } } diff --git a/modules/regexplru/regexplru_test.go b/modules/regexplru/regexplru_test.go index 9c24b23fa9..4b539c31e9 100644 --- a/modules/regexplru/regexplru_test.go +++ b/modules/regexplru/regexplru_test.go @@ -18,9 +18,9 @@ func TestRegexpLru(t *testing.T) { assert.NoError(t, err) assert.True(t, r.MatchString("a")) - assert.EqualValues(t, 1, lruCache.Len()) + assert.Equal(t, 1, lruCache.Len()) _, err = GetCompiled("(") assert.Error(t, err) - assert.EqualValues(t, 2, lruCache.Len()) + assert.Equal(t, 2, lruCache.Len()) } diff --git a/modules/repository/branch.go b/modules/repository/branch.go index 2bf9930f19..30aa0a6e85 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -41,11 +41,12 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, if err != nil { return 0, fmt.Errorf("GetObjectFormat: %w", err) } - _, err = db.GetEngine(ctx).ID(repo.ID).Update(&repo_model.Repository{ObjectFormatName: objFmt.Name()}) - if err != nil { - return 0, fmt.Errorf("UpdateRepository: %w", err) + if objFmt.Name() != repo.ObjectFormatName { + repo.ObjectFormatName = objFmt.Name() + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "object_format_name"); err != nil { + return 0, fmt.Errorf("UpdateRepositoryColsWithAutoTime: %w", err) + } } - repo.ObjectFormatName = objFmt.Name() // keep consistent with db allBranches := container.Set[string]{} { diff --git a/modules/repository/branch_test.go b/modules/repository/branch_test.go index acf75a1ac0..ead28aa141 100644 --- a/modules/repository/branch_test.go +++ b/modules/repository/branch_test.go @@ -27,5 +27,5 @@ func TestSyncRepoBranches(t *testing.T) { assert.Equal(t, "sha1", repo.ObjectFormatName) branch, err := git_model.GetBranch(db.DefaultContext, 1, "master") assert.NoError(t, err) - assert.EqualValues(t, "master", branch.Name) + assert.Equal(t, "master", branch.Name) } diff --git a/modules/repository/commits.go b/modules/repository/commits.go index 6e4b75d5ca..878fdc1603 100644 --- a/modules/repository/commits.go +++ b/modules/repository/commits.go @@ -10,8 +10,10 @@ import ( "time" "code.gitea.io/gitea/models/avatars" + repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -43,7 +45,7 @@ func NewPushCommits() *PushCommits { } // ToAPIPayloadCommit converts a single PushCommit to an api.PayloadCommit object. -func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.User, repoPath, repoLink string, commit *PushCommit) (*api.PayloadCommit, error) { +func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.User, repo *repo_model.Repository, commit *PushCommit) (*api.PayloadCommit, error) { var err error authorUsername := "" author, ok := emailUsers[commit.AuthorEmail] @@ -70,7 +72,7 @@ func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.U committerUsername = committer.Name } - fileStatus, err := git.GetCommitFileStatus(ctx, repoPath, commit.Sha1) + fileStatus, err := git.GetCommitFileStatus(ctx, repo.RepoPath(), commit.Sha1) if err != nil { return nil, fmt.Errorf("FileStatus [commit_sha1: %s]: %w", commit.Sha1, err) } @@ -78,7 +80,7 @@ func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.U return &api.PayloadCommit{ ID: commit.Sha1, Message: commit.Message, - URL: fmt.Sprintf("%s/commit/%s", repoLink, url.PathEscape(commit.Sha1)), + URL: fmt.Sprintf("%s/commit/%s", repo.HTMLURL(), url.PathEscape(commit.Sha1)), Author: &api.PayloadUser{ Name: commit.AuthorName, Email: commit.AuthorEmail, @@ -98,14 +100,14 @@ func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.U // ToAPIPayloadCommits converts a PushCommits object to api.PayloadCommit format. // It returns all converted commits and, if provided, the head commit or an error otherwise. -func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repoPath, repoLink string) ([]*api.PayloadCommit, *api.PayloadCommit, error) { +func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repo *repo_model.Repository) ([]*api.PayloadCommit, *api.PayloadCommit, error) { commits := make([]*api.PayloadCommit, len(pc.Commits)) var headCommit *api.PayloadCommit emailUsers := make(map[string]*user_model.User) for i, commit := range pc.Commits { - apiCommit, err := ToAPIPayloadCommit(ctx, emailUsers, repoPath, repoLink, commit) + apiCommit, err := ToAPIPayloadCommit(ctx, emailUsers, repo, commit) if err != nil { return nil, nil, err } @@ -117,7 +119,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repoPath, repoLi } if pc.HeadCommit != nil && headCommit == nil { var err error - headCommit, err = ToAPIPayloadCommit(ctx, emailUsers, repoPath, repoLink, pc.HeadCommit) + headCommit, err = ToAPIPayloadCommit(ctx, emailUsers, repo, pc.HeadCommit) if err != nil { return nil, nil, err } @@ -130,7 +132,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repoPath, repoLi func (pc *PushCommits) AvatarLink(ctx context.Context, email string) string { size := avatars.DefaultAvatarPixelSize * setting.Avatar.RenderedSizeFactor - v, _ := cache.GetWithContextCache(ctx, "push_commits", email, func() (string, error) { + v, _ := cache.GetWithContextCache(ctx, cachegroup.EmailAvatarLink, email, func(ctx context.Context, email string) (string, error) { u, err := user_model.GetUserByEmail(ctx, email) if err != nil { if !user_model.IsErrUserNotExist(err) { diff --git a/modules/repository/commits_test.go b/modules/repository/commits_test.go index 3afc116e68..030cd7714d 100644 --- a/modules/repository/commits_test.go +++ b/modules/repository/commits_test.go @@ -50,54 +50,54 @@ func TestPushCommits_ToAPIPayloadCommits(t *testing.T) { pushCommits.HeadCommit = &PushCommit{Sha1: "69554a6"} repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 16}) - payloadCommits, headCommit, err := pushCommits.ToAPIPayloadCommits(git.DefaultContext, repo.RepoPath(), "/user2/repo16") + payloadCommits, headCommit, err := pushCommits.ToAPIPayloadCommits(git.DefaultContext, repo) assert.NoError(t, err) assert.Len(t, payloadCommits, 3) assert.NotNil(t, headCommit) assert.Equal(t, "69554a6", payloadCommits[0].ID) assert.Equal(t, "not signed commit", payloadCommits[0].Message) - assert.Equal(t, "/user2/repo16/commit/69554a6", payloadCommits[0].URL) + assert.Equal(t, "https://try.gitea.io/user2/repo16/commit/69554a6", payloadCommits[0].URL) assert.Equal(t, "User2", payloadCommits[0].Committer.Name) assert.Equal(t, "user2", payloadCommits[0].Committer.UserName) assert.Equal(t, "User2", payloadCommits[0].Author.Name) assert.Equal(t, "user2", payloadCommits[0].Author.UserName) - assert.EqualValues(t, []string{}, payloadCommits[0].Added) - assert.EqualValues(t, []string{}, payloadCommits[0].Removed) - assert.EqualValues(t, []string{"readme.md"}, payloadCommits[0].Modified) + assert.Equal(t, []string{}, payloadCommits[0].Added) + assert.Equal(t, []string{}, payloadCommits[0].Removed) + assert.Equal(t, []string{"readme.md"}, payloadCommits[0].Modified) assert.Equal(t, "27566bd", payloadCommits[1].ID) assert.Equal(t, "good signed commit (with not yet validated email)", payloadCommits[1].Message) - assert.Equal(t, "/user2/repo16/commit/27566bd", payloadCommits[1].URL) + assert.Equal(t, "https://try.gitea.io/user2/repo16/commit/27566bd", payloadCommits[1].URL) assert.Equal(t, "User2", payloadCommits[1].Committer.Name) assert.Equal(t, "user2", payloadCommits[1].Committer.UserName) assert.Equal(t, "User2", payloadCommits[1].Author.Name) assert.Equal(t, "user2", payloadCommits[1].Author.UserName) - assert.EqualValues(t, []string{}, payloadCommits[1].Added) - assert.EqualValues(t, []string{}, payloadCommits[1].Removed) - assert.EqualValues(t, []string{"readme.md"}, payloadCommits[1].Modified) + assert.Equal(t, []string{}, payloadCommits[1].Added) + assert.Equal(t, []string{}, payloadCommits[1].Removed) + assert.Equal(t, []string{"readme.md"}, payloadCommits[1].Modified) assert.Equal(t, "5099b81", payloadCommits[2].ID) assert.Equal(t, "good signed commit", payloadCommits[2].Message) - assert.Equal(t, "/user2/repo16/commit/5099b81", payloadCommits[2].URL) + assert.Equal(t, "https://try.gitea.io/user2/repo16/commit/5099b81", payloadCommits[2].URL) assert.Equal(t, "User2", payloadCommits[2].Committer.Name) assert.Equal(t, "user2", payloadCommits[2].Committer.UserName) assert.Equal(t, "User2", payloadCommits[2].Author.Name) assert.Equal(t, "user2", payloadCommits[2].Author.UserName) - assert.EqualValues(t, []string{"readme.md"}, payloadCommits[2].Added) - assert.EqualValues(t, []string{}, payloadCommits[2].Removed) - assert.EqualValues(t, []string{}, payloadCommits[2].Modified) + assert.Equal(t, []string{"readme.md"}, payloadCommits[2].Added) + assert.Equal(t, []string{}, payloadCommits[2].Removed) + assert.Equal(t, []string{}, payloadCommits[2].Modified) assert.Equal(t, "69554a6", headCommit.ID) assert.Equal(t, "not signed commit", headCommit.Message) - assert.Equal(t, "/user2/repo16/commit/69554a6", headCommit.URL) + assert.Equal(t, "https://try.gitea.io/user2/repo16/commit/69554a6", headCommit.URL) assert.Equal(t, "User2", headCommit.Committer.Name) assert.Equal(t, "user2", headCommit.Committer.UserName) assert.Equal(t, "User2", headCommit.Author.Name) assert.Equal(t, "user2", headCommit.Author.UserName) - assert.EqualValues(t, []string{}, headCommit.Added) - assert.EqualValues(t, []string{}, headCommit.Removed) - assert.EqualValues(t, []string{"readme.md"}, headCommit.Modified) + assert.Equal(t, []string{}, headCommit.Added) + assert.Equal(t, []string{}, headCommit.Removed) + assert.Equal(t, []string{"readme.md"}, headCommit.Modified) } func TestPushCommits_AvatarLink(t *testing.T) { @@ -200,5 +200,3 @@ func TestListToPushCommits(t *testing.T) { assert.Equal(t, now, pushCommits.Commits[1].Timestamp) } } - -// TODO TestPushUpdate diff --git a/modules/repository/create.go b/modules/repository/create.go index b4f7033bd7..a75598a84b 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -7,19 +7,10 @@ import ( "context" "fmt" "os" - "path" "path/filepath" - "strings" - activities_model "code.gitea.io/gitea/models/activities" - "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" - access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" - issue_indexer "code.gitea.io/gitea/modules/indexer/issues" - "code.gitea.io/gitea/modules/log" - api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/util" ) const notRegularFileMode = os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular @@ -64,97 +55,3 @@ func UpdateRepoSize(ctx context.Context, repo *repo_model.Repository) error { return repo_model.UpdateRepoSize(ctx, repo.ID, size, lfsSize) } - -// CheckDaemonExportOK creates/removes git-daemon-export-ok for git-daemon... -func CheckDaemonExportOK(ctx context.Context, repo *repo_model.Repository) error { - if err := repo.LoadOwner(ctx); err != nil { - return err - } - - // Create/Remove git-daemon-export-ok for git-daemon... - daemonExportFile := path.Join(repo.RepoPath(), `git-daemon-export-ok`) - - isExist, err := util.IsExist(daemonExportFile) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err) - return err - } - - isPublic := !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePublic - if !isPublic && isExist { - if err = util.Remove(daemonExportFile); err != nil { - log.Error("Failed to remove %s: %v", daemonExportFile, err) - } - } else if isPublic && !isExist { - if f, err := os.Create(daemonExportFile); err != nil { - log.Error("Failed to create %s: %v", daemonExportFile, err) - } else { - f.Close() - } - } - - return nil -} - -// UpdateRepository updates a repository with db context -func UpdateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { - repo.LowerName = strings.ToLower(repo.Name) - - e := db.GetEngine(ctx) - - if _, err = e.ID(repo.ID).AllCols().Update(repo); err != nil { - return fmt.Errorf("update: %w", err) - } - - if err = UpdateRepoSize(ctx, repo); err != nil { - log.Error("Failed to update size for repository: %v", err) - } - - if visibilityChanged { - if err = repo.LoadOwner(ctx); err != nil { - return fmt.Errorf("LoadOwner: %w", err) - } - if repo.Owner.IsOrganization() { - // Organization repository need to recalculate access table when visibility is changed. - if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { - return fmt.Errorf("recalculateTeamAccesses: %w", err) - } - } - - // If repo has become private, we need to set its actions to private. - if repo.IsPrivate { - _, err = e.Where("repo_id = ?", repo.ID).Cols("is_private").Update(&activities_model.Action{ - IsPrivate: true, - }) - if err != nil { - return err - } - - if err = repo_model.ClearRepoStars(ctx, repo.ID); err != nil { - return err - } - } - - // Create/Remove git-daemon-export-ok for git-daemon... - if err := CheckDaemonExportOK(ctx, repo); err != nil { - return err - } - - forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) - if err != nil { - return fmt.Errorf("getRepositoriesByForkID: %w", err) - } - for i := range forkRepos { - forkRepos[i].IsPrivate = repo.IsPrivate || repo.Owner.Visibility == api.VisibleTypePrivate - if err = UpdateRepository(ctx, forkRepos[i], true); err != nil { - return fmt.Errorf("updateRepository[%d]: %w", forkRepos[i].ID, err) - } - } - - // If visibility is changed, we need to update the issue indexer. - // Since the data in the issue indexer have field to indicate if the repo is public or not. - issue_indexer.UpdateRepoIndexer(ctx, repo.ID) - } - - return nil -} diff --git a/modules/repository/create_test.go b/modules/repository/create_test.go index a9151482b4..b85a10adad 100644 --- a/modules/repository/create_test.go +++ b/modules/repository/create_test.go @@ -6,7 +6,6 @@ package repository import ( "testing" - activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" @@ -14,26 +13,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestUpdateRepositoryVisibilityChanged(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - - // Get sample repo and change visibility - repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 9) - assert.NoError(t, err) - repo.IsPrivate = true - - // Update it - err = UpdateRepository(db.DefaultContext, repo, true) - assert.NoError(t, err) - - // Check visibility of action has become private - act := activities_model.Action{} - _, err = db.GetEngine(db.DefaultContext).ID(3).Get(&act) - - assert.NoError(t, err) - assert.True(t, act.IsPrivate) -} - func TestGetDirectorySize(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 1) @@ -41,5 +20,5 @@ func TestGetDirectorySize(t *testing.T) { size, err := getDirectorySize(repo.RepoPath()) assert.NoError(t, err) repo.Size = 8165 // real size on the disk - assert.EqualValues(t, repo.Size, size) + assert.Equal(t, repo.Size, size) } diff --git a/modules/repository/env.go b/modules/repository/env.go index e4f32092fc..78e06f86fb 100644 --- a/modules/repository/env.go +++ b/modules/repository/env.go @@ -4,8 +4,8 @@ package repository import ( - "fmt" "os" + "strconv" "strings" repo_model "code.gitea.io/gitea/models/repo" @@ -72,9 +72,9 @@ func FullPushingEnvironment(author, committer *user_model.User, repo *repo_model EnvRepoUsername+"="+repo.OwnerName, EnvRepoIsWiki+"="+isWiki, EnvPusherName+"="+committer.Name, - EnvPusherID+"="+fmt.Sprintf("%d", committer.ID), - EnvRepoID+"="+fmt.Sprintf("%d", repo.ID), - EnvPRID+"="+fmt.Sprintf("%d", prID), + EnvPusherID+"="+strconv.FormatInt(committer.ID, 10), + EnvRepoID+"="+strconv.FormatInt(repo.ID, 10), + EnvPRID+"="+strconv.FormatInt(prID, 10), EnvAppURL+"="+setting.AppURL, "SSH_ORIGINAL_COMMAND=gitea-internal", ) diff --git a/modules/repository/init.go b/modules/repository/init.go index 24602ae090..12e9606c74 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -11,10 +11,7 @@ import ( "strings" issues_model "code.gitea.io/gitea/models/issues" - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/label" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -120,30 +117,6 @@ func LoadRepoConfig() error { return nil } -func CheckInitRepository(ctx context.Context, owner, name, objectFormatName string) (err error) { - // Somehow the directory could exist. - repoPath := repo_model.RepoPath(owner, name) - isExist, err := util.IsExist(repoPath) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return err - } - if isExist { - return repo_model.ErrRepoFilesAlreadyExist{ - Uname: owner, - Name: name, - } - } - - // Init git bare new repository. - if err = git.InitRepository(ctx, repoPath, true, objectFormatName); err != nil { - return fmt.Errorf("git.InitRepository: %w", err) - } else if err = CreateDelegateHooks(repoPath); err != nil { - return fmt.Errorf("createDelegateHooks: %w", err) - } - return nil -} - // InitializeLabels adds a label set to a repository using a template func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg bool) error { list, err := LoadTemplateLabelsByDisplayName(labelTemplate) @@ -152,12 +125,13 @@ func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg } labels := make([]*issues_model.Label, len(list)) - for i := 0; i < len(list); i++ { + for i := range list { labels[i] = &issues_model.Label{ - Name: list[i].Name, - Exclusive: list[i].Exclusive, - Description: list[i].Description, - Color: list[i].Color, + Name: list[i].Name, + Exclusive: list[i].Exclusive, + ExclusiveOrder: list[i].ExclusiveOrder, + Description: list[i].Description, + Color: list[i].Color, } if isOrg { labels[i].OrgID = id diff --git a/modules/repository/init_test.go b/modules/repository/init_test.go index 227efdc1db..1fa928105c 100644 --- a/modules/repository/init_test.go +++ b/modules/repository/init_test.go @@ -14,17 +14,17 @@ func TestMergeCustomLabels(t *testing.T) { all: []string{"a", "a.yaml", "a.yml"}, custom: nil, }) - assert.EqualValues(t, []string{"a.yaml"}, files, "yaml file should win") + assert.Equal(t, []string{"a.yaml"}, files, "yaml file should win") files = mergeCustomLabelFiles(optionFileList{ all: []string{"a", "a.yaml"}, custom: []string{"a"}, }) - assert.EqualValues(t, []string{"a"}, files, "custom file should win") + assert.Equal(t, []string{"a"}, files, "custom file should win") files = mergeCustomLabelFiles(optionFileList{ all: []string{"a", "a.yml", "a.yaml"}, custom: []string{"a", "a.yml"}, }) - assert.EqualValues(t, []string{"a.yml"}, files, "custom yml file should win if no yaml") + assert.Equal(t, []string{"a.yml"}, files, "custom yml file should win if no yaml") } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 97b0343381..ad4a53b858 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -9,13 +9,10 @@ import ( "fmt" "io" "strings" - "time" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" @@ -59,118 +56,6 @@ func SyncRepoTags(ctx context.Context, repoID int64) error { return SyncReleasesWithTags(ctx, repo, gitRepo) } -// SyncReleasesWithTags synchronizes release table with repository tags -func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) - - // optimized procedure for pull-mirrors which saves a lot of time (in - // particular for repos with many tags). - if repo.IsMirror { - return pullMirrorReleaseSync(ctx, repo, gitRepo) - } - - existingRelTags := make(container.Set[string]) - opts := repo_model.FindReleasesOptions{ - IncludeDrafts: true, - IncludeTags: true, - ListOptions: db.ListOptions{PageSize: 50}, - RepoID: repo.ID, - } - for page := 1; ; page++ { - opts.Page = page - rels, err := db.Find[repo_model.Release](gitRepo.Ctx, opts) - if err != nil { - return fmt.Errorf("unable to GetReleasesByRepoID in Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) - } - if len(rels) == 0 { - break - } - for _, rel := range rels { - if rel.IsDraft { - continue - } - commitID, err := gitRepo.GetTagCommitID(rel.TagName) - if err != nil && !git.IsErrNotExist(err) { - return fmt.Errorf("unable to GetTagCommitID for %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) - } - if git.IsErrNotExist(err) || commitID != rel.Sha1 { - if err := repo_model.PushUpdateDeleteTag(ctx, repo, rel.TagName); err != nil { - return fmt.Errorf("unable to PushUpdateDeleteTag: %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) - } - } else { - existingRelTags.Add(strings.ToLower(rel.TagName)) - } - } - } - - _, err := gitRepo.WalkReferences(git.ObjectTag, 0, 0, func(sha1, refname string) error { - tagName := strings.TrimPrefix(refname, git.TagPrefix) - if existingRelTags.Contains(strings.ToLower(tagName)) { - return nil - } - - if err := PushUpdateAddTag(ctx, repo, gitRepo, tagName, sha1, refname); err != nil { - // sometimes, some tags will be sync failed. i.e. https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tag/?h=v2.6.11 - // this is a tree object, not a tag object which created before git - log.Error("unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %v", tagName, repo.ID, repo.OwnerName, repo.Name, err) - } - - return nil - }) - return err -} - -// PushUpdateAddTag must be called for any push actions to add tag -func PushUpdateAddTag(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tagName, sha1, refname string) error { - tag, err := gitRepo.GetTagWithID(sha1, tagName) - if err != nil { - return fmt.Errorf("unable to GetTag: %w", err) - } - commit, err := tag.Commit(gitRepo) - if err != nil { - return fmt.Errorf("unable to get tag Commit: %w", err) - } - - sig := tag.Tagger - if sig == nil { - sig = commit.Author - } - if sig == nil { - sig = commit.Committer - } - - var author *user_model.User - createdAt := time.Unix(1, 0) - - if sig != nil { - author, err = user_model.GetUserByEmail(ctx, sig.Email) - if err != nil && !user_model.IsErrUserNotExist(err) { - return fmt.Errorf("unable to GetUserByEmail for %q: %w", sig.Email, err) - } - createdAt = sig.When - } - - commitsCount, err := commit.CommitsCount() - if err != nil { - return fmt.Errorf("unable to get CommitsCount: %w", err) - } - - rel := repo_model.Release{ - RepoID: repo.ID, - TagName: tagName, - LowerTagName: strings.ToLower(tagName), - Sha1: commit.ID.String(), - NumCommits: commitsCount, - CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), - IsTag: true, - } - if author != nil { - rel.PublisherID = author.ID - } - - return repo_model.SaveOrUpdateTag(ctx, repo, &rel) -} - // StoreMissingLfsObjectsInRepository downloads missing LFS objects func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error { contentStore := lfs.NewContentStore() @@ -286,18 +171,19 @@ func (shortRelease) TableName() string { return "release" } -// pullMirrorReleaseSync is a pull-mirror specific tag<->release table +// SyncReleasesWithTags is a tag<->release table // synchronization which overwrites all Releases from the repository tags. This // can be relied on since a pull-mirror is always identical to its -// upstream. Hence, after each sync we want the pull-mirror release set to be +// upstream. Hence, after each sync we want the release set to be // identical to the upstream tag set. This is much more efficient for // repositories like https://github.com/vim/vim (with over 13000 tags). -func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) - tags, numTags, err := gitRepo.GetTagInfos(0, 0) +func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) + tags, _, err := gitRepo.GetTagInfos(0, 0) if err != nil { return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) } + var added, deleted, updated int err = db.WithTx(ctx, func(ctx context.Context) error { dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{ RepoID: repo.ID, @@ -318,9 +204,7 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git TagName: tag.Name, LowerTagName: strings.ToLower(tag.Name), Sha1: tag.Object.String(), - // NOTE: ignored, since NumCommits are unused - // for pull-mirrors (only relevant when - // displaying releases, IsTag: false) + // NOTE: ignored, The NumCommits value is calculated and cached on demand when the UI requires it. NumCommits: -1, CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()), IsTag: true, @@ -349,13 +233,14 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err) } } + added, deleted, updated = len(deletes), len(updates), len(inserts) return nil }) if err != nil { return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) } - log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags) + log.Trace("SyncReleasesWithTags: %d tags added, %d tags deleted, %d tags updated", added, deleted, updated) return nil } diff --git a/modules/repository/repo_test.go b/modules/repository/repo_test.go index f3e7be6d7d..f79a79ccbd 100644 --- a/modules/repository/repo_test.go +++ b/modules/repository/repo_test.go @@ -63,7 +63,7 @@ func Test_calcSync(t *testing.T) { inserts, deletes, updates := calcSync(gitTags, dbReleases) if assert.Len(t, inserts, 1, "inserts") { - assert.EqualValues(t, *gitTags[2], *inserts[0], "inserts equal") + assert.Equal(t, *gitTags[2], *inserts[0], "inserts equal") } if assert.Len(t, deletes, 1, "deletes") { @@ -71,6 +71,6 @@ func Test_calcSync(t *testing.T) { } if assert.Len(t, updates, 1, "updates") { - assert.EqualValues(t, *gitTags[1], *updates[0], "updates equal") + assert.Equal(t, *gitTags[1], *updates[0], "updates equal") } } diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 04faa9db3d..d7253d9e02 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -4,42 +4,19 @@ package repository import ( + "context" "fmt" - "os" - "path" - "path/filepath" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) -// LocalCopyPath returns the local repository temporary copy path. -func LocalCopyPath() string { - if filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { - return setting.Repository.Local.LocalCopyPath - } - return path.Join(setting.AppDataPath, setting.Repository.Local.LocalCopyPath) -} - // CreateTemporaryPath creates a temporary path -func CreateTemporaryPath(prefix string) (string, error) { - if err := os.MkdirAll(LocalCopyPath(), os.ModePerm); err != nil { - log.Error("Unable to create localcopypath directory: %s (%v)", LocalCopyPath(), err) - return "", fmt.Errorf("Failed to create localcopypath directory %s: %w", LocalCopyPath(), err) - } - basePath, err := os.MkdirTemp(LocalCopyPath(), prefix+".git") +func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { + basePath, cleanup, err := setting.AppDataTempDir("local-repo").MkdirTempRandom(prefix + ".git") if err != nil { log.Error("Unable to create temporary directory: %s-*.git (%v)", prefix, err) - return "", fmt.Errorf("Failed to create dir %s-*.git: %w", prefix, err) - } - return basePath, nil -} - -// RemoveTemporaryPath removes the temporary path -func RemoveTemporaryPath(basePath string) error { - if _, err := os.Stat(basePath); !os.IsNotExist(err) { - return util.RemoveAll(basePath) + return "", nil, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) } - return nil + return basePath, cleanup, nil } diff --git a/modules/reqctx/datastore.go b/modules/reqctx/datastore.go index 94232450f3..1d4bee613f 100644 --- a/modules/reqctx/datastore.go +++ b/modules/reqctx/datastore.go @@ -6,6 +6,7 @@ package reqctx import ( "context" "io" + "maps" "sync" "code.gitea.io/gitea/modules/process" @@ -22,9 +23,7 @@ func (ds ContextData) GetData() ContextData { } func (ds ContextData) MergeFrom(other ContextData) ContextData { - for k, v := range other { - ds[k] = v - } + maps.Copy(ds, other) return ds } @@ -94,6 +93,9 @@ type RequestContext interface { } func FromContext(ctx context.Context) RequestContext { + if rc, ok := ctx.(RequestContext); ok { + return rc + } // here we must use the current ctx and the underlying store // the current ctx guarantees that the ctx deadline/cancellation/values are respected // the underlying store guarantees that the request-specific data is available @@ -134,6 +136,6 @@ func NewRequestContext(parentCtx context.Context, profDesc string) (_ context.Co // NewRequestContextForTest creates a new RequestContext for testing purposes // It doesn't add the context to the process manager, nor do cleanup -func NewRequestContextForTest(parentCtx context.Context) context.Context { +func NewRequestContextForTest(parentCtx context.Context) RequestContext { return &requestContext{Context: parentCtx, RequestDataStore: &requestDataStore{values: make(map[any]any)}} } diff --git a/modules/secret/secret.go b/modules/secret/secret.go index e70ae1839c..af894a054c 100644 --- a/modules/secret/secret.go +++ b/modules/secret/secret.go @@ -16,6 +16,7 @@ import ( ) // AesEncrypt encrypts text and given key with AES. +// It is only internally used at the moment to use "SECRET_KEY" for some database values. func AesEncrypt(key, text []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { @@ -27,12 +28,13 @@ func AesEncrypt(key, text []byte) ([]byte, error) { if _, err = io.ReadFull(rand.Reader, iv); err != nil { return nil, fmt.Errorf("AesEncrypt unable to read IV: %w", err) } - cfb := cipher.NewCFBEncrypter(block, iv) + cfb := cipher.NewCFBEncrypter(block, iv) //nolint:staticcheck // need to migrate and refactor to a new approach cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b)) return ciphertext, nil } // AesDecrypt decrypts text and given key with AES. +// It is only internally used at the moment to use "SECRET_KEY" for some database values. func AesDecrypt(key, text []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { @@ -43,7 +45,7 @@ func AesDecrypt(key, text []byte) ([]byte, error) { } iv := text[:aes.BlockSize] text = text[aes.BlockSize:] - cfb := cipher.NewCFBDecrypter(block, iv) + cfb := cipher.NewCFBDecrypter(block, iv) //nolint:staticcheck // need to migrate and refactor to a new approach cfb.XORKeyStream(text, text) data, err := base64.StdEncoding.DecodeString(string(text)) if err != nil { diff --git a/modules/session/key.go b/modules/session/key.go new file mode 100644 index 0000000000..c3da997c67 --- /dev/null +++ b/modules/session/key.go @@ -0,0 +1,11 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package session + +const ( + KeyUID = "uid" + KeyUname = "uname" + + KeyUserHasTwoFactorAuth = "userHasTwoFactorAuth" +) diff --git a/modules/setting/actions_test.go b/modules/setting/actions_test.go index 3645a3f5da..353cc657fa 100644 --- a/modules/setting/actions_test.go +++ b/modules/setting/actions_test.go @@ -21,9 +21,9 @@ func Test_getStorageInheritNameSectionTypeForActions(t *testing.T) { assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) iniStr = ` [storage.actions_log] @@ -34,9 +34,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + assert.Equal(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) iniStr = ` [storage.actions_log] @@ -50,9 +50,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + assert.Equal(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) iniStr = ` [storage.actions_artifacts] @@ -66,9 +66,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "local", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.Equal(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) iniStr = ` [storage.actions_artifacts] @@ -82,9 +82,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "local", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.Equal(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) iniStr = `` cfg, err = NewConfigProviderFromData(iniStr) @@ -92,9 +92,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "local", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.Equal(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + assert.Equal(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) } func Test_getDefaultActionsURLForActions(t *testing.T) { @@ -175,7 +175,7 @@ DEFAULT_ACTIONS_URL = gitea if !tt.wantErr(t, loadActionsFrom(cfg)) { return } - assert.EqualValues(t, tt.wantURL, Actions.DefaultActionsURL.URL()) + assert.Equal(t, tt.wantURL, Actions.DefaultActionsURL.URL()) }) } } diff --git a/modules/setting/api.go b/modules/setting/api.go index c36f05cfd1..cdad474cb9 100644 --- a/modules/setting/api.go +++ b/modules/setting/api.go @@ -18,6 +18,7 @@ var API = struct { DefaultPagingNum int DefaultGitTreesPerPage int DefaultMaxBlobSize int64 + DefaultMaxResponseSize int64 }{ EnableSwagger: true, SwaggerURL: "", @@ -25,6 +26,7 @@ var API = struct { DefaultPagingNum: 30, DefaultGitTreesPerPage: 1000, DefaultMaxBlobSize: 10485760, + DefaultMaxResponseSize: 104857600, } func loadAPIFrom(rootCfg ConfigProvider) { diff --git a/modules/setting/attachment_test.go b/modules/setting/attachment_test.go index 3e8d2da4d9..c566dfa60c 100644 --- a/modules/setting/attachment_test.go +++ b/modules/setting/attachment_test.go @@ -25,9 +25,9 @@ MINIO_ENDPOINT = my_minio:9000 assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint) - assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint) + assert.Equal(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) { @@ -47,8 +47,8 @@ MINIO_BUCKET = gitea assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } func Test_getStorageSpecificOverridesStorage(t *testing.T) { @@ -69,8 +69,8 @@ STORAGE_TYPE = local assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } func Test_getStorageGetDefaults(t *testing.T) { @@ -80,7 +80,7 @@ func Test_getStorageGetDefaults(t *testing.T) { assert.NoError(t, loadAttachmentFrom(cfg)) // default storage is local, so bucket is empty - assert.EqualValues(t, "", Attachment.Storage.MinioConfig.Bucket) + assert.Empty(t, Attachment.Storage.MinioConfig.Bucket) } func Test_getStorageInheritNameSectionType(t *testing.T) { @@ -115,7 +115,7 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Attachment.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) } func Test_AttachmentStorage1(t *testing.T) { @@ -128,6 +128,6 @@ STORAGE_TYPE = minio assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "gitea", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } diff --git a/modules/setting/config_env.go b/modules/setting/config_env.go index 5d94a9641f..409588dc44 100644 --- a/modules/setting/config_env.go +++ b/modules/setting/config_env.go @@ -97,7 +97,7 @@ func decodeEnvSectionKey(encoded string) (ok bool, section, key string) { // decodeEnvironmentKey decode the environment key to section and key // The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE -func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) { //nolint:unparam +func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) { if !strings.HasPrefix(envKey, prefixGitea) { return false, "", "", false } diff --git a/modules/setting/config_env_test.go b/modules/setting/config_env_test.go index 7d07c479a1..7d270ac21a 100644 --- a/modules/setting/config_env_test.go +++ b/modules/setting/config_env_test.go @@ -28,8 +28,8 @@ func TestDecodeEnvSectionKey(t *testing.T) { ok, section, key = decodeEnvSectionKey("SEC") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) } func TestDecodeEnvironmentKey(t *testing.T) { @@ -38,19 +38,19 @@ func TestDecodeEnvironmentKey(t *testing.T) { ok, section, key, file := decodeEnvironmentKey(prefix, suffix, "SEC__KEY") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) assert.False(t, file) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) assert.False(t, file) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA____KEY") assert.True(t, ok) - assert.Equal(t, "", section) + assert.Empty(t, section) assert.Equal(t, "KEY", key) assert.False(t, file) @@ -64,8 +64,8 @@ func TestDecodeEnvironmentKey(t *testing.T) { // but it could be fixed in the future by adding a new suffix like "__VALUE" (no such key VALUE is used in Gitea either) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__FILE") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) assert.True(t, file) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY__FILE") @@ -73,6 +73,9 @@ func TestDecodeEnvironmentKey(t *testing.T) { assert.Equal(t, "sec", section) assert.Equal(t, "KEY", key) assert.True(t, file) + + ok, _, _, _ = decodeEnvironmentKey("PREFIX__", "", "PREFIX__SEC__KEY") + assert.True(t, ok) } func TestEnvironmentToConfig(t *testing.T) { diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index 3138f8a63e..09eaaefdaf 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -15,7 +15,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" - "gopkg.in/ini.v1" //nolint:depguard + "gopkg.in/ini.v1" //nolint:depguard // wrapper for this package ) type ConfigKey interface { @@ -26,6 +26,7 @@ type ConfigKey interface { In(defaultVal string, candidates []string) string String() string Strings(delim string) []string + Bool() (bool, error) MustString(defaultVal string) string MustBool(defaultVal ...bool) bool @@ -257,7 +258,7 @@ func (p *iniConfigProvider) Save() error { } filename := p.file if filename == "" { - return fmt.Errorf("config file path must not be empty") + return errors.New("config file path must not be empty") } if p.loadedFromEmpty { if err := os.MkdirAll(filepath.Dir(filename), os.ModePerm); err != nil { diff --git a/modules/setting/config_provider_test.go b/modules/setting/config_provider_test.go index a666d124c7..63121f0074 100644 --- a/modules/setting/config_provider_test.go +++ b/modules/setting/config_provider_test.go @@ -62,17 +62,17 @@ key = 123 // test default behavior assert.Equal(t, "123", ConfigSectionKeyString(sec, "key")) - assert.Equal(t, "", ConfigSectionKeyString(secSub, "key")) + assert.Empty(t, ConfigSectionKeyString(secSub, "key")) assert.Equal(t, "def", ConfigSectionKeyString(secSub, "key", "def")) assert.Equal(t, "123", ConfigInheritedKeyString(secSub, "key")) // Workaround for ini package's BuggyKeyOverwritten behavior - assert.Equal(t, "", ConfigSectionKeyString(sec, "empty")) - assert.Equal(t, "", ConfigSectionKeyString(secSub, "empty")) + assert.Empty(t, ConfigSectionKeyString(sec, "empty")) + assert.Empty(t, ConfigSectionKeyString(secSub, "empty")) assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("def")) assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("xyz")) - assert.Equal(t, "", ConfigSectionKeyString(sec, "empty")) + assert.Empty(t, ConfigSectionKeyString(sec, "empty")) assert.Equal(t, "def", ConfigSectionKeyString(secSub, "empty")) } diff --git a/modules/setting/cron_test.go b/modules/setting/cron_test.go index 55244d7075..39a228068a 100644 --- a/modules/setting/cron_test.go +++ b/modules/setting/cron_test.go @@ -38,6 +38,6 @@ EXTEND = true _, err = getCronSettings(cfg, "test", extended) assert.NoError(t, err) assert.True(t, extended.Base) - assert.EqualValues(t, "white rabbit", extended.Second) + assert.Equal(t, "white rabbit", extended.Second) assert.True(t, extended.Extend) } diff --git a/modules/setting/git_test.go b/modules/setting/git_test.go index 441c514d8c..0d7f634abf 100644 --- a/modules/setting/git_test.go +++ b/modules/setting/git_test.go @@ -6,6 +6,8 @@ package setting import ( "testing" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" ) @@ -23,8 +25,8 @@ a.b = 1 `) assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "1", GitConfig.Options["a.b"]) - assert.EqualValues(t, "histogram", GitConfig.Options["diff.algorithm"]) + assert.Equal(t, "1", GitConfig.Options["a.b"]) + assert.Equal(t, "histogram", GitConfig.Options["diff.algorithm"]) cfg, err = NewConfigProviderFromData(` [git.config] @@ -32,24 +34,20 @@ diff.algorithm = other `) assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "other", GitConfig.Options["diff.algorithm"]) + assert.Equal(t, "other", GitConfig.Options["diff.algorithm"]) } func TestGitReflog(t *testing.T) { - oldGit := Git - oldGitConfig := GitConfig - defer func() { - Git = oldGit - GitConfig = oldGitConfig - }() + defer test.MockVariableValue(&Git) + defer test.MockVariableValue(&GitConfig) // default reflog config without legacy options cfg, err := NewConfigProviderFromData(``) assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "true", GitConfig.GetOption("core.logAllRefUpdates")) - assert.EqualValues(t, "90", GitConfig.GetOption("gc.reflogExpire")) + assert.Equal(t, "true", GitConfig.GetOption("core.logAllRefUpdates")) + assert.Equal(t, "90", GitConfig.GetOption("gc.reflogExpire")) // custom reflog config by legacy options cfg, err = NewConfigProviderFromData(` @@ -60,6 +58,6 @@ EXPIRATION = 123 assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "false", GitConfig.GetOption("core.logAllRefUpdates")) - assert.EqualValues(t, "123", GitConfig.GetOption("gc.reflogExpire")) + assert.Equal(t, "false", GitConfig.GetOption("core.logAllRefUpdates")) + assert.Equal(t, "123", GitConfig.GetOption("gc.reflogExpire")) } diff --git a/modules/setting/global_lock_test.go b/modules/setting/global_lock_test.go index 5eeb275523..5e15eb3483 100644 --- a/modules/setting/global_lock_test.go +++ b/modules/setting/global_lock_test.go @@ -16,7 +16,7 @@ func TestLoadGlobalLockConfig(t *testing.T) { assert.NoError(t, err) loadGlobalLockFrom(cfg) - assert.EqualValues(t, "memory", GlobalLock.ServiceType) + assert.Equal(t, "memory", GlobalLock.ServiceType) }) t.Run("RedisGlobalLockConfig", func(t *testing.T) { @@ -29,7 +29,7 @@ SERVICE_CONN_STR = addrs=127.0.0.1:6379 db=0 assert.NoError(t, err) loadGlobalLockFrom(cfg) - assert.EqualValues(t, "redis", GlobalLock.ServiceType) - assert.EqualValues(t, "addrs=127.0.0.1:6379 db=0", GlobalLock.ServiceConnStr) + assert.Equal(t, "redis", GlobalLock.ServiceType) + assert.Equal(t, "addrs=127.0.0.1:6379 db=0", GlobalLock.ServiceConnStr) }) } diff --git a/modules/setting/incoming_email.go b/modules/setting/incoming_email.go index bf81f292a2..4e433dde60 100644 --- a/modules/setting/incoming_email.go +++ b/modules/setting/incoming_email.go @@ -4,6 +4,7 @@ package setting import ( + "errors" "fmt" "net/mail" "strings" @@ -50,7 +51,7 @@ func checkReplyToAddress() error { } if parsed.Name != "" { - return fmt.Errorf("name must not be set") + return errors.New("name must not be set") } c := strings.Count(IncomingEmail.ReplyToAddress, IncomingEmail.TokenPlaceholder) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index e34baae012..ace7eec70e 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -96,7 +96,7 @@ func loadIndexerFrom(rootCfg ConfigProvider) { // IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing func IndexerGlobFromString(globstr string) []*GlobMatcher { extarr := make([]*GlobMatcher, 0, 10) - for _, expr := range strings.Split(strings.ToLower(globstr), ",") { + for expr := range strings.SplitSeq(strings.ToLower(globstr), ",") { expr = strings.TrimSpace(expr) if expr != "" { if g, err := GlobMatcherCompile(expr, '.', '/'); err != nil { diff --git a/modules/setting/lfs_test.go b/modules/setting/lfs_test.go index d27dd7c5bf..1b829d8839 100644 --- a/modules/setting/lfs_test.go +++ b/modules/setting/lfs_test.go @@ -19,7 +19,7 @@ func Test_getStorageInheritNameSectionTypeForLFS(t *testing.T) { assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) iniStr = ` [server] @@ -54,7 +54,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) iniStr = ` [lfs] @@ -68,7 +68,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) iniStr = ` [lfs] @@ -83,7 +83,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath) } func Test_LFSStorage1(t *testing.T) { @@ -96,8 +96,8 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "gitea", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", LFS.Storage.MinioConfig.Bucket) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) } func Test_LFSClientServerConfigs(t *testing.T) { @@ -112,9 +112,9 @@ BATCH_SIZE = 0 assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, 100, LFS.MaxBatchSize) - assert.EqualValues(t, 20, LFSClient.BatchSize) - assert.EqualValues(t, 8, LFSClient.BatchOperationConcurrency) + assert.Equal(t, 100, LFS.MaxBatchSize) + assert.Equal(t, 20, LFSClient.BatchSize) + assert.Equal(t, 8, LFSClient.BatchOperationConcurrency) iniStr = ` [lfs_client] @@ -125,6 +125,6 @@ BATCH_OPERATION_CONCURRENCY = 10 assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, 50, LFSClient.BatchSize) - assert.EqualValues(t, 10, LFSClient.BatchOperationConcurrency) + assert.Equal(t, 50, LFSClient.BatchSize) + assert.Equal(t, 10, LFSClient.BatchOperationConcurrency) } diff --git a/modules/setting/log.go b/modules/setting/log.go index 50c5779994..59866c7605 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -7,7 +7,6 @@ import ( "fmt" golog "log" "os" - "path" "path/filepath" "strings" @@ -41,7 +40,7 @@ func loadLogGlobalFrom(rootCfg ConfigProvider) { Log.BufferLen = sec.Key("BUFFER_LEN").MustInt(10000) Log.Mode = sec.Key("MODE").MustString("console") - Log.RootPath = sec.Key("ROOT_PATH").MustString(path.Join(AppWorkPath, "log")) + Log.RootPath = sec.Key("ROOT_PATH").MustString(filepath.Join(AppWorkPath, "log")) if !filepath.IsAbs(Log.RootPath) { Log.RootPath = filepath.Join(AppWorkPath, Log.RootPath) } @@ -228,8 +227,8 @@ func initLoggerByName(manager *log.LoggerManager, rootCfg ConfigProvider, logger } var eventWriters []log.EventWriter - modes := strings.Split(modeVal, ",") - for _, modeName := range modes { + modes := strings.SplitSeq(modeVal, ",") + for modeName := range modes { modeName = strings.TrimSpace(modeName) if modeName == "" { continue diff --git a/modules/setting/mailer.go b/modules/setting/mailer.go index 4c3dff6850..e79ff30447 100644 --- a/modules/setting/mailer.go +++ b/modules/setting/mailer.go @@ -13,7 +13,7 @@ import ( "code.gitea.io/gitea/modules/log" - shellquote "github.com/kballard/go-shellquote" + "github.com/kballard/go-shellquote" ) // Mailer represents mail service. @@ -29,6 +29,9 @@ type Mailer struct { SubjectPrefix string `ini:"SUBJECT_PREFIX"` OverrideHeader map[string][]string `ini:"-"` + // Embed attachment images as inline base64 img src attribute + EmbedAttachmentImages bool + // SMTP sender Protocol string `ini:"PROTOCOL"` SMTPAddr string `ini:"SMTP_ADDR"` diff --git a/modules/setting/mailer_test.go b/modules/setting/mailer_test.go index fbabf11378..ceef35b051 100644 --- a/modules/setting/mailer_test.go +++ b/modules/setting/mailer_test.go @@ -34,8 +34,8 @@ func Test_loadMailerFrom(t *testing.T) { // Check mailer setting loadMailerFrom(cfg) - assert.EqualValues(t, kase.SMTPAddr, MailService.SMTPAddr) - assert.EqualValues(t, kase.SMTPPort, MailService.SMTPPort) + assert.Equal(t, kase.SMTPAddr, MailService.SMTPAddr) + assert.Equal(t, kase.SMTPPort, MailService.SMTPPort) }) } } diff --git a/modules/setting/markup.go b/modules/setting/markup.go index dfce8afa77..057b0650c3 100644 --- a/modules/setting/markup.go +++ b/modules/setting/markup.go @@ -8,6 +8,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" ) // ExternalMarkupRenderers represents the external markup renderers @@ -23,18 +24,33 @@ const ( RenderContentModeIframe = "iframe" ) +type MarkdownRenderOptions struct { + NewLineHardBreak bool + ShortIssuePattern bool // Actually it is a "markup" option because it is used in "post processor" +} + +type MarkdownMathCodeBlockOptions struct { + ParseInlineDollar bool + ParseInlineParentheses bool + ParseBlockDollar bool + ParseBlockSquareBrackets bool +} + // Markdown settings var Markdown = struct { - EnableHardLineBreakInComments bool - EnableHardLineBreakInDocuments bool - CustomURLSchemes []string `ini:"CUSTOM_URL_SCHEMES"` - FileExtensions []string - EnableMath bool + RenderOptionsComment MarkdownRenderOptions `ini:"-"` + RenderOptionsWiki MarkdownRenderOptions `ini:"-"` + RenderOptionsRepoFile MarkdownRenderOptions `ini:"-"` + + CustomURLSchemes []string `ini:"CUSTOM_URL_SCHEMES"` // Actually it is a "markup" option because it is used in "post processor" + FileExtensions []string + + EnableMath bool + MathCodeBlockDetection []string + MathCodeBlockOptions MarkdownMathCodeBlockOptions `ini:"-"` }{ - EnableHardLineBreakInComments: true, - EnableHardLineBreakInDocuments: false, - FileExtensions: strings.Split(".md,.markdown,.mdown,.mkd,.livemd", ","), - EnableMath: true, + FileExtensions: strings.Split(".md,.markdown,.mdown,.mkd,.livemd", ","), + EnableMath: true, } // MarkupRenderer defines the external parser configured in ini @@ -60,8 +76,58 @@ type MarkupSanitizerRule struct { func loadMarkupFrom(rootCfg ConfigProvider) { mustMapSetting(rootCfg, "markdown", &Markdown) + const none = "none" + + const renderOptionShortIssuePattern = "short-issue-pattern" + const renderOptionNewLineHardBreak = "new-line-hard-break" + cfgMarkdown := rootCfg.Section("markdown") + parseMarkdownRenderOptions := func(key string, defaults []string) (ret MarkdownRenderOptions) { + options := cfgMarkdown.Key(key).Strings(",") + options = util.IfEmpty(options, defaults) + for _, opt := range options { + switch opt { + case renderOptionShortIssuePattern: + ret.ShortIssuePattern = true + case renderOptionNewLineHardBreak: + ret.NewLineHardBreak = true + case none: + ret = MarkdownRenderOptions{} + case "": + default: + log.Error("Unknown markdown render option in %s: %s", key, opt) + } + } + return ret + } + Markdown.RenderOptionsComment = parseMarkdownRenderOptions("RENDER_OPTIONS_COMMENT", []string{renderOptionShortIssuePattern, renderOptionNewLineHardBreak}) + Markdown.RenderOptionsWiki = parseMarkdownRenderOptions("RENDER_OPTIONS_WIKI", []string{renderOptionShortIssuePattern}) + Markdown.RenderOptionsRepoFile = parseMarkdownRenderOptions("RENDER_OPTIONS_REPO_FILE", nil) + + const mathCodeInlineDollar = "inline-dollar" + const mathCodeInlineParentheses = "inline-parentheses" + const mathCodeBlockDollar = "block-dollar" + const mathCodeBlockSquareBrackets = "block-square-brackets" + Markdown.MathCodeBlockDetection = util.IfEmpty(Markdown.MathCodeBlockDetection, []string{mathCodeInlineDollar, mathCodeBlockDollar}) + Markdown.MathCodeBlockOptions = MarkdownMathCodeBlockOptions{} + for _, s := range Markdown.MathCodeBlockDetection { + switch s { + case mathCodeInlineDollar: + Markdown.MathCodeBlockOptions.ParseInlineDollar = true + case mathCodeInlineParentheses: + Markdown.MathCodeBlockOptions.ParseInlineParentheses = true + case mathCodeBlockDollar: + Markdown.MathCodeBlockOptions.ParseBlockDollar = true + case mathCodeBlockSquareBrackets: + Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets = true + case none: + Markdown.MathCodeBlockOptions = MarkdownMathCodeBlockOptions{} + case "": + default: + log.Error("Unknown math code block detection option: %s", s) + } + } - MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(5000) + MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(50000) ExternalMarkupRenderers = make([]*MarkupRenderer, 0, 10) ExternalSanitizerRules = make([]MarkupSanitizerRule, 0, 10) @@ -83,8 +149,8 @@ func loadMarkupFrom(rootCfg ConfigProvider) { func newMarkupSanitizer(name string, sec ConfigSection) { rule, ok := createMarkupSanitizerRule(name, sec) if ok { - if strings.HasPrefix(name, "sanitizer.") { - names := strings.SplitN(strings.TrimPrefix(name, "sanitizer."), ".", 2) + if after, found := strings.CutPrefix(name, "sanitizer."); found { + names := strings.SplitN(after, ".", 2) name = names[0] } for _, renderer := range ExternalMarkupRenderers { diff --git a/modules/setting/markup_test.go b/modules/setting/markup_test.go new file mode 100644 index 0000000000..c47a38ce15 --- /dev/null +++ b/modules/setting/markup_test.go @@ -0,0 +1,51 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLoadMarkup(t *testing.T) { + cfg, _ := NewConfigProviderFromData(``) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseBlockDollar: true}, Markdown.MathCodeBlockOptions) + assert.Equal(t, MarkdownRenderOptions{NewLineHardBreak: true, ShortIssuePattern: true}, Markdown.RenderOptionsComment) + assert.Equal(t, MarkdownRenderOptions{ShortIssuePattern: true}, Markdown.RenderOptionsWiki) + assert.Equal(t, MarkdownRenderOptions{}, Markdown.RenderOptionsRepoFile) + + t.Run("Math", func(t *testing.T) { + cfg, _ = NewConfigProviderFromData(` +[markdown] +MATH_CODE_BLOCK_DETECTION = none +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownMathCodeBlockOptions{}, Markdown.MathCodeBlockOptions) + + cfg, _ = NewConfigProviderFromData(` +[markdown] +MATH_CODE_BLOCK_DETECTION = inline-dollar, inline-parentheses, block-dollar, block-square-brackets +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseInlineParentheses: true, ParseBlockDollar: true, ParseBlockSquareBrackets: true}, Markdown.MathCodeBlockOptions) + }) + + t.Run("Render", func(t *testing.T) { + cfg, _ = NewConfigProviderFromData(` +[markdown] +RENDER_OPTIONS_COMMENT = none +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownRenderOptions{}, Markdown.RenderOptionsComment) + + cfg, _ = NewConfigProviderFromData(` +[markdown] +RENDER_OPTIONS_REPO_FILE = short-issue-pattern, new-line-hard-break +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownRenderOptions{NewLineHardBreak: true, ShortIssuePattern: true}, Markdown.RenderOptionsRepoFile) + }) +} diff --git a/modules/setting/mirror.go b/modules/setting/mirror.go index 3aa530a1f4..300711789d 100644 --- a/modules/setting/mirror.go +++ b/modules/setting/mirror.go @@ -48,11 +48,7 @@ func loadMirrorFrom(rootCfg ConfigProvider) { Mirror.MinInterval = 1 * time.Minute } if Mirror.DefaultInterval < Mirror.MinInterval { - if time.Hour*8 < Mirror.MinInterval { - Mirror.DefaultInterval = Mirror.MinInterval - } else { - Mirror.DefaultInterval = time.Hour * 8 - } + Mirror.DefaultInterval = max(time.Hour*8, Mirror.MinInterval) log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval, set to %s", Mirror.DefaultInterval.String()) } } diff --git a/modules/setting/oauth2_test.go b/modules/setting/oauth2_test.go index d0e5ccf13d..c6e66cad02 100644 --- a/modules/setting/oauth2_test.go +++ b/modules/setting/oauth2_test.go @@ -31,7 +31,7 @@ JWT_SECRET = BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB actual := GetGeneralTokenSigningSecret() expected, _ := generate.DecodeJwtSecretBase64("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB") assert.Len(t, actual, 32) - assert.EqualValues(t, expected, actual) + assert.Equal(t, expected, actual) } func TestGetGeneralSigningSecretSave(t *testing.T) { diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 3f618cfd64..b598424064 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -6,8 +6,6 @@ package setting import ( "fmt" "math" - "os" - "path/filepath" "github.com/dustin/go-humanize" ) @@ -15,9 +13,8 @@ import ( // Package registry settings var ( Packages = struct { - Storage *Storage - Enabled bool - ChunkedUploadPath string + Storage *Storage + Enabled bool LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 @@ -67,17 +64,6 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { return err } - Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload")) - if !filepath.IsAbs(Packages.ChunkedUploadPath) { - Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath)) - } - - if HasInstallLock(rootCfg) { - if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { - return fmt.Errorf("unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) - } - } - Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeAlpine = mustBytes(sec, "LIMIT_SIZE_ALPINE") Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH") diff --git a/modules/setting/packages_test.go b/modules/setting/packages_test.go index 87de276041..47378f35ad 100644 --- a/modules/setting/packages_test.go +++ b/modules/setting/packages_test.go @@ -41,7 +41,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) // we can also configure packages storage directly iniStr = ` @@ -53,7 +53,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) // or we can indicate the storage type in the packages section iniStr = ` @@ -68,7 +68,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) // or we can indicate the storage type and minio base path in the packages section iniStr = ` @@ -84,7 +84,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "my_packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_packages/", Packages.Storage.MinioConfig.BasePath) } func Test_PackageStorage1(t *testing.T) { @@ -109,8 +109,8 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } @@ -136,8 +136,8 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } @@ -164,8 +164,8 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "my_packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } @@ -192,7 +192,7 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "my_packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } diff --git a/modules/setting/path.go b/modules/setting/path.go index 0fdc305aa1..f51457a620 100644 --- a/modules/setting/path.go +++ b/modules/setting/path.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/tempdir" ) var ( @@ -196,3 +197,18 @@ func InitWorkPathAndCfgProvider(getEnvFn func(name string) string, args ArgWorkP CustomPath = tmpCustomPath.Value CustomConf = tmpCustomConf.Value } + +// AppDataTempDir returns a managed temporary directory for the application data. +// Using empty sub will get the managed base temp directory, and it's safe to delete it. +// Gitea only creates subdirectories under it, but not the APP_TEMP_PATH directory itself. +// * When APP_TEMP_PATH="/tmp": the managed temp directory is "/tmp/gitea-tmp" +// * When APP_TEMP_PATH is not set: the managed temp directory is "/{APP_DATA_PATH}/tmp" +func AppDataTempDir(sub string) *tempdir.TempDir { + if appTempPathInternal != "" { + return tempdir.New(appTempPathInternal, "gitea-tmp/"+sub) + } + if AppDataPath == "" { + panic("setting.AppDataPath is not set") + } + return tempdir.New(AppDataPath, "tmp/"+sub) +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index c5619d0f04..318cf41108 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -5,7 +5,6 @@ package setting import ( "os/exec" - "path" "path/filepath" "strings" @@ -63,17 +62,11 @@ var ( // Repository upload settings Upload struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int } `ini:"-"` - // Repository local settings - Local struct { - LocalCopyPath string - } `ini:"-"` - // Pull request settings PullRequest struct { WorkInProgressPrefixes []string @@ -89,6 +82,7 @@ var ( AddCoCommitterTrailers bool TestConflictingPatchesWithGitApply bool RetargetChildrenOnMerge bool + DelayCheckForInactiveDays int } `ini:"repository.pull-request"` // Issue Setting @@ -106,11 +100,13 @@ var ( SigningKey string SigningName string SigningEmail string + SigningFormat string InitialCommit []string CRUDActions []string `ini:"CRUD_ACTIONS"` Merges []string Wiki []string DefaultTrustModel string + TrustedSSHKeys []string `ini:"TRUSTED_SSH_KEYS"` } `ini:"repository.signing"` }{ DetectedCharsetsOrder: []string{ @@ -182,25 +178,16 @@ var ( // Repository upload settings Upload: struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int }{ Enabled: true, - TempPath: "data/tmp/uploads", AllowedTypes: "", FileMaxSize: 50, MaxFiles: 5, }, - // Repository local settings - Local: struct { - LocalCopyPath string - }{ - LocalCopyPath: "tmp/local-repo", - }, - // Pull request settings PullRequest: struct { WorkInProgressPrefixes []string @@ -216,6 +203,7 @@ var ( AddCoCommitterTrailers bool TestConflictingPatchesWithGitApply bool RetargetChildrenOnMerge bool + DelayCheckForInactiveDays int }{ WorkInProgressPrefixes: []string{"WIP:", "[WIP]"}, // Same as GitHub. See @@ -231,6 +219,7 @@ var ( PopulateSquashCommentWithCommitMessages: false, AddCoCommitterTrailers: true, RetargetChildrenOnMerge: true, + DelayCheckForInactiveDays: 7, }, // Issue settings @@ -255,20 +244,24 @@ var ( SigningKey string SigningName string SigningEmail string + SigningFormat string InitialCommit []string CRUDActions []string `ini:"CRUD_ACTIONS"` Merges []string Wiki []string DefaultTrustModel string + TrustedSSHKeys []string `ini:"TRUSTED_SSH_KEYS"` }{ SigningKey: "default", SigningName: "", SigningEmail: "", + SigningFormat: "openpgp", // git.SigningKeyFormatOpenPGP InitialCommit: []string{"always"}, CRUDActions: []string{"pubkey", "twofa", "parentsigned"}, Merges: []string{"pubkey", "twofa", "basesigned", "commitssigned"}, Wiki: []string{"never"}, DefaultTrustModel: "collaborator", + TrustedSSHKeys: []string{}, }, } RepoRootPath string @@ -284,7 +277,7 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { Repository.GoGetCloneURLProtocol = sec.Key("GO_GET_CLONE_URL_PROTOCOL").MustString("https") Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) Repository.DefaultBranch = sec.Key("DEFAULT_BRANCH").MustString(Repository.DefaultBranch) - RepoRootPath = sec.Key("ROOT").MustString(path.Join(AppDataPath, "gitea-repositories")) + RepoRootPath = sec.Key("ROOT").MustString(filepath.Join(AppDataPath, "gitea-repositories")) if !filepath.IsAbs(RepoRootPath) { RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath) } else { @@ -309,8 +302,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { log.Fatal("Failed to map Repository.Editor settings: %v", err) } else if err = rootCfg.Section("repository.upload").MapTo(&Repository.Upload); err != nil { log.Fatal("Failed to map Repository.Upload settings: %v", err) - } else if err = rootCfg.Section("repository.local").MapTo(&Repository.Local); err != nil { - log.Fatal("Failed to map Repository.Local settings: %v", err) } else if err = rootCfg.Section("repository.pull-request").MapTo(&Repository.PullRequest); err != nil { log.Fatal("Failed to map Repository.PullRequest settings: %v", err) } @@ -362,10 +353,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { } } - if !filepath.IsAbs(Repository.Upload.TempPath) { - Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath) - } - if err := loadRepoArchiveFrom(rootCfg); err != nil { log.Fatal("loadRepoArchiveFrom: %v", err) } diff --git a/modules/setting/repository_archive_test.go b/modules/setting/repository_archive_test.go index a0f91f0da1..d5b95272d6 100644 --- a/modules/setting/repository_archive_test.go +++ b/modules/setting/repository_archive_test.go @@ -20,7 +20,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) // we can also configure packages storage directly iniStr = ` @@ -32,7 +32,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) // or we can indicate the storage type in the packages section iniStr = ` @@ -47,7 +47,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) // or we can indicate the storage type and minio base path in the packages section iniStr = ` @@ -63,7 +63,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath) } func Test_RepoArchiveStorage(t *testing.T) { @@ -85,7 +85,7 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := RepoArchive.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) iniStr = ` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -107,5 +107,5 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage = RepoArchive.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) } diff --git a/modules/setting/security.go b/modules/setting/security.go index 2f798b75c7..153b6bc944 100644 --- a/modules/setting/security.go +++ b/modules/setting/security.go @@ -39,6 +39,7 @@ var ( CSRFCookieName = "_csrf" CSRFCookieHTTPOnly = true RecordUserSignupMetadata = false + TwoFactorAuthEnforced = false ) // loadSecret load the secret from ini by uriKey or verbatimKey, only one of them could be set @@ -110,7 +111,7 @@ func loadSecurityFrom(rootCfg ConfigProvider) { if SecretKey == "" { // FIXME: https://github.com/go-gitea/gitea/issues/16832 // Until it supports rotating an existing secret key, we shouldn't move users off of the widely used default value - SecretKey = "!#@FDEWREWR&*(" //nolint:gosec + SecretKey = "!#@FDEWREWR&*(" } CookieRememberName = sec.Key("COOKIE_REMEMBER_NAME").MustString("gitea_incredible") @@ -142,6 +143,15 @@ func loadSecurityFrom(rootCfg ConfigProvider) { PasswordCheckPwn = sec.Key("PASSWORD_CHECK_PWN").MustBool(false) SuccessfulTokensCacheSize = sec.Key("SUCCESSFUL_TOKENS_CACHE_SIZE").MustInt(20) + twoFactorAuth := sec.Key("TWO_FACTOR_AUTH").String() + switch twoFactorAuth { + case "": + case "enforced": + TwoFactorAuthEnforced = true + default: + log.Fatal("Invalid two-factor auth option: %s", twoFactorAuth) + } + InternalToken = loadSecret(sec, "INTERNAL_TOKEN_URI", "INTERNAL_TOKEN") if InstallLock && InternalToken == "" { // if Gitea has been installed but the InternalToken hasn't been generated (upgrade from an old release), we should generate diff --git a/modules/setting/server.go b/modules/setting/server.go index d7a71578d4..8a22f6a844 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "net" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -40,28 +41,47 @@ const ( LandingPageLogin LandingPage = "/user/login" ) +const ( + PublicURLAuto = "auto" + PublicURLLegacy = "legacy" +) + // Server settings var ( // AppURL is the Application ROOT_URL. It always has a '/' suffix // It maps to ini:"ROOT_URL" AppURL string - // AppSubURL represents the sub-url mounting point for gitea. It is either "" or starts with '/' and ends without '/', such as '/{subpath}'. + + // PublicURLDetection controls how to use the HTTP request headers to detect public URL + PublicURLDetection string + + // AppSubURL represents the sub-url mounting point for gitea, parsed from "ROOT_URL" + // It is either "" or starts with '/' and ends without '/', such as '/{sub-path}'. // This value is empty if site does not have sub-url. AppSubURL string - // UseSubURLPath makes Gitea handle requests with sub-path like "/sub-path/owner/repo/...", to make it easier to debug sub-path related problems without a reverse proxy. + + // UseSubURLPath makes Gitea handle requests with sub-path like "/sub-path/owner/repo/...", + // to make it easier to debug sub-path related problems without a reverse proxy. UseSubURLPath bool + // AppDataPath is the default path for storing data. // It maps to ini:"APP_DATA_PATH" in [server] and defaults to AppWorkPath + "/data" AppDataPath string + // LocalURL is the url for locally running applications to contact Gitea. It always has a '/' suffix // It maps to ini:"LOCAL_ROOT_URL" in [server] LocalURL string - // AssetVersion holds a opaque value that is used for cache-busting assets + + // AssetVersion holds an opaque value that is used for cache-busting assets AssetVersion string + // appTempPathInternal is the temporary path for the app, it is only an internal variable + // DO NOT use it directly, always use AppDataTempDir + appTempPathInternal string + Protocol Scheme - UseProxyProtocol bool // `ini:"USE_PROXY_PROTOCOL"` - ProxyProtocolTLSBridging bool //`ini:"PROXY_PROTOCOL_TLS_BRIDGING"` + UseProxyProtocol bool + ProxyProtocolTLSBridging bool ProxyProtocolHeaderTimeout time.Duration ProxyProtocolAcceptUnknown bool Domain string @@ -169,20 +189,25 @@ func loadServerFrom(rootCfg ConfigProvider) { HTTPAddr = sec.Key("HTTP_ADDR").MustString("0.0.0.0") HTTPPort = sec.Key("HTTP_PORT").MustString("3000") - Protocol = HTTP + // DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version + // if these are removed, the warning will not be shown + if sec.HasKey("ENABLE_ACME") { + EnableAcme = sec.Key("ENABLE_ACME").MustBool(false) + } else { + deprecatedSetting(rootCfg, "server", "ENABLE_LETSENCRYPT", "server", "ENABLE_ACME", "v1.19.0") + EnableAcme = sec.Key("ENABLE_LETSENCRYPT").MustBool(false) + } + protocolCfg := sec.Key("PROTOCOL").String() + if protocolCfg != "https" && EnableAcme { + log.Fatal("ACME could only be used with HTTPS protocol") + } + switch protocolCfg { + case "", "http": + Protocol = HTTP case "https": Protocol = HTTPS - - // DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version - // if these are removed, the warning will not be shown - if sec.HasKey("ENABLE_ACME") { - EnableAcme = sec.Key("ENABLE_ACME").MustBool(false) - } else { - deprecatedSetting(rootCfg, "server", "ENABLE_LETSENCRYPT", "server", "ENABLE_ACME", "v1.19.0") - EnableAcme = sec.Key("ENABLE_LETSENCRYPT").MustBool(false) - } if EnableAcme { AcmeURL = sec.Key("ACME_URL").MustString("") AcmeCARoot = sec.Key("ACME_CA_ROOT").MustString("") @@ -210,6 +235,9 @@ func loadServerFrom(rootCfg ConfigProvider) { deprecatedSetting(rootCfg, "server", "LETSENCRYPT_EMAIL", "server", "ACME_EMAIL", "v1.19.0") AcmeEmail = sec.Key("LETSENCRYPT_EMAIL").MustString("") } + if AcmeEmail == "" { + log.Fatal("ACME Email is not set (ACME_EMAIL).") + } } else { CertFile = sec.Key("CERT_FILE").String() KeyFile = sec.Key("KEY_FILE").String() @@ -233,7 +261,7 @@ func loadServerFrom(rootCfg ConfigProvider) { case "unix": log.Warn("unix PROTOCOL value is deprecated, please use http+unix") fallthrough - case "http+unix": + default: // "http+unix" Protocol = HTTPUnix } UnixSocketPermissionRaw := sec.Key("UNIX_SOCKET_PERMISSION").MustString("666") @@ -246,6 +274,8 @@ func loadServerFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(HTTPAddr) { HTTPAddr = filepath.Join(AppWorkPath, HTTPAddr) } + default: + log.Fatal("Invalid PROTOCOL %q", Protocol) } UseProxyProtocol = sec.Key("USE_PROXY_PROTOCOL").MustBool(false) ProxyProtocolTLSBridging = sec.Key("PROXY_PROTOCOL_TLS_BRIDGING").MustBool(false) @@ -259,11 +289,15 @@ func loadServerFrom(rootCfg ConfigProvider) { defaultAppURL := string(Protocol) + "://" + Domain + ":" + HTTPPort AppURL = sec.Key("ROOT_URL").MustString(defaultAppURL) + PublicURLDetection = sec.Key("PUBLIC_URL_DETECTION").MustString(PublicURLLegacy) + if PublicURLDetection != PublicURLAuto && PublicURLDetection != PublicURLLegacy { + log.Fatal("Invalid PUBLIC_URL_DETECTION value: %s", PublicURLDetection) + } // Check validity of AppURL appURL, err := url.Parse(AppURL) if err != nil { - log.Fatal("Invalid ROOT_URL '%s': %s", AppURL, err) + log.Fatal("Invalid ROOT_URL %q: %s", AppURL, err) } // Remove default ports from AppURL. // (scheme-based URL normalization, RFC 3986 section 6.2.3) @@ -299,13 +333,15 @@ func loadServerFrom(rootCfg ConfigProvider) { defaultLocalURL = AppURL case FCGIUnix: defaultLocalURL = AppURL - default: + case HTTP, HTTPS: defaultLocalURL = string(Protocol) + "://" if HTTPAddr == "0.0.0.0" { defaultLocalURL += net.JoinHostPort("localhost", HTTPPort) + "/" } else { defaultLocalURL += net.JoinHostPort(HTTPAddr, HTTPPort) + "/" } + default: + log.Fatal("Invalid PROTOCOL %q", Protocol) } LocalURL = sec.Key("LOCAL_ROOT_URL").MustString(defaultLocalURL) LocalURL = strings.TrimRight(LocalURL, "/") + "/" @@ -323,6 +359,19 @@ func loadServerFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(AppDataPath) { AppDataPath = filepath.ToSlash(filepath.Join(AppWorkPath, AppDataPath)) } + if IsInTesting && HasInstallLock(rootCfg) { + // FIXME: in testing, the "app data" directory is not correctly initialized before loading settings + if _, err := os.Stat(AppDataPath); err != nil { + _ = os.MkdirAll(AppDataPath, os.ModePerm) + } + } + + appTempPathInternal = sec.Key("APP_TEMP_PATH").String() + if appTempPathInternal != "" { + if _, err := os.Stat(appTempPathInternal); err != nil { + log.Fatal("APP_TEMP_PATH %q is not accessible: %v", appTempPathInternal, err) + } + } EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) diff --git a/modules/setting/service.go b/modules/setting/service.go index 526ad64eb4..b1b9fedd62 100644 --- a/modules/setting/service.go +++ b/modules/setting/service.go @@ -5,6 +5,7 @@ package setting import ( "regexp" + "runtime" "strings" "time" @@ -43,9 +44,11 @@ var Service = struct { ShowRegistrationButton bool EnablePasswordSignInForm bool ShowMilestonesDashboardPage bool - RequireSignInView bool + RequireSignInViewStrict bool + BlockAnonymousAccessExpensive bool EnableNotifyMail bool EnableBasicAuth bool + EnablePasskeyAuth bool EnableReverseProxyAuth bool EnableReverseProxyAuthAPI bool EnableReverseProxyAutoRegister bool @@ -96,6 +99,13 @@ var Service = struct { DisableOrganizationsPage bool `ini:"DISABLE_ORGANIZATIONS_PAGE"` DisableCodePage bool `ini:"DISABLE_CODE_PAGE"` } `ini:"service.explore"` + + QoS struct { + Enabled bool + MaxInFlightRequests int + MaxWaitingRequests int + TargetWaitTime time.Duration + } }{ AllowedUserVisibilityModesSlice: []bool{true, true, true}, } @@ -158,9 +168,21 @@ func loadServiceFrom(rootCfg ConfigProvider) { Service.EmailDomainBlockList = CompileEmailGlobList(sec, "EMAIL_DOMAIN_BLOCKLIST") Service.ShowRegistrationButton = sec.Key("SHOW_REGISTRATION_BUTTON").MustBool(!(Service.DisableRegistration || Service.AllowOnlyExternalRegistration)) Service.ShowMilestonesDashboardPage = sec.Key("SHOW_MILESTONES_DASHBOARD_PAGE").MustBool(true) - Service.RequireSignInView = sec.Key("REQUIRE_SIGNIN_VIEW").MustBool() + + // boolean values are considered as "strict" + var err error + Service.RequireSignInViewStrict, err = sec.Key("REQUIRE_SIGNIN_VIEW").Bool() + if s := sec.Key("REQUIRE_SIGNIN_VIEW").String(); err != nil && s != "" { + // non-boolean value only supports "expensive" at the moment + Service.BlockAnonymousAccessExpensive = s == "expensive" + if !Service.BlockAnonymousAccessExpensive { + log.Fatal("Invalid config option: REQUIRE_SIGNIN_VIEW = %s", s) + } + } + Service.EnableBasicAuth = sec.Key("ENABLE_BASIC_AUTHENTICATION").MustBool(true) Service.EnablePasswordSignInForm = sec.Key("ENABLE_PASSWORD_SIGNIN_FORM").MustBool(true) + Service.EnablePasskeyAuth = sec.Key("ENABLE_PASSKEY_AUTHENTICATION").MustBool(true) Service.EnableReverseProxyAuth = sec.Key("ENABLE_REVERSE_PROXY_AUTHENTICATION").MustBool() Service.EnableReverseProxyAuthAPI = sec.Key("ENABLE_REVERSE_PROXY_AUTHENTICATION_API").MustBool() Service.EnableReverseProxyAutoRegister = sec.Key("ENABLE_REVERSE_PROXY_AUTO_REGISTRATION").MustBool() @@ -241,6 +263,7 @@ func loadServiceFrom(rootCfg ConfigProvider) { mustMapSetting(rootCfg, "service.explore", &Service.Explore) loadOpenIDSetting(rootCfg) + loadQosSetting(rootCfg) } func loadOpenIDSetting(rootCfg ConfigProvider) { @@ -262,3 +285,11 @@ func loadOpenIDSetting(rootCfg ConfigProvider) { } } } + +func loadQosSetting(rootCfg ConfigProvider) { + sec := rootCfg.Section("qos") + Service.QoS.Enabled = sec.Key("ENABLED").MustBool(false) + Service.QoS.MaxInFlightRequests = sec.Key("MAX_INFLIGHT").MustInt(4 * runtime.NumCPU()) + Service.QoS.MaxWaitingRequests = sec.Key("MAX_WAITING").MustInt(100) + Service.QoS.TargetWaitTime = sec.Key("TARGET_WAIT_TIME").MustDuration(250 * time.Millisecond) +} diff --git a/modules/setting/service_test.go b/modules/setting/service_test.go index 1647bcec16..73736b793a 100644 --- a/modules/setting/service_test.go +++ b/modules/setting/service_test.go @@ -7,16 +7,14 @@ import ( "testing" "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/test" "github.com/gobwas/glob" "github.com/stretchr/testify/assert" ) func TestLoadServices(t *testing.T) { - oldService := Service - defer func() { - Service = oldService - }() + defer test.MockVariableValue(&Service)() cfg, err := NewConfigProviderFromData(` [service] @@ -48,10 +46,7 @@ EMAIL_DOMAIN_BLOCKLIST = d3, *.b } func TestLoadServiceVisibilityModes(t *testing.T) { - oldService := Service - defer func() { - Service = oldService - }() + defer test.MockVariableValue(&Service)() kases := map[string]func(){ ` @@ -130,3 +125,33 @@ ALLOWED_USER_VISIBILITY_MODES = public, limit, privated }) } } + +func TestLoadServiceRequireSignInView(t *testing.T) { + defer test.MockVariableValue(&Service)() + + cfg, err := NewConfigProviderFromData(` +[service] +`) + assert.NoError(t, err) + loadServiceFrom(cfg) + assert.False(t, Service.RequireSignInViewStrict) + assert.False(t, Service.BlockAnonymousAccessExpensive) + + cfg, err = NewConfigProviderFromData(` +[service] +REQUIRE_SIGNIN_VIEW = true +`) + assert.NoError(t, err) + loadServiceFrom(cfg) + assert.True(t, Service.RequireSignInViewStrict) + assert.False(t, Service.BlockAnonymousAccessExpensive) + + cfg, err = NewConfigProviderFromData(` +[service] +REQUIRE_SIGNIN_VIEW = expensive +`) + assert.NoError(t, err) + loadServiceFrom(cfg) + assert.False(t, Service.RequireSignInViewStrict) + assert.True(t, Service.BlockAnonymousAccessExpensive) +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 20da796b58..e14997801f 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -12,8 +12,8 @@ import ( "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/user" - "code.gitea.io/gitea/modules/util" ) // settings @@ -159,7 +159,7 @@ func loadRunModeFrom(rootCfg ConfigProvider) { // The following is a purposefully undocumented option. Please do not run Gitea as root. It will only cause future headaches. // Please don't use root as a bandaid to "fix" something that is broken, instead the broken thing should instead be fixed properly. unsafeAllowRunAsRoot := ConfigSectionKeyBool(rootSec, "I_AM_BEING_UNSAFE_RUNNING_AS_ROOT") - unsafeAllowRunAsRoot = unsafeAllowRunAsRoot || util.OptionalBoolParse(os.Getenv("GITEA_I_AM_BEING_UNSAFE_RUNNING_AS_ROOT")).Value() + unsafeAllowRunAsRoot = unsafeAllowRunAsRoot || optional.ParseBool(os.Getenv("GITEA_I_AM_BEING_UNSAFE_RUNNING_AS_ROOT")).Value() RunMode = os.Getenv("GITEA_RUN_MODE") if RunMode == "" { RunMode = rootSec.Key("RUN_MODE").MustString("prod") diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index ea387e521f..900fc6ade2 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -4,8 +4,6 @@ package setting import ( - "os" - "path" "path/filepath" "strings" "text/template" @@ -32,8 +30,6 @@ var SSH = struct { ServerKeyExchanges []string `ini:"SSH_SERVER_KEY_EXCHANGES"` ServerMACs []string `ini:"SSH_SERVER_MACS"` ServerHostKeys []string `ini:"SSH_SERVER_HOST_KEYS"` - KeyTestPath string `ini:"SSH_KEY_TEST_PATH"` - KeygenPath string `ini:"SSH_KEYGEN_PATH"` AuthorizedKeysBackup bool `ini:"SSH_AUTHORIZED_KEYS_BACKUP"` AuthorizedPrincipalsBackup bool `ini:"SSH_AUTHORIZED_PRINCIPALS_BACKUP"` AuthorizedKeysCommandTemplate string `ini:"SSH_AUTHORIZED_KEYS_COMMAND_TEMPLATE"` @@ -55,10 +51,6 @@ var SSH = struct { StartBuiltinServer: false, Domain: "", Port: 22, - ServerCiphers: []string{"chacha20-poly1305@openssh.com", "aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "aes256-gcm@openssh.com"}, - ServerKeyExchanges: []string{"curve25519-sha256", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521", "diffie-hellman-group14-sha256", "diffie-hellman-group14-sha1"}, - ServerMACs: []string{"hmac-sha2-256-etm@openssh.com", "hmac-sha2-256", "hmac-sha1"}, - KeygenPath: "", MinimumKeySizeCheck: true, MinimumKeySizes: map[string]int{"ed25519": 256, "ed25519-sk": 256, "ecdsa": 256, "ecdsa-sk": 256, "rsa": 3071}, ServerHostKeys: []string{"ssh/gitea.rsa", "ssh/gogs.rsa"}, @@ -111,30 +103,27 @@ func loadSSHFrom(rootCfg ConfigProvider) { } homeDir = strings.ReplaceAll(homeDir, "\\", "/") - SSH.RootPath = path.Join(homeDir, ".ssh") - serverCiphers := sec.Key("SSH_SERVER_CIPHERS").Strings(",") - if len(serverCiphers) > 0 { - SSH.ServerCiphers = serverCiphers - } - serverKeyExchanges := sec.Key("SSH_SERVER_KEY_EXCHANGES").Strings(",") - if len(serverKeyExchanges) > 0 { - SSH.ServerKeyExchanges = serverKeyExchanges - } - serverMACs := sec.Key("SSH_SERVER_MACS").Strings(",") - if len(serverMACs) > 0 { - SSH.ServerMACs = serverMACs - } - SSH.KeyTestPath = os.TempDir() + SSH.RootPath = filepath.Join(homeDir, ".ssh") + if err = sec.MapTo(&SSH); err != nil { log.Fatal("Failed to map SSH settings: %v", err) } + + serverCiphers := sec.Key("SSH_SERVER_CIPHERS").Strings(",") + SSH.ServerCiphers = util.Iif(len(serverCiphers) > 0, serverCiphers, nil) + + serverKeyExchanges := sec.Key("SSH_SERVER_KEY_EXCHANGES").Strings(",") + SSH.ServerKeyExchanges = util.Iif(len(serverKeyExchanges) > 0, serverKeyExchanges, nil) + + serverMACs := sec.Key("SSH_SERVER_MACS").Strings(",") + SSH.ServerMACs = util.Iif(len(serverMACs) > 0, serverMACs, nil) + for i, key := range SSH.ServerHostKeys { if !filepath.IsAbs(key) { SSH.ServerHostKeys[i] = filepath.Join(AppDataPath, key) } } - SSH.KeygenPath = sec.Key("SSH_KEYGEN_PATH").String() SSH.Port = sec.Key("SSH_PORT").MustInt(22) SSH.ListenPort = sec.Key("SSH_LISTEN_PORT").MustInt(SSH.Port) SSH.UseProxyProtocol = sec.Key("SSH_SERVER_USE_PROXY_PROTOCOL").MustBool(false) diff --git a/modules/setting/storage.go b/modules/setting/storage.go index d3d1fb9f30..ee246158d9 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "path/filepath" + "slices" "strings" ) @@ -30,12 +31,7 @@ var storageTypes = []StorageType{ // IsValidStorageType returns true if the given storage type is valid func IsValidStorageType(storageType StorageType) bool { - for _, t := range storageTypes { - if t == storageType { - return true - } - } - return false + return slices.Contains(storageTypes, storageType) } // MinioStorageConfig represents the configuration for a minio storage @@ -162,7 +158,7 @@ const ( targetSecIsSec // target section is from the name seciont [name] ) -func getStorageSectionByType(rootCfg ConfigProvider, typ string) (ConfigSection, targetSecType, error) { //nolint:unparam +func getStorageSectionByType(rootCfg ConfigProvider, typ string) (ConfigSection, targetSecType, error) { //nolint:unparam // FIXME: targetSecType is always 0, wrong design? targetSec, err := rootCfg.GetSection(storageSectionName + "." + typ) if err != nil { if !IsValidStorageType(StorageType(typ)) { @@ -210,8 +206,8 @@ func getStorageTargetSection(rootCfg ConfigProvider, name, typ string, sec Confi targetSec, _ := rootCfg.GetSection(storageSectionName + "." + name) if targetSec != nil { targetType := targetSec.Key("STORAGE_TYPE").String() - switch { - case targetType == "": + switch targetType { + case "": if targetSec.Key("PATH").String() == "" { // both storage type and path are empty, use default return getDefaultStorageSection(rootCfg), targetSecIsDefault, nil } @@ -287,7 +283,7 @@ func getStorageForLocal(targetSec, overrideSec ConfigSection, tp targetSecType, return &storage, nil } -func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl +func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl // duplicates azure setup var storage Storage storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String()) if err := targetSec.MapTo(&storage.MinioConfig); err != nil { @@ -316,7 +312,7 @@ func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, return &storage, nil } -func getStorageForAzureBlob(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl +func getStorageForAzureBlob(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl // duplicates minio setup var storage Storage storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String()) if err := targetSec.MapTo(&storage.AzureBlobConfig); err != nil { diff --git a/modules/setting/storage_test.go b/modules/setting/storage_test.go index afff85537e..6f5a54c41c 100644 --- a/modules/setting/storage_test.go +++ b/modules/setting/storage_test.go @@ -26,16 +26,16 @@ MINIO_BUCKET = gitea-storage assert.NoError(t, err) assert.NoError(t, loadAttachmentFrom(cfg)) - assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) assert.NoError(t, loadAvatarsFrom(cfg)) - assert.EqualValues(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket) + assert.Equal(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) } func Test_getStorageUseOtherNameAsType(t *testing.T) { @@ -51,12 +51,12 @@ MINIO_BUCKET = gitea-storage assert.NoError(t, err) assert.NoError(t, loadAttachmentFrom(cfg)) - assert.EqualValues(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) } func Test_getStorageInheritStorageType(t *testing.T) { @@ -69,32 +69,32 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "gitea", Packages.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Packages.Storage.MinioConfig.Bucket) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "gitea", Actions.LogStorage.MinioConfig.Bucket) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Actions.LogStorage.MinioConfig.Bucket) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) assert.NoError(t, loadAvatarsFrom(cfg)) assert.EqualValues(t, "minio", Avatar.Storage.Type) - assert.EqualValues(t, "gitea", Avatar.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Avatar.Storage.MinioConfig.Bucket) + assert.Equal(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) assert.NoError(t, loadRepoAvatarFrom(cfg)) assert.EqualValues(t, "minio", RepoAvatar.Storage.Type) - assert.EqualValues(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket) + assert.Equal(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath) } func Test_getStorageInheritStorageTypeAzureBlob(t *testing.T) { @@ -107,32 +107,32 @@ STORAGE_TYPE = azureblob assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "azureblob", Packages.Storage.Type) - assert.EqualValues(t, "gitea", Packages.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "packages/", Packages.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Packages.Storage.AzureBlobConfig.Container) + assert.Equal(t, "packages/", Packages.Storage.AzureBlobConfig.BasePath) assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "azureblob", RepoArchive.Storage.Type) - assert.EqualValues(t, "gitea", RepoArchive.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", RepoArchive.Storage.AzureBlobConfig.Container) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "azureblob", Actions.LogStorage.Type) - assert.EqualValues(t, "gitea", Actions.LogStorage.AzureBlobConfig.Container) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Actions.LogStorage.AzureBlobConfig.Container) + assert.Equal(t, "actions_log/", Actions.LogStorage.AzureBlobConfig.BasePath) assert.EqualValues(t, "azureblob", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "gitea", Actions.ArtifactStorage.AzureBlobConfig.Container) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Actions.ArtifactStorage.AzureBlobConfig.Container) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.AzureBlobConfig.BasePath) assert.NoError(t, loadAvatarsFrom(cfg)) assert.EqualValues(t, "azureblob", Avatar.Storage.Type) - assert.EqualValues(t, "gitea", Avatar.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "avatars/", Avatar.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Avatar.Storage.AzureBlobConfig.Container) + assert.Equal(t, "avatars/", Avatar.Storage.AzureBlobConfig.BasePath) assert.NoError(t, loadRepoAvatarFrom(cfg)) assert.EqualValues(t, "azureblob", RepoAvatar.Storage.Type) - assert.EqualValues(t, "gitea", RepoAvatar.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", RepoAvatar.Storage.AzureBlobConfig.Container) + assert.Equal(t, "repo-avatars/", RepoAvatar.Storage.AzureBlobConfig.BasePath) } type testLocalStoragePathCase struct { @@ -151,7 +151,7 @@ func testLocalStoragePath(t *testing.T, appDataPath, iniStr string, cases []test assert.EqualValues(t, "local", storage.Type) assert.True(t, filepath.IsAbs(storage.Path)) - assert.EqualValues(t, filepath.Clean(c.expectedPath), filepath.Clean(storage.Path)) + assert.Equal(t, filepath.Clean(c.expectedPath), filepath.Clean(storage.Path)) } } @@ -389,8 +389,8 @@ MINIO_SECRET_ACCESS_KEY = my_secret_key assert.NoError(t, loadRepoArchiveFrom(cfg)) cp := RepoArchive.Storage.ToShadowCopy() - assert.EqualValues(t, "******", cp.MinioConfig.AccessKeyID) - assert.EqualValues(t, "******", cp.MinioConfig.SecretAccessKey) + assert.Equal(t, "******", cp.MinioConfig.AccessKeyID) + assert.Equal(t, "******", cp.MinioConfig.SecretAccessKey) } func Test_getStorageConfiguration24(t *testing.T) { @@ -445,10 +445,10 @@ MINIO_USE_SSL = true `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) assert.True(t, RepoArchive.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) } func Test_getStorageConfiguration28(t *testing.T) { @@ -462,10 +462,10 @@ MINIO_BASE_PATH = /prefix `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) assert.True(t, RepoArchive.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -476,9 +476,9 @@ MINIO_BASE_PATH = /prefix `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "127.0.0.1", RepoArchive.Storage.MinioConfig.IamEndpoint) + assert.Equal(t, "127.0.0.1", RepoArchive.Storage.MinioConfig.IamEndpoint) assert.True(t, RepoArchive.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -493,10 +493,10 @@ MINIO_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) assert.True(t, LFS.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "/lfs", LFS.Storage.MinioConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -511,10 +511,10 @@ MINIO_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) assert.True(t, LFS.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "/lfs", LFS.Storage.MinioConfig.BasePath) } func Test_getStorageConfiguration29(t *testing.T) { @@ -539,9 +539,9 @@ AZURE_BLOB_ACCOUNT_KEY = my_account_key `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) } func Test_getStorageConfiguration31(t *testing.T) { @@ -554,9 +554,9 @@ AZURE_BLOB_BASE_PATH = /prefix `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "/prefix/repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -570,9 +570,9 @@ AZURE_BLOB_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -586,7 +586,7 @@ AZURE_BLOB_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) } diff --git a/modules/setting/ui.go b/modules/setting/ui.go index 20fc612b43..3d9c916bf7 100644 --- a/modules/setting/ui.go +++ b/modules/setting/ui.go @@ -28,6 +28,7 @@ var UI = struct { DefaultShowFullName bool DefaultTheme string Themes []string + FileIconTheme string Reactions []string ReactionsLookup container.Set[string] `ini:"-"` CustomEmojis []string @@ -84,6 +85,7 @@ var UI = struct { ReactionMaxUserNum: 10, MaxDisplayFileSize: 8388608, DefaultTheme: `gitea-auto`, + FileIconTheme: `material`, Reactions: []string{`+1`, `-1`, `laugh`, `hooray`, `confused`, `heart`, `rocket`, `eyes`}, CustomEmojis: []string{`git`, `gitea`, `codeberg`, `gitlab`, `github`, `gogs`}, CustomEmojisMap: map[string]string{"git": ":git:", "gitea": ":gitea:", "codeberg": ":codeberg:", "gitlab": ":gitlab:", "github": ":github:", "gogs": ":gogs:"}, diff --git a/modules/ssh/init.go b/modules/ssh/init.go index 21d4f89936..cfb0d5693a 100644 --- a/modules/ssh/init.go +++ b/modules/ssh/init.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) func Init() error { @@ -23,20 +24,17 @@ func Init() error { if setting.SSH.StartBuiltinServer { Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs) - log.Info("SSH server started on %s. Cipher list (%v), key exchange algorithms (%v), MACs (%v)", + log.Info("SSH server started on %q. Ciphers: %v, key exchange algorithms: %v, MACs: %v", net.JoinHostPort(setting.SSH.ListenHost, strconv.Itoa(setting.SSH.ListenPort)), - setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs, + util.Iif[any](setting.SSH.ServerCiphers == nil, "default", setting.SSH.ServerCiphers), + util.Iif[any](setting.SSH.ServerKeyExchanges == nil, "default", setting.SSH.ServerKeyExchanges), + util.Iif[any](setting.SSH.ServerMACs == nil, "default", setting.SSH.ServerMACs), ) return nil } builtinUnused() - // FIXME: why 0o644 for a directory ..... - if err := os.MkdirAll(setting.SSH.KeyTestPath, 0o644); err != nil { - return fmt.Errorf("failed to create directory %q for ssh key test: %w", setting.SSH.KeyTestPath, err) - } - if len(setting.SSH.TrustedUserCAKeys) > 0 && setting.SSH.AuthorizedPrincipalsEnabled { caKeysFileName := setting.SSH.TrustedUserCAKeysFile caKeysFileDir := filepath.Dir(caKeysFileName) diff --git a/modules/ssh/ssh.go b/modules/ssh/ssh.go index 7479cfbd95..3fea4851c7 100644 --- a/modules/ssh/ssh.go +++ b/modules/ssh/ssh.go @@ -11,7 +11,6 @@ import ( "crypto/x509" "encoding/pem" "errors" - "fmt" "io" "net" "os" @@ -216,7 +215,7 @@ func publicKeyHandler(ctx ssh.Context, key ssh.PublicKey) bool { ctx.Permissions().Permissions = &gossh.Permissions{} setPermExt := func(keyID int64) { ctx.Permissions().Permissions.Extensions = map[string]string{ - giteaPermissionExtensionKeyID: fmt.Sprint(keyID), + giteaPermissionExtensionKeyID: strconv.FormatInt(keyID, 10), } } @@ -334,7 +333,7 @@ func sshConnectionFailed(conn net.Conn, err error) { log.Warn("Failed authentication attempt from %s", conn.RemoteAddr()) } -// Listen starts a SSH server listens on given port. +// Listen starts an SSH server listening on given port. func Listen(host string, port int, ciphers, keyExchanges, macs []string) { srv := ssh.Server{ Addr: net.JoinHostPort(host, strconv.Itoa(port)), diff --git a/modules/storage/azureblob_test.go b/modules/storage/azureblob_test.go index 6905db5008..b3791b4916 100644 --- a/modules/storage/azureblob_test.go +++ b/modules/storage/azureblob_test.go @@ -4,9 +4,9 @@ package storage import ( - "bytes" "io" "os" + "strings" "testing" "code.gitea.io/gitea/modules/setting" @@ -33,14 +33,14 @@ func TestAzureBlobStorageIterator(t *testing.T) { func TestAzureBlobStoragePath(t *testing.T) { m := &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: ""}} - assert.Equal(t, "", m.buildAzureBlobPath("/")) - assert.Equal(t, "", m.buildAzureBlobPath(".")) + assert.Empty(t, m.buildAzureBlobPath("/")) + assert.Empty(t, m.buildAzureBlobPath(".")) assert.Equal(t, "a", m.buildAzureBlobPath("/a")) assert.Equal(t, "a/b", m.buildAzureBlobPath("/a/b/")) m = &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: "/"}} - assert.Equal(t, "", m.buildAzureBlobPath("/")) - assert.Equal(t, "", m.buildAzureBlobPath(".")) + assert.Empty(t, m.buildAzureBlobPath("/")) + assert.Empty(t, m.buildAzureBlobPath(".")) assert.Equal(t, "a", m.buildAzureBlobPath("/a")) assert.Equal(t, "a/b", m.buildAzureBlobPath("/a/b/")) @@ -76,7 +76,7 @@ func Test_azureBlobObject(t *testing.T) { assert.NoError(t, err) data := "Q2xTckt6Y1hDOWh0" - _, err = s.Save("test.txt", bytes.NewBufferString(data), int64(len(data))) + _, err = s.Save("test.txt", strings.NewReader(data), int64(len(data))) assert.NoError(t, err) obj, err := s.Open("test.txt") assert.NoError(t, err) @@ -86,7 +86,7 @@ func Test_azureBlobObject(t *testing.T) { buf1 := make([]byte, 3) read, err := obj.Read(buf1) assert.NoError(t, err) - assert.EqualValues(t, 3, read) + assert.Equal(t, 3, read) assert.Equal(t, data[2:5], string(buf1)) offset, err = obj.Seek(-5, io.SeekEnd) assert.NoError(t, err) @@ -94,7 +94,7 @@ func Test_azureBlobObject(t *testing.T) { buf2 := make([]byte, 4) read, err = obj.Read(buf2) assert.NoError(t, err) - assert.EqualValues(t, 4, read) + assert.Equal(t, 4, read) assert.Equal(t, data[11:15], string(buf2)) assert.NoError(t, obj.Close()) assert.NoError(t, s.Delete("test.txt")) diff --git a/modules/storage/local_test.go b/modules/storage/local_test.go index e230323f67..0592fd716b 100644 --- a/modules/storage/local_test.go +++ b/modules/storage/local_test.go @@ -4,8 +4,6 @@ package storage import ( - "os" - "path/filepath" "testing" "code.gitea.io/gitea/modules/setting" @@ -50,12 +48,11 @@ func TestBuildLocalPath(t *testing.T) { t.Run(k.path, func(t *testing.T) { l := LocalStorage{dir: k.localDir} - assert.EqualValues(t, k.expected, l.buildLocalPath(k.path)) + assert.Equal(t, k.expected, l.buildLocalPath(k.path)) }) } } func TestLocalStorageIterator(t *testing.T) { - dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir") - testStorageIterator(t, setting.LocalStorageType, &setting.Storage{Path: dir}) + testStorageIterator(t, setting.LocalStorageType, &setting.Storage{Path: t.TempDir()}) } diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 6b92be61fb..1c5d25b2d4 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -86,13 +86,14 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath) var lookup minio.BucketLookupType - if config.BucketLookUpType == "auto" || config.BucketLookUpType == "" { + switch config.BucketLookUpType { + case "auto", "": lookup = minio.BucketLookupAuto - } else if config.BucketLookUpType == "dns" { + case "dns": lookup = minio.BucketLookupDNS - } else if config.BucketLookUpType == "path" { + case "path": lookup = minio.BucketLookupPath - } else { + default: return nil, fmt.Errorf("invalid minio bucket lookup type: %s", config.BucketLookUpType) } diff --git a/modules/storage/minio_test.go b/modules/storage/minio_test.go index 395da051e8..2726d765dd 100644 --- a/modules/storage/minio_test.go +++ b/modules/storage/minio_test.go @@ -34,19 +34,19 @@ func TestMinioStorageIterator(t *testing.T) { func TestMinioStoragePath(t *testing.T) { m := &MinioStorage{basePath: ""} - assert.Equal(t, "", m.buildMinioPath("/")) - assert.Equal(t, "", m.buildMinioPath(".")) + assert.Empty(t, m.buildMinioPath("/")) + assert.Empty(t, m.buildMinioPath(".")) assert.Equal(t, "a", m.buildMinioPath("/a")) assert.Equal(t, "a/b", m.buildMinioPath("/a/b/")) - assert.Equal(t, "", m.buildMinioDirPrefix("")) + assert.Empty(t, m.buildMinioDirPrefix("")) assert.Equal(t, "a/", m.buildMinioDirPrefix("/a/")) m = &MinioStorage{basePath: "/"} - assert.Equal(t, "", m.buildMinioPath("/")) - assert.Equal(t, "", m.buildMinioPath(".")) + assert.Empty(t, m.buildMinioPath("/")) + assert.Empty(t, m.buildMinioPath(".")) assert.Equal(t, "a", m.buildMinioPath("/a")) assert.Equal(t, "a/b", m.buildMinioPath("/a/b/")) - assert.Equal(t, "", m.buildMinioDirPrefix("")) + assert.Empty(t, m.buildMinioDirPrefix("")) assert.Equal(t, "a/", m.buildMinioDirPrefix("/a/")) m = &MinioStorage{basePath: "/base"} diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 750ecdfe0d..b0529941e7 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -93,7 +93,7 @@ func Clean(storage ObjectStorage) error { } // SaveFrom saves data to the ObjectStorage with path p from the callback -func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) error) error { +func SaveFrom(objStorage ObjectStorage, path string, callback func(w io.Writer) error) error { pr, pw := io.Pipe() defer pr.Close() go func() { @@ -103,7 +103,7 @@ func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) err } }() - _, err := objStorage.Save(p, pr, -1) + _, err := objStorage.Save(path, pr, -1) return err } diff --git a/modules/storage/storage_test.go b/modules/storage/storage_test.go index 7edde558f3..08f274e74b 100644 --- a/modules/storage/storage_test.go +++ b/modules/storage/storage_test.go @@ -4,7 +4,7 @@ package storage import ( - "bytes" + "strings" "testing" "code.gitea.io/gitea/modules/setting" @@ -26,7 +26,7 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) { {"b/x 4.txt", "bx4"}, } for _, f := range testFiles { - _, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1) + _, err = l.Save(f[0], strings.NewReader(f[1]), -1) assert.NoError(t, err) } diff --git a/modules/structs/admin_user.go b/modules/structs/admin_user.go index f7c6d10ba0..c68b59a897 100644 --- a/modules/structs/admin_user.go +++ b/modules/structs/admin_user.go @@ -8,8 +8,11 @@ import "time" // CreateUserOption create user options type CreateUserOption struct { - SourceID int64 `json:"source_id"` + SourceID int64 `json:"source_id"` + // identifier of the user, provided by the external authenticator (if configured) + // default: empty LoginName string `json:"login_name"` + // username of the user // required: true Username string `json:"username" binding:"Required;Username;MaxSize(40)"` FullName string `json:"full_name" binding:"MaxSize(100)"` @@ -32,6 +35,8 @@ type CreateUserOption struct { type EditUserOption struct { // required: true SourceID int64 `json:"source_id"` + // identifier of the user, provided by the external authenticator (if configured) + // default: empty // required: true LoginName string `json:"login_name" binding:"Required"` // swagger:strfmt email diff --git a/modules/structs/commit_status_test.go b/modules/structs/commit_status_test.go deleted file mode 100644 index 88e09aadc1..0000000000 --- a/modules/structs/commit_status_test.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package structs - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestNoBetterThan(t *testing.T) { - type args struct { - css CommitStatusState - css2 CommitStatusState - } - var unExpectedState CommitStatusState - tests := []struct { - name string - args args - want bool - }{ - { - name: "success is no better than success", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "success is no better than pending", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusPending, - }, - want: false, - }, - { - name: "success is no better than failure", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusFailure, - }, - want: false, - }, - { - name: "success is no better than error", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusError, - }, - want: false, - }, - { - name: "pending is no better than success", - args: args{ - css: CommitStatusPending, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "pending is no better than pending", - args: args{ - css: CommitStatusPending, - css2: CommitStatusPending, - }, - want: true, - }, - { - name: "pending is no better than failure", - args: args{ - css: CommitStatusPending, - css2: CommitStatusFailure, - }, - want: false, - }, - { - name: "pending is no better than error", - args: args{ - css: CommitStatusPending, - css2: CommitStatusError, - }, - want: false, - }, - { - name: "failure is no better than success", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "failure is no better than pending", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusPending, - }, - want: true, - }, - { - name: "failure is no better than failure", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusFailure, - }, - want: true, - }, - { - name: "failure is no better than error", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusError, - }, - want: false, - }, - { - name: "error is no better than success", - args: args{ - css: CommitStatusError, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "error is no better than pending", - args: args{ - css: CommitStatusError, - css2: CommitStatusPending, - }, - want: true, - }, - { - name: "error is no better than failure", - args: args{ - css: CommitStatusError, - css2: CommitStatusFailure, - }, - want: true, - }, - { - name: "error is no better than error", - args: args{ - css: CommitStatusError, - css2: CommitStatusError, - }, - want: true, - }, - { - name: "unExpectedState is no better than success", - args: args{ - css: unExpectedState, - css2: CommitStatusSuccess, - }, - want: false, - }, - { - name: "unExpectedState is no better than unExpectedState", - args: args{ - css: unExpectedState, - css2: unExpectedState, - }, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := tt.args.css.NoBetterThan(tt.args.css2) - assert.Equal(t, tt.want, result) - }) - } -} diff --git a/modules/structs/git_blob.go b/modules/structs/git_blob.go index 96c7a271a9..643b69ed37 100644 --- a/modules/structs/git_blob.go +++ b/modules/structs/git_blob.go @@ -5,9 +5,12 @@ package structs // GitBlobResponse represents a git blob type GitBlobResponse struct { - Content string `json:"content"` - Encoding string `json:"encoding"` - URL string `json:"url"` - SHA string `json:"sha"` - Size int64 `json:"size"` + Content *string `json:"content"` + Encoding *string `json:"encoding"` + URL string `json:"url"` + SHA string `json:"sha"` + Size int64 `json:"size"` + + LfsOid *string `json:"lfs_oid,omitempty"` + LfsSize *int64 `json:"lfs_size,omitempty"` } diff --git a/modules/structs/hook.go b/modules/structs/hook.go index ce5742e5c7..ac779a5740 100644 --- a/modules/structs/hook.go +++ b/modules/structs/hook.go @@ -71,7 +71,8 @@ type PayloadUser struct { // Full name of the commit author Name string `json:"name"` // swagger:strfmt email - Email string `json:"email"` + Email string `json:"email"` + // username of the user UserName string `json:"username"` } @@ -116,14 +117,7 @@ var ( _ Payloader = &PackagePayload{} ) -// _________ __ -// \_ ___ \_______ ____ _____ _/ |_ ____ -// / \ \/\_ __ \_/ __ \\__ \\ __\/ __ \ -// \ \____| | \/\ ___/ / __ \| | \ ___/ -// \______ /|__| \___ >____ /__| \___ > -// \/ \/ \/ \/ - -// CreatePayload FIXME +// CreatePayload represents a payload information of create event. type CreatePayload struct { Sha string `json:"sha"` Ref string `json:"ref"` @@ -157,13 +151,6 @@ func ParseCreateHook(raw []byte) (*CreatePayload, error) { return hook, nil } -// ________ .__ __ -// \______ \ ____ | | _____/ |_ ____ -// | | \_/ __ \| | _/ __ \ __\/ __ \ -// | ` \ ___/| |_\ ___/| | \ ___/ -// /_______ /\___ >____/\___ >__| \___ > -// \/ \/ \/ \/ - // PusherType define the type to push type PusherType string @@ -186,13 +173,6 @@ func (p *DeletePayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } -// ___________ __ -// \_ _____/__________| | __ -// | __)/ _ \_ __ \ |/ / -// | \( <_> ) | \/ < -// \___ / \____/|__| |__|_ \ -// \/ \/ - // ForkPayload represents fork payload type ForkPayload struct { Forkee *Repository `json:"forkee"` @@ -232,13 +212,6 @@ func (p *IssueCommentPayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } -// __________ .__ -// \______ \ ____ | | ____ _____ ______ ____ -// | _// __ \| | _/ __ \\__ \ / ___// __ \ -// | | \ ___/| |_\ ___/ / __ \_\___ \\ ___/ -// |____|_ /\___ >____/\___ >____ /____ >\___ > -// \/ \/ \/ \/ \/ \/ - // HookReleaseAction defines hook release action type type HookReleaseAction string @@ -302,13 +275,6 @@ func (p *PushPayload) Branch() string { return strings.ReplaceAll(p.Ref, "refs/heads/", "") } -// .___ -// | | ______ ________ __ ____ -// | |/ ___// ___/ | \_/ __ \ -// | |\___ \ \___ \| | /\ ___/ -// |___/____ >____ >____/ \___ > -// \/ \/ \/ - // HookIssueAction FIXME type HookIssueAction string @@ -321,6 +287,8 @@ const ( HookIssueReOpened HookIssueAction = "reopened" // HookIssueEdited edited HookIssueEdited HookIssueAction = "edited" + // HookIssueDeleted is an issue action for deleting an issue + HookIssueDeleted HookIssueAction = "deleted" // HookIssueAssigned assigned HookIssueAssigned HookIssueAction = "assigned" // HookIssueUnassigned unassigned @@ -371,13 +339,6 @@ type ChangesPayload struct { Ref *ChangesFromPayload `json:"ref,omitempty"` } -// __________ .__ .__ __________ __ -// \______ \__ __| | | | \______ \ ____ ________ __ ____ _______/ |_ -// | ___/ | \ | | | | _// __ \/ ____/ | \_/ __ \ / ___/\ __\ -// | | | | / |_| |__ | | \ ___< <_| | | /\ ___/ \___ \ | | -// |____| |____/|____/____/ |____|_ /\___ >__ |____/ \___ >____ > |__| -// \/ \/ |__| \/ \/ - // PullRequestPayload represents a payload information of pull request event. type PullRequestPayload struct { Action HookIssueAction `json:"action"` @@ -402,13 +363,6 @@ type ReviewPayload struct { Content string `json:"content"` } -// __ __.__ __ .__ -// / \ / \__| | _|__| -// \ \/\/ / | |/ / | -// \ /| | <| | -// \__/\ / |__|__|_ \__| -// \/ \/ - // HookWikiAction an action that happens to a wiki page type HookWikiAction string @@ -435,13 +389,6 @@ func (p *WikiPayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } -//__________ .__ __ -//\______ \ ____ ______ ____ _____|__|/ |_ ___________ ___.__. -// | _// __ \\____ \ / _ \/ ___/ \ __\/ _ \_ __ < | | -// | | \ ___/| |_> > <_> )___ \| || | ( <_> ) | \/\___ | -// |____|_ /\___ > __/ \____/____ >__||__| \____/|__| / ____| -// \/ \/|__| \/ \/ - // HookRepoAction an action that happens to a repo type HookRepoAction string @@ -480,7 +427,7 @@ type PackagePayload struct { Action HookPackageAction `json:"action"` Repository *Repository `json:"repository"` Package *Package `json:"package"` - Organization *User `json:"organization"` + Organization *Organization `json:"organization"` Sender *User `json:"sender"` } @@ -525,3 +472,34 @@ type CommitStatusPayload struct { func (p *CommitStatusPayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } + +// WorkflowRunPayload represents a payload information of workflow run event. +type WorkflowRunPayload struct { + Action string `json:"action"` + Workflow *ActionWorkflow `json:"workflow"` + WorkflowRun *ActionWorkflowRun `json:"workflow_run"` + PullRequest *PullRequest `json:"pull_request,omitempty"` + Organization *Organization `json:"organization,omitempty"` + Repo *Repository `json:"repository"` + Sender *User `json:"sender"` +} + +// JSONPayload implements Payload +func (p *WorkflowRunPayload) JSONPayload() ([]byte, error) { + return json.MarshalIndent(p, "", " ") +} + +// WorkflowJobPayload represents a payload information of workflow job event. +type WorkflowJobPayload struct { + Action string `json:"action"` + WorkflowJob *ActionWorkflowJob `json:"workflow_job"` + PullRequest *PullRequest `json:"pull_request,omitempty"` + Organization *Organization `json:"organization,omitempty"` + Repo *Repository `json:"repository"` + Sender *User `json:"sender"` +} + +// JSONPayload implements Payload +func (p *WorkflowJobPayload) JSONPayload() ([]byte, error) { + return json.MarshalIndent(p, "", " ") +} diff --git a/modules/structs/issue.go b/modules/structs/issue.go index 3682191be5..df0be8f9ec 100644 --- a/modules/structs/issue.go +++ b/modules/structs/issue.go @@ -203,7 +203,7 @@ func (l *IssueTemplateStringSlice) UnmarshalYAML(value *yaml.Node) error { if err != nil { return err } - for _, v := range strings.Split(str, ",") { + for v := range strings.SplitSeq(str, ",") { if v = strings.TrimSpace(v); v == "" { continue } @@ -266,3 +266,8 @@ type IssueMeta struct { Owner string `json:"owner"` Name string `json:"repo"` } + +// LockIssueOption options to lock an issue +type LockIssueOption struct { + Reason string `json:"lock_reason"` +} diff --git a/modules/structs/issue_tracked_time.go b/modules/structs/issue_tracked_time.go index a3904af80e..befcfb323d 100644 --- a/modules/structs/issue_tracked_time.go +++ b/modules/structs/issue_tracked_time.go @@ -14,7 +14,7 @@ type AddTimeOption struct { Time int64 `json:"time" binding:"Required"` // swagger:strfmt date-time Created time.Time `json:"created"` - // User who spent the time (optional) + // username of the user who spent the time working on the issue (optional) User string `json:"user_name"` } @@ -26,7 +26,8 @@ type TrackedTime struct { // Time in seconds Time int64 `json:"time"` // deprecated (only for backwards compatibility) - UserID int64 `json:"user_id"` + UserID int64 `json:"user_id"` + // username of the user UserName string `json:"user_name"` // deprecated (only for backwards compatibility) IssueID int64 `json:"issue_id"` diff --git a/modules/structs/org.go b/modules/structs/org.go index c0a545ac1c..33b45c6344 100644 --- a/modules/structs/org.go +++ b/modules/structs/org.go @@ -15,6 +15,7 @@ type Organization struct { Location string `json:"location"` Visibility string `json:"visibility"` RepoAdminChangeTeamAccess bool `json:"repo_admin_change_team_access"` + // username of the organization // deprecated UserName string `json:"username"` } @@ -30,6 +31,7 @@ type OrganizationPermissions struct { // CreateOrgOption options for creating an organization type CreateOrgOption struct { + // username of the organization // required: true UserName string `json:"username" binding:"Required;Username;MaxSize(40)"` FullName string `json:"full_name" binding:"MaxSize(100)"` @@ -57,3 +59,12 @@ type EditOrgOption struct { Visibility string `json:"visibility" binding:"In(,public,limited,private)"` RepoAdminChangeTeamAccess *bool `json:"repo_admin_change_team_access"` } + +// RenameOrgOption options when renaming an organization +type RenameOrgOption struct { + // New username for this org. This name cannot be in use yet by any other user. + // + // required: true + // unique: true + NewName string `json:"new_name" binding:"Required"` +} diff --git a/modules/structs/package.go b/modules/structs/package.go index a9a9429de2..1973f925a5 100644 --- a/modules/structs/package.go +++ b/modules/structs/package.go @@ -23,8 +23,8 @@ type Package struct { // PackageFile represents a package file type PackageFile struct { - ID int64 `json:"id"` - Size int64 + ID int64 `json:"id"` + Size int64 `json:"size"` Name string `json:"name"` HashMD5 string `json:"md5"` HashSHA1 string `json:"sha1"` diff --git a/modules/structs/pull.go b/modules/structs/pull.go index 55831e642c..f53d6adafc 100644 --- a/modules/structs/pull.go +++ b/modules/structs/pull.go @@ -25,11 +25,13 @@ type PullRequest struct { Draft bool `json:"draft"` IsLocked bool `json:"is_locked"` Comments int `json:"comments"` + // number of review comments made on the diff of a PR review (not including comments on commits or issues in a PR) - ReviewComments int `json:"review_comments"` - Additions int `json:"additions"` - Deletions int `json:"deletions"` - ChangedFiles int `json:"changed_files"` + ReviewComments int `json:"review_comments,omitempty"` + + Additions *int `json:"additions,omitempty"` + Deletions *int `json:"deletions,omitempty"` + ChangedFiles *int `json:"changed_files,omitempty"` HTMLURL string `json:"html_url"` DiffURL string `json:"diff_url"` diff --git a/modules/structs/release.go b/modules/structs/release.go index c7378645c2..fac86ca7a2 100644 --- a/modules/structs/release.go +++ b/modules/structs/release.go @@ -33,6 +33,7 @@ type Release struct { type CreateReleaseOption struct { // required: true TagName string `json:"tag_name" binding:"Required"` + TagMessage string `json:"tag_message"` Target string `json:"target_commitish"` Title string `json:"name"` Note string `json:"body"` diff --git a/modules/structs/repo.go b/modules/structs/repo.go index fb784bd8b3..abc8076387 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -101,6 +101,8 @@ type Repository struct { AllowSquash bool `json:"allow_squash_merge"` AllowFastForwardOnly bool `json:"allow_fast_forward_only_merge"` AllowRebaseUpdate bool `json:"allow_rebase_update"` + AllowManualMerge bool `json:"allow_manual_merge"` + AutodetectManualMerge bool `json:"autodetect_manual_merge"` DefaultDeleteBranchAfterMerge bool `json:"default_delete_branch_after_merge"` DefaultMergeStyle string `json:"default_merge_style"` DefaultAllowMaintainerEdit bool `json:"default_allow_maintainer_edit"` @@ -111,7 +113,7 @@ type Repository struct { // enum: sha1,sha256 ObjectFormatName string `json:"object_format_name"` // swagger:strfmt date-time - MirrorUpdated time.Time `json:"mirror_updated,omitempty"` + MirrorUpdated time.Time `json:"mirror_updated"` RepoTransfer *RepoTransfer `json:"repo_transfer"` Topics []string `json:"topics"` Licenses []string `json:"licenses"` @@ -357,7 +359,7 @@ type MigrateRepoOptions struct { // required: true RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"` - // enum: git,github,gitea,gitlab,gogs,onedev,gitbucket,codebase + // enum: git,github,gitea,gitlab,gogs,onedev,gitbucket,codebase,codecommit Service string `json:"service"` AuthUsername string `json:"auth_username"` AuthPassword string `json:"auth_password"` diff --git a/modules/structs/repo_actions.go b/modules/structs/repo_actions.go index b13f344738..ac1c288270 100644 --- a/modules/structs/repo_actions.go +++ b/modules/structs/repo_actions.go @@ -32,3 +32,157 @@ type ActionTaskResponse struct { Entries []*ActionTask `json:"workflow_runs"` TotalCount int64 `json:"total_count"` } + +// CreateActionWorkflowDispatch represents the payload for triggering a workflow dispatch event +// swagger:model +type CreateActionWorkflowDispatch struct { + // required: true + // example: refs/heads/main + Ref string `json:"ref" binding:"Required"` + // required: false + Inputs map[string]string `json:"inputs,omitempty"` +} + +// ActionWorkflow represents a ActionWorkflow +type ActionWorkflow struct { + ID string `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + State string `json:"state"` + // swagger:strfmt date-time + CreatedAt time.Time `json:"created_at"` + // swagger:strfmt date-time + UpdatedAt time.Time `json:"updated_at"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + BadgeURL string `json:"badge_url"` + // swagger:strfmt date-time + DeletedAt time.Time `json:"deleted_at"` +} + +// ActionWorkflowResponse returns a ActionWorkflow +type ActionWorkflowResponse struct { + Workflows []*ActionWorkflow `json:"workflows"` + TotalCount int64 `json:"total_count"` +} + +// ActionArtifact represents a ActionArtifact +type ActionArtifact struct { + ID int64 `json:"id"` + Name string `json:"name"` + SizeInBytes int64 `json:"size_in_bytes"` + URL string `json:"url"` + ArchiveDownloadURL string `json:"archive_download_url"` + Expired bool `json:"expired"` + WorkflowRun *ActionWorkflowRun `json:"workflow_run"` + + // swagger:strfmt date-time + CreatedAt time.Time `json:"created_at"` + // swagger:strfmt date-time + UpdatedAt time.Time `json:"updated_at"` + // swagger:strfmt date-time + ExpiresAt time.Time `json:"expires_at"` +} + +// ActionWorkflowRun represents a WorkflowRun +type ActionWorkflowRun struct { + ID int64 `json:"id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + DisplayTitle string `json:"display_title"` + Path string `json:"path"` + Event string `json:"event"` + RunAttempt int64 `json:"run_attempt"` + RunNumber int64 `json:"run_number"` + RepositoryID int64 `json:"repository_id,omitempty"` + HeadSha string `json:"head_sha"` + HeadBranch string `json:"head_branch,omitempty"` + Status string `json:"status"` + Actor *User `json:"actor,omitempty"` + TriggerActor *User `json:"trigger_actor,omitempty"` + Repository *Repository `json:"repository,omitempty"` + HeadRepository *Repository `json:"head_repository,omitempty"` + Conclusion string `json:"conclusion,omitempty"` + // swagger:strfmt date-time + StartedAt time.Time `json:"started_at"` + // swagger:strfmt date-time + CompletedAt time.Time `json:"completed_at"` +} + +// ActionWorkflowRunsResponse returns ActionWorkflowRuns +type ActionWorkflowRunsResponse struct { + Entries []*ActionWorkflowRun `json:"workflow_runs"` + TotalCount int64 `json:"total_count"` +} + +// ActionWorkflowJobsResponse returns ActionWorkflowJobs +type ActionWorkflowJobsResponse struct { + Entries []*ActionWorkflowJob `json:"jobs"` + TotalCount int64 `json:"total_count"` +} + +// ActionArtifactsResponse returns ActionArtifacts +type ActionArtifactsResponse struct { + Entries []*ActionArtifact `json:"artifacts"` + TotalCount int64 `json:"total_count"` +} + +// ActionWorkflowStep represents a step of a WorkflowJob +type ActionWorkflowStep struct { + Name string `json:"name"` + Number int64 `json:"number"` + Status string `json:"status"` + Conclusion string `json:"conclusion,omitempty"` + // swagger:strfmt date-time + StartedAt time.Time `json:"started_at"` + // swagger:strfmt date-time + CompletedAt time.Time `json:"completed_at"` +} + +// ActionWorkflowJob represents a WorkflowJob +type ActionWorkflowJob struct { + ID int64 `json:"id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + RunID int64 `json:"run_id"` + RunURL string `json:"run_url"` + Name string `json:"name"` + Labels []string `json:"labels"` + RunAttempt int64 `json:"run_attempt"` + HeadSha string `json:"head_sha"` + HeadBranch string `json:"head_branch,omitempty"` + Status string `json:"status"` + Conclusion string `json:"conclusion,omitempty"` + RunnerID int64 `json:"runner_id,omitempty"` + RunnerName string `json:"runner_name,omitempty"` + Steps []*ActionWorkflowStep `json:"steps"` + // swagger:strfmt date-time + CreatedAt time.Time `json:"created_at"` + // swagger:strfmt date-time + StartedAt time.Time `json:"started_at"` + // swagger:strfmt date-time + CompletedAt time.Time `json:"completed_at"` +} + +// ActionRunnerLabel represents a Runner Label +type ActionRunnerLabel struct { + ID int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` +} + +// ActionRunner represents a Runner +type ActionRunner struct { + ID int64 `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Busy bool `json:"busy"` + Ephemeral bool `json:"ephemeral"` + Labels []*ActionRunnerLabel `json:"labels"` +} + +// ActionRunnersResponse returns Runners +type ActionRunnersResponse struct { + Entries []*ActionRunner `json:"runners"` + TotalCount int64 `json:"total_count"` +} diff --git a/modules/structs/repo_branch.go b/modules/structs/repo_branch.go index 55c98d60b9..5416f43b0d 100644 --- a/modules/structs/repo_branch.go +++ b/modules/structs/repo_branch.go @@ -136,6 +136,7 @@ type UpdateBranchProtectionPriories struct { type MergeUpstreamRequest struct { Branch string `json:"branch"` + FfOnly bool `json:"ff_only"` } type MergeUpstreamResponse struct { diff --git a/modules/structs/repo_file.go b/modules/structs/repo_file.go index 82bde96ab6..5a86db868b 100644 --- a/modules/structs/repo_file.go +++ b/modules/structs/repo_file.go @@ -4,6 +4,8 @@ package structs +import "time" + // FileOptions options for all file APIs type FileOptions struct { // message (optional) for the commit of this file. if not supplied, a default message will be used @@ -20,6 +22,23 @@ type FileOptions struct { Signoff bool `json:"signoff"` } +type FileOptionsWithSHA struct { + FileOptions + // the blob ID (SHA) for the file that already exists, it is required for changing existing files + // required: true + SHA string `json:"sha" binding:"Required"` +} + +func (f *FileOptions) GetFileOptions() *FileOptions { + return f +} + +type FileOptionsInterface interface { + GetFileOptions() *FileOptions +} + +var _ FileOptionsInterface = (*FileOptions)(nil) + // CreateFileOptions options for creating files // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type CreateFileOptions struct { @@ -29,29 +48,16 @@ type CreateFileOptions struct { ContentBase64 string `json:"content"` } -// Branch returns branch name -func (o *CreateFileOptions) Branch() string { - return o.FileOptions.BranchName -} - // DeleteFileOptions options for deleting files (used for other File structs below) // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type DeleteFileOptions struct { - FileOptions - // sha is the SHA for the file that already exists - // required: true - SHA string `json:"sha" binding:"Required"` -} - -// Branch returns branch name -func (o *DeleteFileOptions) Branch() string { - return o.FileOptions.BranchName + FileOptionsWithSHA } // UpdateFileOptions options for updating files // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type UpdateFileOptions struct { - DeleteFileOptions + FileOptionsWithSHA // content must be base64 encoded // required: true ContentBase64 string `json:"content"` @@ -59,23 +65,21 @@ type UpdateFileOptions struct { FromPath string `json:"from_path" binding:"MaxSize(500)"` } -// Branch returns branch name -func (o *UpdateFileOptions) Branch() string { - return o.FileOptions.BranchName -} +// FIXME: there is no LastCommitID in FileOptions, actually it should be an alternative to the SHA in ChangeFileOperation // ChangeFileOperation for creating, updating or deleting a file type ChangeFileOperation struct { - // indicates what to do with the file + // indicates what to do with the file: "create" for creating a new file, "update" for updating an existing file, + // "upload" for creating or updating a file, "rename" for renaming a file, and "delete" for deleting an existing file. // required: true - // enum: create,update,delete + // enum: create,update,upload,rename,delete Operation string `json:"operation" binding:"Required"` // path to the existing or new file // required: true Path string `json:"path" binding:"Required;MaxSize(500)"` - // new or updated file content, must be base64 encoded + // new or updated file content, it must be base64 encoded ContentBase64 string `json:"content"` - // sha is the SHA for the file that already exists, required for update or delete + // the blob ID (SHA) for the file that already exists, required for changing existing files SHA string `json:"sha"` // old path of the file to move FromPath string `json:"from_path"` @@ -90,20 +94,10 @@ type ChangeFilesOptions struct { Files []*ChangeFileOperation `json:"files" binding:"Required"` } -// Branch returns branch name -func (o *ChangeFilesOptions) Branch() string { - return o.FileOptions.BranchName -} - -// FileOptionInterface provides a unified interface for the different file options -type FileOptionInterface interface { - Branch() string -} - // ApplyDiffPatchFileOptions options for applying a diff patch // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type ApplyDiffPatchFileOptions struct { - DeleteFileOptions + FileOptions // required: true Content string `json:"content"` } @@ -115,12 +109,24 @@ type FileLinksResponse struct { HTMLURL *string `json:"html"` } +type ContentsExtResponse struct { + FileContents *ContentsResponse `json:"file_contents,omitempty"` + DirContents []*ContentsResponse `json:"dir_contents,omitempty"` +} + // ContentsResponse contains information about a repo's entry's (dir, file, symlink, submodule) metadata and content type ContentsResponse struct { - Name string `json:"name"` - Path string `json:"path"` - SHA string `json:"sha"` - LastCommitSHA string `json:"last_commit_sha"` + Name string `json:"name"` + Path string `json:"path"` + SHA string `json:"sha"` + + LastCommitSHA *string `json:"last_commit_sha,omitempty"` + // swagger:strfmt date-time + LastCommitterDate *time.Time `json:"last_committer_date,omitempty"` + // swagger:strfmt date-time + LastAuthorDate *time.Time `json:"last_author_date,omitempty"` + LastCommitMessage *string `json:"last_commit_message,omitempty"` + // `type` will be `file`, `dir`, `symlink`, or `submodule` Type string `json:"type"` Size int64 `json:"size"` @@ -137,6 +143,9 @@ type ContentsResponse struct { // `submodule_git_url` is populated when `type` is `submodule`, otherwise null SubmoduleGitURL *string `json:"submodule_git_url"` Links *FileLinksResponse `json:"_links"` + + LfsOid *string `json:"lfs_oid,omitempty"` + LfsSize *int64 `json:"lfs_size,omitempty"` } // FileCommitResponse contains information generated from a Git commit for a repo's file. @@ -170,3 +179,8 @@ type FileDeleteResponse struct { Commit *FileCommitResponse `json:"commit"` Verification *PayloadCommitVerification `json:"verification"` } + +// GetFilesOptions options for retrieving metadate and content of multiple files +type GetFilesOptions struct { + Files []string `json:"files" binding:"Required"` +} diff --git a/modules/structs/repo_tag.go b/modules/structs/repo_tag.go index 5722513f4f..bb8bfd10cb 100644 --- a/modules/structs/repo_tag.go +++ b/modules/structs/repo_tag.go @@ -11,8 +11,8 @@ type Tag struct { Message string `json:"message"` ID string `json:"id"` Commit *CommitMeta `json:"commit"` - ZipballURL string `json:"zipball_url"` - TarballURL string `json:"tarball_url"` + ZipballURL string `json:"zipball_url,omitempty"` + TarballURL string `json:"tarball_url,omitempty"` } // AnnotatedTag represents an annotated tag diff --git a/modules/structs/secret.go b/modules/structs/secret.go index a0673ca08c..2afb41ec43 100644 --- a/modules/structs/secret.go +++ b/modules/structs/secret.go @@ -10,6 +10,8 @@ import "time" type Secret struct { // the secret's name Name string `json:"name"` + // the secret's description + Description string `json:"description"` // swagger:strfmt date-time Created time.Time `json:"created_at"` } @@ -21,4 +23,9 @@ type CreateOrUpdateSecretOption struct { // // required: true Data string `json:"data" binding:"Required"` + + // Description of the secret to update + // + // required: false + Description string `json:"description"` } diff --git a/modules/structs/settings.go b/modules/structs/settings.go index e48b1a493d..59176210e6 100644 --- a/modules/structs/settings.go +++ b/modules/structs/settings.go @@ -26,6 +26,7 @@ type GeneralAPISettings struct { DefaultPagingNum int `json:"default_paging_num"` DefaultGitTreesPerPage int `json:"default_git_trees_per_page"` DefaultMaxBlobSize int64 `json:"default_max_blob_size"` + DefaultMaxResponseSize int64 `json:"default_max_response_size"` } // GeneralAttachmentSettings contains global Attachment settings exposed by API diff --git a/modules/structs/status.go b/modules/structs/status.go index c1d8b902ec..a9779541ff 100644 --- a/modules/structs/status.go +++ b/modules/structs/status.go @@ -5,17 +5,19 @@ package structs import ( "time" + + "code.gitea.io/gitea/modules/commitstatus" ) // CommitStatus holds a single status of a single Commit type CommitStatus struct { - ID int64 `json:"id"` - State CommitStatusState `json:"status"` - TargetURL string `json:"target_url"` - Description string `json:"description"` - URL string `json:"url"` - Context string `json:"context"` - Creator *User `json:"creator"` + ID int64 `json:"id"` + State commitstatus.CommitStatusState `json:"status"` + TargetURL string `json:"target_url"` + Description string `json:"description"` + URL string `json:"url"` + Context string `json:"context"` + Creator *User `json:"creator"` // swagger:strfmt date-time Created time.Time `json:"created_at"` // swagger:strfmt date-time @@ -24,19 +26,19 @@ type CommitStatus struct { // CombinedStatus holds the combined state of several statuses for a single commit type CombinedStatus struct { - State CommitStatusState `json:"state"` - SHA string `json:"sha"` - TotalCount int `json:"total_count"` - Statuses []*CommitStatus `json:"statuses"` - Repository *Repository `json:"repository"` - CommitURL string `json:"commit_url"` - URL string `json:"url"` + State commitstatus.CommitStatusState `json:"state"` + SHA string `json:"sha"` + TotalCount int `json:"total_count"` + Statuses []*CommitStatus `json:"statuses"` + Repository *Repository `json:"repository"` + CommitURL string `json:"commit_url"` + URL string `json:"url"` } // CreateStatusOption holds the information needed to create a new CommitStatus for a Commit type CreateStatusOption struct { - State CommitStatusState `json:"state"` - TargetURL string `json:"target_url"` - Description string `json:"description"` - Context string `json:"context"` + State commitstatus.CommitStatusState `json:"state"` + TargetURL string `json:"target_url"` + Description string `json:"description"` + Context string `json:"context"` } diff --git a/modules/structs/user.go b/modules/structs/user.go index 5ed677f239..89349cda2c 100644 --- a/modules/structs/user.go +++ b/modules/structs/user.go @@ -15,9 +15,9 @@ import ( type User struct { // the user's id ID int64 `json:"id"` - // the user's username + // login of the user, same as `username` UserName string `json:"login"` - // the user's authentication sign-in name. + // identifier of the user, provided by the external authenticator (if configured) // default: empty LoginName string `json:"login_name"` // The ID of the user's Authentication Source @@ -35,9 +35,9 @@ type User struct { // Is the user an administrator IsAdmin bool `json:"is_admin"` // swagger:strfmt date-time - LastLogin time.Time `json:"last_login,omitempty"` + LastLogin time.Time `json:"last_login"` // swagger:strfmt date-time - Created time.Time `json:"created,omitempty"` + Created time.Time `json:"created"` // Is user restricted Restricted bool `json:"restricted"` // Is user active diff --git a/modules/structs/user_app.go b/modules/structs/user_app.go index a7d2e28b41..15811ceb66 100644 --- a/modules/structs/user_app.go +++ b/modules/structs/user_app.go @@ -11,11 +11,13 @@ import ( // AccessToken represents an API access token. // swagger:response AccessToken type AccessToken struct { - ID int64 `json:"id"` - Name string `json:"name"` - Token string `json:"sha1"` - TokenLastEight string `json:"token_last_eight"` - Scopes []string `json:"scopes"` + ID int64 `json:"id"` + Name string `json:"name"` + Token string `json:"sha1"` + TokenLastEight string `json:"token_last_eight"` + Scopes []string `json:"scopes"` + Created time.Time `json:"created_at"` + Updated time.Time `json:"last_used_at"` } // AccessTokenList represents a list of API access token. @@ -23,9 +25,11 @@ type AccessToken struct { type AccessTokenList []*AccessToken // CreateAccessTokenOption options when create access token +// swagger:model CreateAccessTokenOption type CreateAccessTokenOption struct { // required: true - Name string `json:"name" binding:"Required"` + Name string `json:"name" binding:"Required"` + // example: ["all", "read:activitypub","read:issue", "write:misc", "read:notification", "read:organization", "read:package", "read:repository", "read:user"] Scopes []string `json:"scopes"` } diff --git a/modules/structs/user_email.go b/modules/structs/user_email.go index 9319667e8f..01895a0058 100644 --- a/modules/structs/user_email.go +++ b/modules/structs/user_email.go @@ -11,6 +11,7 @@ type Email struct { Verified bool `json:"verified"` Primary bool `json:"primary"` UserID int64 `json:"user_id"` + // username of the user UserName string `json:"username"` } diff --git a/modules/structs/user_gpgkey.go b/modules/structs/user_gpgkey.go index ff9b0aea1d..deae70de33 100644 --- a/modules/structs/user_gpgkey.go +++ b/modules/structs/user_gpgkey.go @@ -21,9 +21,9 @@ type GPGKey struct { CanCertify bool `json:"can_certify"` Verified bool `json:"verified"` // swagger:strfmt date-time - Created time.Time `json:"created_at,omitempty"` + Created time.Time `json:"created_at"` // swagger:strfmt date-time - Expires time.Time `json:"expires_at,omitempty"` + Expires time.Time `json:"expires_at"` } // GPGKeyEmail an email attached to a GPGKey diff --git a/modules/structs/user_key.go b/modules/structs/user_key.go index 08eed59a89..16225a852a 100644 --- a/modules/structs/user_key.go +++ b/modules/structs/user_key.go @@ -15,7 +15,8 @@ type PublicKey struct { Title string `json:"title,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` // swagger:strfmt date-time - Created time.Time `json:"created_at,omitempty"` + Created time.Time `json:"created_at"` + Updated time.Time `json:"last_used_at"` Owner *User `json:"user,omitempty"` ReadOnly bool `json:"read_only,omitempty"` KeyType string `json:"key_type,omitempty"` diff --git a/modules/structs/variable.go b/modules/structs/variable.go index cc846cf0ec..5198937303 100644 --- a/modules/structs/variable.go +++ b/modules/structs/variable.go @@ -10,6 +10,11 @@ type CreateVariableOption struct { // // required: true Value string `json:"value" binding:"Required"` + + // Description of the variable to create + // + // required: false + Description string `json:"description"` } // UpdateVariableOption the option when updating variable @@ -21,6 +26,11 @@ type UpdateVariableOption struct { // // required: true Value string `json:"value" binding:"Required"` + + // Description of the variable to update + // + // required: false + Description string `json:"description"` } // ActionVariable return value of the query API @@ -34,4 +44,6 @@ type ActionVariable struct { Name string `json:"name"` // the value of the variable Data string `json:"data"` + // the description of the variable + Description string `json:"description"` } diff --git a/modules/system/appstate_test.go b/modules/system/appstate_test.go index 911319d00a..b5c057cf88 100644 --- a/modules/system/appstate_test.go +++ b/modules/system/appstate_test.go @@ -38,8 +38,8 @@ func TestAppStateDB(t *testing.T) { item1 := new(testItem1) assert.NoError(t, as.Get(db.DefaultContext, item1)) - assert.Equal(t, "", item1.Val1) - assert.EqualValues(t, 0, item1.Val2) + assert.Empty(t, item1.Val1) + assert.Equal(t, 0, item1.Val2) item1 = new(testItem1) item1.Val1 = "a" @@ -53,7 +53,7 @@ func TestAppStateDB(t *testing.T) { item1 = new(testItem1) assert.NoError(t, as.Get(db.DefaultContext, item1)) assert.Equal(t, "a", item1.Val1) - assert.EqualValues(t, 2, item1.Val2) + assert.Equal(t, 2, item1.Val2) item2 = new(testItem2) assert.NoError(t, as.Get(db.DefaultContext, item2)) diff --git a/modules/tailmsg/talimsg.go b/modules/tailmsg/talimsg.go new file mode 100644 index 0000000000..aafc98e2d2 --- /dev/null +++ b/modules/tailmsg/talimsg.go @@ -0,0 +1,73 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tailmsg + +import ( + "sync" + "time" +) + +type MsgRecord struct { + Time time.Time + Content string +} + +type MsgRecorder interface { + Record(content string) + GetRecords() []*MsgRecord +} + +type memoryMsgRecorder struct { + mu sync.RWMutex + msgs []*MsgRecord + limit int +} + +// TODO: use redis for a clustered environment + +func (m *memoryMsgRecorder) Record(content string) { + m.mu.Lock() + defer m.mu.Unlock() + m.msgs = append(m.msgs, &MsgRecord{ + Time: time.Now(), + Content: content, + }) + if len(m.msgs) > m.limit { + m.msgs = m.msgs[len(m.msgs)-m.limit:] + } +} + +func (m *memoryMsgRecorder) GetRecords() []*MsgRecord { + m.mu.RLock() + defer m.mu.RUnlock() + ret := make([]*MsgRecord, len(m.msgs)) + copy(ret, m.msgs) + return ret +} + +func NewMsgRecorder(limit int) MsgRecorder { + return &memoryMsgRecorder{ + limit: limit, + } +} + +type Manager struct { + traceRecorder MsgRecorder + logRecorder MsgRecorder +} + +func (m *Manager) GetTraceRecorder() MsgRecorder { + return m.traceRecorder +} + +func (m *Manager) GetLogRecorder() MsgRecorder { + return m.logRecorder +} + +var GetManager = sync.OnceValue(func() *Manager { + return &Manager{ + traceRecorder: NewMsgRecorder(100), + logRecorder: NewMsgRecorder(1000), + } +}) diff --git a/modules/tempdir/tempdir.go b/modules/tempdir/tempdir.go new file mode 100644 index 0000000000..22c2e4ea16 --- /dev/null +++ b/modules/tempdir/tempdir.go @@ -0,0 +1,112 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" +) + +type TempDir struct { + // base is the base directory for temporary files, it must exist before accessing and won't be created automatically. + // for example: base="/system-tmpdir", sub="gitea-tmp" + base, sub string +} + +func (td *TempDir) JoinPath(elems ...string) string { + return filepath.Join(append([]string{td.base, td.sub}, elems...)...) +} + +// MkdirAllSub works like os.MkdirAll, but the base directory must exist +func (td *TempDir) MkdirAllSub(dir string) (string, error) { + if _, err := os.Stat(td.base); err != nil { + return "", err + } + full := filepath.Join(td.base, td.sub, dir) + if err := os.MkdirAll(full, os.ModePerm); err != nil { + return "", err + } + return full, nil +} + +func (td *TempDir) prepareDirWithPattern(elems ...string) (dir, pattern string, err error) { + if _, err = os.Stat(td.base); err != nil { + return "", "", err + } + dir, pattern = filepath.Split(filepath.Join(append([]string{td.base, td.sub}, elems...)...)) + if err = os.MkdirAll(dir, os.ModePerm); err != nil { + return "", "", err + } + return dir, pattern, nil +} + +// MkdirTempRandom works like os.MkdirTemp, the last path field is the "pattern" +func (td *TempDir) MkdirTempRandom(elems ...string) (string, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return "", nil, err + } + dir, err = os.MkdirTemp(dir, pattern) + if err != nil { + return "", nil, err + } + return dir, func() { + if err := util.RemoveAll(dir); err != nil { + log.Error("Failed to remove temp directory %s: %v", dir, err) + } + }, nil +} + +// CreateTempFileRandom works like os.CreateTemp, the last path field is the "pattern" +func (td *TempDir) CreateTempFileRandom(elems ...string) (*os.File, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return nil, nil, err + } + f, err := os.CreateTemp(dir, pattern) + if err != nil { + return nil, nil, err + } + filename := f.Name() + return f, func() { + _ = f.Close() + if err := util.Remove(filename); err != nil { + log.Error("Unable to remove temporary file: %s: Error: %v", filename, err) + } + }, err +} + +func (td *TempDir) RemoveOutdated(d time.Duration) { + var remove func(path string) + remove = func(path string) { + entries, _ := os.ReadDir(path) + for _, entry := range entries { + full := filepath.Join(path, entry.Name()) + if entry.IsDir() { + remove(full) + _ = os.Remove(full) + continue + } + info, err := entry.Info() + if err == nil && time.Since(info.ModTime()) > d { + _ = os.Remove(full) + } + } + } + remove(td.JoinPath("")) +} + +// New create a new TempDir instance, "base" must be an existing directory, +// "sub" could be a multi-level directory and will be created if not exist +func New(base, sub string) *TempDir { + return &TempDir{base: base, sub: sub} +} + +func OsTempDir(sub string) *TempDir { + return New(os.TempDir(), sub) +} diff --git a/modules/tempdir/tempdir_test.go b/modules/tempdir/tempdir_test.go new file mode 100644 index 0000000000..d6afcb7bed --- /dev/null +++ b/modules/tempdir/tempdir_test.go @@ -0,0 +1,75 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestTempDir(t *testing.T) { + base := t.TempDir() + + t.Run("Create", func(t *testing.T) { + td := New(base, "sub1/sub2") // make sure the sub dir supports "/" in the path + assert.Equal(t, filepath.Join(base, "sub1", "sub2"), td.JoinPath()) + assert.Equal(t, filepath.Join(base, "sub1", "sub2/test"), td.JoinPath("test")) + + t.Run("MkdirTempRandom", func(t *testing.T) { + s, cleanup, err := td.MkdirTempRandom("foo") + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(s, filepath.Join(base, "sub1/sub2", "foo"))) + + _, err = os.Stat(s) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(s) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("CreateTempFileRandom", func(t *testing.T) { + f, cleanup, err := td.CreateTempFileRandom("foo", "bar") + filename := f.Name() + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(filename, filepath.Join(base, "sub1/sub2", "foo", "bar"))) + _, err = os.Stat(filename) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(filename) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("RemoveOutDated", func(t *testing.T) { + fa1, _, err := td.CreateTempFileRandom("dir-a", "f1") + assert.NoError(t, err) + fa2, _, err := td.CreateTempFileRandom("dir-a", "f2") + assert.NoError(t, err) + _ = os.Chtimes(fa2.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + fb1, _, err := td.CreateTempFileRandom("dir-b", "f1") + assert.NoError(t, err) + _ = os.Chtimes(fb1.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + _, _, _ = fa1.Close(), fa2.Close(), fb1.Close() + + td.RemoveOutdated(time.Minute) + + _, err = os.Stat(fa1.Name()) + assert.NoError(t, err) + _, err = os.Stat(fa2.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(fb1.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + }) + + t.Run("BaseNotExist", func(t *testing.T) { + td := New(filepath.Join(base, "not-exist"), "sub") + _, _, err := td.MkdirTempRandom("foo") + assert.ErrorIs(t, err, os.ErrNotExist) + }) +} diff --git a/modules/templates/eval/eval_test.go b/modules/templates/eval/eval_test.go index c9e514b5eb..f956f6cbdf 100644 --- a/modules/templates/eval/eval_test.go +++ b/modules/templates/eval/eval_test.go @@ -12,7 +12,7 @@ import ( ) func tokens(s string) (a []any) { - for _, v := range strings.Fields(s) { + for v := range strings.FieldsSeq(s) { a = append(a, v) } return a diff --git a/modules/templates/helper.go b/modules/templates/helper.go index a2cc166de9..ff3f7cfda1 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -6,9 +6,9 @@ package templates import ( "fmt" - "html" "html/template" "net/url" + "strconv" "strings" "time" @@ -37,9 +37,7 @@ func NewFuncMap() template.FuncMap { "dict": dict, // it's lowercase because this name has been widely used. Our other functions should have uppercase names. "Iif": iif, "Eval": evalTokens, - "SafeHTML": safeHTML, "HTMLFormat": htmlFormat, - "HTMLEscape": htmlEscape, "QueryEscape": queryEscape, "QueryBuild": QueryBuild, "JSEscape": jsEscapeSafe, @@ -59,7 +57,6 @@ func NewFuncMap() template.FuncMap { // ----------------------------------------------------------------- // svg / avatar / icon / color "svg": svg.RenderHTML, - "EntryIcon": base.EntryIcon, "MigrationIcon": migrationIcon, "ActionIcon": actionIcon, "SortArrow": sortArrow, @@ -69,12 +66,12 @@ func NewFuncMap() template.FuncMap { // time / number / format "FileSize": base.FileSize, "CountFmt": countFmt, - "Sec2Time": util.SecToHours, + "Sec2Hour": util.SecToHours, "TimeEstimateString": timeEstimateString, "LoadTimes": func(startTime time.Time) string { - return fmt.Sprint(time.Since(startTime).Nanoseconds()/1e6) + "ms" + return strconv.FormatInt(time.Since(startTime).Nanoseconds()/1e6, 10) + "ms" }, // ----------------------------------------------------------------- @@ -162,49 +159,12 @@ func NewFuncMap() template.FuncMap { "FilenameIsImage": filenameIsImage, "TabSizeClass": tabSizeClass, - - // for backward compatibility only, do not use them anymore - "TimeSince": timeSinceLegacy, - "TimeSinceUnix": timeSinceLegacy, - "DateTime": dateTimeLegacy, - - "RenderEmoji": renderEmojiLegacy, - "RenderLabel": renderLabelLegacy, - "RenderLabels": renderLabelsLegacy, - "RenderIssueTitle": renderIssueTitleLegacy, - - "RenderMarkdownToHtml": renderMarkdownToHtmlLegacy, - - "RenderCommitMessage": renderCommitMessageLegacy, - "RenderCommitMessageLinkSubject": renderCommitMessageLinkSubjectLegacy, - "RenderCommitBody": renderCommitBodyLegacy, - } -} - -// safeHTML render raw as HTML -func safeHTML(s any) template.HTML { - switch v := s.(type) { - case string: - return template.HTML(v) - case template.HTML: - return v } - panic(fmt.Sprintf("unexpected type %T", s)) } -// SanitizeHTML sanitizes the input by pre-defined markdown rules +// SanitizeHTML sanitizes the input by default sanitization rules. func SanitizeHTML(s string) template.HTML { - return template.HTML(markup.Sanitize(s)) -} - -func htmlEscape(s any) template.HTML { - switch v := s.(type) { - case string: - return template.HTML(html.EscapeString(v)) - case template.HTML: - return v - } - panic(fmt.Sprintf("unexpected type %T", s)) + return markup.Sanitize(s) } func htmlFormat(s any, args ...any) template.HTML { @@ -367,7 +327,3 @@ func QueryBuild(a ...any) template.URL { } return template.URL(s) } - -func panicIfDevOrTesting() { - setting.PanicInDevOrTesting("legacy template functions are for backward compatibility only, do not use them in new code") -} diff --git a/modules/templates/helper_test.go b/modules/templates/helper_test.go index 5d7bc93622..81f8235bd2 100644 --- a/modules/templates/helper_test.go +++ b/modules/templates/helper_test.go @@ -15,7 +15,7 @@ import ( func TestSubjectBodySeparator(t *testing.T) { test := func(input, subject, body string) { - loc := mailSubjectSplit.FindIndex([]byte(input)) + loc := mailSubjectSplit.FindStringIndex(input) if loc == nil { assert.Empty(t, subject, "no subject found, but one expected") assert.Equal(t, body, input) @@ -120,8 +120,8 @@ func TestTemplateEscape(t *testing.T) { func TestQueryBuild(t *testing.T) { t.Run("construct", func(t *testing.T) { - assert.Equal(t, "", string(QueryBuild())) - assert.Equal(t, "", string(QueryBuild("a", nil, "b", false, "c", 0, "d", ""))) + assert.Empty(t, string(QueryBuild())) + assert.Empty(t, string(QueryBuild("a", nil, "b", false, "c", 0, "d", ""))) assert.Equal(t, "a=1&b=true", string(QueryBuild("a", 1, "b", "true"))) // path with query parameters @@ -136,9 +136,9 @@ func TestQueryBuild(t *testing.T) { // only query parameters assert.Equal(t, "&k=1", string(QueryBuild("&", "k", 1))) - assert.Equal(t, "", string(QueryBuild("&", "k", 0))) - assert.Equal(t, "", string(QueryBuild("&k=a", "k", 0))) - assert.Equal(t, "", string(QueryBuild("k=a&", "k", 0))) + assert.Empty(t, string(QueryBuild("&", "k", 0))) + assert.Empty(t, string(QueryBuild("&k=a", "k", 0))) + assert.Empty(t, string(QueryBuild("k=a&", "k", 0))) assert.Equal(t, "a=1&b=2", string(QueryBuild("a=1", "b", 2))) assert.Equal(t, "&a=1&b=2", string(QueryBuild("&a=1", "b", 2))) assert.Equal(t, "a=1&b=2&", string(QueryBuild("a=1&", "b", 2))) diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index 529284f7e8..8073a6e5f5 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -42,7 +42,7 @@ var ( var ErrTemplateNotInitialized = errors.New("template system is not initialized, check your log for errors") -func (h *HTMLRender) HTML(w io.Writer, status int, tplName TplName, data any, ctx context.Context) error { //nolint:revive +func (h *HTMLRender) HTML(w io.Writer, status int, tplName TplName, data any, ctx context.Context) error { //nolint:revive // we don't use ctx, only pass it to the template executor name := string(tplName) if respWriter, ok := w.(http.ResponseWriter); ok { if respWriter.Header().Get("Content-Type") == "" { @@ -57,7 +57,7 @@ func (h *HTMLRender) HTML(w io.Writer, status int, tplName TplName, data any, ct return t.Execute(w, data) } -func (h *HTMLRender) TemplateLookup(name string, ctx context.Context) (TemplateExecutor, error) { //nolint:revive +func (h *HTMLRender) TemplateLookup(name string, ctx context.Context) (TemplateExecutor, error) { //nolint:revive // we don't use ctx, only pass it to the template executor tmpls := h.templates.Load() if tmpls == nil { return nil, ErrTemplateNotInitialized @@ -251,7 +251,7 @@ func extractErrorLine(code []byte, lineNum, posNum int, target string) string { b := bufio.NewReader(bytes.NewReader(code)) var line []byte var err error - for i := 0; i < lineNum; i++ { + for i := range lineNum { if line, err = b.ReadBytes('\n'); err != nil { if i == lineNum-1 && errors.Is(err, io.EOF) { err = nil diff --git a/modules/templates/htmlrenderer_test.go b/modules/templates/htmlrenderer_test.go index 2a74b74c23..e8b01c30fe 100644 --- a/modules/templates/htmlrenderer_test.go +++ b/modules/templates/htmlrenderer_test.go @@ -65,7 +65,7 @@ func TestHandleError(t *testing.T) { _, err = tmpl.Parse(s) assert.Error(t, err) msg := h(err) - assert.EqualValues(t, strings.TrimSpace(expect), strings.TrimSpace(msg)) + assert.Equal(t, strings.TrimSpace(expect), strings.TrimSpace(msg)) } test("{{", p.handleGenericTemplateError, ` @@ -102,5 +102,5 @@ god knows XXX ---------------------------------------------------------------------- ` actualMsg := p.handleExpectedEndError(errors.New("template: test:1: expected end; found XXX")) - assert.EqualValues(t, strings.TrimSpace(expectedMsg), strings.TrimSpace(actualMsg)) + assert.Equal(t, strings.TrimSpace(expectedMsg), strings.TrimSpace(actualMsg)) } diff --git a/modules/templates/scopedtmpl/scopedtmpl.go b/modules/templates/scopedtmpl/scopedtmpl.go index 2722ba97a2..34e8b9ad70 100644 --- a/modules/templates/scopedtmpl/scopedtmpl.go +++ b/modules/templates/scopedtmpl/scopedtmpl.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "io" + "maps" "reflect" "sync" texttemplate "text/template" @@ -40,9 +41,7 @@ func (t *ScopedTemplate) Funcs(funcMap template.FuncMap) { panic("cannot add new functions to frozen template set") } t.all.Funcs(funcMap) - for k, v := range funcMap { - t.parseFuncs[k] = v - } + maps.Copy(t.parseFuncs, funcMap) } func (t *ScopedTemplate) New(name string) *template.Template { @@ -103,31 +102,28 @@ func escapeTemplate(t *template.Template) error { return nil } -//nolint:unused type htmlTemplate struct { - escapeErr error - text *texttemplate.Template + _/*escapeErr*/ error + text *texttemplate.Template } -//nolint:unused type textTemplateCommon struct { - tmpl map[string]*template.Template // Map from name to defined templates. - muTmpl sync.RWMutex // protects tmpl - option struct { + _/*tmpl*/ map[string]*template.Template + _/*muTmpl*/ sync.RWMutex + _/*option*/ struct { missingKey int } - muFuncs sync.RWMutex // protects parseFuncs and execFuncs - parseFuncs texttemplate.FuncMap - execFuncs map[string]reflect.Value + muFuncs sync.RWMutex + _/*parseFuncs*/ texttemplate.FuncMap + execFuncs map[string]reflect.Value } -//nolint:unused type textTemplate struct { - name string + _/*name*/ string *parse.Tree *textTemplateCommon - leftDelim string - rightDelim string + _/*leftDelim*/ string + _/*rightDelim*/ string } func ptr[T, P any](ptr *P) *T { @@ -159,9 +155,7 @@ func newScopedTemplateSet(all *template.Template, name string) (*scopedTemplateS textTmplPtr.muFuncs.Lock() ts.execFuncs = map[string]reflect.Value{} - for k, v := range textTmplPtr.execFuncs { - ts.execFuncs[k] = v - } + maps.Copy(ts.execFuncs, textTmplPtr.execFuncs) textTmplPtr.muFuncs.Unlock() var collectTemplates func(nodes []parse.Node) @@ -220,9 +214,7 @@ func (ts *scopedTemplateSet) newExecutor(funcMap map[string]any) TemplateExecuto tmpl := texttemplate.New("") tmplPtr := ptr[textTemplate](tmpl) tmplPtr.execFuncs = map[string]reflect.Value{} - for k, v := range ts.execFuncs { - tmplPtr.execFuncs[k] = v - } + maps.Copy(tmplPtr.execFuncs, ts.execFuncs) if funcMap != nil { tmpl.Funcs(funcMap) } diff --git a/modules/templates/static.go b/modules/templates/static.go deleted file mode 100644 index b5a7e561ec..0000000000 --- a/modules/templates/static.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package templates - -import ( - "time" - - "code.gitea.io/gitea/modules/assetfs" - "code.gitea.io/gitea/modules/timeutil" -) - -// GlobalModTime provide a global mod time for embedded asset files -func GlobalModTime(filename string) time.Time { - return timeutil.GetExecutableModTime() -} - -func BuiltinAssets() *assetfs.Layer { - return assetfs.Bindata("builtin(bindata)", Assets) -} diff --git a/modules/templates/templates_bindata.go b/modules/templates/templates_bindata.go index 6f1d3cf539..a919591ecf 100644 --- a/modules/templates/templates_bindata.go +++ b/modules/templates/templates_bindata.go @@ -3,6 +3,21 @@ //go:build bindata +//go:generate go run ../../build/generate-bindata.go ../../templates bindata.dat + package templates -//go:generate go run ../../build/generate-bindata.go ../../templates templates bindata.go true +import ( + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", assetfs.NewEmbeddedFS(bindata)) +}) diff --git a/modules/templates/dynamic.go b/modules/templates/templates_dynamic.go index e1babd83c9..e1babd83c9 100644 --- a/modules/templates/dynamic.go +++ b/modules/templates/templates_dynamic.go diff --git a/modules/templates/util_avatar.go b/modules/templates/util_avatar.go index f7dd408ee2..ee9994ab0b 100644 --- a/modules/templates/util_avatar.go +++ b/modules/templates/util_avatar.go @@ -5,9 +5,9 @@ package templates import ( "context" - "fmt" "html" "html/template" + "strconv" activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/avatars" @@ -28,13 +28,14 @@ func NewAvatarUtils(ctx context.Context) *AvatarUtils { // AvatarHTML creates the HTML for an avatar func AvatarHTML(src string, size int, class, name string) template.HTML { - sizeStr := fmt.Sprintf(`%d`, size) + sizeStr := strconv.Itoa(size) if name == "" { name = "avatar" } - return template.HTML(`<img loading="lazy" class="` + class + `" src="` + src + `" title="` + html.EscapeString(name) + `" width="` + sizeStr + `" height="` + sizeStr + `"/>`) + // use empty alt, otherwise if the image fails to load, the width will follow the "alt" text's width + return template.HTML(`<img loading="lazy" alt class="` + class + `" src="` + src + `" title="` + html.EscapeString(name) + `" width="` + sizeStr + `" height="` + sizeStr + `"/>`) } // Avatar renders user avatars. args: user, size (int), class (string) diff --git a/modules/templates/util_date.go b/modules/templates/util_date.go index 658691ee40..fc3f3f2339 100644 --- a/modules/templates/util_date.go +++ b/modules/templates/util_date.go @@ -99,7 +99,7 @@ func dateTimeFormat(format string, datetime any) template.HTML { attrs = append(attrs, `format="datetime"`, `month="short"`, `day="numeric"`, `hour="numeric"`, `minute="numeric"`, `second="numeric"`, `data-tooltip-content`, `data-tooltip-interactive="true"`) return template.HTML(fmt.Sprintf(`<relative-time %s datetime="%s">%s</relative-time>`, strings.Join(attrs, " "), datetimeEscaped, textEscaped)) default: - panic(fmt.Sprintf("Unsupported format %s", format)) + panic("Unsupported format " + format) } } diff --git a/modules/templates/util_date_legacy.go b/modules/templates/util_date_legacy.go deleted file mode 100644 index ceefb00447..0000000000 --- a/modules/templates/util_date_legacy.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package templates - -import ( - "html/template" - - "code.gitea.io/gitea/modules/translation" -) - -func dateTimeLegacy(format string, datetime any, _ ...string) template.HTML { - panicIfDevOrTesting() - if s, ok := datetime.(string); ok { - datetime = parseLegacy(s) - } - return dateTimeFormat(format, datetime) -} - -func timeSinceLegacy(time any, _ translation.Locale) template.HTML { - panicIfDevOrTesting() - return TimeSince(time) -} diff --git a/modules/templates/util_date_test.go b/modules/templates/util_date_test.go index f3a2409a9f..2c1f2d242e 100644 --- a/modules/templates/util_date_test.go +++ b/modules/templates/util_date_test.go @@ -17,12 +17,12 @@ import ( func TestDateTime(t *testing.T) { testTz, _ := time.LoadLocation("America/New_York") defer test.MockVariableValue(&setting.DefaultUILocation, testTz)() + defer test.MockVariableValue(&setting.IsProd, true)() defer test.MockVariableValue(&setting.IsInTesting, false)() du := NewDateUtils() refTimeStr := "2018-01-01T00:00:00Z" - refDateStr := "2018-01-01" refTime, _ := time.Parse(time.RFC3339, refTimeStr) refTimeStamp := timeutil.TimeStamp(refTime.Unix()) @@ -31,18 +31,9 @@ func TestDateTime(t *testing.T) { assert.EqualValues(t, "-", du.AbsoluteShort(time.Time{})) assert.EqualValues(t, "-", du.AbsoluteShort(timeutil.TimeStamp(0))) - actual := dateTimeLegacy("short", "invalid") - assert.EqualValues(t, `-`, actual) - - actual = dateTimeLegacy("short", refTimeStr) - assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2018-01-01T00:00:00Z">2018-01-01</absolute-date>`, actual) - - actual = du.AbsoluteShort(refTime) + actual := du.AbsoluteShort(refTime) assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2018-01-01T00:00:00Z">2018-01-01</absolute-date>`, actual) - actual = dateTimeLegacy("short", refDateStr) - assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2018-01-01T00:00:00-05:00">2018-01-01</absolute-date>`, actual) - actual = du.AbsoluteShort(refTimeStamp) assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2017-12-31T19:00:00-05:00">2017-12-31</absolute-date>`, actual) @@ -53,6 +44,7 @@ func TestDateTime(t *testing.T) { func TestTimeSince(t *testing.T) { testTz, _ := time.LoadLocation("America/New_York") defer test.MockVariableValue(&setting.DefaultUILocation, testTz)() + defer test.MockVariableValue(&setting.IsProd, true)() defer test.MockVariableValue(&setting.IsInTesting, false)() du := NewDateUtils() @@ -67,6 +59,6 @@ func TestTimeSince(t *testing.T) { actual = timeSinceTo(&refTime, time.Time{}) assert.EqualValues(t, `<relative-time prefix="" tense="future" datetime="2018-01-01T00:00:00Z" data-tooltip-content data-tooltip-interactive="true">2018-01-01 00:00:00 +00:00</relative-time>`, actual) - actual = timeSinceLegacy(timeutil.TimeStampNano(refTime.UnixNano()), nil) + actual = du.TimeSince(timeutil.TimeStampNano(refTime.UnixNano())) assert.EqualValues(t, `<relative-time prefix="" tense="past" datetime="2017-12-31T19:00:00-05:00" data-tooltip-content data-tooltip-interactive="true">2017-12-31 19:00:00 -05:00</relative-time>`, actual) } diff --git a/modules/templates/util_dict.go b/modules/templates/util_dict.go index 8d6376b522..cc3018a71c 100644 --- a/modules/templates/util_dict.go +++ b/modules/templates/util_dict.go @@ -4,6 +4,7 @@ package templates import ( + "errors" "fmt" "html" "html/template" @@ -33,7 +34,7 @@ func dictMerge(base map[string]any, arg any) bool { // The dot syntax is highly discouraged because it might cause unclear key conflicts. It's always good to use explicit keys. func dict(args ...any) (map[string]any, error) { if len(args)%2 != 0 { - return nil, fmt.Errorf("invalid dict constructor syntax: must have key-value pairs") + return nil, errors.New("invalid dict constructor syntax: must have key-value pairs") } m := make(map[string]any, len(args)/2) for i := 0; i < len(args); i += 2 { diff --git a/modules/templates/util_format.go b/modules/templates/util_format.go index bee6fb7b75..3485e3251e 100644 --- a/modules/templates/util_format.go +++ b/modules/templates/util_format.go @@ -5,6 +5,7 @@ package templates import ( "fmt" + "strconv" "code.gitea.io/gitea/modules/util" ) @@ -24,7 +25,7 @@ func countFmt(data any) string { return "" } if num < 1000 { - return fmt.Sprintf("%d", num) + return strconv.FormatInt(num, 10) } else if num < 1_000_000 { num2 := float32(num) / 1000.0 return fmt.Sprintf("%.1fk", num2) diff --git a/modules/templates/util_format_test.go b/modules/templates/util_format_test.go index 8d466faff0..13a57c24e2 100644 --- a/modules/templates/util_format_test.go +++ b/modules/templates/util_format_test.go @@ -14,5 +14,5 @@ func TestCountFmt(t *testing.T) { assert.Equal(t, "1.3k", countFmt(int64(1317))) assert.Equal(t, "21.3M", countFmt(21317675)) assert.Equal(t, "45.7G", countFmt(45721317675)) - assert.Equal(t, "", countFmt("test")) + assert.Empty(t, countFmt("test")) } diff --git a/modules/templates/util_json.go b/modules/templates/util_json.go index 71a4e23d36..29a04290fa 100644 --- a/modules/templates/util_json.go +++ b/modules/templates/util_json.go @@ -9,11 +9,11 @@ import ( "code.gitea.io/gitea/modules/json" ) -type JsonUtils struct{} //nolint:revive +type JsonUtils struct{} //nolint:revive // variable naming triggers on Json, wants JSON var jsonUtils = JsonUtils{} -func NewJsonUtils() *JsonUtils { //nolint:revive +func NewJsonUtils() *JsonUtils { //nolint:revive // variable naming triggers on Json, wants JSON return &jsonUtils } diff --git a/modules/templates/util_misc.go b/modules/templates/util_misc.go index 2d42bc76b5..cc5bf67b42 100644 --- a/modules/templates/util_misc.go +++ b/modules/templates/util_misc.go @@ -38,10 +38,11 @@ func sortArrow(normSort, revSort, urlSort string, isDefault bool) template.HTML } else { // if sort arg is in url test if it correlates with column header sort arguments // the direction of the arrow should indicate the "current sort order", up means ASC(normal), down means DESC(rev) - if urlSort == normSort { + switch urlSort { + case normSort: // the table is sorted with this header normal return svg.RenderHTML("octicon-triangle-up", 16) - } else if urlSort == revSort { + case revSort: // the table is sorted with this header reverse return svg.RenderHTML("octicon-triangle-down", 16) } diff --git a/modules/templates/util_render.go b/modules/templates/util_render.go index 1800747f48..1056c42643 100644 --- a/modules/templates/util_render.go +++ b/modules/templates/util_render.go @@ -4,7 +4,6 @@ package templates import ( - "context" "encoding/hex" "fmt" "html/template" @@ -15,44 +14,47 @@ import ( "unicode" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/renderhelper" + "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/emoji" "code.gitea.io/gitea/modules/htmlutil" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" + "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" ) type RenderUtils struct { - ctx context.Context + ctx reqctx.RequestContext } -func NewRenderUtils(ctx context.Context) *RenderUtils { +func NewRenderUtils(ctx reqctx.RequestContext) *RenderUtils { return &RenderUtils{ctx: ctx} } // RenderCommitMessage renders commit message with XSS-safe and special links. -func (ut *RenderUtils) RenderCommitMessage(msg string, metas map[string]string) template.HTML { +func (ut *RenderUtils) RenderCommitMessage(msg string, repo *repo.Repository) template.HTML { cleanMsg := template.HTMLEscapeString(msg) - // we can safely assume that it will not return any error, since there - // shouldn't be any special HTML. - fullMessage, err := markup.PostProcessCommitMessage(markup.NewRenderContext(ut.ctx).WithMetas(metas), cleanMsg) + // we can safely assume that it will not return any error, since there shouldn't be any special HTML. + // "repo" can be nil when rendering commit messages for deleted repositories in a user's dashboard feed. + fullMessage, err := markup.PostProcessCommitMessage(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), cleanMsg) if err != nil { log.Error("PostProcessCommitMessage: %v", err) return "" } msgLines := strings.Split(strings.TrimSpace(fullMessage), "\n") if len(msgLines) == 0 { - return template.HTML("") + return "" } return renderCodeBlock(template.HTML(msgLines[0])) } // RenderCommitMessageLinkSubject renders commit message as a XSS-safe link to // the provided default url, handling for special links without email to links. -func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, metas map[string]string) template.HTML { +func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, repo *repo.Repository) template.HTML { msgLine := strings.TrimLeftFunc(msg, unicode.IsSpace) lineEnd := strings.IndexByte(msgLine, '\n') if lineEnd > 0 { @@ -63,9 +65,8 @@ func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, me return "" } - // we can safely assume that it will not return any error, since there - // shouldn't be any special HTML. - renderedMessage, err := markup.PostProcessCommitMessageSubject(markup.NewRenderContext(ut.ctx).WithMetas(metas), urlDefault, template.HTMLEscapeString(msgLine)) + // we can safely assume that it will not return any error, since there shouldn't be any special HTML. + renderedMessage, err := markup.PostProcessCommitMessageSubject(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), urlDefault, template.HTMLEscapeString(msgLine)) if err != nil { log.Error("PostProcessCommitMessageSubject: %v", err) return "" @@ -74,7 +75,7 @@ func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, me } // RenderCommitBody extracts the body of a commit message without its title. -func (ut *RenderUtils) RenderCommitBody(msg string, metas map[string]string) template.HTML { +func (ut *RenderUtils) RenderCommitBody(msg string, repo *repo.Repository) template.HTML { msgLine := strings.TrimSpace(msg) lineEnd := strings.IndexByte(msgLine, '\n') if lineEnd > 0 { @@ -87,7 +88,7 @@ func (ut *RenderUtils) RenderCommitBody(msg string, metas map[string]string) tem return "" } - renderedMessage, err := markup.PostProcessCommitMessage(markup.NewRenderContext(ut.ctx).WithMetas(metas), template.HTMLEscapeString(msgLine)) + renderedMessage, err := markup.PostProcessCommitMessage(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), template.HTMLEscapeString(msgLine)) if err != nil { log.Error("PostProcessCommitMessage: %v", err) return "" @@ -105,8 +106,8 @@ func renderCodeBlock(htmlEscapedTextToRender template.HTML) template.HTML { } // RenderIssueTitle renders issue/pull title with defined post processors -func (ut *RenderUtils) RenderIssueTitle(text string, metas map[string]string) template.HTML { - renderedText, err := markup.PostProcessIssueTitle(markup.NewRenderContext(ut.ctx).WithMetas(metas), template.HTMLEscapeString(text)) +func (ut *RenderUtils) RenderIssueTitle(text string, repo *repo.Repository) template.HTML { + renderedText, err := markup.PostProcessIssueTitle(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), template.HTMLEscapeString(text)) if err != nil { log.Error("PostProcessIssueTitle: %v", err) return "" @@ -121,8 +122,23 @@ func (ut *RenderUtils) RenderIssueSimpleTitle(text string) template.HTML { return ret } -// RenderLabel renders a label +func (ut *RenderUtils) RenderLabelWithLink(label *issues_model.Label, link any) template.HTML { + var attrHref template.HTML + switch link.(type) { + case template.URL, string: + attrHref = htmlutil.HTMLFormat(`href="%s"`, link) + default: + panic(fmt.Sprintf("unexpected type %T for link", link)) + } + return ut.renderLabelWithTag(label, "a", attrHref) +} + func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { + return ut.renderLabelWithTag(label, "span", "") +} + +// RenderLabel renders a label +func (ut *RenderUtils) renderLabelWithTag(label *issues_model.Label, tagName, tagAttrs template.HTML) template.HTML { locale := ut.ctx.Value(translation.ContextKey).(translation.Locale) var extraCSSClasses string textColor := util.ContrastColor(label.Color) @@ -136,8 +152,8 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { if labelScope == "" { // Regular label - return htmlutil.HTMLFormat(`<div class="ui label %s" style="color: %s !important; background-color: %s !important;" data-tooltip-content title="%s">%s</div>`, - extraCSSClasses, textColor, label.Color, descriptionText, ut.RenderEmoji(label.Name)) + return htmlutil.HTMLFormat(`<%s %s class="ui label %s" style="color: %s !important; background-color: %s !important;" data-tooltip-content title="%s"><span class="gt-ellipsis">%s</span></%s>`, + tagName, tagAttrs, extraCSSClasses, textColor, label.Color, descriptionText, ut.RenderEmoji(label.Name), tagName) } // Scoped label @@ -151,7 +167,7 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { // Ensure we add the same amount of contrast also near 0 and 1. darken := contrast + math.Max(luminance+contrast-1.0, 0.0) lighten := contrast + math.Max(contrast-luminance, 0.0) - // Compute factor to keep RGB values proportional. + // Compute the factor to keep RGB values proportional. darkenFactor := math.Max(luminance-darken, 0.0) / math.Max(luminance, 1.0/255.0) lightenFactor := math.Min(luminance+lighten, 1.0) / math.Max(luminance, 1.0/255.0) @@ -170,13 +186,31 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { itemColor := "#" + hex.EncodeToString(itemBytes) scopeColor := "#" + hex.EncodeToString(scopeBytes) - return htmlutil.HTMLFormat(`<span class="ui label %s scope-parent" data-tooltip-content title="%s">`+ + if label.ExclusiveOrder > 0 { + // <scope> | <label> | <order> + return htmlutil.HTMLFormat(`<%s %s class="ui label %s scope-parent" data-tooltip-content title="%s">`+ + `<div class="ui label scope-left" style="color: %s !important; background-color: %s !important">%s</div>`+ + `<div class="ui label scope-middle" style="color: %s !important; background-color: %s !important">%s</div>`+ + `<div class="ui label scope-right">%d</div>`+ + `</%s>`, + tagName, tagAttrs, + extraCSSClasses, descriptionText, + textColor, scopeColor, scopeHTML, + textColor, itemColor, itemHTML, + label.ExclusiveOrder, + tagName) + } + + // <scope> | <label> + return htmlutil.HTMLFormat(`<%s %s class="ui label %s scope-parent" data-tooltip-content title="%s">`+ `<div class="ui label scope-left" style="color: %s !important; background-color: %s !important">%s</div>`+ `<div class="ui label scope-right" style="color: %s !important; background-color: %s !important">%s</div>`+ - `</span>`, + `</%s>`, + tagName, tagAttrs, extraCSSClasses, descriptionText, textColor, scopeColor, scopeHTML, - textColor, itemColor, itemHTML) + textColor, itemColor, itemHTML, + tagName) } // RenderEmoji renders html text with emoji post processors @@ -202,7 +236,7 @@ func reactionToEmoji(reaction string) template.HTML { return template.HTML(fmt.Sprintf(`<img alt=":%s:" src="%s/assets/img/emoji/%s.png"></img>`, reaction, setting.StaticURLPrefix, url.PathEscape(reaction))) } -func (ut *RenderUtils) MarkdownToHtml(input string) template.HTML { //nolint:revive +func (ut *RenderUtils) MarkdownToHtml(input string) template.HTML { //nolint:revive // variable naming triggers on Html, wants HTML output, err := markdown.RenderString(markup.NewRenderContext(ut.ctx).WithMetas(markup.ComposeSimpleDocumentMetas()), input) if err != nil { log.Error("RenderString: %v", err) @@ -219,7 +253,8 @@ func (ut *RenderUtils) RenderLabels(labels []*issues_model.Label, repoLink strin if label == nil { continue } - htmlCode += fmt.Sprintf(`<a href="%s?labels=%d">%s</a>`, baseLink, label.ID, ut.RenderLabel(label)) + link := fmt.Sprintf("%s?labels=%d", baseLink, label.ID) + htmlCode += string(ut.RenderLabelWithLink(label, template.URL(link))) } htmlCode += "</span>" return template.HTML(htmlCode) diff --git a/modules/templates/util_render_legacy.go b/modules/templates/util_render_legacy.go deleted file mode 100644 index 994f2fa064..0000000000 --- a/modules/templates/util_render_legacy.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package templates - -import ( - "context" - "html/template" - - issues_model "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/modules/translation" -) - -func renderEmojiLegacy(ctx context.Context, text string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderEmoji(text) -} - -func renderLabelLegacy(ctx context.Context, locale translation.Locale, label *issues_model.Label) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderLabel(label) -} - -func renderLabelsLegacy(ctx context.Context, locale translation.Locale, labels []*issues_model.Label, repoLink string, issue *issues_model.Issue) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderLabels(labels, repoLink, issue) -} - -func renderMarkdownToHtmlLegacy(ctx context.Context, input string) template.HTML { //nolint:revive - panicIfDevOrTesting() - return NewRenderUtils(ctx).MarkdownToHtml(input) -} - -func renderCommitMessageLegacy(ctx context.Context, msg string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderCommitMessage(msg, metas) -} - -func renderCommitMessageLinkSubjectLegacy(ctx context.Context, msg, urlDefault string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderCommitMessageLinkSubject(msg, urlDefault, metas) -} - -func renderIssueTitleLegacy(ctx context.Context, text string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderIssueTitle(text, metas) -} - -func renderCommitBodyLegacy(ctx context.Context, msg string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderCommitBody(msg, metas) -} diff --git a/modules/templates/util_render_test.go b/modules/templates/util_render_test.go index 80094ab26e..5c37f084df 100644 --- a/modules/templates/util_render_test.go +++ b/modules/templates/util_render_test.go @@ -11,10 +11,11 @@ import ( "testing" "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/reqctx" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/translation" @@ -46,19 +47,8 @@ mail@domain.com return strings.ReplaceAll(s, "<SPACE>", " ") } -var testMetas = map[string]string{ - "user": "user13", - "repo": "repo11", - "repoPath": "../../tests/gitea-repositories-meta/user13/repo11.git/", - "markdownLineBreakStyle": "comment", - "markupAllowShortIssuePattern": "true", -} - func TestMain(m *testing.M) { - unittest.InitSettings() - if err := git.InitSimple(context.Background()); err != nil { - log.Fatal("git init failed, err: %v", err) - } + setting.Markdown.RenderOptionsComment.ShortIssuePattern = true markup.Init(&markup.RenderHelperFuncs{ IsUsernameMentionable: func(ctx context.Context, username string) bool { return username == "mention-user" @@ -67,52 +57,58 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func newTestRenderUtils() *RenderUtils { - ctx := context.Background() - ctx = context.WithValue(ctx, translation.ContextKey, &translation.MockLocale{}) +func newTestRenderUtils(t *testing.T) *RenderUtils { + ctx := reqctx.NewRequestContextForTest(t.Context()) + ctx.SetContextValue(translation.ContextKey, &translation.MockLocale{}) return NewRenderUtils(ctx) } -func TestRenderCommitBody(t *testing.T) { - defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() - type args struct { - msg string +func TestRenderRepoComment(t *testing.T) { + mockRepo := &repo.Repository{ + ID: 1, OwnerName: "user13", Name: "repo11", + Owner: &user_model.User{ID: 13, Name: "user13"}, + Units: []*repo.RepoUnit{}, } - tests := []struct { - name string - args args - want template.HTML - }{ - { - name: "multiple lines", - args: args{ - msg: "first line\nsecond line", + t.Run("RenderCommitBody", func(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + type args struct { + msg string + } + tests := []struct { + name string + args args + want template.HTML + }{ + { + name: "multiple lines", + args: args{ + msg: "first line\nsecond line", + }, + want: "second line", }, - want: "second line", - }, - { - name: "multiple lines with leading newlines", - args: args{ - msg: "\n\n\n\nfirst line\nsecond line", + { + name: "multiple lines with leading newlines", + args: args{ + msg: "\n\n\n\nfirst line\nsecond line", + }, + want: "second line", }, - want: "second line", - }, - { - name: "multiple lines with trailing newlines", - args: args{ - msg: "first line\nsecond line\n\n\n", + { + name: "multiple lines with trailing newlines", + args: args{ + msg: "first line\nsecond line\n\n\n", + }, + want: "second line", }, - want: "second line", - }, - } - ut := newTestRenderUtils() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, ut.RenderCommitBody(tt.args.msg, nil), "RenderCommitBody(%v, %v)", tt.args.msg, nil) - }) - } - - expected := `/just/a/path.bin + } + ut := newTestRenderUtils(t) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, ut.RenderCommitBody(tt.args.msg, mockRepo), "RenderCommitBody(%v, %v)", tt.args.msg, nil) + }) + } + + expected := `/just/a/path.bin <a href="https://example.com/file.bin">https://example.com/file.bin</a> [local link](file.bin) [remote link](<a href="https://example.com">https://example.com</a>) @@ -122,31 +118,31 @@ func TestRenderCommitBody(t *testing.T) { ![remote image](<a href="https://example.com/image.jpg">https://example.com/image.jpg</a>) [[local image|image.jpg]] [[remote link|<a href="https://example.com/image.jpg">https://example.com/image.jpg</a>]] -<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" class="compare"><code class="nohighlight">88fc37a3c0...12fc37a3c0 (hash)</code></a> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" class="compare"><code>88fc37a3c0...12fc37a3c0 (hash)</code></a> com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare -<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" class="commit"><code class="nohighlight">88fc37a3c0</code></a> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" class="commit"><code>88fc37a3c0</code></a> com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit <span class="emoji" aria-label="thumbs up">👍</span> <a href="mailto:mail@domain.com">mail@domain.com</a> <a href="/mention-user">@mention-user</a> test <a href="/user13/repo11/issues/123" class="ref-issue">#123</a> space` - assert.EqualValues(t, expected, string(newTestRenderUtils().RenderCommitBody(testInput(), testMetas))) -} + assert.Equal(t, expected, string(newTestRenderUtils(t).RenderCommitBody(testInput(), mockRepo))) + }) -func TestRenderCommitMessage(t *testing.T) { - expected := `space <a href="/mention-user" data-markdown-generated-content="">@mention-user</a> ` - assert.EqualValues(t, expected, newTestRenderUtils().RenderCommitMessage(testInput(), testMetas)) -} + t.Run("RenderCommitMessage", func(t *testing.T) { + expected := `space <a href="/mention-user" data-markdown-generated-content="">@mention-user</a> ` + assert.EqualValues(t, expected, newTestRenderUtils(t).RenderCommitMessage(testInput(), mockRepo)) + }) -func TestRenderCommitMessageLinkSubject(t *testing.T) { - expected := `<a href="https://example.com/link" class="muted">space </a><a href="/mention-user" data-markdown-generated-content="">@mention-user</a>` - assert.EqualValues(t, expected, newTestRenderUtils().RenderCommitMessageLinkSubject(testInput(), "https://example.com/link", testMetas)) -} + t.Run("RenderCommitMessageLinkSubject", func(t *testing.T) { + expected := `<a href="https://example.com/link" class="muted">space </a><a href="/mention-user" data-markdown-generated-content="">@mention-user</a>` + assert.EqualValues(t, expected, newTestRenderUtils(t).RenderCommitMessageLinkSubject(testInput(), "https://example.com/link", mockRepo)) + }) -func TestRenderIssueTitle(t *testing.T) { - defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() - expected := ` space @mention-user<SPACE><SPACE> + t.Run("RenderIssueTitle", func(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + expected := ` space @mention-user<SPACE><SPACE> /just/a/path.bin https://example.com/file.bin [local link](file.bin) @@ -167,8 +163,9 @@ mail@domain.com <a href="/user13/repo11/issues/123" class="ref-issue">#123</a> space<SPACE><SPACE> ` - expected = strings.ReplaceAll(expected, "<SPACE>", " ") - assert.EqualValues(t, expected, string(newTestRenderUtils().RenderIssueTitle(testInput(), testMetas))) + expected = strings.ReplaceAll(expected, "<SPACE>", " ") + assert.Equal(t, expected, string(newTestRenderUtils(t).RenderIssueTitle(testInput(), mockRepo))) + }) } func TestRenderMarkdownToHtml(t *testing.T) { @@ -194,11 +191,11 @@ com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit #123 space</p> ` - assert.Equal(t, expected, string(newTestRenderUtils().MarkdownToHtml(testInput()))) + assert.Equal(t, expected, string(newTestRenderUtils(t).MarkdownToHtml(testInput()))) } func TestRenderLabels(t *testing.T) { - ut := newTestRenderUtils() + ut := newTestRenderUtils(t) label := &issues.Label{ID: 123, Name: "label-name", Color: "label-color"} issue := &issues.Issue{} expected := `/owner/repo/issues?labels=123` @@ -208,10 +205,21 @@ func TestRenderLabels(t *testing.T) { issue = &issues.Issue{IsPull: true} expected = `/owner/repo/pulls?labels=123` assert.Contains(t, ut.RenderLabels([]*issues.Label{label}, "/owner/repo", issue), expected) + + expectedLabel := `<a href="<>" class="ui label " style="color: #fff !important; background-color: label-color !important;" data-tooltip-content title=""><span class="gt-ellipsis">label-name</span></a>` + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, "<>"))) + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, template.URL("<>")))) + + label = &issues.Label{ID: 123, Name: "</>", Exclusive: true} + expectedLabel = `<a href="" class="ui label scope-parent" data-tooltip-content title=""><div class="ui label scope-left" style="color: #fff !important; background-color: #000000 !important"><</div><div class="ui label scope-right" style="color: #fff !important; background-color: #000000 !important">></div></a>` + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, ""))) + label = &issues.Label{ID: 123, Name: "</>", Exclusive: true, ExclusiveOrder: 1} + expectedLabel = `<a href="" class="ui label scope-parent" data-tooltip-content title=""><div class="ui label scope-left" style="color: #fff !important; background-color: #000000 !important"><</div><div class="ui label scope-middle" style="color: #fff !important; background-color: #000000 !important">></div><div class="ui label scope-right">1</div></a>` + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, ""))) } func TestUserMention(t *testing.T) { markup.RenderBehaviorForTesting.DisableAdditionalAttributes = true - rendered := newTestRenderUtils().MarkdownToHtml("@no-such-user @mention-user @mention-user") - assert.EqualValues(t, `<p>@no-such-user <a href="/mention-user" rel="nofollow">@mention-user</a> <a href="/mention-user" rel="nofollow">@mention-user</a></p>`, strings.TrimSpace(string(rendered))) + rendered := newTestRenderUtils(t).MarkdownToHtml("@no-such-user @mention-user @mention-user") + assert.Equal(t, `<p>@no-such-user <a href="/mention-user" rel="nofollow">@mention-user</a> <a href="/mention-user" rel="nofollow">@mention-user</a></p>`, strings.TrimSpace(string(rendered))) } diff --git a/modules/templates/util_test.go b/modules/templates/util_test.go index febaf7fa88..a6448a6ff2 100644 --- a/modules/templates/util_test.go +++ b/modules/templates/util_test.go @@ -28,7 +28,7 @@ func TestDict(t *testing.T) { for _, c := range cases { got, err := dict(c.args...) if assert.NoError(t, err) { - assert.EqualValues(t, c.want, got) + assert.Equal(t, c.want, got) } } diff --git a/modules/templates/vars/vars.go b/modules/templates/vars/vars.go index cc9d0e976f..500078d4b8 100644 --- a/modules/templates/vars/vars.go +++ b/modules/templates/vars/vars.go @@ -16,7 +16,7 @@ type ErrWrongSyntax struct { } func (err ErrWrongSyntax) Error() string { - return fmt.Sprintf("wrong syntax found in %s", err.Template) + return "wrong syntax found in " + err.Template } // ErrVarMissing represents an error that no matched variable diff --git a/modules/templates/vars/vars_test.go b/modules/templates/vars/vars_test.go index 8f421d9e4b..9b48167237 100644 --- a/modules/templates/vars/vars_test.go +++ b/modules/templates/vars/vars_test.go @@ -60,7 +60,7 @@ func TestExpandVars(t *testing.T) { for _, kase := range kases { t.Run(kase.tmpl, func(t *testing.T) { res, err := Expand(kase.tmpl, kase.data) - assert.EqualValues(t, kase.out, res) + assert.Equal(t, kase.out, res) if kase.error { assert.Error(t, err) } else { diff --git a/modules/test/logchecker.go b/modules/test/logchecker.go index 7bf234f560..829f735c7c 100644 --- a/modules/test/logchecker.go +++ b/modules/test/logchecker.go @@ -5,7 +5,7 @@ package test import ( "context" - "fmt" + "strconv" "strings" "sync" "sync/atomic" @@ -58,7 +58,7 @@ var checkerIndex int64 func NewLogChecker(namePrefix string) (logChecker *LogChecker, cancel func()) { logger := log.GetManager().GetLogger(namePrefix) newCheckerIndex := atomic.AddInt64(&checkerIndex, 1) - writerName := namePrefix + "-" + fmt.Sprint(newCheckerIndex) + writerName := namePrefix + "-" + strconv.FormatInt(newCheckerIndex, 10) lc := &LogChecker{} lc.EventWriterBaseImpl = log.NewEventWriterBase(writerName, "test-log-checker", log.WriterMode{}) diff --git a/modules/test/utils.go b/modules/test/utils.go index ec4c976388..53c6a3ed52 100644 --- a/modules/test/utils.go +++ b/modules/test/utils.go @@ -4,7 +4,6 @@ package test import ( - "fmt" "net/http" "net/http/httptest" "os" @@ -18,6 +17,7 @@ import ( // RedirectURL returns the redirect URL of a http response. // It also works for JSONRedirect: `{"redirect": "..."}` +// FIXME: it should separate the logic of checking from header and JSON body func RedirectURL(resp http.ResponseWriter) string { loc := resp.Header().Get("Location") if loc != "" { @@ -35,6 +35,15 @@ func RedirectURL(resp http.ResponseWriter) string { return "" } +func ParseJSONError(buf []byte) (ret struct { + ErrorMessage string `json:"errorMessage"` + RenderFormat string `json:"renderFormat"` +}, +) { + _ = json.Unmarshal(buf, &ret) + return ret +} + func IsNormalPageCompleted(s string) bool { return strings.Contains(s, `<footer class="page-footer"`) && strings.Contains(s, `</html>`) } @@ -57,7 +66,7 @@ func SetupGiteaRoot() string { giteaRoot = filepath.Dir(filepath.Dir(filepath.Dir(filename))) fixturesDir := filepath.Join(giteaRoot, "models", "fixtures") if exist, _ := util.IsDir(fixturesDir); !exist { - panic(fmt.Sprintf("fixtures directory not found: %s", fixturesDir)) + panic("fixtures directory not found: " + fixturesDir) } _ = os.Setenv("GITEA_ROOT", giteaRoot) return giteaRoot diff --git a/modules/testlogger/testlogger.go b/modules/testlogger/testlogger.go index 2fbfce6b03..60e281d403 100644 --- a/modules/testlogger/testlogger.go +++ b/modules/testlogger/testlogger.go @@ -4,7 +4,6 @@ package testlogger import ( - "context" "fmt" "os" "runtime" @@ -93,7 +92,7 @@ func (w *testLoggerWriterCloser) Reset() { // Printf takes a format and args and prints the string to os.Stdout func Printf(format string, args ...any) { if !log.CanColorStdout { - for i := 0; i < len(args); i++ { + for i := range args { if c, ok := args[i].(*log.ColoredValue); ok { args[i] = c.Value() } @@ -131,7 +130,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { slowFlushChecker := time.AfterFunc(TestSlowFlush, func() { Printf("+++ %s ... still flushing after %v ...\n", log.NewColoredValue(t.Name(), log.Bold, log.FgRed), TestSlowFlush) }) - if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil { + if err := queue.GetManager().FlushAll(t.Context(), -1); err != nil { t.Errorf("Flushing queues failed with error %v", err) } slowFlushChecker.Stop() diff --git a/modules/timeutil/executable.go b/modules/timeutil/executable.go deleted file mode 100644 index 57ae8b2a9d..0000000000 --- a/modules/timeutil/executable.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package timeutil - -import ( - "os" - "path/filepath" - "sync" - "time" - - "code.gitea.io/gitea/modules/log" -) - -var ( - executablModTime = time.Now() - executablModTimeOnce sync.Once -) - -// GetExecutableModTime get executable file modified time of current process. -func GetExecutableModTime() time.Time { - executablModTimeOnce.Do(func() { - exePath, err := os.Executable() - if err != nil { - log.Error("os.Executable: %v", err) - return - } - - exePath, err = filepath.Abs(exePath) - if err != nil { - log.Error("filepath.Abs: %v", err) - return - } - - exePath, err = filepath.EvalSymlinks(exePath) - if err != nil { - log.Error("filepath.EvalSymlinks: %v", err) - return - } - - st, err := os.Stat(exePath) - if err != nil { - log.Error("os.Stat: %v", err) - return - } - - executablModTime = st.ModTime() - }) - return executablModTime -} diff --git a/modules/translation/i18n/errors.go b/modules/translation/i18n/errors.go deleted file mode 100644 index 7f64ccf908..0000000000 --- a/modules/translation/i18n/errors.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package i18n - -import ( - "code.gitea.io/gitea/modules/util" -) - -var ( - ErrLocaleAlreadyExist = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist} - ErrUncertainArguments = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument} -) diff --git a/modules/translation/i18n/format.go b/modules/translation/i18n/format.go index e5e221831f..450509333f 100644 --- a/modules/translation/i18n/format.go +++ b/modules/translation/i18n/format.go @@ -4,6 +4,7 @@ package i18n import ( + "errors" "fmt" "reflect" ) @@ -30,7 +31,7 @@ func Format(format string, args ...any) (msg string, err error) { fmtArgs = append(fmtArgs, val.Index(i).Interface()) } } else { - err = ErrUncertainArguments + err = errors.New("arguments to i18n should not contain uncertain slices") break } } else { diff --git a/modules/translation/i18n/localestore.go b/modules/translation/i18n/localestore.go index b422996984..4f1ae7e13d 100644 --- a/modules/translation/i18n/localestore.go +++ b/modules/translation/i18n/localestore.go @@ -4,6 +4,7 @@ package i18n import ( + "errors" "fmt" "html/template" "slices" @@ -41,7 +42,7 @@ func NewLocaleStore() LocaleStore { // AddLocaleByIni adds locale by ini into the store func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error { if _, ok := store.localeMap[langName]; ok { - return ErrLocaleAlreadyExist + return errors.New("lang has already been added") } store.langNames = append(store.langNames, langName) diff --git a/modules/translation/translation_test.go b/modules/translation/translation_test.go index 464aa32661..87df9eb825 100644 --- a/modules/translation/translation_test.go +++ b/modules/translation/translation_test.go @@ -20,13 +20,13 @@ func TestPrettyNumber(t *testing.T) { allLangMap["id-ID"] = &LangType{Lang: "id-ID", Name: "Bahasa Indonesia"} l := NewLocale("id-ID") - assert.EqualValues(t, "1.000.000", l.PrettyNumber(1000000)) - assert.EqualValues(t, "1.000.000,1", l.PrettyNumber(1000000.1)) - assert.EqualValues(t, "1.000.000", l.PrettyNumber("1000000")) - assert.EqualValues(t, "1.000.000", l.PrettyNumber("1000000.0")) - assert.EqualValues(t, "1.000.000,1", l.PrettyNumber("1000000.1")) + assert.Equal(t, "1.000.000", l.PrettyNumber(1000000)) + assert.Equal(t, "1.000.000,1", l.PrettyNumber(1000000.1)) + assert.Equal(t, "1.000.000", l.PrettyNumber("1000000")) + assert.Equal(t, "1.000.000", l.PrettyNumber("1000000.0")) + assert.Equal(t, "1.000.000,1", l.PrettyNumber("1000000.1")) l = NewLocale("nosuch") - assert.EqualValues(t, "1,000,000", l.PrettyNumber(1000000)) - assert.EqualValues(t, "1,000,000.1", l.PrettyNumber(1000000.1)) + assert.Equal(t, "1,000,000", l.PrettyNumber(1000000)) + assert.Equal(t, "1,000,000.1", l.PrettyNumber(1000000.1)) } diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go index 8cb3d278ce..2e8d9c4a1e 100644 --- a/modules/typesniffer/typesniffer.go +++ b/modules/typesniffer/typesniffer.go @@ -6,18 +6,14 @@ package typesniffer import ( "bytes" "encoding/binary" - "fmt" - "io" "net/http" "regexp" "slices" "strings" - - "code.gitea.io/gitea/modules/util" + "sync" ) -// Use at most this many bytes to determine Content Type. -const sniffLen = 1024 +const SniffContentSize = 1024 const ( MimeTypeImageSvg = "image/svg+xml" @@ -26,22 +22,30 @@ const ( MimeTypeApplicationOctetStream = "application/octet-stream" ) -var ( - svgComment = regexp.MustCompile(`(?s)<!--.*?-->`) - svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) - svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) -) - -// SniffedType contains information about a blobs type. +var globalVars = sync.OnceValue(func() (ret struct { + svgComment, svgTagRegex, svgTagInXMLRegex *regexp.Regexp +}, +) { + ret.svgComment = regexp.MustCompile(`(?s)<!--.*?-->`) + ret.svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) + ret.svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) + return ret +}) + +// SniffedType contains information about a blob's type. type SniffedType struct { contentType string } -// IsText etects if content format is plain text. +// IsText detects if the content format is text family, including text/plain, text/html, text/css, etc. func (ct SniffedType) IsText() bool { return strings.Contains(ct.contentType, "text/") } +func (ct SniffedType) IsTextPlain() bool { + return strings.Contains(ct.contentType, "text/plain") +} + // IsImage detects if data is an image format func (ct SniffedType) IsImage() bool { return strings.Contains(ct.contentType, "image/") @@ -57,12 +61,12 @@ func (ct SniffedType) IsPDF() bool { return strings.Contains(ct.contentType, "application/pdf") } -// IsVideo detects if data is an video format +// IsVideo detects if data is a video format func (ct SniffedType) IsVideo() bool { return strings.Contains(ct.contentType, "video/") } -// IsAudio detects if data is an video format +// IsAudio detects if data is a video format func (ct SniffedType) IsAudio() bool { return strings.Contains(ct.contentType, "audio/") } @@ -103,33 +107,34 @@ func detectFileTypeBox(data []byte) (brands []string, found bool) { return brands, true } -// DetectContentType extends http.DetectContentType with more content types. Defaults to text/unknown if input is empty. +// DetectContentType extends http.DetectContentType with more content types. Defaults to text/plain if input is empty. func DetectContentType(data []byte) SniffedType { if len(data) == 0 { - return SniffedType{"text/unknown"} + return SniffedType{"text/plain"} } ct := http.DetectContentType(data) - if len(data) > sniffLen { - data = data[:sniffLen] + if len(data) > SniffContentSize { + data = data[:SniffContentSize] } + vars := globalVars() // SVG is unsupported by http.DetectContentType, https://github.com/golang/go/issues/15888 detectByHTML := strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html") detectByXML := strings.Contains(ct, "text/xml") if detectByHTML || detectByXML { - dataProcessed := svgComment.ReplaceAll(data, nil) + dataProcessed := vars.svgComment.ReplaceAll(data, nil) dataProcessed = bytes.TrimSpace(dataProcessed) - if detectByHTML && svgTagRegex.Match(dataProcessed) || - detectByXML && svgTagInXMLRegex.Match(dataProcessed) { + if detectByHTML && vars.svgTagRegex.Match(dataProcessed) || + detectByXML && vars.svgTagInXMLRegex.Match(dataProcessed) { ct = MimeTypeImageSvg } } if strings.HasPrefix(ct, "audio/") && bytes.HasPrefix(data, []byte("ID3")) { // The MP3 detection is quite inaccurate, any content with "ID3" prefix will result in "audio/mpeg". - // So remove the "ID3" prefix and detect again, if result is text, then it must be text content. + // So remove the "ID3" prefix and detect again, then if the result is "text", it must be text content. // This works especially because audio files contain many unprintable/invalid characters like `0x00` ct2 := http.DetectContentType(data[3:]) if strings.HasPrefix(ct2, "text/") { @@ -155,15 +160,3 @@ func DetectContentType(data []byte) SniffedType { } return SniffedType{ct} } - -// DetectContentTypeFromReader guesses the content type contained in the reader. -func DetectContentTypeFromReader(r io.Reader) (SniffedType, error) { - buf := make([]byte, sniffLen) - n, err := util.ReadAtMost(r, buf) - if err != nil { - return SniffedType{}, fmt.Errorf("DetectContentTypeFromReader io error: %w", err) - } - buf = buf[:n] - - return DetectContentType(buf), nil -} diff --git a/modules/typesniffer/typesniffer_test.go b/modules/typesniffer/typesniffer_test.go index 3e5db3308b..a0c824b912 100644 --- a/modules/typesniffer/typesniffer_test.go +++ b/modules/typesniffer/typesniffer_test.go @@ -4,7 +4,6 @@ package typesniffer import ( - "bytes" "encoding/base64" "encoding/hex" "strings" @@ -17,7 +16,7 @@ func TestDetectContentTypeLongerThanSniffLen(t *testing.T) { // Pre-condition: Shorter than sniffLen detects SVG. assert.Equal(t, "image/svg+xml", DetectContentType([]byte(`<!-- Comment --><svg></svg>`)).contentType) // Longer than sniffLen detects something else. - assert.NotEqual(t, "image/svg+xml", DetectContentType([]byte(`<!-- `+strings.Repeat("x", sniffLen)+` --><svg></svg>`)).contentType) + assert.NotEqual(t, "image/svg+xml", DetectContentType([]byte(`<!-- `+strings.Repeat("x", SniffContentSize)+` --><svg></svg>`)).contentType) } func TestIsTextFile(t *testing.T) { @@ -116,22 +115,13 @@ func TestIsAudio(t *testing.T) { assert.True(t, DetectContentType([]byte("ID3Toy\n====\t* hi 🌞, ..."+"🌛"[0:2])).IsText()) // test ID3 tag with incomplete UTF8 char } -func TestDetectContentTypeFromReader(t *testing.T) { - mp3, _ := base64.StdEncoding.DecodeString("SUQzBAAAAAABAFRYWFgAAAASAAADbWFqb3JfYnJhbmQAbXA0MgBUWFhYAAAAEQAAA21pbm9yX3Zl") - st, err := DetectContentTypeFromReader(bytes.NewReader(mp3)) - assert.NoError(t, err) - assert.True(t, st.IsAudio()) -} - func TestDetectContentTypeOgg(t *testing.T) { oggAudio, _ := hex.DecodeString("4f67675300020000000000000000352f0000000000007dc39163011e01766f72626973000000000244ac0000000000000071020000000000b8014f6767530000") - st, err := DetectContentTypeFromReader(bytes.NewReader(oggAudio)) - assert.NoError(t, err) + st := DetectContentType(oggAudio) assert.True(t, st.IsAudio()) oggVideo, _ := hex.DecodeString("4f676753000200000000000000007d9747ef000000009b59daf3012a807468656f7261030201001e00110001e000010e00020000001e00000001000001000001") - st, err = DetectContentTypeFromReader(bytes.NewReader(oggVideo)) - assert.NoError(t, err) + st = DetectContentType(oggVideo) assert.True(t, st.IsVideo()) } diff --git a/modules/updatechecker/update_checker.go b/modules/updatechecker/update_checker.go index 3c1e05d060..f0686c0f78 100644 --- a/modules/updatechecker/update_checker.go +++ b/modules/updatechecker/update_checker.go @@ -34,7 +34,7 @@ func GiteaUpdateChecker(httpEndpoint string) error { }, } - req, err := http.NewRequest("GET", httpEndpoint, nil) + req, err := http.NewRequest(http.MethodGet, httpEndpoint, nil) if err != nil { return err } diff --git a/modules/util/error.go b/modules/util/error.go index 0f3597147c..6b2721618e 100644 --- a/modules/util/error.go +++ b/modules/util/error.go @@ -10,56 +10,88 @@ import ( // Common Errors forming the base of our error system // -// Many Errors returned by Gitea can be tested against these errors -// using errors.Is. +// Many Errors returned by Gitea can be tested against these errors using "errors.Is". var ( - ErrInvalidArgument = errors.New("invalid argument") - ErrPermissionDenied = errors.New("permission denied") - ErrAlreadyExist = errors.New("resource already exists") - ErrNotExist = errors.New("resource does not exist") + ErrInvalidArgument = errors.New("invalid argument") // also implies HTTP 400 + ErrPermissionDenied = errors.New("permission denied") // also implies HTTP 403 + ErrNotExist = errors.New("resource does not exist") // also implies HTTP 404 + ErrAlreadyExist = errors.New("resource already exists") // also implies HTTP 409 + + // ErrUnprocessableContent implies HTTP 422, the syntax of the request content is correct, + // but the server is unable to process the contained instructions + ErrUnprocessableContent = errors.New("unprocessable content") ) -// SilentWrap provides a simple wrapper for a wrapped error where the wrapped error message plays no part in the error message +// errorWrapper provides a simple wrapper for a wrapped error where the wrapped error message plays no part in the error message // Especially useful for "untyped" errors created with "errors.New(…)" that can be classified as 'invalid argument', 'permission denied', 'exists already', or 'does not exist' -type SilentWrap struct { +type errorWrapper struct { Message string Err error } // Error returns the message -func (w SilentWrap) Error() string { +func (w errorWrapper) Error() string { return w.Message } // Unwrap returns the underlying error -func (w SilentWrap) Unwrap() error { +func (w errorWrapper) Unwrap() error { return w.Err } -// NewSilentWrapErrorf returns an error that formats as the given text but unwraps as the provided error -func NewSilentWrapErrorf(unwrap error, message string, args ...any) error { +type LocaleWrapper struct { + err error + TrKey string + TrArgs []any +} + +// Error returns the message +func (w LocaleWrapper) Error() string { + return w.err.Error() +} + +// Unwrap returns the underlying error +func (w LocaleWrapper) Unwrap() error { + return w.err +} + +// ErrorWrap returns an error that formats as the given text but unwraps as the provided error +func ErrorWrap(unwrap error, message string, args ...any) error { if len(args) == 0 { - return SilentWrap{Message: message, Err: unwrap} + return errorWrapper{Message: message, Err: unwrap} } - return SilentWrap{Message: fmt.Sprintf(message, args...), Err: unwrap} + return errorWrapper{Message: fmt.Sprintf(message, args...), Err: unwrap} } // NewInvalidArgumentErrorf returns an error that formats as the given text but unwraps as an ErrInvalidArgument func NewInvalidArgumentErrorf(message string, args ...any) error { - return NewSilentWrapErrorf(ErrInvalidArgument, message, args...) + return ErrorWrap(ErrInvalidArgument, message, args...) } // NewPermissionDeniedErrorf returns an error that formats as the given text but unwraps as an ErrPermissionDenied func NewPermissionDeniedErrorf(message string, args ...any) error { - return NewSilentWrapErrorf(ErrPermissionDenied, message, args...) + return ErrorWrap(ErrPermissionDenied, message, args...) } // NewAlreadyExistErrorf returns an error that formats as the given text but unwraps as an ErrAlreadyExist func NewAlreadyExistErrorf(message string, args ...any) error { - return NewSilentWrapErrorf(ErrAlreadyExist, message, args...) + return ErrorWrap(ErrAlreadyExist, message, args...) } // NewNotExistErrorf returns an error that formats as the given text but unwraps as an ErrNotExist func NewNotExistErrorf(message string, args ...any) error { - return NewSilentWrapErrorf(ErrNotExist, message, args...) + return ErrorWrap(ErrNotExist, message, args...) +} + +// ErrorWrapLocale wraps an err with a translation key and arguments +func ErrorWrapLocale(err error, trKey string, trArgs ...any) error { + return LocaleWrapper{err: err, TrKey: trKey, TrArgs: trArgs} +} + +func ErrorAsLocale(err error) *LocaleWrapper { + var e LocaleWrapper + if errors.As(err, &e) { + return &e + } + return nil } diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index 739543e297..0731ba30c8 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -7,16 +7,10 @@ import ( "bytes" "errors" "io" - "math" "os" ) -var ( - // ErrInvalidMemorySize occurs if the memory size is not in a valid range - ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") - // ErrWriteAfterRead occurs if Write is called after a read operation - ErrWriteAfterRead = errors.New("Write is unsupported after a read operation") -) +var ErrWriteAfterRead = errors.New("write is unsupported after a read operation") // occurs if Write is called after a read operation type readAtSeeker interface { io.ReadSeeker @@ -30,34 +24,17 @@ type FileBackedBuffer struct { maxMemorySize int64 size int64 buffer bytes.Buffer + tempDir string file *os.File reader readAtSeeker } // New creates a file backed buffer with a specific maximum memory size -func New(maxMemorySize int) (*FileBackedBuffer, error) { - if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 { - return nil, ErrInvalidMemorySize - } - +func New(maxMemorySize int, tempDir string) *FileBackedBuffer { return &FileBackedBuffer{ maxMemorySize: int64(maxMemorySize), - }, nil -} - -// CreateFromReader creates a file backed buffer and copies the provided reader data into it. -func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) { - b, err := New(maxMemorySize) - if err != nil { - return nil, err + tempDir: tempDir, } - - _, err = io.Copy(b, r) - if err != nil { - return nil, err - } - - return b, nil } // Write implements io.Writer @@ -73,7 +50,7 @@ func (b *FileBackedBuffer) Write(p []byte) (int, error) { n, err = b.file.Write(p) } else { if b.size+int64(len(p)) > b.maxMemorySize { - b.file, err = os.CreateTemp("", "gitea-buffer-") + b.file, err = os.CreateTemp(b.tempDir, "gitea-buffer-") if err != nil { return 0, err } @@ -148,7 +125,7 @@ func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { func (b *FileBackedBuffer) Close() error { if b.file != nil { err := b.file.Close() - os.Remove(b.file.Name()) + _ = os.Remove(b.file.Name()) b.file = nil return err } diff --git a/modules/util/filebuffer/file_backed_buffer_test.go b/modules/util/filebuffer/file_backed_buffer_test.go index 16d5a1965f..3f13c6ac7b 100644 --- a/modules/util/filebuffer/file_backed_buffer_test.go +++ b/modules/util/filebuffer/file_backed_buffer_test.go @@ -21,7 +21,8 @@ func TestFileBackedBuffer(t *testing.T) { } for _, c := range cases { - buf, err := CreateFromReader(strings.NewReader(c.Data), c.MaxMemorySize) + buf := New(c.MaxMemorySize, t.TempDir()) + _, err := io.Copy(buf, strings.NewReader(c.Data)) assert.NoError(t, err) assert.EqualValues(t, len(c.Data), buf.Size()) diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go index e732094c29..565fb7f284 100644 --- a/modules/util/legacy_test.go +++ b/modules/util/legacy_test.go @@ -17,7 +17,7 @@ import ( func TestCopyFile(t *testing.T) { testContent := []byte("hello") - tmpDir := os.TempDir() + tmpDir := t.TempDir() now := time.Now() srcFile := fmt.Sprintf("%s/copy-test-%d-src.txt", tmpDir, now.UnixMicro()) dstFile := fmt.Sprintf("%s/copy-test-%d-dst.txt", tmpDir, now.UnixMicro()) diff --git a/modules/util/map.go b/modules/util/map.go new file mode 100644 index 0000000000..f307faad1f --- /dev/null +++ b/modules/util/map.go @@ -0,0 +1,13 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +func GetMapValueOrDefault[T any](m map[string]any, key string, defaultValue T) T { + if value, ok := m[key]; ok { + if v, ok := value.(T); ok { + return v + } + } + return defaultValue +} diff --git a/modules/util/map_test.go b/modules/util/map_test.go new file mode 100644 index 0000000000..1a141cec88 --- /dev/null +++ b/modules/util/map_test.go @@ -0,0 +1,26 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetMapValueOrDefault(t *testing.T) { + testMap := map[string]any{ + "key1": "value1", + "key2": 42, + "key3": nil, + } + + assert.Equal(t, "value1", GetMapValueOrDefault(testMap, "key1", "default")) + assert.Equal(t, 42, GetMapValueOrDefault(testMap, "key2", 0)) + + assert.Equal(t, "default", GetMapValueOrDefault(testMap, "key4", "default")) + assert.Equal(t, 100, GetMapValueOrDefault(testMap, "key5", 100)) + + assert.Equal(t, "default", GetMapValueOrDefault(testMap, "key3", "default")) +} diff --git a/modules/util/paginate_test.go b/modules/util/paginate_test.go index 6e69dd19cc..3dc5095071 100644 --- a/modules/util/paginate_test.go +++ b/modules/util/paginate_test.go @@ -13,23 +13,23 @@ func TestPaginateSlice(t *testing.T) { stringSlice := []string{"a", "b", "c", "d", "e"} result, ok := PaginateSlice(stringSlice, 1, 2).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"a", "b"}, result) + assert.Equal(t, []string{"a", "b"}, result) result, ok = PaginateSlice(stringSlice, 100, 2).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{}, result) + assert.Equal(t, []string{}, result) result, ok = PaginateSlice(stringSlice, 3, 2).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"e"}, result) + assert.Equal(t, []string{"e"}, result) result, ok = PaginateSlice(stringSlice, 1, 0).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"a", "b", "c", "d", "e"}, result) + assert.Equal(t, []string{"a", "b", "c", "d", "e"}, result) result, ok = PaginateSlice(stringSlice, 1, -1).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"a", "b", "c", "d", "e"}, result) + assert.Equal(t, []string{"a", "b", "c", "d", "e"}, result) type Test struct { Val int @@ -38,9 +38,9 @@ func TestPaginateSlice(t *testing.T) { testVar := []*Test{{Val: 2}, {Val: 3}, {Val: 4}} testVar, ok = PaginateSlice(testVar, 1, 50).([]*Test) assert.True(t, ok) - assert.EqualValues(t, []*Test{{Val: 2}, {Val: 3}, {Val: 4}}, testVar) + assert.Equal(t, []*Test{{Val: 2}, {Val: 3}, {Val: 4}}, testVar) testVar, ok = PaginateSlice(testVar, 2, 2).([]*Test) assert.True(t, ok) - assert.EqualValues(t, []*Test{{Val: 4}}, testVar) + assert.Equal(t, []*Test{{Val: 4}}, testVar) } diff --git a/modules/util/path.go b/modules/util/path.go index d9f17bd124..0e56348978 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -36,9 +36,10 @@ func PathJoinRel(elem ...string) string { elems[i] = path.Clean("/" + e) } p := path.Join(elems...) - if p == "" { + switch p { + case "": return "" - } else if p == "/" { + case "/": return "." } return p[1:] diff --git a/modules/util/remove.go b/modules/util/remove.go index d1e38faf5f..3db0b5a796 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -15,7 +15,7 @@ const windowsSharingViolationError syscall.Errno = 32 // Remove removes the named file or (empty) directory with at most 5 attempts. func Remove(name string) error { var err error - for i := 0; i < 5; i++ { + for range 5 { err = os.Remove(name) if err == nil { break @@ -44,7 +44,7 @@ func Remove(name string) error { // RemoveAll removes the named file or (empty) directory with at most 5 attempts. func RemoveAll(name string) error { var err error - for i := 0; i < 5; i++ { + for range 5 { err = os.RemoveAll(name) if err == nil { break @@ -73,7 +73,7 @@ func RemoveAll(name string) error { // Rename renames (moves) oldpath to newpath with at most 5 attempts. func Rename(oldpath, newpath string) error { var err error - for i := 0; i < 5; i++ { + for i := range 5 { err = os.Rename(oldpath, newpath) if err == nil { break diff --git a/modules/util/rotatingfilewriter/writer_test.go b/modules/util/rotatingfilewriter/writer_test.go index 88392797b3..f6ea1d50ae 100644 --- a/modules/util/rotatingfilewriter/writer_test.go +++ b/modules/util/rotatingfilewriter/writer_test.go @@ -23,7 +23,7 @@ func TestCompressOldFile(t *testing.T) { ng, err := os.OpenFile(nonGzip, os.O_CREATE|os.O_WRONLY, 0o660) assert.NoError(t, err) - for i := 0; i < 999; i++ { + for range 999 { f.WriteString("This is a test file\n") ng.WriteString("This is a test file\n") } diff --git a/modules/util/runtime_test.go b/modules/util/runtime_test.go index 20f9063b0b..01dd034cea 100644 --- a/modules/util/runtime_test.go +++ b/modules/util/runtime_test.go @@ -18,14 +18,14 @@ func TestCallerFuncName(t *testing.T) { func BenchmarkCallerFuncName(b *testing.B) { // BenchmarkCaller/sprintf-12 12744829 95.49 ns/op b.Run("sprintf", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { _ = fmt.Sprintf("aaaaaaaaaaaaaaaa %s %s %s", "bbbbbbbbbbbbbbbbbbb", b.Name(), "ccccccccccccccccccccc") } }) // BenchmarkCaller/caller-12 10625133 113.6 ns/op // It is almost as fast as fmt.Sprintf b.Run("caller", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { CallerFuncName(1) } }) diff --git a/modules/util/sec_to_time.go b/modules/util/sec_to_time.go index 73667d723e..646f33c82a 100644 --- a/modules/util/sec_to_time.go +++ b/modules/util/sec_to_time.go @@ -11,16 +11,20 @@ import ( // SecToHours converts an amount of seconds to a human-readable hours string. // This is stable for planning and managing timesheets. // Here it only supports hours and minutes, because a work day could contain 6 or 7 or 8 hours. +// If the duration is less than 1 minute, it will be shown as seconds. func SecToHours(durationVal any) string { - duration, _ := ToInt64(durationVal) - hours := duration / 3600 - minutes := (duration / 60) % 60 + seconds, _ := ToInt64(durationVal) + hours := seconds / 3600 + minutes := (seconds / 60) % 60 formattedTime := "" formattedTime = formatTime(hours, "hour", formattedTime) formattedTime = formatTime(minutes, "minute", formattedTime) // The formatTime() function always appends a space at the end. This will be trimmed + if formattedTime == "" && seconds > 0 { + formattedTime = formatTime(seconds, "second", "") + } return strings.TrimRight(formattedTime, " ") } diff --git a/modules/util/sec_to_time_test.go b/modules/util/sec_to_time_test.go index 71a8801d4f..84e767c6e0 100644 --- a/modules/util/sec_to_time_test.go +++ b/modules/util/sec_to_time_test.go @@ -22,4 +22,7 @@ func TestSecToHours(t *testing.T) { assert.Equal(t, "156 hours 30 minutes", SecToHours(6*day+12*hour+30*minute+18*second)) assert.Equal(t, "98 hours 16 minutes", SecToHours(4*day+2*hour+16*minute+58*second)) assert.Equal(t, "672 hours", SecToHours(4*7*day)) + assert.Equal(t, "1 second", SecToHours(1)) + assert.Equal(t, "2 seconds", SecToHours(2)) + assert.Empty(t, SecToHours(nil)) // old behavior, empty means no output } diff --git a/modules/util/slice.go b/modules/util/slice.go index 9c878c24be..da6886491e 100644 --- a/modules/util/slice.go +++ b/modules/util/slice.go @@ -71,3 +71,10 @@ func KeysOfMap[K comparable, V any](m map[K]V) []K { } return keys } + +func SliceNilAsEmpty[T any](a []T) []T { + if a == nil { + return []T{} + } + return a +} diff --git a/modules/util/string.go b/modules/util/string.go index 19cf75b8b3..b9b59df3ef 100644 --- a/modules/util/string.go +++ b/modules/util/string.go @@ -103,10 +103,31 @@ func UnsafeStringToBytes(s string) []byte { func SplitTrimSpace(input, sep string) []string { input = strings.TrimSpace(input) var stringList []string - for _, s := range strings.Split(input, sep) { + for s := range strings.SplitSeq(input, sep) { if s = strings.TrimSpace(s); s != "" { stringList = append(stringList, s) } } return stringList } + +func asciiLower(b byte) byte { + if 'A' <= b && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +// AsciiEqualFold is from Golang https://cs.opensource.google/go/go/+/refs/tags/go1.24.4:src/net/http/internal/ascii/print.go +// ASCII only. In most cases for protocols, we should only use this but not [strings.EqualFold] +func AsciiEqualFold(s, t string) bool { //nolint:revive // PascalCase + if len(s) != len(t) { + return false + } + for i := 0; i < len(s); i++ { + if asciiLower(s[i]) != asciiLower(t[i]) { + return false + } + } + return true +} diff --git a/modules/util/truncate_test.go b/modules/util/truncate_test.go index 8789c824f5..9f4ad7dc20 100644 --- a/modules/util/truncate_test.go +++ b/modules/util/truncate_test.go @@ -5,6 +5,7 @@ package util import ( "fmt" + "strconv" "strings" "testing" @@ -100,7 +101,7 @@ func TestEllipsisString(t *testing.T) { {limit: 7, left: "\xef\x03\xfe\xef\x03\xfe", right: ""}, } for _, c := range invalidCases { - t.Run(fmt.Sprintf("%d", c.limit), func(t *testing.T) { + t.Run(strconv.Itoa(c.limit), func(t *testing.T) { left, right := EllipsisDisplayStringX("\xef\x03\xfe\xef\x03\xfe", c.limit) assert.Equal(t, c.left, left, "left") assert.Equal(t, c.right, right, "right") @@ -115,15 +116,15 @@ func TestEllipsisString(t *testing.T) { } func TestTruncateRunes(t *testing.T) { - assert.Equal(t, "", TruncateRunes("", 0)) - assert.Equal(t, "", TruncateRunes("", 1)) + assert.Empty(t, TruncateRunes("", 0)) + assert.Empty(t, TruncateRunes("", 1)) - assert.Equal(t, "", TruncateRunes("ab", 0)) + assert.Empty(t, TruncateRunes("ab", 0)) assert.Equal(t, "a", TruncateRunes("ab", 1)) assert.Equal(t, "ab", TruncateRunes("ab", 2)) assert.Equal(t, "ab", TruncateRunes("ab", 3)) - assert.Equal(t, "", TruncateRunes("测试", 0)) + assert.Empty(t, TruncateRunes("测试", 0)) assert.Equal(t, "测", TruncateRunes("测试", 1)) assert.Equal(t, "测试", TruncateRunes("测试", 2)) assert.Equal(t, "测试", TruncateRunes("测试", 3)) diff --git a/modules/util/util.go b/modules/util/util.go index 1fb4cb21cb..dd8e073888 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -11,21 +11,10 @@ import ( "strconv" "strings" - "code.gitea.io/gitea/modules/optional" - "golang.org/x/text/cases" "golang.org/x/text/language" ) -// OptionalBoolParse get the corresponding optional.Option[bool] of a string using strconv.ParseBool -func OptionalBoolParse(s string) optional.Option[bool] { - v, e := strconv.ParseBool(s) - if e != nil { - return optional.None[bool]() - } - return optional.Some(v) -} - // IsEmptyString checks if the provided string is empty func IsEmptyString(s string) bool { return len(strings.TrimSpace(s)) == 0 @@ -230,6 +219,13 @@ func IfZero[T comparable](v, def T) T { return v } +func IfEmpty[T any](v, def []T) []T { + if len(v) == 0 { + return def + } + return v +} + // OptionalArg helps the "optional argument" in Golang: // // func foo(optArg ...int) { return OptionalArg(optArg) } diff --git a/modules/util/util_test.go b/modules/util/util_test.go index 52b05acc5b..fe4125cdb5 100644 --- a/modules/util/util_test.go +++ b/modules/util/util_test.go @@ -8,8 +8,6 @@ import ( "strings" "testing" - "code.gitea.io/gitea/modules/optional" - "github.com/stretchr/testify/assert" ) @@ -175,19 +173,6 @@ func Test_RandomBytes(t *testing.T) { assert.NotEqual(t, bytes3, bytes4) } -func TestOptionalBoolParse(t *testing.T) { - assert.Equal(t, optional.None[bool](), OptionalBoolParse("")) - assert.Equal(t, optional.None[bool](), OptionalBoolParse("x")) - - assert.Equal(t, optional.Some(false), OptionalBoolParse("0")) - assert.Equal(t, optional.Some(false), OptionalBoolParse("f")) - assert.Equal(t, optional.Some(false), OptionalBoolParse("False")) - - assert.Equal(t, optional.Some(true), OptionalBoolParse("1")) - assert.Equal(t, optional.Some(true), OptionalBoolParse("t")) - assert.Equal(t, optional.Some(true), OptionalBoolParse("True")) -} - // Test case for any function which accepts and returns a single string. type StringTest struct { in, out string @@ -215,7 +200,7 @@ func TestToUpperASCII(t *testing.T) { func BenchmarkToUpper(b *testing.B) { for _, tc := range upperTests { b.Run(tc.in, func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { ToUpperASCII(tc.in) } }) diff --git a/modules/validation/binding_test.go b/modules/validation/binding_test.go index 28d0f57b5c..0cd328f312 100644 --- a/modules/validation/binding_test.go +++ b/modules/validation/binding_test.go @@ -47,7 +47,7 @@ func performValidationTest(t *testing.T, testCase validationTestCase) { assert.Equal(t, testCase.expectedErrors, actual) }) - req, err := http.NewRequest("POST", testRoute, nil) + req, err := http.NewRequest(http.MethodPost, testRoute, nil) if err != nil { panic(err) } diff --git a/modules/validation/glob_pattern_test.go b/modules/validation/glob_pattern_test.go index 1bf622e61d..7f3e609acf 100644 --- a/modules/validation/glob_pattern_test.go +++ b/modules/validation/glob_pattern_test.go @@ -19,39 +19,39 @@ func getGlobPatternErrorString(pattern string) string { return "" } -var globValidationTestCases = []validationTestCase{ - { - description: "Empty glob pattern", - data: TestForm{ - GlobPattern: "", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Valid glob", - data: TestForm{ - GlobPattern: "{master,release*}", - }, - expectedErrors: binding.Errors{}, - }, +func Test_GlobPatternValidation(t *testing.T) { + AddBindingRules() - { - description: "Invalid glob", - data: TestForm{ - GlobPattern: "[a-", + globValidationTestCases := []validationTestCase{ + { + description: "Empty glob pattern", + data: TestForm{ + GlobPattern: "", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"GlobPattern"}, - Classification: ErrGlobPattern, - Message: getGlobPatternErrorString("[a-"), + { + description: "Valid glob", + data: TestForm{ + GlobPattern: "{master,release*}", }, + expectedErrors: binding.Errors{}, }, - }, -} -func Test_GlobPatternValidation(t *testing.T) { - AddBindingRules() + { + description: "Invalid glob", + data: TestForm{ + GlobPattern: "[a-", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"GlobPattern"}, + Classification: ErrGlobPattern, + Message: getGlobPatternErrorString("[a-"), + }, + }, + }, + } for _, testCase := range globValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/validation/helpers.go b/modules/validation/helpers.go index f6e00f3887..ba383ba195 100644 --- a/modules/validation/helpers.go +++ b/modules/validation/helpers.go @@ -7,14 +7,28 @@ import ( "net" "net/url" "regexp" + "slices" "strings" + "sync" "code.gitea.io/gitea/modules/setting" "github.com/gobwas/glob" ) -var externalTrackerRegex = regexp.MustCompile(`({?)(?:user|repo|index)+?(}?)`) +type globalVarsStruct struct { + externalTrackerRegex *regexp.Regexp + validUsernamePattern *regexp.Regexp + invalidUsernamePattern *regexp.Regexp +} + +var globalVars = sync.OnceValue(func() *globalVarsStruct { + return &globalVarsStruct{ + externalTrackerRegex: regexp.MustCompile(`({?)(?:user|repo|index)+?(}?)`), + validUsernamePattern: regexp.MustCompile(`^[\da-zA-Z][-.\w]*$`), + invalidUsernamePattern: regexp.MustCompile(`[-._]{2,}|[-._]$`), // No consecutive or trailing non-alphanumeric chars + } +}) func isLoopbackIP(ip string) bool { return net.ParseIP(ip).IsLoopback() @@ -42,12 +56,7 @@ func IsValidSiteURL(uri string) bool { return false } - for _, scheme := range setting.Service.ValidSiteURLSchemes { - if scheme == u.Scheme { - return true - } - } - return false + return slices.Contains(setting.Service.ValidSiteURLSchemes, u.Scheme) } // IsEmailDomainListed checks whether the domain of an email address @@ -105,9 +114,9 @@ func IsValidExternalTrackerURLFormat(uri string) bool { if !IsValidExternalURL(uri) { return false } - + vars := globalVars() // check for typoed variables like /{index/ or /[repo} - for _, match := range externalTrackerRegex.FindAllStringSubmatch(uri, -1) { + for _, match := range vars.externalTrackerRegex.FindAllStringSubmatch(uri, -1) { if (match[1] == "{" || match[2] == "}") && (match[1] != "{" || match[2] != "}") { return false } @@ -116,14 +125,10 @@ func IsValidExternalTrackerURLFormat(uri string) bool { return true } -var ( - validUsernamePattern = regexp.MustCompile(`^[\da-zA-Z][-.\w]*$`) - invalidUsernamePattern = regexp.MustCompile(`[-._]{2,}|[-._]$`) // No consecutive or trailing non-alphanumeric chars -) - // IsValidUsername checks if username is valid func IsValidUsername(name string) bool { // It is difficult to find a single pattern that is both readable and effective, // but it's easier to use positive and negative checks. - return validUsernamePattern.MatchString(name) && !invalidUsernamePattern.MatchString(name) + vars := globalVars() + return vars.validUsernamePattern.MatchString(name) && !vars.invalidUsernamePattern.MatchString(name) } diff --git a/modules/validation/helpers_test.go b/modules/validation/helpers_test.go index 52f383f698..6a982965f6 100644 --- a/modules/validation/helpers_test.go +++ b/modules/validation/helpers_test.go @@ -7,6 +7,7 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" ) @@ -47,7 +48,7 @@ func Test_IsValidURL(t *testing.T) { } func Test_IsValidExternalURL(t *testing.T) { - setting.AppURL = "https://try.gitea.io/" + defer test.MockVariableValue(&setting.AppURL, "https://try.gitea.io/")() cases := []struct { description string @@ -89,7 +90,7 @@ func Test_IsValidExternalURL(t *testing.T) { } func Test_IsValidExternalTrackerURLFormat(t *testing.T) { - setting.AppURL = "https://try.gitea.io/" + defer test.MockVariableValue(&setting.AppURL, "https://try.gitea.io/")() cases := []struct { description string diff --git a/modules/validation/refname_test.go b/modules/validation/refname_test.go index 3af7387c47..93703761cf 100644 --- a/modules/validation/refname_test.go +++ b/modules/validation/refname_test.go @@ -9,253 +9,252 @@ import ( "gitea.com/go-chi/binding" ) -var gitRefNameValidationTestCases = []validationTestCase{ - { - description: "Reference name contains only characters", - data: TestForm{ - BranchName: "test", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Reference name contains single slash", - data: TestForm{ - BranchName: "feature/test", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Reference name has allowed special characters", - data: TestForm{ - BranchName: "debian/1%1.6.0-2", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Reference name contains backslash", - data: TestForm{ - BranchName: "feature\\test", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", +func Test_GitRefNameValidation(t *testing.T) { + AddBindingRules() + gitRefNameValidationTestCases := []validationTestCase{ + { + description: "Reference name contains only characters", + data: TestForm{ + BranchName: "test", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Reference name starts with dot", - data: TestForm{ - BranchName: ".test", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name contains single slash", + data: TestForm{ + BranchName: "feature/test", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Reference name ends with dot", - data: TestForm{ - BranchName: "test.", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has allowed special characters", + data: TestForm{ + BranchName: "debian/1%1.6.0-2", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Reference name starts with slash", - data: TestForm{ - BranchName: "/test", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name contains backslash", + data: TestForm{ + BranchName: "feature\\test", }, - }, - }, - { - description: "Reference name ends with slash", - data: TestForm{ - BranchName: "test/", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name ends with .lock", - data: TestForm{ - BranchName: "test.lock", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name starts with dot", + data: TestForm{ + BranchName: ".test", }, - }, - }, - { - description: "Reference name contains multiple consecutive dots", - data: TestForm{ - BranchName: "te..st", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name contains multiple consecutive slashes", - data: TestForm{ - BranchName: "te//st", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name ends with dot", + data: TestForm{ + BranchName: "test.", }, - }, - }, - { - description: "Reference name is single @", - data: TestForm{ - BranchName: "@", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has @{", - data: TestForm{ - BranchName: "branch@{", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name starts with slash", + data: TestForm{ + BranchName: "/test", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character ~", - data: TestForm{ - BranchName: "~debian/1%1.6.0-2", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name ends with slash", + data: TestForm{ + BranchName: "test/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character *", - data: TestForm{ - BranchName: "*debian/1%1.6.0-2", + { + description: "Reference name ends with .lock", + data: TestForm{ + BranchName: "test.lock", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name contains multiple consecutive dots", + data: TestForm{ + BranchName: "te..st", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character ?", - data: TestForm{ - BranchName: "?debian/1%1.6.0-2", + { + description: "Reference name contains multiple consecutive slashes", + data: TestForm{ + BranchName: "te//st", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name is single @", + data: TestForm{ + BranchName: "@", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character ^", - data: TestForm{ - BranchName: "^debian/1%1.6.0-2", + { + description: "Reference name has @{", + data: TestForm{ + BranchName: "branch@{", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has unallowed special character ~", + data: TestForm{ + BranchName: "~debian/1%1.6.0-2", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character :", - data: TestForm{ - BranchName: "debian:jessie", + { + description: "Reference name has unallowed special character *", + data: TestForm{ + BranchName: "*debian/1%1.6.0-2", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has unallowed special character ?", + data: TestForm{ + BranchName: "?debian/1%1.6.0-2", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character (whitespace)", - data: TestForm{ - BranchName: "debian jessie", + { + description: "Reference name has unallowed special character ^", + data: TestForm{ + BranchName: "^debian/1%1.6.0-2", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has unallowed special character :", + data: TestForm{ + BranchName: "debian:jessie", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character [", - data: TestForm{ - BranchName: "debian[jessie", + { + description: "Reference name has unallowed special character (whitespace)", + data: TestForm{ + BranchName: "debian jessie", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has unallowed special character [", + data: TestForm{ + BranchName: "debian[jessie", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, -} - -func Test_GitRefNameValidation(t *testing.T) { - AddBindingRules() + } for _, testCase := range gitRefNameValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/validation/regex_pattern_test.go b/modules/validation/regex_pattern_test.go index efcb276734..80790a23b1 100644 --- a/modules/validation/regex_pattern_test.go +++ b/modules/validation/regex_pattern_test.go @@ -17,39 +17,39 @@ func getRegexPatternErrorString(pattern string) string { return "" } -var regexValidationTestCases = []validationTestCase{ - { - description: "Empty regex pattern", - data: TestForm{ - RegexPattern: "", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Valid regex", - data: TestForm{ - RegexPattern: `(\d{1,3})+`, - }, - expectedErrors: binding.Errors{}, - }, +func Test_RegexPatternValidation(t *testing.T) { + AddBindingRules() - { - description: "Invalid regex", - data: TestForm{ - RegexPattern: "[a-", + regexValidationTestCases := []validationTestCase{ + { + description: "Empty regex pattern", + data: TestForm{ + RegexPattern: "", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"RegexPattern"}, - Classification: ErrRegexPattern, - Message: getRegexPatternErrorString("[a-"), + { + description: "Valid regex", + data: TestForm{ + RegexPattern: `(\d{1,3})+`, }, + expectedErrors: binding.Errors{}, }, - }, -} -func Test_RegexPatternValidation(t *testing.T) { - AddBindingRules() + { + description: "Invalid regex", + data: TestForm{ + RegexPattern: "[a-", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"RegexPattern"}, + Classification: ErrRegexPattern, + Message: getRegexPatternErrorString("[a-"), + }, + }, + }, + } for _, testCase := range regexValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/validation/validurl_test.go b/modules/validation/validurl_test.go index 39f7fa5d65..ce4898b271 100644 --- a/modules/validation/validurl_test.go +++ b/modules/validation/validurl_test.go @@ -9,98 +9,98 @@ import ( "gitea.com/go-chi/binding" ) -var urlValidationTestCases = []validationTestCase{ - { - description: "Empty URL", - data: TestForm{ - URL: "", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL without port", - data: TestForm{ - URL: "http://test.lan/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with port", - data: TestForm{ - URL: "http://test.lan:3000/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with IPv6 address without port", - data: TestForm{ - URL: "http://[::1]/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with IPv6 address with port", - data: TestForm{ - URL: "http://[::1]:3000/", +func Test_ValidURLValidation(t *testing.T) { + AddBindingRules() + + urlValidationTestCases := []validationTestCase{ + { + description: "Empty URL", + data: TestForm{ + URL: "", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{}, - }, - { - description: "Invalid URL", - data: TestForm{ - URL: "http//test.lan/", + { + description: "URL without port", + data: TestForm{ + URL: "http://test.lan/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URL"}, - Classification: binding.ERR_URL, - Message: "Url", + { + description: "URL with port", + data: TestForm{ + URL: "http://test.lan:3000/", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid schema", - data: TestForm{ - URL: "ftp://test.lan/", + { + description: "URL with IPv6 address without port", + data: TestForm{ + URL: "http://[::1]/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URL"}, - Classification: binding.ERR_URL, - Message: "Url", + { + description: "URL with IPv6 address with port", + data: TestForm{ + URL: "http://[::1]:3000/", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid port", - data: TestForm{ - URL: "http://test.lan:3x4/", + { + description: "Invalid URL", + data: TestForm{ + URL: "http//test.lan/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URL"}, + Classification: binding.ERR_URL, + Message: "Url", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URL"}, - Classification: binding.ERR_URL, - Message: "Url", + { + description: "Invalid schema", + data: TestForm{ + URL: "ftp://test.lan/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URL"}, + Classification: binding.ERR_URL, + Message: "Url", + }, }, }, - }, - { - description: "Invalid port with IPv6 address", - data: TestForm{ - URL: "http://[::1]:3x4/", + { + description: "Invalid port", + data: TestForm{ + URL: "http://test.lan:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URL"}, + Classification: binding.ERR_URL, + Message: "Url", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URL"}, - Classification: binding.ERR_URL, - Message: "Url", + { + description: "Invalid port with IPv6 address", + data: TestForm{ + URL: "http://[::1]:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URL"}, + Classification: binding.ERR_URL, + Message: "Url", + }, }, }, - }, -} - -func Test_ValidURLValidation(t *testing.T) { - AddBindingRules() + } for _, testCase := range urlValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/validation/validurllist_test.go b/modules/validation/validurllist_test.go index c6f862a962..cccc570a1a 100644 --- a/modules/validation/validurllist_test.go +++ b/modules/validation/validurllist_test.go @@ -9,145 +9,145 @@ import ( "gitea.com/go-chi/binding" ) -// This is a copy of all the URL tests cases, plus additional ones to -// account for multiple URLs -var urlListValidationTestCases = []validationTestCase{ - { - description: "Empty URL", - data: TestForm{ - URLs: "", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL without port", - data: TestForm{ - URLs: "http://test.lan/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with port", - data: TestForm{ - URLs: "http://test.lan:3000/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with IPv6 address without port", - data: TestForm{ - URLs: "http://[::1]/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with IPv6 address with port", - data: TestForm{ - URLs: "http://[::1]:3000/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Invalid URL", - data: TestForm{ - URLs: "http//test.lan/", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http//test.lan/", +func Test_ValidURLListValidation(t *testing.T) { + AddBindingRules() + + // This is a copy of all the URL tests cases, plus additional ones to + // account for multiple URLs + urlListValidationTestCases := []validationTestCase{ + { + description: "Empty URL", + data: TestForm{ + URLs: "", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid schema", - data: TestForm{ - URLs: "ftp://test.lan/", + { + description: "URL without port", + data: TestForm{ + URLs: "http://test.lan/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "ftp://test.lan/", + { + description: "URL with port", + data: TestForm{ + URLs: "http://test.lan:3000/", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid port", - data: TestForm{ - URLs: "http://test.lan:3x4/", + { + description: "URL with IPv6 address without port", + data: TestForm{ + URLs: "http://[::1]/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http://test.lan:3x4/", + { + description: "URL with IPv6 address with port", + data: TestForm{ + URLs: "http://[::1]:3000/", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid port with IPv6 address", - data: TestForm{ - URLs: "http://[::1]:3x4/", + { + description: "Invalid URL", + data: TestForm{ + URLs: "http//test.lan/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http//test.lan/", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http://[::1]:3x4/", + { + description: "Invalid schema", + data: TestForm{ + URLs: "ftp://test.lan/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "ftp://test.lan/", + }, }, }, - }, - { - description: "Multi URLs", - data: TestForm{ - URLs: "http://test.lan:3000/\nhttp://test.local/", + { + description: "Invalid port", + data: TestForm{ + URLs: "http://test.lan:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http://test.lan:3x4/", + }, + }, }, - expectedErrors: binding.Errors{}, - }, - { - description: "Multi URLs with newline", - data: TestForm{ - URLs: "http://test.lan:3000/\nhttp://test.local/\n", + { + description: "Invalid port with IPv6 address", + data: TestForm{ + URLs: "http://[::1]:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http://[::1]:3x4/", + }, + }, }, - expectedErrors: binding.Errors{}, - }, - { - description: "List with invalid entry", - data: TestForm{ - URLs: "http://test.lan:3000/\nhttp://[::1]:3x4/", + { + description: "Multi URLs", + data: TestForm{ + URLs: "http://test.lan:3000/\nhttp://test.local/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http://[::1]:3x4/", + { + description: "Multi URLs with newline", + data: TestForm{ + URLs: "http://test.lan:3000/\nhttp://test.local/\n", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "List with two invalid entries", - data: TestForm{ - URLs: "ftp://test.lan:3000/\nhttp://[::1]:3x4/\n", + { + description: "List with invalid entry", + data: TestForm{ + URLs: "http://test.lan:3000/\nhttp://[::1]:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http://[::1]:3x4/", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "ftp://test.lan:3000/", + { + description: "List with two invalid entries", + data: TestForm{ + URLs: "ftp://test.lan:3000/\nhttp://[::1]:3x4/\n", }, - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http://[::1]:3x4/", + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "ftp://test.lan:3000/", + }, + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http://[::1]:3x4/", + }, }, }, - }, -} - -func Test_ValidURLListValidation(t *testing.T) { - AddBindingRules() + } for _, testCase := range urlListValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/web/middleware/binding.go b/modules/web/middleware/binding.go index 43e1bbc70e..ee4eca976e 100644 --- a/modules/web/middleware/binding.go +++ b/modules/web/middleware/binding.go @@ -50,7 +50,7 @@ func AssignForm(form any, data map[string]any) { } func getRuleBody(field reflect.StructField, prefix string) string { - for _, rule := range strings.Split(field.Tag.Get("binding"), ";") { + for rule := range strings.SplitSeq(field.Tag.Get("binding"), ";") { if strings.HasPrefix(rule, prefix) { return rule[len(prefix) : len(rule)-1] } @@ -78,7 +78,7 @@ func GetInclude(field reflect.StructField) string { return getRuleBody(field, "Include(") } -// Validate validate TODO: +// Validate validate func Validate(errs binding.Errors, data map[string]any, f Form, l translation.Locale) binding.Errors { if errs.Len() == 0 { return errs diff --git a/modules/web/middleware/flash.go b/modules/web/middleware/flash.go index 0caaa8c036..0e848c7902 100644 --- a/modules/web/middleware/flash.go +++ b/modules/web/middleware/flash.go @@ -6,6 +6,7 @@ package middleware import ( "fmt" "html/template" + "net/http" "net/url" "code.gitea.io/gitea/modules/reqctx" @@ -65,3 +66,27 @@ func (f *Flash) Success(msg any, current ...bool) { f.SuccessMsg = flashMsgStringOrHTML(msg) f.set("success", f.SuccessMsg, current...) } + +func ParseCookieFlashMessage(val string) *Flash { + if vals, _ := url.ParseQuery(val); len(vals) > 0 { + return &Flash{ + Values: vals, + ErrorMsg: vals.Get("error"), + SuccessMsg: vals.Get("success"), + InfoMsg: vals.Get("info"), + WarningMsg: vals.Get("warning"), + } + } + return nil +} + +func GetSiteCookieFlashMessage(dataStore reqctx.RequestDataStore, req *http.Request, cookieName string) (string, *Flash) { + // Get the last flash message from cookie + lastFlashCookie := GetSiteCookie(req, cookieName) + lastFlashMsg := ParseCookieFlashMessage(lastFlashCookie) + if lastFlashMsg != nil { + lastFlashMsg.DataStore = dataStore + return lastFlashCookie, lastFlashMsg + } + return lastFlashCookie, nil +} diff --git a/modules/web/middleware/request.go b/modules/web/middleware/request.go deleted file mode 100644 index 0bb155df70..0000000000 --- a/modules/web/middleware/request.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package middleware - -import ( - "net/http" - "strings" -) - -// IsAPIPath returns true if the specified URL is an API path -func IsAPIPath(req *http.Request) bool { - return strings.HasPrefix(req.URL.Path, "/api/") -} diff --git a/modules/web/routemock_test.go b/modules/web/routemock_test.go index 89cfaacdd1..a0949bf622 100644 --- a/modules/web/routemock_test.go +++ b/modules/web/routemock_test.go @@ -30,13 +30,13 @@ func TestRouteMock(t *testing.T) { // normal request recorder := httptest.NewRecorder() - req, err := http.NewRequest("GET", "http://localhost:8000/foo", nil) + req, err := http.NewRequest(http.MethodGet, "http://localhost:8000/foo", nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Len(t, recorder.Header(), 3) - assert.EqualValues(t, "m1", recorder.Header().Get("X-Test-Middleware1")) - assert.EqualValues(t, "m2", recorder.Header().Get("X-Test-Middleware2")) - assert.EqualValues(t, "h", recorder.Header().Get("X-Test-Handler")) + assert.Equal(t, "m1", recorder.Header().Get("X-Test-Middleware1")) + assert.Equal(t, "m2", recorder.Header().Get("X-Test-Middleware2")) + assert.Equal(t, "h", recorder.Header().Get("X-Test-Handler")) RouteMockReset() // mock at "mock-point" @@ -45,12 +45,12 @@ func TestRouteMock(t *testing.T) { resp.WriteHeader(http.StatusOK) }) recorder = httptest.NewRecorder() - req, err = http.NewRequest("GET", "http://localhost:8000/foo", nil) + req, err = http.NewRequest(http.MethodGet, "http://localhost:8000/foo", nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Len(t, recorder.Header(), 2) - assert.EqualValues(t, "m1", recorder.Header().Get("X-Test-Middleware1")) - assert.EqualValues(t, "a", recorder.Header().Get("X-Test-MockPoint")) + assert.Equal(t, "m1", recorder.Header().Get("X-Test-Middleware1")) + assert.Equal(t, "a", recorder.Header().Get("X-Test-MockPoint")) RouteMockReset() // mock at MockAfterMiddlewares @@ -59,12 +59,12 @@ func TestRouteMock(t *testing.T) { resp.WriteHeader(http.StatusOK) }) recorder = httptest.NewRecorder() - req, err = http.NewRequest("GET", "http://localhost:8000/foo", nil) + req, err = http.NewRequest(http.MethodGet, "http://localhost:8000/foo", nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Len(t, recorder.Header(), 3) - assert.EqualValues(t, "m1", recorder.Header().Get("X-Test-Middleware1")) - assert.EqualValues(t, "m2", recorder.Header().Get("X-Test-Middleware2")) - assert.EqualValues(t, "b", recorder.Header().Get("X-Test-MockPoint")) + assert.Equal(t, "m1", recorder.Header().Get("X-Test-Middleware1")) + assert.Equal(t, "m2", recorder.Header().Get("X-Test-Middleware2")) + assert.Equal(t, "b", recorder.Header().Get("X-Test-MockPoint")) RouteMockReset() } diff --git a/modules/web/router.go b/modules/web/router.go index da06b955b1..5812ff69d4 100644 --- a/modules/web/router.go +++ b/modules/web/router.go @@ -125,8 +125,8 @@ func (r *Router) Methods(methods, pattern string, h ...any) { middlewares, handlerFunc := wrapMiddlewareAndHandler(r.curMiddlewares, h) fullPattern := r.getPattern(pattern) if strings.Contains(methods, ",") { - methods := strings.Split(methods, ",") - for _, method := range methods { + methods := strings.SplitSeq(methods, ",") + for method := range methods { r.chiRouter.With(middlewares...).Method(strings.TrimSpace(method), fullPattern, handlerFunc) } } else { diff --git a/modules/web/router_path.go b/modules/web/router_path.go index b59948581a..ce041eedab 100644 --- a/modules/web/router_path.go +++ b/modules/web/router_path.go @@ -4,9 +4,9 @@ package web import ( - "fmt" "net/http" "regexp" + "slices" "strings" "code.gitea.io/gitea/modules/container" @@ -37,11 +37,21 @@ func (g *RouterPathGroup) ServeHTTP(resp http.ResponseWriter, req *http.Request) g.r.chiRouter.NotFoundHandler().ServeHTTP(resp, req) } +type RouterPathGroupPattern struct { + re *regexp.Regexp + params []routerPathParam + middlewares []any +} + // MatchPath matches the request method, and uses regexp to match the path. -// The pattern uses "<...>" to define path parameters, for example: "/<name>" (different from chi router) -// It is only designed to resolve some special cases which chi router can't handle. +// The pattern uses "<...>" to define path parameters, for example, "/<name>" (different from chi router) +// It is only designed to resolve some special cases that chi router can't handle. // For most cases, it shouldn't be used because it needs to iterate all rules to find the matched one (inefficient). func (g *RouterPathGroup) MatchPath(methods, pattern string, h ...any) { + g.MatchPattern(methods, g.PatternRegexp(pattern), h...) +} + +func (g *RouterPathGroup) MatchPattern(methods string, pattern *RouterPathGroupPattern, h ...any) { g.matchers = append(g.matchers, newRouterPathMatcher(methods, pattern, h...)) } @@ -97,29 +107,35 @@ func isValidMethod(name string) bool { return false } -func newRouterPathMatcher(methods, pattern string, h ...any) *routerPathMatcher { - middlewares, handlerFunc := wrapMiddlewareAndHandler(nil, h) +func newRouterPathMatcher(methods string, patternRegexp *RouterPathGroupPattern, h ...any) *routerPathMatcher { + middlewares, handlerFunc := wrapMiddlewareAndHandler(patternRegexp.middlewares, h) p := &routerPathMatcher{methods: make(container.Set[string]), middlewares: middlewares, handlerFunc: handlerFunc} - for _, method := range strings.Split(methods, ",") { + for method := range strings.SplitSeq(methods, ",") { method = strings.TrimSpace(method) if !isValidMethod(method) { - panic(fmt.Sprintf("invalid HTTP method: %s", method)) + panic("invalid HTTP method: " + method) } p.methods.Add(method) } + p.re, p.params = patternRegexp.re, patternRegexp.params + return p +} + +func patternRegexp(pattern string, h ...any) *RouterPathGroupPattern { + p := &RouterPathGroupPattern{middlewares: slices.Clone(h)} re := []byte{'^'} lastEnd := 0 for lastEnd < len(pattern) { start := strings.IndexByte(pattern[lastEnd:], '<') if start == -1 { - re = append(re, pattern[lastEnd:]...) + re = append(re, regexp.QuoteMeta(pattern[lastEnd:])...) break } end := strings.IndexByte(pattern[lastEnd+start:], '>') if end == -1 { - panic(fmt.Sprintf("invalid pattern: %s", pattern)) + panic("invalid pattern: " + pattern) } - re = append(re, pattern[lastEnd:lastEnd+start]...) + re = append(re, regexp.QuoteMeta(pattern[lastEnd:lastEnd+start])...) partName, partExp, _ := strings.Cut(pattern[lastEnd+start+1:lastEnd+start+end], ":") lastEnd += start + end + 1 @@ -141,7 +157,10 @@ func newRouterPathMatcher(methods, pattern string, h ...any) *routerPathMatcher p.params = append(p.params, param) } re = append(re, '$') - reStr := string(re) - p.re = regexp.MustCompile(reStr) + p.re = regexp.MustCompile(string(re)) return p } + +func (g *RouterPathGroup) PatternRegexp(pattern string, h ...any) *RouterPathGroupPattern { + return patternRegexp(pattern, h...) +} diff --git a/modules/web/router_test.go b/modules/web/router_test.go index 582980a27a..1cee2b879b 100644 --- a/modules/web/router_test.go +++ b/modules/web/router_test.go @@ -34,7 +34,7 @@ func TestPathProcessor(t *testing.T) { testProcess := func(pattern, uri string, expectedPathParams map[string]string) { chiCtx := chi.NewRouteContext() chiCtx.RouteMethod = "GET" - p := newRouterPathMatcher("GET", pattern, http.NotFound) + p := newRouterPathMatcher("GET", patternRegexp(pattern), http.NotFound) assert.True(t, p.matchPath(chiCtx, uri), "use pattern %s to process uri %s", pattern, uri) assert.Equal(t, expectedPathParams, chiURLParamsToMap(chiCtx), "use pattern %s to process uri %s", pattern, uri) } @@ -51,23 +51,25 @@ func TestPathProcessor(t *testing.T) { } func TestRouter(t *testing.T) { - buff := bytes.NewBufferString("") + buff := &bytes.Buffer{} recorder := httptest.NewRecorder() recorder.Body = buff type resultStruct struct { - method string - pathParams map[string]string - handlerMark string + method string + pathParams map[string]string + handlerMarks []string } - var res resultStruct + var res resultStruct h := func(optMark ...string) func(resp http.ResponseWriter, req *http.Request) { mark := util.OptionalArg(optMark, "") return func(resp http.ResponseWriter, req *http.Request) { res.method = req.Method res.pathParams = chiURLParamsToMap(chi.RouteContext(req.Context())) - res.handlerMark = mark + if mark != "" { + res.handlerMarks = append(res.handlerMarks, mark) + } } } @@ -77,6 +79,8 @@ func TestRouter(t *testing.T) { if stop := req.FormValue("stop"); stop != "" && (mark == "" || mark == stop) { h(stop)(resp, req) resp.WriteHeader(http.StatusOK) + } else if mark != "" { + res.handlerMarks = append(res.handlerMarks, mark) } } } @@ -108,7 +112,7 @@ func TestRouter(t *testing.T) { m.Delete("", h()) }) m.PathGroup("/*", func(g *RouterPathGroup) { - g.MatchPath("GET", `/<dir:*>/<file:[a-z]{1,2}>`, stopMark("s2"), h("match-path")) + g.MatchPattern("GET", g.PatternRegexp(`/<dir:*>/<file:[a-z]{1,2}>`, stopMark("s2")), stopMark("s3"), h("match-path")) }, stopMark("s1")) }) }) @@ -121,36 +125,36 @@ func TestRouter(t *testing.T) { req, err := http.NewRequest(methodPathFields[0], methodPathFields[1], nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) - assert.EqualValues(t, expected, res) + assert.Equal(t, expected, res) }) } t.Run("RootRouter", func(t *testing.T) { - testRoute(t, "GET /the-user/the-repo/other", resultStruct{method: "GET", handlerMark: "not-found:/"}) + testRoute(t, "GET /the-user/the-repo/other", resultStruct{method: "GET", handlerMarks: []string{"not-found:/"}}) testRoute(t, "GET /the-user/the-repo/pulls", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "pulls"}, - handlerMark: "list-issues-b", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "pulls"}, + handlerMarks: []string{"list-issues-b"}, }) testRoute(t, "GET /the-user/the-repo/issues/123", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, - handlerMark: "view-issue", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, + handlerMarks: []string{"view-issue"}, }) testRoute(t, "GET /the-user/the-repo/issues/123?stop=hijack", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, - handlerMark: "hijack", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, + handlerMarks: []string{"hijack"}, }) testRoute(t, "POST /the-user/the-repo/issues/123/update", resultStruct{ - method: "POST", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "index": "123"}, - handlerMark: "update-issue", + method: "POST", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "index": "123"}, + handlerMarks: []string{"update-issue"}, }) }) t.Run("Sub Router", func(t *testing.T) { - testRoute(t, "GET /api/v1/other", resultStruct{method: "GET", handlerMark: "not-found:/api/v1"}) + testRoute(t, "GET /api/v1/other", resultStruct{method: "GET", handlerMarks: []string{"not-found:/api/v1"}}) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches", resultStruct{ method: "GET", pathParams: map[string]string{"username": "the-user", "reponame": "the-repo"}, @@ -179,31 +183,37 @@ func TestRouter(t *testing.T) { t.Run("MatchPath", func(t *testing.T) { testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, - handlerMark: "match-path", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3", "match-path"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1%2fd2/fn", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1%2fd2/fn", "dir": "d1%2fd2", "file": "fn"}, - handlerMark: "match-path", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1%2fd2/fn", "dir": "d1%2fd2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3", "match-path"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/000", resultStruct{ - method: "GET", - pathParams: map[string]string{"reponame": "the-repo", "username": "the-user", "*": "d1/d2/000"}, - handlerMark: "not-found:/api/v1", + method: "GET", + pathParams: map[string]string{"reponame": "the-repo", "username": "the-user", "*": "d1/d2/000"}, + handlerMarks: []string{"s1", "not-found:/api/v1"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s1", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn"}, - handlerMark: "s1", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn"}, + handlerMarks: []string{"s1"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s2", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, - handlerMark: "s2", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2"}, + }) + + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s3", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3"}, }) }) } @@ -224,7 +234,7 @@ func TestRouteNormalizePath(t *testing.T) { actualPaths.Path = req.URL.Path }) - req, err := http.NewRequest("GET", reqPath, nil) + req, err := http.NewRequest(http.MethodGet, reqPath, nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Equal(t, expectedPaths, actualPaths, "req path = %q", reqPath) diff --git a/modules/web/routing/context.go b/modules/web/routing/context.go index fbf371b839..d3eb98f83d 100644 --- a/modules/web/routing/context.go +++ b/modules/web/routing/context.go @@ -6,6 +6,9 @@ package routing import ( "context" "net/http" + + "code.gitea.io/gitea/modules/gtprof" + "code.gitea.io/gitea/modules/reqctx" ) type contextKeyType struct{} @@ -14,10 +17,12 @@ var contextKey contextKeyType // RecordFuncInfo records a func info into context func RecordFuncInfo(ctx context.Context, funcInfo *FuncInfo) (end func()) { - // TODO: reqCtx := reqctx.FromContext(ctx), add trace support end = func() {} - - // save the func info into the context record + if reqCtx := reqctx.FromContext(ctx); reqCtx != nil { + var traceSpan *gtprof.TraceSpan + traceSpan, end = gtprof.GetTracer().StartInContext(reqCtx, "http.func") + traceSpan.SetAttributeString("func", funcInfo.shortName) + } if record, ok := ctx.Value(contextKey).(*requestRecord); ok { record.lock.Lock() record.funcInfo = funcInfo diff --git a/modules/web/routing/logger.go b/modules/web/routing/logger.go index 5f3a7592af..3bca9b3420 100644 --- a/modules/web/routing/logger.go +++ b/modules/web/routing/logger.go @@ -35,6 +35,19 @@ var ( ) func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { + const callerName = "HTTPRequest" + logTrace := func(fmt string, args ...any) { + logger.Log(2, &log.Event{Level: log.TRACE, Caller: callerName}, fmt, args...) + } + logInfo := func(fmt string, args ...any) { + logger.Log(2, &log.Event{Level: log.INFO, Caller: callerName}, fmt, args...) + } + logWarn := func(fmt string, args ...any) { + logger.Log(2, &log.Event{Level: log.WARN, Caller: callerName}, fmt, args...) + } + logError := func(fmt string, args ...any) { + logger.Log(2, &log.Event{Level: log.ERROR, Caller: callerName}, fmt, args...) + } return func(trigger Event, record *requestRecord) { if trigger == StartEvent { if !logger.LevelEnabled(log.TRACE) { @@ -44,7 +57,7 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { } // when a request starts, we have no information about the handler function information, we only have the request path req := record.request - logger.Trace("router: %s %v %s for %s", startMessage, log.ColoredMethod(req.Method), req.RequestURI, req.RemoteAddr) + logTrace("router: %s %v %s for %s", startMessage, log.ColoredMethod(req.Method), req.RequestURI, req.RemoteAddr) return } @@ -60,9 +73,9 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { if trigger == StillExecutingEvent { message := slowMessage - logf := logger.Warn + logf := logWarn if isLongPolling { - logf = logger.Info + logf = logInfo message = pollingMessage } logf("router: %s %v %s for %s, elapsed %v @ %s", @@ -75,7 +88,7 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { } if panicErr != nil { - logger.Warn("router: %s %v %s for %s, panic in %v @ %s, err=%v", + logWarn("router: %s %v %s for %s, panic in %v @ %s, err=%v", failedMessage, log.ColoredMethod(req.Method), req.RequestURI, req.RemoteAddr, log.ColoredTime(time.Since(record.startTime)), @@ -89,13 +102,16 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { if v, ok := record.responseWriter.(types.ResponseStatusProvider); ok { status = v.WrittenStatus() } - logf := logger.Info - if strings.HasPrefix(req.RequestURI, "/assets/") { - logf = logger.Trace + logf := logInfo + // lower the log level for some specific requests, in most cases these logs are not useful + if strings.HasPrefix(req.RequestURI, "/assets/") /* static assets */ || + req.RequestURI == "/user/events" /* Server-Sent Events (SSE) handler */ || + req.RequestURI == "/api/actions/runner.v1.RunnerService/FetchTask" /* Actions Runner polling */ { + logf = logTrace } message := completedMessage if isUnknownHandler { - logf = logger.Error + logf = logError message = unknownHandlerMessage } diff --git a/modules/webhook/events.go b/modules/webhook/events.go new file mode 100644 index 0000000000..f4dfff0294 --- /dev/null +++ b/modules/webhook/events.go @@ -0,0 +1,20 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package webhook + +type HookEvents map[HookEventType]bool + +func (he HookEvents) Get(evt HookEventType) bool { + return he[evt] +} + +// HookEvent represents events that will delivery hook. +type HookEvent struct { + PushOnly bool `json:"push_only"` + SendEverything bool `json:"send_everything"` + ChooseEvents bool `json:"choose_events"` + BranchFilter string `json:"branch_filter"` + + HookEvents `json:"events"` +} diff --git a/modules/webhook/structs.go b/modules/webhook/structs.go deleted file mode 100644 index 927a91a74c..0000000000 --- a/modules/webhook/structs.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package webhook - -// HookEvents is a set of web hook events -type HookEvents struct { - Create bool `json:"create"` - Delete bool `json:"delete"` - Fork bool `json:"fork"` - Issues bool `json:"issues"` - IssueAssign bool `json:"issue_assign"` - IssueLabel bool `json:"issue_label"` - IssueMilestone bool `json:"issue_milestone"` - IssueComment bool `json:"issue_comment"` - Push bool `json:"push"` - PullRequest bool `json:"pull_request"` - PullRequestAssign bool `json:"pull_request_assign"` - PullRequestLabel bool `json:"pull_request_label"` - PullRequestMilestone bool `json:"pull_request_milestone"` - PullRequestComment bool `json:"pull_request_comment"` - PullRequestReview bool `json:"pull_request_review"` - PullRequestSync bool `json:"pull_request_sync"` - PullRequestReviewRequest bool `json:"pull_request_review_request"` - Wiki bool `json:"wiki"` - Repository bool `json:"repository"` - Release bool `json:"release"` - Package bool `json:"package"` -} - -// HookEvent represents events that will delivery hook. -type HookEvent struct { - PushOnly bool `json:"push_only"` - SendEverything bool `json:"send_everything"` - ChooseEvents bool `json:"choose_events"` - BranchFilter string `json:"branch_filter"` - - HookEvents `json:"events"` -} diff --git a/modules/webhook/type.go b/modules/webhook/type.go index aa4de45eb4..89c6a4bfe5 100644 --- a/modules/webhook/type.go +++ b/modules/webhook/type.go @@ -31,21 +31,51 @@ const ( HookEventRepository HookEventType = "repository" HookEventRelease HookEventType = "release" HookEventPackage HookEventType = "package" - HookEventSchedule HookEventType = "schedule" HookEventStatus HookEventType = "status" + // once a new event added here, please also added to AllEvents() function + + // FIXME: This event should be a group of pull_request_review_xxx events + HookEventPullRequestReview HookEventType = "pull_request_review" + // Actions event only + HookEventSchedule HookEventType = "schedule" + HookEventWorkflowRun HookEventType = "workflow_run" + HookEventWorkflowJob HookEventType = "workflow_job" ) +func AllEvents() []HookEventType { + return []HookEventType{ + HookEventCreate, + HookEventDelete, + HookEventFork, + HookEventPush, + HookEventIssues, + HookEventIssueAssign, + HookEventIssueLabel, + HookEventIssueMilestone, + HookEventIssueComment, + HookEventPullRequest, + HookEventPullRequestAssign, + HookEventPullRequestLabel, + HookEventPullRequestMilestone, + HookEventPullRequestComment, + HookEventPullRequestReviewApproved, + HookEventPullRequestReviewRejected, + HookEventPullRequestReviewComment, + HookEventPullRequestSync, + HookEventPullRequestReviewRequest, + HookEventWiki, + HookEventRepository, + HookEventRelease, + HookEventPackage, + HookEventStatus, + HookEventWorkflowRun, + HookEventWorkflowJob, + } +} + // Event returns the HookEventType as an event string func (h HookEventType) Event() string { switch h { - case HookEventCreate: - return "create" - case HookEventDelete: - return "delete" - case HookEventFork: - return "fork" - case HookEventPush: - return "push" case HookEventIssues, HookEventIssueAssign, HookEventIssueLabel, HookEventIssueMilestone: return "issues" case HookEventPullRequest, HookEventPullRequestAssign, HookEventPullRequestLabel, HookEventPullRequestMilestone, @@ -59,14 +89,9 @@ func (h HookEventType) Event() string { return "pull_request_rejected" case HookEventPullRequestReviewComment: return "pull_request_comment" - case HookEventWiki: - return "wiki" - case HookEventRepository: - return "repository" - case HookEventRelease: - return "release" + default: + return string(h) } - return "" } func (h HookEventType) IsPullRequest() bool { diff --git a/modules/zstd/zstd_test.go b/modules/zstd/zstd_test.go index c3ca8e78f7..7fd30484ca 100644 --- a/modules/zstd/zstd_test.go +++ b/modules/zstd/zstd_test.go @@ -16,7 +16,7 @@ import ( ) func TestWriterReader(t *testing.T) { - testData := prepareTestData(t, 20_000_000) + testData := prepareTestData(t, 1_000_000) result := bytes.NewBuffer(nil) @@ -64,7 +64,7 @@ func TestWriterReader(t *testing.T) { } func TestSeekableWriterReader(t *testing.T) { - testData := prepareTestData(t, 20_000_000) + testData := prepareTestData(t, 2_000_000) result := bytes.NewBuffer(nil) @@ -109,7 +109,7 @@ func TestSeekableWriterReader(t *testing.T) { reader, err := NewSeekableReader(assertReader) require.NoError(t, err) - _, err = reader.Seek(10_000_000, io.SeekStart) + _, err = reader.Seek(1_000_000, io.SeekStart) require.NoError(t, err) data := make([]byte, 1000) @@ -117,7 +117,7 @@ func TestSeekableWriterReader(t *testing.T) { require.NoError(t, err) require.NoError(t, reader.Close()) - assert.Equal(t, testData[10_000_000:10_000_000+1000], data) + assert.Equal(t, testData[1_000_000:1_000_000+1000], data) // Should seek 3 times, // the first two times are for getting the index, |