diff options
Diffstat (limited to 'modules')
567 files changed, 11304 insertions, 8412 deletions
diff --git a/modules/actions/artifacts.go b/modules/actions/artifacts.go new file mode 100644 index 0000000000..d28726e899 --- /dev/null +++ b/modules/actions/artifacts.go @@ -0,0 +1,48 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "net/http" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/services/context" +) + +// Artifacts using the v4 backend are stored as a single combined zip file per artifact on the backend +// The v4 backend ensures ContentEncoding is set to "application/zip", which is not the case for the old backend +func IsArtifactV4(art *actions_model.ActionArtifact) bool { + return art.ArtifactName+".zip" == art.ArtifactPath && art.ContentEncoding == "application/zip" +} + +func DownloadArtifactV4ServeDirectOnly(ctx *context.Base, art *actions_model.ActionArtifact) (bool, error) { + if setting.Actions.ArtifactStorage.ServeDirect() { + u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath, ctx.Req.Method, nil) + if u != nil && err == nil { + ctx.Redirect(u.String(), http.StatusFound) + return true, nil + } + } + return false, nil +} + +func DownloadArtifactV4Fallback(ctx *context.Base, art *actions_model.ActionArtifact) error { + f, err := storage.ActionsArtifacts.Open(art.StoragePath) + if err != nil { + return err + } + defer f.Close() + http.ServeContent(ctx.Resp, ctx.Req, art.ArtifactName+".zip", art.CreatedUnix.AsLocalTime(), f) + return nil +} + +func DownloadArtifactV4(ctx *context.Base, art *actions_model.ActionArtifact) error { + ok, err := DownloadArtifactV4ServeDirectOnly(ctx, art) + if ok || err != nil { + return err + } + return DownloadArtifactV4Fallback(ctx, art) +} diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 0d2b0dd919..27bcafa649 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -6,6 +6,7 @@ package actions import ( "bytes" "io" + "slices" "strings" "code.gitea.io/gitea/modules/git" @@ -43,21 +44,23 @@ func IsWorkflow(path string) bool { return strings.HasPrefix(path, ".gitea/workflows") || strings.HasPrefix(path, ".github/workflows") } -func ListWorkflows(commit *git.Commit) (git.Entries, error) { - tree, err := commit.SubTree(".gitea/workflows") +func ListWorkflows(commit *git.Commit) (string, git.Entries, error) { + rpath := ".gitea/workflows" + tree, err := commit.SubTree(rpath) if _, ok := err.(git.ErrNotExist); ok { - tree, err = commit.SubTree(".github/workflows") + rpath = ".github/workflows" + tree, err = commit.SubTree(rpath) } if _, ok := err.(git.ErrNotExist); ok { - return nil, nil + return "", nil, nil } if err != nil { - return nil, err + return "", nil, err } entries, err := tree.ListEntriesRecursiveFast() if err != nil { - return nil, err + return "", nil, err } ret := make(git.Entries, 0, len(entries)) @@ -66,7 +69,7 @@ func ListWorkflows(commit *git.Commit) (git.Entries, error) { ret = append(ret, entry) } } - return ret, nil + return rpath, ret, nil } func GetContentFromEntry(entry *git.TreeEntry) ([]byte, error) { @@ -102,7 +105,7 @@ func DetectWorkflows( payload api.Payloader, detectSchedule bool, ) ([]*DetectedWorkflow, []*DetectedWorkflow, error) { - entries, err := ListWorkflows(commit) + _, entries, err := ListWorkflows(commit) if err != nil { return nil, nil, err } @@ -147,7 +150,7 @@ func DetectWorkflows( } func DetectScheduledWorkflows(gitRepo *git.Repository, commit *git.Commit) ([]*DetectedWorkflow, error) { - entries, err := ListWorkflows(commit) + _, entries, err := ListWorkflows(commit) if err != nil { return nil, err } @@ -243,6 +246,10 @@ func detectMatched(gitRepo *git.Repository, commit *git.Commit, triggedEvent web webhook_module.HookEventPackage: return matchPackageEvent(payload.(*api.PackagePayload), evt) + case // workflow_run + webhook_module.HookEventWorkflowRun: + return matchWorkflowRunEvent(payload.(*api.WorkflowRunPayload), evt) + default: log.Warn("unsupported event %q", triggedEvent) return false @@ -311,6 +318,10 @@ func matchPushEvent(commit *git.Commit, pushPayload *api.PushPayload, evt *jobpa matchTimes++ } case "paths": + if refName.IsTag() { + matchTimes++ + break + } filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err) @@ -324,6 +335,10 @@ func matchPushEvent(commit *git.Commit, pushPayload *api.PushPayload, evt *jobpa } } case "paths-ignore": + if refName.IsTag() { + matchTimes++ + break + } filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err) @@ -463,7 +478,7 @@ func matchPullRequestEvent(gitRepo *git.Repository, commit *git.Commit, prPayloa matchTimes++ } case "paths": - filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.Base.Ref) + filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err) } else { @@ -476,7 +491,7 @@ func matchPullRequestEvent(gitRepo *git.Repository, commit *git.Commit, prPayloa } } case "paths-ignore": - filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.Base.Ref) + filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err) } else { @@ -554,21 +569,12 @@ func matchPullRequestReviewEvent(prPayload *api.PullRequestPayload, evt *jobpars actions = append(actions, "submitted", "edited") } - matched := false for _, val := range vals { - for _, action := range actions { - if glob.MustCompile(val, '/').Match(action) { - matched = true - break - } - } - if matched { + if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) { + matchTimes++ break } } - if matched { - matchTimes++ - } default: log.Warn("pull request review event unsupported condition %q", cond) } @@ -603,21 +609,12 @@ func matchPullRequestReviewCommentEvent(prPayload *api.PullRequestPayload, evt * actions = append(actions, "created", "edited") } - matched := false for _, val := range vals { - for _, action := range actions { - if glob.MustCompile(val, '/').Match(action) { - matched = true - break - } - } - if matched { + if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) { + matchTimes++ break } } - if matched { - matchTimes++ - } default: log.Warn("pull request review comment event unsupported condition %q", cond) } @@ -698,3 +695,53 @@ func matchPackageEvent(payload *api.PackagePayload, evt *jobparser.Event) bool { } return matchTimes == len(evt.Acts()) } + +func matchWorkflowRunEvent(payload *api.WorkflowRunPayload, evt *jobparser.Event) bool { + // with no special filter parameters + if len(evt.Acts()) == 0 { + return true + } + + matchTimes := 0 + // all acts conditions should be satisfied + for cond, vals := range evt.Acts() { + switch cond { + case "types": + action := payload.Action + for _, val := range vals { + if glob.MustCompile(val, '/').Match(action) { + matchTimes++ + break + } + } + case "workflows": + workflow := payload.Workflow + patterns, err := workflowpattern.CompilePatterns(vals...) + if err != nil { + break + } + if !workflowpattern.Skip(patterns, []string{workflow.Name}, &workflowpattern.EmptyTraceWriter{}) { + matchTimes++ + } + case "branches": + patterns, err := workflowpattern.CompilePatterns(vals...) + if err != nil { + break + } + if !workflowpattern.Skip(patterns, []string{payload.WorkflowRun.HeadBranch}, &workflowpattern.EmptyTraceWriter{}) { + matchTimes++ + } + case "branches-ignore": + patterns, err := workflowpattern.CompilePatterns(vals...) + if err != nil { + break + } + if !workflowpattern.Filter(patterns, []string{payload.WorkflowRun.HeadBranch}, &workflowpattern.EmptyTraceWriter{}) { + matchTimes++ + } + default: + log.Warn("workflow run event unsupported condition %q", cond) + } + } + return matchTimes == len(evt.Acts()) +} diff --git a/modules/actions/workflows_test.go b/modules/actions/workflows_test.go index c8e1e553fe..e23431651d 100644 --- a/modules/actions/workflows_test.go +++ b/modules/actions/workflows_test.go @@ -125,6 +125,24 @@ func TestDetectMatched(t *testing.T) { yamlOn: "on: schedule", expected: true, }, + { + desc: "push to tag matches workflow with paths condition (should skip paths check)", + triggedEvent: webhook_module.HookEventPush, + payload: &api.PushPayload{ + Ref: "refs/tags/v1.0.0", + Before: "0000000", + Commits: []*api.PayloadCommit{ + { + ID: "abcdef123456", + Added: []string{"src/main.go"}, + Message: "Release v1.0.0", + }, + }, + }, + commit: nil, + yamlOn: "on:\n push:\n paths:\n - src/**", + expected: true, + }, } for _, tc := range testCases { diff --git a/modules/analyze/vendor_test.go b/modules/analyze/vendor_test.go index aafd3c431b..02a51d4c8f 100644 --- a/modules/analyze/vendor_test.go +++ b/modules/analyze/vendor_test.go @@ -3,7 +3,11 @@ package analyze -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestIsVendor(t *testing.T) { tests := []struct { @@ -33,9 +37,8 @@ func TestIsVendor(t *testing.T) { } for _, tt := range tests { t.Run(tt.path, func(t *testing.T) { - if got := IsVendor(tt.path); got != tt.want { - t.Errorf("IsVendor() = %v, want %v", got, tt.want) - } + got := IsVendor(tt.path) + assert.Equal(t, tt.want, got) }) } } diff --git a/modules/assetfs/embed.go b/modules/assetfs/embed.go new file mode 100644 index 0000000000..95176372d1 --- /dev/null +++ b/modules/assetfs/embed.go @@ -0,0 +1,375 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "bytes" + "compress/gzip" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" +) + +type EmbeddedFile interface { + io.ReadSeeker + fs.ReadDirFile + ReadDir(n int) ([]fs.DirEntry, error) +} + +type EmbeddedFileInfo interface { + fs.FileInfo + fs.DirEntry + GetGzipContent() ([]byte, bool) +} + +type decompressor interface { + io.Reader + Close() error + Reset(io.Reader) error +} + +type embeddedFileInfo struct { + fs *embeddedFS + fullName string + data []byte + + BaseName string `json:"n"` + OriginSize int64 `json:"s,omitempty"` + DataBegin int64 `json:"b,omitempty"` + DataLen int64 `json:"l,omitempty"` + Children []*embeddedFileInfo `json:"c,omitempty"` +} + +func (fi *embeddedFileInfo) GetGzipContent() ([]byte, bool) { + // when generating the bindata, if the compressed data equals or is larger than the original data, we store the original data + if fi.DataLen == fi.OriginSize { + return nil, false + } + return fi.data, true +} + +type EmbeddedFileBase struct { + info *embeddedFileInfo + dataReader io.ReadSeeker + seekPos int64 +} + +func (f *EmbeddedFileBase) ReadDir(n int) ([]fs.DirEntry, error) { + // this method is used to satisfy the "func (f ioFile) ReadDir(...)" in httpfs + l, err := f.info.fs.ReadDir(f.info.fullName) + if err != nil { + return nil, err + } + if n < 0 || n > len(l) { + return l, nil + } + return l[:n], nil +} + +type EmbeddedOriginFile struct { + EmbeddedFileBase +} + +type EmbeddedCompressedFile struct { + EmbeddedFileBase + decompressor decompressor + decompressorPos int64 +} + +type embeddedFS struct { + meta func() *EmbeddedMeta + + files map[string]*embeddedFileInfo + filesMu sync.RWMutex + + data []byte +} + +type EmbeddedMeta struct { + Root *embeddedFileInfo +} + +func NewEmbeddedFS(data []byte) fs.ReadDirFS { + efs := &embeddedFS{data: data, files: make(map[string]*embeddedFileInfo)} + efs.meta = sync.OnceValue(func() *EmbeddedMeta { + var meta EmbeddedMeta + p := bytes.LastIndexByte(data, '\n') + if p < 0 { + return &meta + } + if err := json.Unmarshal(data[p+1:], &meta); err != nil { + panic("embedded file is not valid") + } + return &meta + }) + return efs +} + +var _ fs.ReadDirFS = (*embeddedFS)(nil) + +func (e *embeddedFS) ReadDir(name string) (l []fs.DirEntry, err error) { + fi, err := e.getFileInfo(name) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fs.ErrNotExist + } + l = make([]fs.DirEntry, len(fi.Children)) + for i, child := range fi.Children { + l[i], err = e.getFileInfo(name + "/" + child.BaseName) + if err != nil { + return nil, err + } + } + return l, nil +} + +func (e *embeddedFS) getFileInfo(fullName string) (*embeddedFileInfo, error) { + // no need to do heavy "path.Clean()" because we don't want to support "foo/../bar" or absolute paths + fullName = strings.TrimPrefix(fullName, "./") + if fullName == "" { + fullName = "." + } + + e.filesMu.RLock() + fi := e.files[fullName] + e.filesMu.RUnlock() + if fi != nil { + return fi, nil + } + + fields := strings.Split(fullName, "/") + fi = e.meta().Root + if fullName != "." { + found := true + for _, field := range fields { + for _, child := range fi.Children { + if found = child.BaseName == field; found { + fi = child + break + } + } + if !found { + return nil, fs.ErrNotExist + } + } + } + + e.filesMu.Lock() + defer e.filesMu.Unlock() + if fi != nil { + fi.fs = e + fi.fullName = fullName + fi.data = e.data[fi.DataBegin : fi.DataBegin+fi.DataLen] + e.files[fullName] = fi // do not cache nil, otherwise keeping accessing random non-existing file will cause OOM + return fi, nil + } + return nil, fs.ErrNotExist +} + +func (e *embeddedFS) Open(name string) (fs.File, error) { + info, err := e.getFileInfo(name) + if err != nil { + return nil, err + } + base := EmbeddedFileBase{info: info} + base.dataReader = bytes.NewReader(base.info.data) + if info.DataLen != info.OriginSize { + decomp, err := gzip.NewReader(base.dataReader) + if err != nil { + return nil, err + } + return &EmbeddedCompressedFile{EmbeddedFileBase: base, decompressor: decomp}, nil + } + return &EmbeddedOriginFile{base}, nil +} + +var ( + _ EmbeddedFileInfo = (*embeddedFileInfo)(nil) + _ EmbeddedFile = (*EmbeddedOriginFile)(nil) + _ EmbeddedFile = (*EmbeddedCompressedFile)(nil) +) + +func (f *EmbeddedOriginFile) Read(p []byte) (n int, err error) { + return f.dataReader.Read(p) +} + +func (f *EmbeddedCompressedFile) Read(p []byte) (n int, err error) { + if f.decompressorPos > f.seekPos { + if err = f.decompressor.Reset(bytes.NewReader(f.info.data)); err != nil { + return 0, err + } + f.decompressorPos = 0 + } + if f.decompressorPos < f.seekPos { + if _, err = io.CopyN(io.Discard, f.decompressor, f.seekPos-f.decompressorPos); err != nil { + return 0, err + } + f.decompressorPos = f.seekPos + } + n, err = f.decompressor.Read(p) + f.decompressorPos += int64(n) + f.seekPos = f.decompressorPos + return n, err +} + +func (f *EmbeddedFileBase) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + f.seekPos = offset + case io.SeekCurrent: + f.seekPos += offset + case io.SeekEnd: + f.seekPos = f.info.OriginSize + offset + } + return f.seekPos, nil +} + +func (f *EmbeddedFileBase) Stat() (fs.FileInfo, error) { + return f.info, nil +} + +func (f *EmbeddedOriginFile) Close() error { + return nil +} + +func (f *EmbeddedCompressedFile) Close() error { + return f.decompressor.Close() +} + +func (fi *embeddedFileInfo) Name() string { + return fi.BaseName +} + +func (fi *embeddedFileInfo) Size() int64 { + return fi.OriginSize +} + +func (fi *embeddedFileInfo) Mode() fs.FileMode { + return util.Iif(fi.IsDir(), fs.ModeDir|0o555, 0o444) +} + +func (fi *embeddedFileInfo) ModTime() time.Time { + return getExecutableModTime() +} + +func (fi *embeddedFileInfo) IsDir() bool { + return fi.Children != nil +} + +func (fi *embeddedFileInfo) Sys() any { + return nil +} + +func (fi *embeddedFileInfo) Type() fs.FileMode { + return util.Iif(fi.IsDir(), fs.ModeDir, 0) +} + +func (fi *embeddedFileInfo) Info() (fs.FileInfo, error) { + return fi, nil +} + +// getExecutableModTime returns the modification time of the executable file. +// In bindata, we can't use the ModTime of the files because we need to make the build reproducible +var getExecutableModTime = sync.OnceValue(func() (modTime time.Time) { + exePath, err := os.Executable() + if err != nil { + return modTime + } + exePath, err = filepath.Abs(exePath) + if err != nil { + return modTime + } + exePath, err = filepath.EvalSymlinks(exePath) + if err != nil { + return modTime + } + st, err := os.Stat(exePath) + if err != nil { + return modTime + } + return st.ModTime() +}) + +func GenerateEmbedBindata(fsRootPath, outputFile string) error { + output, err := os.OpenFile(outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer output.Close() + + meta := &EmbeddedMeta{} + meta.Root = &embeddedFileInfo{} + var outputOffset int64 + var embedFiles func(parent *embeddedFileInfo, fsPath, embedPath string) error + embedFiles = func(parent *embeddedFileInfo, fsPath, embedPath string) error { + dirEntries, err := os.ReadDir(fsPath) + if err != nil { + return err + } + for _, dirEntry := range dirEntries { + if err != nil { + return err + } + if dirEntry.IsDir() { + child := &embeddedFileInfo{ + BaseName: dirEntry.Name(), + Children: []*embeddedFileInfo{}, // non-nil means it's a directory + } + parent.Children = append(parent.Children, child) + if err = embedFiles(child, filepath.Join(fsPath, dirEntry.Name()), path.Join(embedPath, dirEntry.Name())); err != nil { + return err + } + } else { + data, err := os.ReadFile(filepath.Join(fsPath, dirEntry.Name())) + if err != nil { + return err + } + var compressed bytes.Buffer + gz, _ := gzip.NewWriterLevel(&compressed, gzip.BestCompression) + if _, err = gz.Write(data); err != nil { + return err + } + if err = gz.Close(); err != nil { + return err + } + + // only use the compressed data if it is smaller than the original data + outputBytes := util.Iif(len(compressed.Bytes()) < len(data), compressed.Bytes(), data) + child := &embeddedFileInfo{ + BaseName: dirEntry.Name(), + OriginSize: int64(len(data)), + DataBegin: outputOffset, + DataLen: int64(len(outputBytes)), + } + if _, err = output.Write(outputBytes); err != nil { + return err + } + outputOffset += child.DataLen + parent.Children = append(parent.Children, child) + } + } + return nil + } + + if err = embedFiles(meta.Root, fsRootPath, ""); err != nil { + return err + } + jsonBuf, err := json.Marshal(meta) // can't use json.NewEncoder here because it writes extra EOL + if err != nil { + return err + } + _, _ = output.Write([]byte{'\n'}) + _, err = output.Write(jsonBuf) + return err +} diff --git a/modules/assetfs/embed_test.go b/modules/assetfs/embed_test.go new file mode 100644 index 0000000000..06598da4c4 --- /dev/null +++ b/modules/assetfs/embed_test.go @@ -0,0 +1,98 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "bytes" + "io/fs" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEmbed(t *testing.T) { + tmpDir := t.TempDir() + tmpDataDir := tmpDir + "/data" + _ = os.MkdirAll(tmpDataDir+"/foo/bar", 0o755) + _ = os.WriteFile(tmpDataDir+"/a.txt", []byte("a"), 0o644) + _ = os.WriteFile(tmpDataDir+"/foo/bar/b.txt", bytes.Repeat([]byte("a"), 1000), 0o644) + _ = os.WriteFile(tmpDataDir+"/foo/c.txt", []byte("c"), 0o644) + require.NoError(t, GenerateEmbedBindata(tmpDataDir, tmpDir+"/out.dat")) + + data, err := os.ReadFile(tmpDir + "/out.dat") + require.NoError(t, err) + efs := NewEmbeddedFS(data) + + // test a non-existing file + _, err = fs.ReadFile(efs, "not exist") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // test a normal file (no compression) + content, err := fs.ReadFile(efs, "a.txt") + require.NoError(t, err) + assert.Equal(t, "a", string(content)) + fi, err := fs.Stat(efs, "a.txt") + require.NoError(t, err) + _, ok := fi.(EmbeddedFileInfo).GetGzipContent() + assert.False(t, ok) + + // test a compressed file + content, err = fs.ReadFile(efs, "foo/bar/b.txt") + require.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte("a"), 1000), content) + fi, err = fs.Stat(efs, "foo/bar/b.txt") + require.NoError(t, err) + assert.False(t, fi.Mode().IsDir()) + assert.True(t, fi.Mode().IsRegular()) + gzipContent, ok := fi.(EmbeddedFileInfo).GetGzipContent() + assert.True(t, ok) + assert.Greater(t, len(gzipContent), 1) + assert.Less(t, len(gzipContent), 1000) + + // test list root directory + entries, err := fs.ReadDir(efs, ".") + require.NoError(t, err) + assert.Len(t, entries, 2) + assert.Equal(t, "a.txt", entries[0].Name()) + assert.False(t, entries[0].IsDir()) + + // test list subdirectory + entries, err = fs.ReadDir(efs, "foo") + require.NoError(t, err) + require.Len(t, entries, 2) + assert.Equal(t, "bar", entries[0].Name()) + assert.True(t, entries[0].IsDir()) + assert.Equal(t, "c.txt", entries[1].Name()) + assert.False(t, entries[1].IsDir()) + + // test directory mode + fi, err = fs.Stat(efs, "foo") + require.NoError(t, err) + assert.True(t, fi.IsDir()) + assert.True(t, fi.Mode().IsDir()) + assert.False(t, fi.Mode().IsRegular()) + + // test httpfs + hfs := http.FS(efs) + hf, err := hfs.Open("foo/bar/b.txt") + require.NoError(t, err) + hi, err := hf.Stat() + require.NoError(t, err) + fiEmbedded, ok := hi.(EmbeddedFileInfo) + require.True(t, ok) + gzipContent, ok = fiEmbedded.GetGzipContent() + assert.True(t, ok) + assert.Greater(t, len(gzipContent), 1) + assert.Less(t, len(gzipContent), 1000) + + // test httpfs directory listing + hf, err = hfs.Open("foo") + require.NoError(t, err) + dirs, err := hf.Readdir(1) + require.NoError(t, err) + assert.Len(t, dirs, 1) +} diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index 9678d23ad6..ce55475bd9 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -52,8 +52,8 @@ func Local(name, base string, sub ...string) *Layer { } // Bindata returns a new Layer with the given name, it serves files from the given bindata asset. -func Bindata(name string, fs http.FileSystem) *Layer { - return &Layer{name: name, fs: fs} +func Bindata(name string, fs fs.FS) *Layer { + return &Layer{name: name, fs: http.FS(fs)} } // LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers. @@ -103,7 +103,7 @@ func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) { } func shouldInclude(info fs.FileInfo, fileMode ...bool) bool { - if util.CommonSkip(info.Name()) { + if util.IsCommonHiddenFileName(info.Name()) { return false } if len(fileMode) == 0 { diff --git a/modules/assetfs/layered_test.go b/modules/assetfs/layered_test.go index 03a3ae0d7c..b549815ea5 100644 --- a/modules/assetfs/layered_test.go +++ b/modules/assetfs/layered_test.go @@ -52,7 +52,7 @@ func TestLayered(t *testing.T) { assert.NoError(t, err) bs, err := io.ReadAll(f) assert.NoError(t, err) - assert.EqualValues(t, "f1", string(bs)) + assert.Equal(t, "f1", string(bs)) _ = f.Close() assertRead := func(expected string, expectedErr error, elems ...string) { @@ -76,27 +76,27 @@ func TestLayered(t *testing.T) { files, err := assets.ListFiles(".", true) assert.NoError(t, err) - assert.EqualValues(t, []string{"f1", "f2", "fa"}, files) + assert.Equal(t, []string{"f1", "f2", "fa"}, files) files, err = assets.ListFiles(".", false) assert.NoError(t, err) - assert.EqualValues(t, []string{"d1", "d2", "da"}, files) + assert.Equal(t, []string{"d1", "d2", "da"}, files) files, err = assets.ListFiles(".") assert.NoError(t, err) - assert.EqualValues(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files) + assert.Equal(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files) files, err = assets.ListAllFiles(".", true) assert.NoError(t, err) - assert.EqualValues(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files) + assert.Equal(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files) files, err = assets.ListAllFiles(".", false) assert.NoError(t, err) - assert.EqualValues(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files) + assert.Equal(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files) files, err = assets.ListAllFiles(".") assert.NoError(t, err) - assert.EqualValues(t, []string{ + assert.Equal(t, []string{ "d1", "d1/f", "d2", "d2/f", "da", "da/f", "da/sub1", "da/sub2", @@ -104,6 +104,6 @@ func TestLayered(t *testing.T) { }, files) assert.Empty(t, assets.GetFileLayerName("no-such")) - assert.EqualValues(t, "l1", assets.GetFileLayerName("f1")) - assert.EqualValues(t, "l2", assets.GetFileLayerName("f2")) + assert.Equal(t, "l1", assets.GetFileLayerName("f1")) + assert.Equal(t, "l2", assets.GetFileLayerName("f2")) } diff --git a/modules/auth/httpauth/httpauth.go b/modules/auth/httpauth/httpauth.go new file mode 100644 index 0000000000..7f1f1ee152 --- /dev/null +++ b/modules/auth/httpauth/httpauth.go @@ -0,0 +1,47 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httpauth + +import ( + "encoding/base64" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +type BasicAuth struct { + Username, Password string +} + +type BearerToken struct { + Token string +} + +type ParsedAuthorizationHeader struct { + BasicAuth *BasicAuth + BearerToken *BearerToken +} + +func ParseAuthorizationHeader(header string) (ret ParsedAuthorizationHeader, _ bool) { + parts := strings.Fields(header) + if len(parts) != 2 { + return ret, false + } + if util.AsciiEqualFold(parts[0], "basic") { + s, err := base64.StdEncoding.DecodeString(parts[1]) + if err != nil { + return ret, false + } + u, p, ok := strings.Cut(string(s), ":") + if !ok { + return ret, false + } + ret.BasicAuth = &BasicAuth{Username: u, Password: p} + return ret, true + } else if util.AsciiEqualFold(parts[0], "token") || util.AsciiEqualFold(parts[0], "bearer") { + ret.BearerToken = &BearerToken{Token: parts[1]} + return ret, true + } + return ret, false +} diff --git a/modules/auth/httpauth/httpauth_test.go b/modules/auth/httpauth/httpauth_test.go new file mode 100644 index 0000000000..087b86917f --- /dev/null +++ b/modules/auth/httpauth/httpauth_test.go @@ -0,0 +1,43 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httpauth + +import ( + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseAuthorizationHeader(t *testing.T) { + type parsed = ParsedAuthorizationHeader + type basic = BasicAuth + type bearer = BearerToken + cases := []struct { + headerValue string + expected parsed + ok bool + }{ + {"", parsed{}, false}, + {"?", parsed{}, false}, + {"foo", parsed{}, false}, + {"any value", parsed{}, false}, + + {"Basic ?", parsed{}, false}, + {"Basic " + base64.StdEncoding.EncodeToString([]byte("foo")), parsed{}, false}, + {"Basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true}, + {"basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true}, + + {"token value", parsed{BearerToken: &bearer{"value"}}, true}, + {"Token value", parsed{BearerToken: &bearer{"value"}}, true}, + {"bearer value", parsed{BearerToken: &bearer{"value"}}, true}, + {"Bearer value", parsed{BearerToken: &bearer{"value"}}, true}, + {"Bearer wrong value", parsed{}, false}, + } + for _, c := range cases { + ret, ok := ParseAuthorizationHeader(c.headerValue) + assert.Equal(t, c.ok, ok, "header %q", c.headerValue) + assert.Equal(t, c.expected, ret, "header %q", c.headerValue) + } +} diff --git a/modules/auth/openid/discovery_cache_test.go b/modules/auth/openid/discovery_cache_test.go index 5a7f450937..f3d7dd226e 100644 --- a/modules/auth/openid/discovery_cache_test.go +++ b/modules/auth/openid/discovery_cache_test.go @@ -6,6 +6,9 @@ package openid import ( "testing" "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type testDiscoveredInfo struct{} @@ -23,27 +26,24 @@ func (s *testDiscoveredInfo) OpLocalID() string { } func TestTimedDiscoveryCache(t *testing.T) { - dc := newTimedDiscoveryCache(1 * time.Second) + ttl := 50 * time.Millisecond + dc := newTimedDiscoveryCache(ttl) // Put some initial values dc.Put("foo", &testDiscoveredInfo{}) // openid.opEndpoint: "a", openid.opLocalID: "b", openid.claimedID: "c"}) // Make sure we can retrieve them - if di := dc.Get("foo"); di == nil { - t.Errorf("Expected a result, got nil") - } else if di.OpEndpoint() != "opEndpoint" || di.OpLocalID() != "opLocalID" || di.ClaimedID() != "claimedID" { - t.Errorf("Expected opEndpoint opLocalID claimedID, got %v %v %v", di.OpEndpoint(), di.OpLocalID(), di.ClaimedID()) - } + di := dc.Get("foo") + require.NotNil(t, di) + assert.Equal(t, "opEndpoint", di.OpEndpoint()) + assert.Equal(t, "opLocalID", di.OpLocalID()) + assert.Equal(t, "claimedID", di.ClaimedID()) // Attempt to get a non-existent value - if di := dc.Get("bar"); di != nil { - t.Errorf("Expected nil, got %v", di) - } + assert.Nil(t, dc.Get("bar")) - // Sleep one second and try retrieve again - time.Sleep(1 * time.Second) + // Sleep for a while and try to retrieve again + time.Sleep(ttl * 3 / 2) - if di := dc.Get("foo"); di != nil { - t.Errorf("Expected a nil, got a result") - } + assert.Nil(t, dc.Get("foo")) } diff --git a/modules/auth/pam/pam_test.go b/modules/auth/pam/pam_test.go index 7265b5d0c1..d4ab058ec7 100644 --- a/modules/auth/pam/pam_test.go +++ b/modules/auth/pam/pam_test.go @@ -15,5 +15,5 @@ func TestPamAuth(t *testing.T) { result, err := Auth("gitea", "user1", "false-pwd") assert.Error(t, err) assert.EqualError(t, err, "Authentication failure") - assert.Len(t, result) + assert.Empty(t, result) } diff --git a/modules/auth/password/hash/common.go b/modules/auth/password/hash/common.go index 487c0738f4..d5e2c34314 100644 --- a/modules/auth/password/hash/common.go +++ b/modules/auth/password/hash/common.go @@ -18,7 +18,7 @@ func parseIntParam(value, param, algorithmName, config string, previousErr error return parsed, previousErr // <- Keep the previous error as this function should still return an error once everything has been checked if any call failed } -func parseUIntParam(value, param, algorithmName, config string, previousErr error) (uint64, error) { //nolint:unparam +func parseUIntParam(value, param, algorithmName, config string, previousErr error) (uint64, error) { //nolint:unparam // algorithmName is always argon2 parsed, err := strconv.ParseUint(value, 10, 64) if err != nil { log.Error("invalid integer for %s representation in %s hash spec %s", param, algorithmName, config) diff --git a/modules/auth/password/password.go b/modules/auth/password/password.go index c66b62937f..a1e101dd62 100644 --- a/modules/auth/password/password.go +++ b/modules/auth/password/password.go @@ -101,7 +101,7 @@ func Generate(n int) (string, error) { buffer := make([]byte, n) maxInt := big.NewInt(int64(len(validChars))) for { - for j := 0; j < n; j++ { + for j := range n { rnd, err := rand.Int(rand.Reader, maxInt) if err != nil { return "", err diff --git a/modules/auth/password/password_test.go b/modules/auth/password/password_test.go index 6c35dc86bd..0fea593c85 100644 --- a/modules/auth/password/password_test.go +++ b/modules/auth/password/password_test.go @@ -50,7 +50,7 @@ func TestComplexity_Generate(t *testing.T) { test := func(t *testing.T, modes []string) { testComplextity(modes) - for i := 0; i < maxCount; i++ { + for range maxCount { pwd, err := Generate(pwdLen) assert.NoError(t, err) assert.Len(t, pwd, pwdLen) diff --git a/modules/auth/password/pwn/pwn.go b/modules/auth/password/pwn/pwn.go index f77ce9f40b..99a6ca6cea 100644 --- a/modules/auth/password/pwn/pwn.go +++ b/modules/auth/password/pwn/pwn.go @@ -101,7 +101,7 @@ func (c *Client) CheckPassword(pw string, padding bool) (int, error) { } defer resp.Body.Close() - for _, pair := range strings.Split(string(body), "\n") { + for pair := range strings.SplitSeq(string(body), "\n") { parts := strings.Split(pair, ":") if len(parts) != 2 { continue diff --git a/modules/auth/password/pwn/pwn_test.go b/modules/auth/password/pwn/pwn_test.go index b3e7734c3f..ae03fabc57 100644 --- a/modules/auth/password/pwn/pwn_test.go +++ b/modules/auth/password/pwn/pwn_test.go @@ -4,46 +4,57 @@ package pwn import ( + "errors" + "io" "net/http" + "strings" "testing" - "time" - "github.com/h2non/gock" "github.com/stretchr/testify/assert" ) -var client = New(WithHTTP(&http.Client{ - Timeout: time.Second * 2, -})) +type mockTransport struct{} + +func (mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Host != "api.pwnedpasswords.com" { + return nil, errors.New("unsupported host") + } + respMap := map[string]string{ + "/range/5c1d8": "EAF2F254732680E8AC339B84F3266ECCBB5:1\r\nFC446EB88938834178CB9322C1EE273C2A7:2", + "/range/ba189": "FD4CB34F0378BCB15D23F6FFD28F0775C9E:3\r\nFDF342FCD8C3611DAE4D76E8A992A3E4169:4", + "/range/a1733": "C4CE0F1F0062B27B9E2F41AF0C08218017C:1\r\nFC446EB88938834178CB9322C1EE273C2A7:2\r\nFE81480327C992FE62065A827429DD1318B:0", + "/range/5617b": "FD4CB34F0378BCB15D23F6FFD28F0775C9E:3\r\nFDF342FCD8C3611DAE4D76E8A992A3E4169:4\r\nFE81480327C992FE62065A827429DD1318B:0", + "/range/79082": "FDF342FCD8C3611DAE4D76E8A992A3E4169:4\r\nFE81480327C992FE62065A827429DD1318B:0\r\nAFEF386F56EB0B4BE314E07696E5E6E6536:0", + } + if resp, ok := respMap[req.URL.Path]; ok { + return &http.Response{Request: req, Body: io.NopCloser(strings.NewReader(resp))}, nil + } + return nil, errors.New("unsupported path") +} func TestPassword(t *testing.T) { - defer gock.Off() + client := New(WithHTTP(&http.Client{Transport: mockTransport{}})) count, err := client.CheckPassword("", false) assert.ErrorIs(t, err, ErrEmptyPassword, "blank input should return ErrEmptyPassword") assert.Equal(t, -1, count) - gock.New("https://api.pwnedpasswords.com").Get("/range/5c1d8").Times(1).Reply(200).BodyString("EAF2F254732680E8AC339B84F3266ECCBB5:1\r\nFC446EB88938834178CB9322C1EE273C2A7:2") count, err = client.CheckPassword("pwned", false) assert.NoError(t, err) assert.Equal(t, 1, count) - gock.New("https://api.pwnedpasswords.com").Get("/range/ba189").Times(1).Reply(200).BodyString("FD4CB34F0378BCB15D23F6FFD28F0775C9E:3\r\nFDF342FCD8C3611DAE4D76E8A992A3E4169:4") count, err = client.CheckPassword("notpwned", false) assert.NoError(t, err) assert.Equal(t, 0, count) - gock.New("https://api.pwnedpasswords.com").Get("/range/a1733").Times(1).Reply(200).BodyString("C4CE0F1F0062B27B9E2F41AF0C08218017C:1\r\nFC446EB88938834178CB9322C1EE273C2A7:2\r\nFE81480327C992FE62065A827429DD1318B:0") count, err = client.CheckPassword("paddedpwned", true) assert.NoError(t, err) assert.Equal(t, 1, count) - gock.New("https://api.pwnedpasswords.com").Get("/range/5617b").Times(1).Reply(200).BodyString("FD4CB34F0378BCB15D23F6FFD28F0775C9E:3\r\nFDF342FCD8C3611DAE4D76E8A992A3E4169:4\r\nFE81480327C992FE62065A827429DD1318B:0") count, err = client.CheckPassword("paddednotpwned", true) assert.NoError(t, err) assert.Equal(t, 0, count) - gock.New("https://api.pwnedpasswords.com").Get("/range/79082").Times(1).Reply(200).BodyString("FDF342FCD8C3611DAE4D76E8A992A3E4169:4\r\nFE81480327C992FE62065A827429DD1318B:0\r\nAFEF386F56EB0B4BE314E07696E5E6E6536:0") count, err = client.CheckPassword("paddednotpwnedzero", true) assert.NoError(t, err) assert.Equal(t, 0, count) diff --git a/modules/avatar/avatar_test.go b/modules/avatar/avatar_test.go index a721c77868..a5a1a7c1b0 100644 --- a/modules/avatar/avatar_test.go +++ b/modules/avatar/avatar_test.go @@ -94,8 +94,8 @@ func Test_ProcessAvatarImage(t *testing.T) { assert.NotEqual(t, origin, result) decoded, err := png.Decode(bytes.NewReader(result)) assert.NoError(t, err) - assert.EqualValues(t, scaledSize, decoded.Bounds().Max.X) - assert.EqualValues(t, scaledSize, decoded.Bounds().Max.Y) + assert.Equal(t, scaledSize, decoded.Bounds().Max.X) + assert.Equal(t, scaledSize, decoded.Bounds().Max.Y) // if origin image is smaller than the default size, use the origin image origin = newImgData(1) diff --git a/modules/avatar/hash_test.go b/modules/avatar/hash_test.go index 1b8249c696..c518144b47 100644 --- a/modules/avatar/hash_test.go +++ b/modules/avatar/hash_test.go @@ -19,8 +19,8 @@ func Test_HashAvatar(t *testing.T) { var buff bytes.Buffer png.Encode(&buff, myImage) - assert.EqualValues(t, "9ddb5bac41d57e72aa876321d0c09d71090c05f94bc625303801be2f3240d2cb", avatar.HashAvatar(1, buff.Bytes())) - assert.EqualValues(t, "9a5d44e5d637b9582a976676e8f3de1dccd877c2fe3e66ca3fab1629f2f47609", avatar.HashAvatar(8, buff.Bytes())) - assert.EqualValues(t, "ed7399158672088770de6f5211ce15528ebd675e92fc4fc060c025f4b2794ccb", avatar.HashAvatar(1024, buff.Bytes())) - assert.EqualValues(t, "161178642c7d59eb25a61dddced5e6b66eae1c70880d5f148b1b497b767e72d9", avatar.HashAvatar(1024, []byte{})) + assert.Equal(t, "9ddb5bac41d57e72aa876321d0c09d71090c05f94bc625303801be2f3240d2cb", avatar.HashAvatar(1, buff.Bytes())) + assert.Equal(t, "9a5d44e5d637b9582a976676e8f3de1dccd877c2fe3e66ca3fab1629f2f47609", avatar.HashAvatar(8, buff.Bytes())) + assert.Equal(t, "ed7399158672088770de6f5211ce15528ebd675e92fc4fc060c025f4b2794ccb", avatar.HashAvatar(1024, buff.Bytes())) + assert.Equal(t, "161178642c7d59eb25a61dddced5e6b66eae1c70880d5f148b1b497b767e72d9", avatar.HashAvatar(1024, []byte{})) } diff --git a/modules/avatar/identicon/block.go b/modules/avatar/identicon/block.go index cb1803a231..fc8ce90212 100644 --- a/modules/avatar/identicon/block.go +++ b/modules/avatar/identicon/block.go @@ -24,8 +24,8 @@ func drawBlock(img *image.Paletted, x, y, size, angle int, points []int) { rotate(points, m, m, angle) } - for i := 0; i < size; i++ { - for j := 0; j < size; j++ { + for i := range size { + for j := range size { if pointInPolygon(i, j, points) { img.SetColorIndex(x+i, y+j, 1) } diff --git a/modules/avatar/identicon/identicon.go b/modules/avatar/identicon/identicon.go index 63926d5f19..ee92416a53 100644 --- a/modules/avatar/identicon/identicon.go +++ b/modules/avatar/identicon/identicon.go @@ -8,6 +8,7 @@ package identicon import ( "crypto/sha256" + "errors" "fmt" "image" "image/color" @@ -29,7 +30,7 @@ type Identicon struct { // fore all possible foreground colors. only one foreground color will be picked randomly for one image func New(size int, back color.Color, fore ...color.Color) (*Identicon, error) { if len(fore) == 0 { - return nil, fmt.Errorf("foreground is not set") + return nil, errors.New("foreground is not set") } if size < minImageSize { @@ -133,7 +134,7 @@ func drawBlocks(p *image.Paletted, size int, c, b1, b2 blockFunc, b1Angle, b2Ang // then we make it left-right mirror, so we didn't draw 3/6/9 before for x := 0; x < size/2; x++ { - for y := 0; y < size; y++ { + for y := range size { p.SetColorIndex(size-x, y, p.ColorIndexAt(x, y)) } } diff --git a/modules/badge/badge.go b/modules/badge/badge.go index b30d0b4729..d2e9bd9d1b 100644 --- a/modules/badge/badge.go +++ b/modules/badge/badge.go @@ -4,6 +4,10 @@ package badge import ( + "strings" + "sync" + "unicode" + actions_model "code.gitea.io/gitea/models/actions" ) @@ -11,94 +15,115 @@ import ( // We use 10x scale to calculate more precisely // Then scale down to normal size in tmpl file -type Label struct { - text string - width int -} - -func (l Label) Text() string { - return l.text -} - -func (l Label) Width() int { - return l.width -} - -func (l Label) TextLength() int { - return int(float64(l.width-defaultOffset) * 9.5) -} - -func (l Label) X() int { - return l.width*5 + 10 -} - -type Message struct { +type Text struct { text string width int x int } -func (m Message) Text() string { - return m.text +func (t Text) Text() string { + return t.text } -func (m Message) Width() int { - return m.width +func (t Text) Width() int { + return t.width } -func (m Message) X() int { - return m.x +func (t Text) X() int { + return t.x } -func (m Message) TextLength() int { - return int(float64(m.width-defaultOffset) * 9.5) +func (t Text) TextLength() int { + return int(float64(t.width-defaultOffset) * 10) } type Badge struct { - Color string - FontSize int - Label Label - Message Message + IDPrefix string + FontFamily string + Color string + FontSize int + Label Text + Message Text } func (b Badge) Width() int { return b.Label.width + b.Message.width } +// Style follows https://shields.io/badges const ( - defaultOffset = 9 - defaultFontSize = 11 - DefaultColor = "#9f9f9f" // Grey - defaultFontWidth = 7 // approximate speculation + StyleFlat = "flat" + StyleFlatSquare = "flat-square" ) -var StatusColorMap = map[actions_model.Status]string{ - actions_model.StatusSuccess: "#4c1", // Green - actions_model.StatusSkipped: "#dfb317", // Yellow - actions_model.StatusUnknown: "#97ca00", // Light Green - actions_model.StatusFailure: "#e05d44", // Red - actions_model.StatusCancelled: "#fe7d37", // Orange - actions_model.StatusWaiting: "#dfb317", // Yellow - actions_model.StatusRunning: "#dfb317", // Yellow - actions_model.StatusBlocked: "#dfb317", // Yellow -} +const ( + defaultOffset = 10 + defaultFontSize = 11 + DefaultColor = "#9f9f9f" // Grey + DefaultFontFamily = "DejaVu Sans,Verdana,Geneva,sans-serif" + DefaultStyle = StyleFlat +) + +var GlobalVars = sync.OnceValue(func() (ret struct { + StatusColorMap map[actions_model.Status]string + DejaVuGlyphWidthData map[rune]uint8 + AllStyles []string +}, +) { + ret.StatusColorMap = map[actions_model.Status]string{ + actions_model.StatusSuccess: "#4c1", // Green + actions_model.StatusSkipped: "#dfb317", // Yellow + actions_model.StatusUnknown: "#97ca00", // Light Green + actions_model.StatusFailure: "#e05d44", // Red + actions_model.StatusCancelled: "#fe7d37", // Orange + actions_model.StatusWaiting: "#dfb317", // Yellow + actions_model.StatusRunning: "#dfb317", // Yellow + actions_model.StatusBlocked: "#dfb317", // Yellow + } + ret.DejaVuGlyphWidthData = dejaVuGlyphWidthDataFunc() + ret.AllStyles = []string{StyleFlat, StyleFlatSquare} + return ret +}) // GenerateBadge generates badge with given template func GenerateBadge(label, message, color string) Badge { - lw := defaultFontWidth*len(label) + defaultOffset - mw := defaultFontWidth*len(message) + defaultOffset - x := lw*10 + mw*5 - 10 + lw := calculateTextWidth(label) + defaultOffset + mw := calculateTextWidth(message) + defaultOffset + + lx := lw * 5 + mx := lw*10 + mw*5 - 10 return Badge{ - Label: Label{ + FontFamily: DefaultFontFamily, + Label: Text{ text: label, width: lw, + x: lx, }, - Message: Message{ + Message: Text{ text: message, width: mw, - x: x, + x: mx, }, FontSize: defaultFontSize * 10, Color: color, } } + +func calculateTextWidth(text string) int { + width := 0 + widthData := GlobalVars().DejaVuGlyphWidthData + for _, char := range strings.TrimSpace(text) { + charWidth, ok := widthData[char] + if !ok { + // use the width of 'm' in case of missing glyph width data for a printable character + if unicode.IsPrint(char) { + charWidth = widthData['m'] + } else { + charWidth = 0 + } + } + width += int(charWidth) + } + + return width +} diff --git a/modules/badge/badge_glyph_width.go b/modules/badge/badge_glyph_width.go new file mode 100644 index 0000000000..0d950c5a70 --- /dev/null +++ b/modules/badge/badge_glyph_width.go @@ -0,0 +1,206 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package badge + +// DejaVuGlyphWidthData is generated by `sfnt.Face.GlyphAdvance(nil, <rune>, 11, font.HintingNone)` with DejaVu Sans +// v2.37 (https://github.com/dejavu-fonts/dejavu-fonts/releases/download/version_2_37/dejavu-sans-ttf-2.37.zip). +// +// Fonts defined in "DefaultFontFamily" all have similar widths (including "DejaVu Sans"), +// and these widths are fixed and don't seem to change. +// +// A devtest page "/devtest/badge-actions-svg" could be used to check the rendered images. + +func dejaVuGlyphWidthDataFunc() map[rune]uint8 { + return map[rune]uint8{ + 32: 3, + 33: 4, + 34: 5, + 35: 9, + 36: 7, + 37: 10, + 38: 9, + 39: 3, + 40: 4, + 41: 4, + 42: 6, + 43: 9, + 44: 3, + 45: 4, + 46: 3, + 47: 4, + 48: 7, + 49: 7, + 50: 7, + 51: 7, + 52: 7, + 53: 7, + 54: 7, + 55: 7, + 56: 7, + 57: 7, + 58: 4, + 59: 4, + 60: 9, + 61: 9, + 62: 9, + 63: 6, + 64: 11, + 65: 8, + 66: 8, + 67: 8, + 68: 8, + 69: 7, + 70: 6, + 71: 9, + 72: 8, + 73: 3, + 74: 3, + 75: 7, + 76: 6, + 77: 9, + 78: 8, + 79: 9, + 80: 7, + 81: 9, + 82: 8, + 83: 7, + 84: 7, + 85: 8, + 86: 8, + 87: 11, + 88: 8, + 89: 7, + 90: 8, + 91: 4, + 92: 4, + 93: 4, + 94: 9, + 95: 6, + 96: 6, + 97: 7, + 98: 7, + 99: 6, + 100: 7, + 101: 7, + 102: 4, + 103: 7, + 104: 7, + 105: 3, + 106: 3, + 107: 6, + 108: 3, + 109: 11, + 110: 7, + 111: 7, + 112: 7, + 113: 7, + 114: 5, + 115: 6, + 116: 4, + 117: 7, + 118: 7, + 119: 9, + 120: 7, + 121: 7, + 122: 6, + 123: 7, + 124: 4, + 125: 7, + 126: 9, + 161: 4, + 162: 7, + 163: 7, + 164: 7, + 165: 7, + 166: 4, + 167: 6, + 168: 6, + 169: 11, + 170: 5, + 171: 7, + 172: 9, + 174: 11, + 175: 6, + 176: 6, + 177: 9, + 178: 4, + 179: 4, + 180: 6, + 181: 7, + 182: 7, + 183: 3, + 184: 6, + 185: 4, + 186: 5, + 187: 7, + 188: 11, + 189: 11, + 190: 11, + 191: 6, + 192: 8, + 193: 8, + 194: 8, + 195: 8, + 196: 8, + 197: 8, + 198: 11, + 199: 8, + 200: 7, + 201: 7, + 202: 7, + 203: 7, + 204: 3, + 205: 3, + 206: 3, + 207: 3, + 208: 9, + 209: 8, + 210: 9, + 211: 9, + 212: 9, + 213: 9, + 214: 9, + 215: 9, + 216: 9, + 217: 8, + 218: 8, + 219: 8, + 220: 8, + 221: 7, + 222: 7, + 223: 7, + 224: 7, + 225: 7, + 226: 7, + 227: 7, + 228: 7, + 229: 7, + 230: 11, + 231: 6, + 232: 7, + 233: 7, + 234: 7, + 235: 7, + 236: 3, + 237: 3, + 238: 3, + 239: 3, + 240: 7, + 241: 7, + 242: 7, + 243: 7, + 244: 7, + 245: 7, + 246: 7, + 247: 9, + 248: 7, + 249: 7, + 250: 7, + 251: 7, + 252: 7, + 253: 7, + 254: 7, + 255: 7, + } +} diff --git a/modules/base/tool.go b/modules/base/tool.go index 2303e64a68..ed94575e74 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -8,19 +8,14 @@ import ( "crypto/sha1" "crypto/sha256" "crypto/subtle" - "encoding/base64" "encoding/hex" - "errors" "fmt" "hash" "strconv" - "strings" "time" - "unicode/utf8" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "github.com/dustin/go-humanize" ) @@ -35,20 +30,7 @@ func EncodeSha256(str string) string { // ShortSha is basically just truncating. // It is DEPRECATED and will be removed in the future. func ShortSha(sha1 string) string { - return TruncateString(sha1, 10) -} - -// BasicAuthDecode decode basic auth string -func BasicAuthDecode(encoded string) (string, string, error) { - s, err := base64.StdEncoding.DecodeString(encoded) - if err != nil { - return "", "", err - } - - if username, password, ok := strings.Cut(string(s), ":"); ok { - return username, password, nil - } - return "", "", errors.New("invalid basic authentication") + return util.TruncateRunes(sha1, 10) } // VerifyTimeLimitCode verify time limit code @@ -64,10 +46,7 @@ func VerifyTimeLimitCode(now time.Time, data string, minutes int, code string) b // check code retCode := CreateTimeLimitCode(data, aliveTime, startTimeStr, nil) if subtle.ConstantTimeCompare([]byte(retCode), []byte(code)) != 1 { - retCode = CreateTimeLimitCode(data, aliveTime, startTimeStr, sha1.New()) // TODO: this is only for the support of legacy codes, remove this in/after 1.23 - if subtle.ConstantTimeCompare([]byte(retCode), []byte(code)) != 1 { - return false - } + return false } // check time is expired or not: startTime <= now && now < startTime + minutes @@ -116,27 +95,6 @@ func FileSize(s int64) string { return humanize.IBytes(uint64(s)) } -// EllipsisString returns a truncated short string, -// it appends '...' in the end of the length of string is too large. -func EllipsisString(str string, length int) string { - if length <= 3 { - return "..." - } - if utf8.RuneCountInString(str) <= length { - return str - } - return string([]rune(str)[:length-3]) + "..." -} - -// TruncateString returns a truncated string with given limit, -// it returns input string if length is not reached limit. -func TruncateString(str string, limit int) string { - if utf8.RuneCountInString(str) < limit { - return str - } - return string([]rune(str)[:limit]) -} - // StringsToInt64s converts a slice of string to a slice of int64. func StringsToInt64s(strs []string) ([]int64, error) { if strs == nil { @@ -164,25 +122,3 @@ func Int64sToStrings(ints []int64) []string { } return strs } - -// EntryIcon returns the octicon class for displaying files/directories -func EntryIcon(entry *git.TreeEntry) string { - switch { - case entry.IsLink(): - te, err := entry.FollowLink() - if err != nil { - log.Debug(err.Error()) - return "file-symlink-file" - } - if te.IsDir() { - return "file-directory-symlink" - } - return "file-symlink-file" - case entry.IsDir(): - return "file-directory-fill" - case entry.IsSubModule(): - return "file-submodule" - } - - return "file" -} diff --git a/modules/base/tool_test.go b/modules/base/tool_test.go index de6c311856..b7365e40c4 100644 --- a/modules/base/tool_test.go +++ b/modules/base/tool_test.go @@ -26,25 +26,6 @@ func TestShortSha(t *testing.T) { assert.Equal(t, "veryverylo", ShortSha("veryverylong")) } -func TestBasicAuthDecode(t *testing.T) { - _, _, err := BasicAuthDecode("?") - assert.Equal(t, "illegal base64 data at input byte 0", err.Error()) - - user, pass, err := BasicAuthDecode("Zm9vOmJhcg==") - assert.NoError(t, err) - assert.Equal(t, "foo", user) - assert.Equal(t, "bar", pass) - - _, _, err = BasicAuthDecode("aW52YWxpZA==") - assert.Error(t, err) - - _, _, err = BasicAuthDecode("invalid") - assert.Error(t, err) - - _, _, err = BasicAuthDecode("YWxpY2U=") // "alice", no colon - assert.Error(t, err) -} - func TestVerifyTimeLimitCode(t *testing.T) { defer test.MockVariableValue(&setting.InstallLock, true)() initGeneralSecret := func(secret string) { @@ -86,13 +67,10 @@ JWT_SECRET = %s verifyDataCode := func(c string) bool { return VerifyTimeLimitCode(now, "data", 2, c) } - code1 := CreateTimeLimitCode("data", 2, now, sha1.New()) - code2 := CreateTimeLimitCode("data", 2, now, nil) - assert.True(t, verifyDataCode(code1)) - assert.True(t, verifyDataCode(code2)) + code := CreateTimeLimitCode("data", 2, now, nil) + assert.True(t, verifyDataCode(code)) initGeneralSecret("000_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko") - assert.False(t, verifyDataCode(code1)) - assert.False(t, verifyDataCode(code2)) + assert.False(t, verifyDataCode(code)) }) } @@ -113,36 +91,6 @@ func TestFileSize(t *testing.T) { assert.Equal(t, "2.0 EiB", FileSize(size)) } -func TestEllipsisString(t *testing.T) { - assert.Equal(t, "...", EllipsisString("foobar", 0)) - assert.Equal(t, "...", EllipsisString("foobar", 1)) - assert.Equal(t, "...", EllipsisString("foobar", 2)) - assert.Equal(t, "...", EllipsisString("foobar", 3)) - assert.Equal(t, "f...", EllipsisString("foobar", 4)) - assert.Equal(t, "fo...", EllipsisString("foobar", 5)) - assert.Equal(t, "foobar", EllipsisString("foobar", 6)) - assert.Equal(t, "foobar", EllipsisString("foobar", 10)) - assert.Equal(t, "测...", EllipsisString("测试文本一二三四", 4)) - assert.Equal(t, "测试...", EllipsisString("测试文本一二三四", 5)) - assert.Equal(t, "测试文...", EllipsisString("测试文本一二三四", 6)) - assert.Equal(t, "测试文本一二三四", EllipsisString("测试文本一二三四", 10)) -} - -func TestTruncateString(t *testing.T) { - assert.Equal(t, "", TruncateString("foobar", 0)) - assert.Equal(t, "f", TruncateString("foobar", 1)) - assert.Equal(t, "fo", TruncateString("foobar", 2)) - assert.Equal(t, "foo", TruncateString("foobar", 3)) - assert.Equal(t, "foob", TruncateString("foobar", 4)) - assert.Equal(t, "fooba", TruncateString("foobar", 5)) - assert.Equal(t, "foobar", TruncateString("foobar", 6)) - assert.Equal(t, "foobar", TruncateString("foobar", 7)) - assert.Equal(t, "测试文本", TruncateString("测试文本一二三四", 4)) - assert.Equal(t, "测试文本一", TruncateString("测试文本一二三四", 5)) - assert.Equal(t, "测试文本一二", TruncateString("测试文本一二三四", 6)) - assert.Equal(t, "测试文本一二三", TruncateString("测试文本一二三四", 7)) -} - func TestStringsToInt64s(t *testing.T) { testSuccess := func(input []string, expected []int64) { result, err := StringsToInt64s(input) @@ -167,5 +115,3 @@ func TestInt64sToStrings(t *testing.T) { Int64sToStrings([]int64{1, 4, 16, 64, 256}), ) } - -// TODO: Test EntryIcon diff --git a/modules/cache/cache.go b/modules/cache/cache.go index b5400b0bd6..039caa9fbc 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -4,6 +4,8 @@ package cache import ( + "encoding/hex" + "errors" "fmt" "strconv" "time" @@ -22,7 +24,7 @@ func Init() error { if err != nil { return err } - for i := 0; i < 10; i++ { + for range 10 { if err = c.Ping(); err == nil { break } @@ -37,16 +39,21 @@ func Init() error { } const ( - testCacheKey = "DefaultCache.TestKey" - SlowCacheThreshold = 100 * time.Microsecond + testCacheKey = "DefaultCache.TestKey" + // SlowCacheThreshold marks cache tests as slow + // set to 30ms per discussion: https://github.com/go-gitea/gitea/issues/33190 + // TODO: Replace with metrics histogram + SlowCacheThreshold = 30 * time.Millisecond ) +// Test performs delete, put and get operations on a predefined key +// returns func Test() (time.Duration, error) { if defaultCache == nil { - return 0, fmt.Errorf("default cache not initialized") + return 0, errors.New("default cache not initialized") } - testData := fmt.Sprintf("%x", make([]byte, 500)) + testData := hex.EncodeToString(make([]byte, 500)) start := time.Now() @@ -58,10 +65,10 @@ func Test() (time.Duration, error) { } testVal, hit := defaultCache.Get(testCacheKey) if !hit { - return 0, fmt.Errorf("expect cache hit but got none") + return 0, errors.New("expect cache hit but got none") } if testVal != testData { - return 0, fmt.Errorf("expect cache to return same value as stored but got other") + return 0, errors.New("expect cache to return same value as stored but got other") } return time.Since(start), nil diff --git a/modules/cache/cache_redis.go b/modules/cache/cache_redis.go index c5b52a2086..7473c938af 100644 --- a/modules/cache/cache_redis.go +++ b/modules/cache/cache_redis.go @@ -11,7 +11,7 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/nosql" - "gitea.com/go-chi/cache" //nolint:depguard + "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here "github.com/redis/go-redis/v9" ) diff --git a/modules/cache/cache_test.go b/modules/cache/cache_test.go index d0352947a8..d6ea2032ee 100644 --- a/modules/cache/cache_test.go +++ b/modules/cache/cache_test.go @@ -4,7 +4,7 @@ package cache import ( - "fmt" + "errors" "testing" "time" @@ -43,7 +43,8 @@ func TestTest(t *testing.T) { elapsed, err := Test() assert.NoError(t, err) // mem cache should take from 300ns up to 1ms on modern hardware ... - assert.Less(t, elapsed, time.Millisecond) + assert.Positive(t, elapsed) + assert.Less(t, elapsed, SlowCacheThreshold) } func TestGetCache(t *testing.T) { @@ -56,22 +57,22 @@ func TestGetString(t *testing.T) { createTestCache() data, err := GetString("key", func() (string, error) { - return "", fmt.Errorf("some error") + return "", errors.New("some error") }) assert.Error(t, err) - assert.Equal(t, "", data) + assert.Empty(t, data) data, err = GetString("key", func() (string, error) { return "", nil }) assert.NoError(t, err) - assert.Equal(t, "", data) + assert.Empty(t, data) data, err = GetString("key", func() (string, error) { return "some data", nil }) assert.NoError(t, err) - assert.Equal(t, "", data) + assert.Empty(t, data) Remove("key") data, err = GetString("key", func() (string, error) { @@ -81,7 +82,7 @@ func TestGetString(t *testing.T) { assert.Equal(t, "some data", data) data, err = GetString("key", func() (string, error) { - return "", fmt.Errorf("some error") + return "", errors.New("some error") }) assert.NoError(t, err) assert.Equal(t, "some data", data) @@ -92,7 +93,7 @@ func TestGetInt64(t *testing.T) { createTestCache() data, err := GetInt64("key", func() (int64, error) { - return 0, fmt.Errorf("some error") + return 0, errors.New("some error") }) assert.Error(t, err) assert.EqualValues(t, 0, data) @@ -117,7 +118,7 @@ func TestGetInt64(t *testing.T) { assert.EqualValues(t, 100, data) data, err = GetInt64("key", func() (int64, error) { - return 0, fmt.Errorf("some error") + return 0, errors.New("some error") }) assert.NoError(t, err) assert.EqualValues(t, 100, data) diff --git a/modules/cache/cache_twoqueue.go b/modules/cache/cache_twoqueue.go index 1eda2debc4..c8db686e57 100644 --- a/modules/cache/cache_twoqueue.go +++ b/modules/cache/cache_twoqueue.go @@ -10,7 +10,7 @@ import ( "code.gitea.io/gitea/modules/json" - mc "gitea.com/go-chi/cache" //nolint:depguard + mc "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here lru "github.com/hashicorp/golang-lru/v2" ) diff --git a/modules/cache/context.go b/modules/cache/context.go index 484cee659a..23f7c23a52 100644 --- a/modules/cache/context.go +++ b/modules/cache/context.go @@ -5,176 +5,39 @@ package cache import ( "context" - "sync" "time" - - "code.gitea.io/gitea/modules/log" ) -// cacheContext is a context that can be used to cache data in a request level context -// This is useful for caching data that is expensive to calculate and is likely to be -// used multiple times in a request. -type cacheContext struct { - data map[any]map[any]any - lock sync.RWMutex - created time.Time - discard bool -} - -func (cc *cacheContext) Get(tp, key any) any { - cc.lock.RLock() - defer cc.lock.RUnlock() - return cc.data[tp][key] -} - -func (cc *cacheContext) Put(tp, key, value any) { - cc.lock.Lock() - defer cc.lock.Unlock() - - if cc.discard { - return - } - - d := cc.data[tp] - if d == nil { - d = make(map[any]any) - cc.data[tp] = d - } - d[key] = value -} - -func (cc *cacheContext) Delete(tp, key any) { - cc.lock.Lock() - defer cc.lock.Unlock() - delete(cc.data[tp], key) -} - -func (cc *cacheContext) Discard() { - cc.lock.Lock() - defer cc.lock.Unlock() - cc.data = nil - cc.discard = true -} - -func (cc *cacheContext) isDiscard() bool { - cc.lock.RLock() - defer cc.lock.RUnlock() - return cc.discard -} - -// cacheContextLifetime is the max lifetime of cacheContext. -// Since cacheContext is used to cache data in a request level context, 5 minutes is enough. -// If a cacheContext is used more than 5 minutes, it's probably misuse. -const cacheContextLifetime = 5 * time.Minute - -var timeNow = time.Now +type cacheContextKeyType struct{} -func (cc *cacheContext) Expired() bool { - return timeNow().Sub(cc.created) > cacheContextLifetime -} - -var cacheContextKey = struct{}{} - -/* -Since there are both WithCacheContext and WithNoCacheContext, -it may be confusing when there is nesting. - -Some cases to explain the design: - -When: -- A, B or C means a cache context. -- A', B' or C' means a discard cache context. -- ctx means context.Backgrand(). -- A(ctx) means a cache context with ctx as the parent context. -- B(A(ctx)) means a cache context with A(ctx) as the parent context. -- With is alias of WithCacheContext. -- WithNo is alias of WithNoCacheContext. +var cacheContextKey = cacheContextKeyType{} -So: -- With(ctx) -> A(ctx) -- With(With(ctx)) -> A(ctx), not B(A(ctx)), always reuse parent cache context if possible. -- With(With(With(ctx))) -> A(ctx), not C(B(A(ctx))), ditto. -- WithNo(ctx) -> ctx, not A'(ctx), don't create new cache context if we don't have to. -- WithNo(With(ctx)) -> A'(ctx) -- WithNo(WithNo(With(ctx))) -> A'(ctx), not B'(A'(ctx)), don't create new cache context if we don't have to. -- With(WithNo(With(ctx))) -> B(A'(ctx)), not A(ctx), never reuse a discard cache context. -- WithNo(With(WithNo(With(ctx)))) -> B'(A'(ctx)) -- With(WithNo(With(WithNo(With(ctx))))) -> C(B'(A'(ctx))), so there's always only one not-discard cache context. -*/ +// contextCacheLifetime is the max lifetime of context cache. +// Since context cache is used to cache data in a request level context, 5 minutes is enough. +// If a context cache is used more than 5 minutes, it's probably abused. +const contextCacheLifetime = 5 * time.Minute func WithCacheContext(ctx context.Context) context.Context { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if !c.isDiscard() { - // reuse parent context - return ctx - } - } - // FIXME: review the use of this nolint directive - return context.WithValue(ctx, cacheContextKey, &cacheContext{ //nolint:staticcheck - data: make(map[any]map[any]any), - created: timeNow(), - }) -} - -func WithNoCacheContext(ctx context.Context) context.Context { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - // The caller want to run long-life tasks, but the parent context is a cache context. - // So we should disable and clean the cache data, or it will be kept in memory for a long time. - c.Discard() + if c := GetContextCache(ctx); c != nil { return ctx } - - return ctx -} - -func GetContextData(ctx context.Context, tp, key any) any { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if c.Expired() { - // The warning means that the cache context is misused for long-life task, - // it can be resolved with WithNoCacheContext(ctx). - log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c) - return nil - } - return c.Get(tp, key) - } - return nil + return context.WithValue(ctx, cacheContextKey, NewEphemeralCache(contextCacheLifetime)) } -func SetContextData(ctx context.Context, tp, key, value any) { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if c.Expired() { - // The warning means that the cache context is misused for long-life task, - // it can be resolved with WithNoCacheContext(ctx). - log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c) - return - } - c.Put(tp, key, value) - return - } -} - -func RemoveContextData(ctx context.Context, tp, key any) { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if c.Expired() { - // The warning means that the cache context is misused for long-life task, - // it can be resolved with WithNoCacheContext(ctx). - log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c) - return - } - c.Delete(tp, key) - } +func GetContextCache(ctx context.Context) *EphemeralCache { + c, _ := ctx.Value(cacheContextKey).(*EphemeralCache) + return c } // GetWithContextCache returns the cache value of the given key in the given context. -func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error) { - v := GetContextData(ctx, cacheGroupKey, cacheTargetID) - if vv, ok := v.(T); ok { - return vv, nil - } - t, err := f() - if err != nil { - return t, err - } - SetContextData(ctx, cacheGroupKey, cacheTargetID, t) - return t, nil +// FIXME: in some cases, the "context cache" should not be used, because it has uncontrollable behaviors +// For example, these calls: +// * GetWithContextCache(TargetID) -> OtherCodeCreateModel(TargetID) -> GetWithContextCache(TargetID) +// Will cause the second call is not able to get the correct created target. +// UNLESS it is certain that the target won't be changed during the request, DO NOT use it. +func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) { + if c := GetContextCache(ctx); c != nil { + return GetWithEphemeralCache(ctx, c, groupKey, targetKey, f) + } + return f(ctx, targetKey) } diff --git a/modules/cache/context_test.go b/modules/cache/context_test.go index c01b9e8d84..8371c2b908 100644 --- a/modules/cache/context_test.go +++ b/modules/cache/context_test.go @@ -8,71 +8,43 @@ import ( "testing" "time" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" ) func TestWithCacheContext(t *testing.T) { - ctx := WithCacheContext(context.Background()) - - v := GetContextData(ctx, "empty_field", "my_config1") + ctx := WithCacheContext(t.Context()) + c := GetContextCache(ctx) + v, _ := c.Get("empty_field", "my_config1") assert.Nil(t, v) const field = "system_setting" - v = GetContextData(ctx, field, "my_config1") + v, _ = c.Get(field, "my_config1") assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") + c.Put(field, "my_config1", 1) + v, _ = c.Get(field, "my_config1") assert.NotNil(t, v) - assert.EqualValues(t, 1, v.(int)) + assert.Equal(t, 1, v.(int)) - RemoveContextData(ctx, field, "my_config1") - RemoveContextData(ctx, field, "my_config2") // remove a non-exist key + c.Delete(field, "my_config1") + c.Delete(field, "my_config2") // remove a non-exist key - v = GetContextData(ctx, field, "my_config1") + v, _ = c.Get(field, "my_config1") assert.Nil(t, v) - vInt, err := GetWithContextCache(ctx, field, "my_config1", func() (int, error) { + vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) { return 1, nil }) assert.NoError(t, err) - assert.EqualValues(t, 1, vInt) + assert.Equal(t, 1, vInt) - v = GetContextData(ctx, field, "my_config1") + v, _ = c.Get(field, "my_config1") assert.EqualValues(t, 1, v) - now := timeNow - defer func() { - timeNow = now - }() - timeNow = func() time.Time { - return now().Add(5 * time.Minute) - } - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) -} - -func TestWithNoCacheContext(t *testing.T) { - ctx := context.Background() - - const field = "system_setting" - - v := GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) // still no cache - - ctx = WithCacheContext(ctx) - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") - assert.NotNil(t, v) - - ctx = WithNoCacheContext(ctx) - v = GetContextData(ctx, field, "my_config1") + defer test.MockVariableValue(&timeNow, func() time.Time { + return time.Now().Add(5 * time.Minute) + })() + v, _ = c.Get(field, "my_config1") assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) // still no cache } diff --git a/modules/cache/ephemeral.go b/modules/cache/ephemeral.go new file mode 100644 index 0000000000..6996010ac4 --- /dev/null +++ b/modules/cache/ephemeral.go @@ -0,0 +1,90 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cache + +import ( + "context" + "sync" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" +) + +// EphemeralCache is a cache that can be used to store data in a request level context +// This is useful for caching data that is expensive to calculate and is likely to be +// used multiple times in a request. +type EphemeralCache struct { + data map[any]map[any]any + lock sync.RWMutex + created time.Time + checkLifeTime time.Duration +} + +var timeNow = time.Now + +func NewEphemeralCache(checkLifeTime ...time.Duration) *EphemeralCache { + return &EphemeralCache{ + data: make(map[any]map[any]any), + created: timeNow(), + checkLifeTime: util.OptionalArg(checkLifeTime, 0), + } +} + +func (cc *EphemeralCache) checkExceededLifeTime(tp, key any) bool { + if cc.checkLifeTime > 0 && timeNow().Sub(cc.created) > cc.checkLifeTime { + log.Warn("EphemeralCache is expired, is highly likely to be abused for long-life tasks: %v, %v", tp, key) + return true + } + return false +} + +func (cc *EphemeralCache) Get(tp, key any) (any, bool) { + if cc.checkExceededLifeTime(tp, key) { + return nil, false + } + cc.lock.RLock() + defer cc.lock.RUnlock() + ret, ok := cc.data[tp][key] + return ret, ok +} + +func (cc *EphemeralCache) Put(tp, key, value any) { + if cc.checkExceededLifeTime(tp, key) { + return + } + + cc.lock.Lock() + defer cc.lock.Unlock() + + d := cc.data[tp] + if d == nil { + d = make(map[any]any) + cc.data[tp] = d + } + d[key] = value +} + +func (cc *EphemeralCache) Delete(tp, key any) { + if cc.checkExceededLifeTime(tp, key) { + return + } + + cc.lock.Lock() + defer cc.lock.Unlock() + delete(cc.data[tp], key) +} + +func GetWithEphemeralCache[T, K any](ctx context.Context, c *EphemeralCache, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) { + v, has := c.Get(groupKey, targetKey) + if vv, ok := v.(T); has && ok { + return vv, nil + } + t, err := f(ctx, targetKey) + if err != nil { + return t, err + } + c.Put(groupKey, targetKey, t) + return t, nil +} diff --git a/modules/cache/string_cache.go b/modules/cache/string_cache.go index 4f659616f5..3562b7a926 100644 --- a/modules/cache/string_cache.go +++ b/modules/cache/string_cache.go @@ -11,7 +11,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - chi_cache "gitea.com/go-chi/cache" //nolint:depguard + chi_cache "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here ) type GetJSONError struct { diff --git a/modules/cachegroup/cachegroup.go b/modules/cachegroup/cachegroup.go new file mode 100644 index 0000000000..06085f860f --- /dev/null +++ b/modules/cachegroup/cachegroup.go @@ -0,0 +1,12 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cachegroup + +const ( + User = "user" + EmailAvatarLink = "email_avatar_link" + UserEmailAddresses = "user_email_addresses" + GPGKeyWithSubKeys = "gpg_key_with_subkeys" + RepoUserPermission = "repo_user_permission" +) diff --git a/modules/charset/ambiguous_gen_test.go b/modules/charset/ambiguous_gen_test.go index 221c27d0e1..d3be0b1a13 100644 --- a/modules/charset/ambiguous_gen_test.go +++ b/modules/charset/ambiguous_gen_test.go @@ -14,7 +14,7 @@ import ( func TestAmbiguousCharacters(t *testing.T) { for locale, ambiguous := range AmbiguousCharacters { assert.Equal(t, locale, ambiguous.Locale) - assert.Equal(t, len(ambiguous.Confusable), len(ambiguous.With)) + assert.Len(t, ambiguous.With, len(ambiguous.Confusable)) assert.True(t, sort.SliceIsSorted(ambiguous.Confusable, func(i, j int) bool { return ambiguous.Confusable[i] < ambiguous.Confusable[j] })) diff --git a/modules/charset/charset.go b/modules/charset/charset.go index 1855446a98..597ce5120c 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -164,7 +164,7 @@ func DetectEncoding(content []byte) (string, error) { } times := 1024 / len(content) detectContent = make([]byte, 0, times*len(content)) - for i := 0; i < times; i++ { + for range times { detectContent = append(detectContent, content...) } } else { diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index 19b1303365..cd2e3b9aaa 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -242,7 +242,7 @@ func stringMustEndWith(t *testing.T, expected, value string) { func TestToUTF8WithFallbackReader(t *testing.T) { resetDefaultCharsetsOrder() - for testLen := 0; testLen < 2048; testLen++ { + for testLen := range 2048 { pattern := " test { () }\n" input := "" for len(input) < testLen { @@ -252,7 +252,7 @@ func TestToUTF8WithFallbackReader(t *testing.T) { input += "// Выключаем" rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)), ConvertOpts{}) r, _ := io.ReadAll(rd) - assert.EqualValuesf(t, input, string(r), "testing string len=%d", testLen) + assert.Equalf(t, input, string(r), "testing string len=%d", testLen) } truncatedOneByteExtension := failFastBytes diff --git a/modules/structs/commit_status.go b/modules/commitstatus/commit_status.go index dc880ef5eb..a0ab4e7186 100644 --- a/modules/structs/commit_status.go +++ b/modules/commitstatus/commit_status.go @@ -1,11 +1,11 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package structs +package commitstatus // CommitStatusState holds the state of a CommitStatus -// It can be "pending", "success", "error" and "failure" -type CommitStatusState string +// swagger:enum CommitStatusState +type CommitStatusState string //nolint:revive // export stutter const ( // CommitStatusPending is for when the CommitStatus is Pending @@ -18,35 +18,14 @@ const ( CommitStatusFailure CommitStatusState = "failure" // CommitStatusWarning is for when the CommitStatus is Warning CommitStatusWarning CommitStatusState = "warning" + // CommitStatusSkipped is for when CommitStatus is Skipped + CommitStatusSkipped CommitStatusState = "skipped" ) -var commitStatusPriorities = map[CommitStatusState]int{ - CommitStatusError: 0, - CommitStatusFailure: 1, - CommitStatusWarning: 2, - CommitStatusPending: 3, - CommitStatusSuccess: 4, -} - func (css CommitStatusState) String() string { return string(css) } -// NoBetterThan returns true if this State is no better than the given State -// This function only handles the states defined in CommitStatusPriorities -func (css CommitStatusState) NoBetterThan(css2 CommitStatusState) bool { - // NoBetterThan only handles the 5 states above - if _, exist := commitStatusPriorities[css]; !exist { - return false - } - - if _, exist := commitStatusPriorities[css2]; !exist { - return false - } - - return commitStatusPriorities[css] <= commitStatusPriorities[css2] -} - // IsPending represents if commit status state is pending func (css CommitStatusState) IsPending() bool { return css == CommitStatusPending @@ -71,3 +50,32 @@ func (css CommitStatusState) IsFailure() bool { func (css CommitStatusState) IsWarning() bool { return css == CommitStatusWarning } + +// IsSkipped represents if commit status state is skipped +func (css CommitStatusState) IsSkipped() bool { + return css == CommitStatusSkipped +} + +type CommitStatusStates []CommitStatusState //nolint:revive // export stutter + +// According to https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#get-the-combined-status-for-a-specific-reference +// > Additionally, a combined state is returned. The state is one of: +// > failure if any of the contexts report as error or failure +// > pending if there are no statuses or a context is pending +// > success if the latest status for all contexts is success +func (css CommitStatusStates) Combine() CommitStatusState { + successCnt := 0 + for _, state := range css { + switch { + case state.IsError() || state.IsFailure(): + return CommitStatusFailure + case state.IsPending(): + case state.IsSuccess() || state.IsWarning() || state.IsSkipped(): + successCnt++ + } + } + if successCnt > 0 && successCnt == len(css) { + return CommitStatusSuccess + } + return CommitStatusPending +} diff --git a/modules/commitstatus/commit_status_test.go b/modules/commitstatus/commit_status_test.go new file mode 100644 index 0000000000..10d8f20aa4 --- /dev/null +++ b/modules/commitstatus/commit_status_test.go @@ -0,0 +1,201 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package commitstatus + +import "testing" + +func TestCombine(t *testing.T) { + tests := []struct { + name string + states CommitStatusStates + expected CommitStatusState + }{ + // 0 states + { + name: "empty", + states: CommitStatusStates{}, + expected: CommitStatusPending, + }, + // 1 state + { + name: "pending", + states: CommitStatusStates{CommitStatusPending}, + expected: CommitStatusPending, + }, + { + name: "success", + states: CommitStatusStates{CommitStatusSuccess}, + expected: CommitStatusSuccess, + }, + { + name: "error", + states: CommitStatusStates{CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "failure", + states: CommitStatusStates{CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "warning", + states: CommitStatusStates{CommitStatusWarning}, + expected: CommitStatusSuccess, + }, + // 2 states + { + name: "pending and success", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess}, + expected: CommitStatusPending, + }, + { + name: "pending and error", + states: CommitStatusStates{CommitStatusPending, CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "pending and failure", + states: CommitStatusStates{CommitStatusPending, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "pending and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusWarning}, + expected: CommitStatusPending, + }, + { + name: "success and error", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "success and failure", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "success and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusWarning}, + expected: CommitStatusSuccess, + }, + { + name: "error and failure", + states: CommitStatusStates{CommitStatusError, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "error and warning", + states: CommitStatusStates{CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "failure and warning", + states: CommitStatusStates{CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + // 3 states + { + name: "pending, success and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusWarning}, + expected: CommitStatusPending, + }, + { + name: "pending, success and error", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "pending, success and failure", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "pending, error and failure", + states: CommitStatusStates{CommitStatusPending, CommitStatusError, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "success, error and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "success, failure and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "error, failure and warning", + states: CommitStatusStates{CommitStatusError, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "success, warning and skipped", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusWarning, CommitStatusSkipped}, + expected: CommitStatusSuccess, + }, + // All success + { + name: "all success", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusSuccess}, + expected: CommitStatusSuccess, + }, + // All pending + { + name: "all pending", + states: CommitStatusStates{CommitStatusPending, CommitStatusPending, CommitStatusPending}, + expected: CommitStatusPending, + }, + { + name: "all skipped", + states: CommitStatusStates{CommitStatusSkipped, CommitStatusSkipped, CommitStatusSkipped}, + expected: CommitStatusSuccess, + }, + // 4 states + { + name: "pending, success, error and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "pending, success, failure and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "pending, error, failure and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusError, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "success, error, failure and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusError, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "mixed states", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "mixed states with all success", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusPending, CommitStatusWarning}, + expected: CommitStatusPending, + }, + { + name: "all success with warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusSuccess, CommitStatusWarning}, + expected: CommitStatusSuccess, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.states.Combine() + if result != tt.expected { + t.Errorf("expected %v, got %v", tt.expected, result) + } + }) + } +} diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 29ed58db97..be9fc5f823 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -5,7 +5,6 @@ package csv import ( "bytes" - "context" "encoding/csv" "io" "strconv" @@ -100,10 +99,10 @@ j, ,\x20 for n, c := range cases { rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(decodeSlashes(t, c.csv))) assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err) - assert.EqualValues(t, c.expectedDelimiter, rd.Comma, "case %d: delimiter should be '%c', got '%c'", n, c.expectedDelimiter, rd.Comma) + assert.Equal(t, c.expectedDelimiter, rd.Comma, "case %d: delimiter should be '%c', got '%c'", n, c.expectedDelimiter, rd.Comma) rows, err := rd.ReadAll() assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err) - assert.EqualValues(t, c.expectedRows, rows, "case %d: rows should be equal", n) + assert.Equal(t, c.expectedRows, rows, "case %d: rows should be equal", n) } } @@ -231,8 +230,8 @@ John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimiters`, } for n, c := range cases { - delimiter := determineDelimiter(markup.NewRenderContext(context.Background()).WithRelativePath(c.filename), []byte(decodeSlashes(t, c.csv))) - assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + delimiter := determineDelimiter(markup.NewRenderContext(t.Context()).WithRelativePath(c.filename), []byte(decodeSlashes(t, c.csv))) + assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -297,7 +296,7 @@ abc | |123 for n, c := range cases { modifiedText := removeQuotedString(decodeSlashes(t, c.text)) - assert.EqualValues(t, c.expectedText, modifiedText, "case %d: modified text should be equal", n) + assert.Equal(t, c.expectedText, modifiedText, "case %d: modified text should be equal", n) } } @@ -452,7 +451,7 @@ jkl`, for n, c := range cases { delimiter := guessDelimiter([]byte(decodeSlashes(t, c.csv))) - assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -544,7 +543,7 @@ a|"he said, ""here I am"""`, for n, c := range cases { delimiter := guessFromBeforeAfterQuotes([]byte(decodeSlashes(t, c.csv))) - assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -580,7 +579,7 @@ func TestFormatError(t *testing.T) { assert.Error(t, err, "case %d: expected an error to be returned", n) } else { assert.NoError(t, err, "case %d: no error was expected, got error: %v", n, err) - assert.EqualValues(t, c.expectedMessage, message, "case %d: messages should be equal, expected '%s' got '%s'", n, c.expectedMessage, message) + assert.Equal(t, c.expectedMessage, message, "case %d: messages should be equal, expected '%s' got '%s'", n, c.expectedMessage, message) } } } diff --git a/modules/dump/dumper_test.go b/modules/dump/dumper_test.go index 2db3a598a4..8f06c1851d 100644 --- a/modules/dump/dumper_test.go +++ b/modules/dump/dumper_test.go @@ -103,11 +103,11 @@ func TestDumper(t *testing.T) { d.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1")) err := d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")}) assert.NoError(t, err) - assert.EqualValues(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added)) + assert.Equal(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added)) tw = &testWriter{} d = &Dumper{Writer: tw} err = d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil) assert.NoError(t, err) - assert.EqualValues(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added)) + assert.Equal(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added)) } diff --git a/modules/emoji/emoji_test.go b/modules/emoji/emoji_test.go index 2526cd121e..fbf80fe41a 100644 --- a/modules/emoji/emoji_test.go +++ b/modules/emoji/emoji_test.go @@ -5,7 +5,6 @@ package emoji import ( - "reflect" "testing" "github.com/stretchr/testify/assert" @@ -22,32 +21,18 @@ func TestLookup(t *testing.T) { c := FromAlias(":beer:") d := FromAlias("beer") - if !reflect.DeepEqual(a, b) { - t.Errorf("a and b should equal") - } - if !reflect.DeepEqual(b, c) { - t.Errorf("b and c should equal") - } - if !reflect.DeepEqual(c, d) { - t.Errorf("c and d should equal") - } - if !reflect.DeepEqual(a, d) { - t.Errorf("a and d should equal") - } + assert.Equal(t, a, b) + assert.Equal(t, b, c) + assert.Equal(t, c, d) + assert.Equal(t, a, d) m := FromCode("\U0001f44d") n := FromAlias(":thumbsup:") o := FromAlias("+1") - if !reflect.DeepEqual(m, n) { - t.Errorf("m and n should equal") - } - if !reflect.DeepEqual(n, o) { - t.Errorf("n and o should equal") - } - if !reflect.DeepEqual(m, o) { - t.Errorf("m and o should equal") - } + assert.Equal(t, m, n) + assert.Equal(t, m, o) + assert.Equal(t, n, o) } func TestReplacers(t *testing.T) { @@ -61,9 +46,7 @@ func TestReplacers(t *testing.T) { for i, x := range tests { s := x.f(x.v) - if s != x.exp { - t.Errorf("test %d `%s` expected `%s`, got: `%s`", i, x.v, x.exp, s) - } + assert.Equalf(t, x.exp, s, "test %d `%s` expected `%s`, got: `%s`", i, x.v, x.exp, s) } } diff --git a/modules/eventsource/event_test.go b/modules/eventsource/event_test.go index 4c4272880d..a1c3e5c7a8 100644 --- a/modules/eventsource/event_test.go +++ b/modules/eventsource/event_test.go @@ -6,6 +6,9 @@ package eventsource import ( "bytes" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_wrapNewlines(t *testing.T) { @@ -38,16 +41,10 @@ func Test_wrapNewlines(t *testing.T) { t.Run(tt.name, func(t *testing.T) { w := &bytes.Buffer{} gotSum, err := wrapNewlines(w, []byte(tt.prefix), []byte(tt.value)) - if err != nil { - t.Errorf("wrapNewlines() error = %v", err) - return - } - if gotSum != int64(len(tt.output)) { - t.Errorf("wrapNewlines() = %v, want %v", gotSum, int64(len(tt.output))) - } - if gotW := w.String(); gotW != tt.output { - t.Errorf("wrapNewlines() = %v, want %v", gotW, tt.output) - } + require.NoError(t, err) + + assert.EqualValues(t, len(tt.output), gotSum) + assert.Equal(t, tt.output, w.String()) }) } } diff --git a/modules/fileicon/basic.go b/modules/fileicon/basic.go new file mode 100644 index 0000000000..9c513ccbd9 --- /dev/null +++ b/modules/fileicon/basic.go @@ -0,0 +1,31 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import ( + "html/template" + + "code.gitea.io/gitea/modules/svg" + "code.gitea.io/gitea/modules/util" +) + +func BasicEntryIconName(entry *EntryInfo) string { + svgName := "octicon-file" + switch { + case entry.EntryMode.IsLink(): + svgName = "octicon-file-symlink-file" + if entry.SymlinkToMode.IsDir() { + svgName = "octicon-file-directory-symlink" + } + case entry.EntryMode.IsDir(): + svgName = util.Iif(entry.IsOpen, "octicon-file-directory-open-fill", "octicon-file-directory-fill") + case entry.EntryMode.IsSubModule(): + svgName = "octicon-file-submodule" + } + return svgName +} + +func BasicEntryIconHTML(entry *EntryInfo) template.HTML { + return svg.RenderHTML(BasicEntryIconName(entry)) +} diff --git a/modules/fileicon/entry.go b/modules/fileicon/entry.go new file mode 100644 index 0000000000..0326c2bfa8 --- /dev/null +++ b/modules/fileicon/entry.go @@ -0,0 +1,31 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import "code.gitea.io/gitea/modules/git" + +type EntryInfo struct { + BaseName string + EntryMode git.EntryMode + SymlinkToMode git.EntryMode + IsOpen bool +} + +func EntryInfoFromGitTreeEntry(commit *git.Commit, fullPath string, gitEntry *git.TreeEntry) *EntryInfo { + ret := &EntryInfo{BaseName: gitEntry.Name(), EntryMode: gitEntry.Mode()} + if gitEntry.IsLink() { + if res, err := git.EntryFollowLink(commit, fullPath, gitEntry); err == nil && res.TargetEntry.IsDir() { + ret.SymlinkToMode = res.TargetEntry.Mode() + } + } + return ret +} + +func EntryInfoFolder() *EntryInfo { + return &EntryInfo{EntryMode: git.EntryModeTree} +} + +func EntryInfoFolderOpen() *EntryInfo { + return &EntryInfo{EntryMode: git.EntryModeTree, IsOpen: true} +} diff --git a/modules/fileicon/material.go b/modules/fileicon/material.go new file mode 100644 index 0000000000..5361592d8a --- /dev/null +++ b/modules/fileicon/material.go @@ -0,0 +1,162 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import ( + "html/template" + "strings" + "sync" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/options" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/svg" + "code.gitea.io/gitea/modules/util" +) + +type materialIconRulesData struct { + FileNames map[string]string `json:"fileNames"` + FolderNames map[string]string `json:"folderNames"` + FileExtensions map[string]string `json:"fileExtensions"` + LanguageIDs map[string]string `json:"languageIds"` +} + +type MaterialIconProvider struct { + once sync.Once + rules *materialIconRulesData + svgs map[string]string +} + +var materialIconProvider MaterialIconProvider + +func DefaultMaterialIconProvider() *MaterialIconProvider { + materialIconProvider.once.Do(materialIconProvider.loadData) + return &materialIconProvider +} + +func (m *MaterialIconProvider) loadData() { + buf, err := options.AssetFS().ReadFile("fileicon/material-icon-rules.json") + if err != nil { + log.Error("Failed to read material icon rules: %v", err) + return + } + err = json.Unmarshal(buf, &m.rules) + if err != nil { + log.Error("Failed to unmarshal material icon rules: %v", err) + return + } + + buf, err = options.AssetFS().ReadFile("fileicon/material-icon-svgs.json") + if err != nil { + log.Error("Failed to read material icon rules: %v", err) + return + } + err = json.Unmarshal(buf, &m.svgs) + if err != nil { + log.Error("Failed to unmarshal material icon rules: %v", err) + return + } + log.Debug("Loaded material icon rules and SVG images") +} + +func (m *MaterialIconProvider) renderFileIconSVG(p *RenderedIconPool, name, svg, extraClass string) template.HTML { + // This part is a bit hacky, but it works really well. It should be safe to do so because all SVG icons are generated by us. + // Will try to refactor this in the future. + if !strings.HasPrefix(svg, "<svg") { + panic("Invalid SVG icon") + } + svgID := "svg-mfi-" + name + svgCommonAttrs := `class="svg git-entry-icon ` + extraClass + `" width="16" height="16" aria-hidden="true"` + svgHTML := template.HTML(`<svg id="` + svgID + `" ` + svgCommonAttrs + svg[4:]) + if p == nil { + return svgHTML + } + if p.IconSVGs[svgID] == "" { + p.IconSVGs[svgID] = svgHTML + } + return template.HTML(`<svg ` + svgCommonAttrs + `><use xlink:href="#` + svgID + `"></use></svg>`) +} + +func (m *MaterialIconProvider) EntryIconHTML(p *RenderedIconPool, entry *EntryInfo) template.HTML { + if m.rules == nil { + return BasicEntryIconHTML(entry) + } + + if entry.EntryMode.IsLink() { + if entry.SymlinkToMode.IsDir() { + // keep the old "octicon-xxx" class name to make some "theme plugin selector" could still work + return svg.RenderHTML("material-folder-symlink", 16, "octicon-file-directory-symlink") + } + return svg.RenderHTML("octicon-file-symlink-file") // TODO: find some better icons for them + } + + name := m.FindIconName(entry) + iconSVG := m.svgs[name] + if iconSVG == "" { + name = "file" + if entry.EntryMode.IsDir() { + name = util.Iif(entry.IsOpen, "folder-open", "folder") + } + iconSVG = m.svgs[name] + if iconSVG == "" { + setting.PanicInDevOrTesting("missing file icon for %s", name) + } + } + + // keep the old "octicon-xxx" class name to make some "theme plugin selector" could still work + extraClass := "octicon-file" + switch { + case entry.EntryMode.IsDir(): + extraClass = BasicEntryIconName(entry) + case entry.EntryMode.IsSubModule(): + extraClass = "octicon-file-submodule" + } + return m.renderFileIconSVG(p, name, iconSVG, extraClass) +} + +func (m *MaterialIconProvider) findIconNameWithLangID(s string) string { + if _, ok := m.svgs[s]; ok { + return s + } + if s, ok := m.rules.LanguageIDs[s]; ok { + if _, ok = m.svgs[s]; ok { + return s + } + } + return "" +} + +func (m *MaterialIconProvider) FindIconName(entry *EntryInfo) string { + if entry.EntryMode.IsSubModule() { + return "folder-git" + } + + fileNameLower := strings.ToLower(entry.BaseName) + if entry.EntryMode.IsDir() { + if s, ok := m.rules.FolderNames[fileNameLower]; ok { + return s + } + return util.Iif(entry.IsOpen, "folder-open", "folder") + } + + if s, ok := m.rules.FileNames[fileNameLower]; ok { + if s = m.findIconNameWithLangID(s); s != "" { + return s + } + } + + for i := len(fileNameLower) - 1; i >= 0; i-- { + if fileNameLower[i] == '.' { + ext := fileNameLower[i+1:] + if s, ok := m.rules.FileExtensions[ext]; ok { + if s = m.findIconNameWithLangID(s); s != "" { + return s + } + } + } + } + + return "file" +} diff --git a/modules/fileicon/material_test.go b/modules/fileicon/material_test.go new file mode 100644 index 0000000000..d2a769eaac --- /dev/null +++ b/modules/fileicon/material_test.go @@ -0,0 +1,27 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon_test + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/fileicon" + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m, &unittest.TestOptions{FixtureFiles: []string{}}) +} + +func TestFindIconName(t *testing.T) { + unittest.PrepareTestEnv(t) + p := fileicon.DefaultMaterialIconProvider() + assert.Equal(t, "php", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.php", EntryMode: git.EntryModeBlob})) + assert.Equal(t, "php", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.PHP", EntryMode: git.EntryModeBlob})) + assert.Equal(t, "javascript", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.js", EntryMode: git.EntryModeBlob})) + assert.Equal(t, "visualstudio", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.vba", EntryMode: git.EntryModeBlob})) +} diff --git a/modules/fileicon/render.go b/modules/fileicon/render.go new file mode 100644 index 0000000000..8ed86b9ac0 --- /dev/null +++ b/modules/fileicon/render.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import ( + "html/template" + "strings" + + "code.gitea.io/gitea/modules/setting" +) + +type RenderedIconPool struct { + IconSVGs map[string]template.HTML +} + +func NewRenderedIconPool() *RenderedIconPool { + return &RenderedIconPool{ + IconSVGs: make(map[string]template.HTML), + } +} + +func (p *RenderedIconPool) RenderToHTML() template.HTML { + if len(p.IconSVGs) == 0 { + return "" + } + sb := &strings.Builder{} + sb.WriteString(`<div class=tw-hidden>`) + for _, icon := range p.IconSVGs { + sb.WriteString(string(icon)) + } + sb.WriteString(`</div>`) + return template.HTML(sb.String()) +} + +func RenderEntryIconHTML(renderedIconPool *RenderedIconPool, entry *EntryInfo) template.HTML { + if setting.UI.FileIconTheme == "material" { + return DefaultMaterialIconProvider().EntryIconHTML(renderedIconPool, entry) + } + return BasicEntryIconHTML(entry) +} diff --git a/modules/git/attribute.go b/modules/git/attribute.go deleted file mode 100644 index 4dfa510369..0000000000 --- a/modules/git/attribute.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "code.gitea.io/gitea/modules/optional" -) - -const ( - AttributeLinguistVendored = "linguist-vendored" - AttributeLinguistGenerated = "linguist-generated" - AttributeLinguistDocumentation = "linguist-documentation" - AttributeLinguistDetectable = "linguist-detectable" - AttributeLinguistLanguage = "linguist-language" - AttributeGitlabLanguage = "gitlab-language" -) - -// true if "set"/"true", false if "unset"/"false", none otherwise -func AttributeToBool(attr map[string]string, name string) optional.Option[bool] { - switch attr[name] { - case "set", "true": - return optional.Some(true) - case "unset", "false": - return optional.Some(false) - } - return optional.None[bool]() -} - -func AttributeToString(attr map[string]string, name string) optional.Option[string] { - if value, has := attr[name]; has && value != "unspecified" { - return optional.Some(value) - } - return optional.None[string]() -} diff --git a/modules/git/attribute/attribute.go b/modules/git/attribute/attribute.go new file mode 100644 index 0000000000..9c01cb339e --- /dev/null +++ b/modules/git/attribute/attribute.go @@ -0,0 +1,115 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "strings" + + "code.gitea.io/gitea/modules/optional" +) + +type Attribute string + +const ( + LinguistVendored = "linguist-vendored" + LinguistGenerated = "linguist-generated" + LinguistDocumentation = "linguist-documentation" + LinguistDetectable = "linguist-detectable" + LinguistLanguage = "linguist-language" + GitlabLanguage = "gitlab-language" + Lockable = "lockable" + Filter = "filter" + Diff = "diff" +) + +var LinguistAttributes = []string{ + LinguistVendored, + LinguistGenerated, + LinguistDocumentation, + LinguistDetectable, + LinguistLanguage, + GitlabLanguage, +} + +func (a Attribute) IsUnspecified() bool { + return a == "" || a == "unspecified" +} + +func (a Attribute) ToString() optional.Option[string] { + if !a.IsUnspecified() { + return optional.Some(string(a)) + } + return optional.None[string]() +} + +// ToBool converts the attribute value to optional boolean: true if "set"/"true", false if "unset"/"false", none otherwise +func (a Attribute) ToBool() optional.Option[bool] { + switch a { + case "set", "true": + return optional.Some(true) + case "unset", "false": + return optional.Some(false) + } + return optional.None[bool]() +} + +type Attributes struct { + m map[string]Attribute +} + +func NewAttributes() *Attributes { + return &Attributes{m: make(map[string]Attribute)} +} + +func (attrs *Attributes) Get(name string) Attribute { + if value, has := attrs.m[name]; has { + return value + } + return "" +} + +func (attrs *Attributes) GetVendored() optional.Option[bool] { + return attrs.Get(LinguistVendored).ToBool() +} + +func (attrs *Attributes) GetGenerated() optional.Option[bool] { + return attrs.Get(LinguistGenerated).ToBool() +} + +func (attrs *Attributes) GetDocumentation() optional.Option[bool] { + return attrs.Get(LinguistDocumentation).ToBool() +} + +func (attrs *Attributes) GetDetectable() optional.Option[bool] { + return attrs.Get(LinguistDetectable).ToBool() +} + +func (attrs *Attributes) GetLinguistLanguage() optional.Option[string] { + return attrs.Get(LinguistLanguage).ToString() +} + +func (attrs *Attributes) GetGitlabLanguage() optional.Option[string] { + attrStr := attrs.Get(GitlabLanguage).ToString() + if attrStr.Has() { + raw := attrStr.Value() + // gitlab-language may have additional parameters after the language + // ignore them and just use the main language + // https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type + if idx := strings.IndexByte(raw, '?'); idx >= 0 { + return optional.Some(raw[:idx]) + } + } + return attrStr +} + +func (attrs *Attributes) GetLanguage() optional.Option[string] { + // prefer linguist-language over gitlab-language + // if linguist-language is not set, use gitlab-language + // if both are not set, return none + language := attrs.GetLinguistLanguage() + if language.Value() == "" { + language = attrs.GetGitlabLanguage() + } + return language +} diff --git a/modules/git/attribute/attribute_test.go b/modules/git/attribute/attribute_test.go new file mode 100644 index 0000000000..dadb5582a3 --- /dev/null +++ b/modules/git/attribute/attribute_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Attribute(t *testing.T) { + assert.Empty(t, Attribute("").ToString().Value()) + assert.Empty(t, Attribute("unspecified").ToString().Value()) + assert.Equal(t, "python", Attribute("python").ToString().Value()) + assert.Equal(t, "Java", Attribute("Java").ToString().Value()) + + attributes := Attributes{ + m: map[string]Attribute{ + LinguistGenerated: "true", + LinguistDocumentation: "false", + LinguistDetectable: "set", + LinguistLanguage: "Python", + GitlabLanguage: "Java", + "filter": "unspecified", + "test": "", + }, + } + + assert.Empty(t, attributes.Get("test").ToString().Value()) + assert.Empty(t, attributes.Get("filter").ToString().Value()) + assert.Equal(t, "Python", attributes.Get(LinguistLanguage).ToString().Value()) + assert.Equal(t, "Java", attributes.Get(GitlabLanguage).ToString().Value()) + assert.True(t, attributes.Get(LinguistGenerated).ToBool().Value()) + assert.False(t, attributes.Get(LinguistDocumentation).ToBool().Value()) + assert.True(t, attributes.Get(LinguistDetectable).ToBool().Value()) +} diff --git a/modules/git/attribute/batch.go b/modules/git/attribute/batch.go new file mode 100644 index 0000000000..4e31fda575 --- /dev/null +++ b/modules/git/attribute/batch.go @@ -0,0 +1,216 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" +) + +// BatchChecker provides a reader for check-attribute content that can be long running +type BatchChecker struct { + attributesNum int + repo *git.Repository + stdinWriter *os.File + stdOut *nulSeparatedAttributeWriter + ctx context.Context + cancel context.CancelFunc + cmd *git.Command +} + +// NewBatchChecker creates a check attribute reader for the current repository and provided commit ID +// If treeish is empty, then it will use current working directory, otherwise it will use the provided treeish on the bare repo +func NewBatchChecker(repo *git.Repository, treeish string, attributes []string) (checker *BatchChecker, returnedErr error) { + ctx, cancel := context.WithCancel(repo.Ctx) + defer func() { + if returnedErr != nil { + cancel() + } + }() + + cmd, envs, cleanup, err := checkAttrCommand(repo, treeish, nil, attributes) + if err != nil { + return nil, err + } + defer func() { + if returnedErr != nil { + cleanup() + } + }() + + cmd.AddArguments("--stdin") + + checker = &BatchChecker{ + attributesNum: len(attributes), + repo: repo, + ctx: ctx, + cmd: cmd, + cancel: func() { + cancel() + cleanup() + }, + } + + stdinReader, stdinWriter, err := os.Pipe() + if err != nil { + return nil, err + } + checker.stdinWriter = stdinWriter + + lw := new(nulSeparatedAttributeWriter) + lw.attributes = make(chan attributeTriple, len(attributes)) + lw.closed = make(chan struct{}) + checker.stdOut = lw + + go func() { + defer func() { + _ = stdinReader.Close() + _ = lw.Close() + }() + stdErr := new(bytes.Buffer) + err := cmd.Run(ctx, &git.RunOpts{ + Env: envs, + Dir: repo.Path, + Stdin: stdinReader, + Stdout: lw, + Stderr: stdErr, + }) + + if err != nil && !git.IsErrCanceledOrKilled(err) { + log.Error("Attribute checker for commit %s exits with error: %v", treeish, err) + } + checker.cancel() + }() + + return checker, nil +} + +// CheckPath check attr for given path +func (c *BatchChecker) CheckPath(path string) (rs *Attributes, err error) { + defer func() { + if err != nil && err != c.ctx.Err() { + log.Error("Unexpected error when checking path %s in %s, error: %v", path, filepath.Base(c.repo.Path), err) + } + }() + + select { + case <-c.ctx.Done(): + return nil, c.ctx.Err() + default: + } + + if _, err = c.stdinWriter.Write([]byte(path + "\x00")); err != nil { + defer c.Close() + return nil, err + } + + reportTimeout := func() error { + stdOutClosed := false + select { + case <-c.stdOut.closed: + stdOutClosed = true + default: + } + debugMsg := fmt.Sprintf("check path %q in repo %q", path, filepath.Base(c.repo.Path)) + debugMsg += fmt.Sprintf(", stdOut: tmp=%q, pos=%d, closed=%v", string(c.stdOut.tmp), c.stdOut.pos, stdOutClosed) + if c.cmd != nil { + debugMsg += fmt.Sprintf(", process state: %q", c.cmd.ProcessState()) + } + _ = c.Close() + return fmt.Errorf("CheckPath timeout: %s", debugMsg) + } + + rs = NewAttributes() + for i := 0; i < c.attributesNum; i++ { + select { + case <-time.After(5 * time.Second): + // there is no "hang" problem now. This code is just used to catch other potential problems. + return nil, reportTimeout() + case attr, ok := <-c.stdOut.ReadAttribute(): + if !ok { + return nil, c.ctx.Err() + } + rs.m[attr.Attribute] = Attribute(attr.Value) + case <-c.ctx.Done(): + return nil, c.ctx.Err() + } + } + return rs, nil +} + +func (c *BatchChecker) Close() error { + c.cancel() + err := c.stdinWriter.Close() + return err +} + +type attributeTriple struct { + Filename string + Attribute string + Value string +} + +type nulSeparatedAttributeWriter struct { + tmp []byte + attributes chan attributeTriple + closed chan struct{} + working attributeTriple + pos int +} + +func (wr *nulSeparatedAttributeWriter) Write(p []byte) (n int, err error) { + l, read := len(p), 0 + + nulIdx := bytes.IndexByte(p, '\x00') + for nulIdx >= 0 { + wr.tmp = append(wr.tmp, p[:nulIdx]...) + switch wr.pos { + case 0: + wr.working = attributeTriple{ + Filename: string(wr.tmp), + } + case 1: + wr.working.Attribute = string(wr.tmp) + case 2: + wr.working.Value = string(wr.tmp) + } + wr.tmp = wr.tmp[:0] + wr.pos++ + if wr.pos > 2 { + wr.attributes <- wr.working + wr.pos = 0 + } + read += nulIdx + 1 + if l > read { + p = p[nulIdx+1:] + nulIdx = bytes.IndexByte(p, '\x00') + } else { + return l, nil + } + } + wr.tmp = append(wr.tmp, p...) + return l, nil +} + +func (wr *nulSeparatedAttributeWriter) ReadAttribute() <-chan attributeTriple { + return wr.attributes +} + +func (wr *nulSeparatedAttributeWriter) Close() error { + select { + case <-wr.closed: + return nil + default: + } + close(wr.attributes) + close(wr.closed) + return nil +} diff --git a/modules/git/attribute/batch_test.go b/modules/git/attribute/batch_test.go new file mode 100644 index 0000000000..30a3d805fe --- /dev/null +++ b/modules/git/attribute/batch_test.go @@ -0,0 +1,172 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "path/filepath" + "testing" + "time" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { + wr := &nulSeparatedAttributeWriter{ + attributes: make(chan attributeTriple, 5), + } + + testStr := ".gitignore\"\n\x00linguist-vendored\x00unspecified\x00" + + n, err := wr.Write([]byte(testStr)) + + assert.Len(t, testStr, n) + assert.NoError(t, err) + select { + case attr := <-wr.ReadAttribute(): + assert.Equal(t, ".gitignore\"\n", attr.Filename) + assert.Equal(t, LinguistVendored, attr.Attribute) + assert.Equal(t, "unspecified", attr.Value) + case <-time.After(100 * time.Millisecond): + assert.FailNow(t, "took too long to read an attribute from the list") + } + // Write a second attribute again + n, err = wr.Write([]byte(testStr)) + + assert.Len(t, testStr, n) + assert.NoError(t, err) + + select { + case attr := <-wr.ReadAttribute(): + assert.Equal(t, ".gitignore\"\n", attr.Filename) + assert.Equal(t, LinguistVendored, attr.Attribute) + assert.Equal(t, "unspecified", attr.Value) + case <-time.After(100 * time.Millisecond): + assert.FailNow(t, "took too long to read an attribute from the list") + } + + // Write a partial attribute + _, err = wr.Write([]byte("incomplete-file")) + assert.NoError(t, err) + _, err = wr.Write([]byte("name\x00")) + assert.NoError(t, err) + + select { + case <-wr.ReadAttribute(): + assert.FailNow(t, "There should not be an attribute ready to read") + case <-time.After(100 * time.Millisecond): + } + _, err = wr.Write([]byte("attribute\x00")) + assert.NoError(t, err) + select { + case <-wr.ReadAttribute(): + assert.FailNow(t, "There should not be an attribute ready to read") + case <-time.After(100 * time.Millisecond): + } + + _, err = wr.Write([]byte("value\x00")) + assert.NoError(t, err) + + attr := <-wr.ReadAttribute() + assert.Equal(t, "incomplete-filename", attr.Filename) + assert.Equal(t, "attribute", attr.Attribute) + assert.Equal(t, "value", attr.Value) + + _, err = wr.Write([]byte("shouldbe.vendor\x00linguist-vendored\x00set\x00shouldbe.vendor\x00linguist-generated\x00unspecified\x00shouldbe.vendor\x00linguist-language\x00unspecified\x00")) + assert.NoError(t, err) + attr = <-wr.ReadAttribute() + assert.NoError(t, err) + assert.Equal(t, attributeTriple{ + Filename: "shouldbe.vendor", + Attribute: LinguistVendored, + Value: "set", + }, attr) + attr = <-wr.ReadAttribute() + assert.NoError(t, err) + assert.Equal(t, attributeTriple{ + Filename: "shouldbe.vendor", + Attribute: LinguistGenerated, + Value: "unspecified", + }, attr) + attr = <-wr.ReadAttribute() + assert.NoError(t, err) + assert.Equal(t, attributeTriple{ + Filename: "shouldbe.vendor", + Attribute: LinguistLanguage, + Value: "unspecified", + }, attr) +} + +func expectedAttrs() *Attributes { + return &Attributes{ + m: map[string]Attribute{ + LinguistGenerated: "unspecified", + LinguistDetectable: "unspecified", + LinguistDocumentation: "unspecified", + LinguistVendored: "unspecified", + LinguistLanguage: "Python", + GitlabLanguage: "unspecified", + }, + } +} + +func Test_BatchChecker(t *testing.T) { + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + require.NoError(t, err) + defer gitRepo.Close() + + commitID := "8fee858da5796dfb37704761701bb8e800ad9ef3" + + t.Run("Create index file to run git check-attr", func(t *testing.T) { + defer test.MockVariableValue(&git.DefaultFeatures().SupportCheckAttrOnBare, false)() + checker, err := NewBatchChecker(gitRepo, commitID, LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) + + // run git check-attr on work tree + t.Run("Run git check-attr on git work tree", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "test-repo") + err := git.Clone(t.Context(), repoPath, dir, git.CloneRepoOptions{ + Shared: true, + Branch: "master", + }) + assert.NoError(t, err) + + tempRepo, err := git.OpenRepository(t.Context(), dir) + assert.NoError(t, err) + defer tempRepo.Close() + + checker, err := NewBatchChecker(tempRepo, "", LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) + + if !git.DefaultFeatures().SupportCheckAttrOnBare { + t.Skip("git version 2.40 is required to support run check-attr on bare repo") + return + } + + t.Run("Run git check-attr in bare repository", func(t *testing.T) { + checker, err := NewBatchChecker(gitRepo, commitID, LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) +} diff --git a/modules/git/attribute/checker.go b/modules/git/attribute/checker.go new file mode 100644 index 0000000000..167b31416e --- /dev/null +++ b/modules/git/attribute/checker.go @@ -0,0 +1,101 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + + "code.gitea.io/gitea/modules/git" +) + +func checkAttrCommand(gitRepo *git.Repository, treeish string, filenames, attributes []string) (*git.Command, []string, func(), error) { + cancel := func() {} + envs := []string{"GIT_FLUSH=1"} + cmd := git.NewCommand("check-attr", "-z") + if len(attributes) == 0 { + cmd.AddArguments("--all") + } + + // there is treeish, read from bare repo or temp index created by "read-tree" + if treeish != "" { + if git.DefaultFeatures().SupportCheckAttrOnBare { + cmd.AddArguments("--source") + cmd.AddDynamicArguments(treeish) + } else { + indexFilename, worktree, deleteTemporaryFile, err := gitRepo.ReadTreeToTemporaryIndex(treeish) + if err != nil { + return nil, nil, nil, err + } + + cmd.AddArguments("--cached") + envs = append(envs, + "GIT_INDEX_FILE="+indexFilename, + "GIT_WORK_TREE="+worktree, + ) + cancel = deleteTemporaryFile + } + } else { + // Read from existing index, in cases where the repo is bare and has an index, + // or the work tree contains unstaged changes that shouldn't affect the attribute check. + // It is caller's responsibility to add changed ".gitattributes" into the index if they want to respect the new changes. + cmd.AddArguments("--cached") + } + + cmd.AddDynamicArguments(attributes...) + if len(filenames) > 0 { + cmd.AddDashesAndList(filenames...) + } + return cmd, envs, cancel, nil +} + +type CheckAttributeOpts struct { + Filenames []string + Attributes []string +} + +// CheckAttributes return the attributes of the given filenames and attributes in the given treeish. +// If treeish is empty, then it will use current working directory, otherwise it will use the provided treeish on the bare repo +func CheckAttributes(ctx context.Context, gitRepo *git.Repository, treeish string, opts CheckAttributeOpts) (map[string]*Attributes, error) { + cmd, envs, cancel, err := checkAttrCommand(gitRepo, treeish, opts.Filenames, opts.Attributes) + if err != nil { + return nil, err + } + defer cancel() + + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + if err := cmd.Run(ctx, &git.RunOpts{ + Env: append(os.Environ(), envs...), + Dir: gitRepo.Path, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + return nil, fmt.Errorf("failed to run check-attr: %w\n%s\n%s", err, stdOut.String(), stdErr.String()) + } + + fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) + if len(fields)%3 != 1 { + return nil, errors.New("wrong number of fields in return from check-attr") + } + + attributesMap := make(map[string]*Attributes) + for i := 0; i < (len(fields) / 3); i++ { + filename := string(fields[3*i]) + attribute := string(fields[3*i+1]) + info := string(fields[3*i+2]) + attribute2info, ok := attributesMap[filename] + if !ok { + attribute2info = NewAttributes() + attributesMap[filename] = attribute2info + } + attribute2info.m[attribute] = Attribute(info) + } + + return attributesMap, nil +} diff --git a/modules/git/attribute/checker_test.go b/modules/git/attribute/checker_test.go new file mode 100644 index 0000000000..67fbda8918 --- /dev/null +++ b/modules/git/attribute/checker_test.go @@ -0,0 +1,84 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_Checker(t *testing.T) { + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + require.NoError(t, err) + defer gitRepo.Close() + + commitID := "8fee858da5796dfb37704761701bb8e800ad9ef3" + + t.Run("Create index file to run git check-attr", func(t *testing.T) { + defer test.MockVariableValue(&git.DefaultFeatures().SupportCheckAttrOnBare, false)() + attrs, err := CheckAttributes(t.Context(), gitRepo, commitID, CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + // run git check-attr on work tree + t.Run("Run git check-attr on git work tree", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "test-repo") + err := git.Clone(t.Context(), repoPath, dir, git.CloneRepoOptions{ + Shared: true, + Branch: "master", + }) + assert.NoError(t, err) + + tempRepo, err := git.OpenRepository(t.Context(), dir) + assert.NoError(t, err) + defer tempRepo.Close() + + attrs, err := CheckAttributes(t.Context(), tempRepo, "", CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + t.Run("Run git check-attr in bare repository using index", func(t *testing.T) { + attrs, err := CheckAttributes(t.Context(), gitRepo, "", CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + if !git.DefaultFeatures().SupportCheckAttrOnBare { + t.Skip("git version 2.40 is required to support run check-attr on bare repo without using index") + return + } + + t.Run("Run git check-attr in bare repository", func(t *testing.T) { + attrs, err := CheckAttributes(t.Context(), gitRepo, commitID, CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) +} diff --git a/modules/git/attribute/main_test.go b/modules/git/attribute/main_test.go new file mode 100644 index 0000000000..df8241bfb0 --- /dev/null +++ b/modules/git/attribute/main_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "context" + "fmt" + "os" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func testRun(m *testing.M) error { + gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + if err != nil { + return fmt.Errorf("unable to create temp dir: %w", err) + } + defer util.RemoveAll(gitHomePath) + setting.Git.HomePath = gitHomePath + + if err = git.InitFull(context.Background()); err != nil { + return fmt.Errorf("failed to call Init: %w", err) + } + + exitCode := m.Run() + if exitCode != 0 { + return fmt.Errorf("run test failed, ExitCode=%d", exitCode) + } + return nil +} + +func TestMain(m *testing.M) { + if err := testRun(m); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Test failed: %v", err) + os.Exit(1) + } +} diff --git a/modules/git/batch.go b/modules/git/batch.go index 3ec4f1ddcc..f9e1748b54 100644 --- a/modules/git/batch.go +++ b/modules/git/batch.go @@ -14,25 +14,26 @@ type Batch struct { Writer WriteCloserError } -func (repo *Repository) NewBatch(ctx context.Context) (*Batch, error) { +// NewBatch creates a new batch for the given repository, the Close must be invoked before release the batch +func NewBatch(ctx context.Context, repoPath string) (*Batch, error) { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repo.Path); err != nil { + if err := ensureValidGitRepository(ctx, repoPath); err != nil { return nil, err } var batch Batch - batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repo.Path) + batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repoPath) return &batch, nil } -func (repo *Repository) NewBatchCheck(ctx context.Context) (*Batch, error) { +func NewBatchCheck(ctx context.Context, repoPath string) (*Batch, error) { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repo.Path); err != nil { + if err := ensureValidGitRepository(ctx, repoPath); err != nil { return nil, err } var check Batch - check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repo.Path) + check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repoPath) return &check, nil } diff --git a/modules/git/batch_reader.go b/modules/git/batch_reader.go index 532dbad989..7bbab76bb8 100644 --- a/modules/git/batch_reader.go +++ b/modules/git/batch_reader.go @@ -29,8 +29,8 @@ type WriteCloserError interface { // This is needed otherwise the git cat-file will hang for invalid repositories. func ensureValidGitRepository(ctx context.Context, repoPath string) error { stderr := strings.Builder{} - err := NewCommand(ctx, "rev-parse"). - Run(&RunOpts{ + err := NewCommand("rev-parse"). + Run(ctx, &RunOpts{ Dir: repoPath, Stderr: &stderr, }) @@ -61,8 +61,8 @@ func catFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, go func() { stderr := strings.Builder{} - err := NewCommand(ctx, "cat-file", "--batch-check"). - Run(&RunOpts{ + err := NewCommand("cat-file", "--batch-check"). + Run(ctx, &RunOpts{ Dir: repoPath, Stdin: batchStdinReader, Stdout: batchStdoutWriter, @@ -109,8 +109,8 @@ func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufi go func() { stderr := strings.Builder{} - err := NewCommand(ctx, "cat-file", "--batch"). - Run(&RunOpts{ + err := NewCommand("cat-file", "--batch"). + Run(ctx, &RunOpts{ Dir: repoPath, Stdin: batchStdinReader, Stdout: batchStdoutWriter, @@ -242,7 +242,7 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte { return out } -// ParseTreeLine reads an entry from a tree in a cat-file --batch stream +// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream // This carefully avoids allocations - except where fnameBuf is too small. // It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations // @@ -250,7 +250,7 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte { // <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <binary HASH> // // We don't attempt to convert the raw HASH to save a lot of time -func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { +func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { var readBytes []byte // Read the Mode & fname @@ -260,7 +260,7 @@ func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBu } idx := bytes.IndexByte(readBytes, ' ') if idx < 0 { - log.Debug("missing space in readBytes ParseTreeLine: %s", readBytes) + log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes) return mode, fname, sha, n, &ErrNotExist{} } diff --git a/modules/git/blame.go b/modules/git/blame.go index cad720edf4..659dec34a1 100644 --- a/modules/git/blame.go +++ b/modules/git/blame.go @@ -11,7 +11,7 @@ import ( "os" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/setting" ) // BlamePart represents block of blame - continuous lines with one sha @@ -29,12 +29,13 @@ type BlameReader struct { bufferedReader *bufio.Reader done chan error lastSha *string - ignoreRevsFile *string + ignoreRevsFile string objectFormat ObjectFormat + cleanupFuncs []func() } func (r *BlameReader) UsesIgnoreRevs() bool { - return r.ignoreRevsFile != nil + return r.ignoreRevsFile != "" } // NextPart returns next part of blame (sequential code lines with the same commit) @@ -122,40 +123,49 @@ func (r *BlameReader) Close() error { r.bufferedReader = nil _ = r.reader.Close() _ = r.output.Close() - if r.ignoreRevsFile != nil { - _ = util.Remove(*r.ignoreRevsFile) + for _, cleanup := range r.cleanupFuncs { + if cleanup != nil { + cleanup() + } } return err } // CreateBlameReader creates reader for given repository, commit and file -func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (*BlameReader, error) { - var ignoreRevsFile *string +func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (rd *BlameReader, err error) { + var ignoreRevsFileName string + var ignoreRevsFileCleanup func() + defer func() { + if err != nil && ignoreRevsFileCleanup != nil { + ignoreRevsFileCleanup() + } + }() + + cmd := NewCommandNoGlobals("blame", "--porcelain") + if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { - ignoreRevsFile = tryCreateBlameIgnoreRevsFile(commit) + ignoreRevsFileName, ignoreRevsFileCleanup, err = tryCreateBlameIgnoreRevsFile(commit) + if err != nil && !IsErrNotExist(err) { + return nil, err + } + if ignoreRevsFileName != "" { + // Possible improvement: use --ignore-revs-file /dev/stdin on unix + // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. + cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName) + } } - cmd := NewCommandContextNoGlobals(ctx, "blame", "--porcelain") - if ignoreRevsFile != nil { - // Possible improvement: use --ignore-revs-file /dev/stdin on unix - // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. - cmd.AddOptionValues("--ignore-revs-file", *ignoreRevsFile) - } cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file) + + done := make(chan error, 1) reader, stdout, err := os.Pipe() if err != nil { - if ignoreRevsFile != nil { - _ = util.Remove(*ignoreRevsFile) - } return nil, err } - - done := make(chan error, 1) - go func() { stderr := bytes.Buffer{} // TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close" - err := cmd.Run(&RunOpts{ + err := cmd.Run(ctx, &RunOpts{ UseContextTimeout: true, Dir: repoPath, Stdout: stdout, @@ -169,40 +179,40 @@ func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath }() bufferedReader := bufio.NewReader(reader) - return &BlameReader{ output: stdout, reader: reader, bufferedReader: bufferedReader, done: done, - ignoreRevsFile: ignoreRevsFile, + ignoreRevsFile: ignoreRevsFileName, objectFormat: objectFormat, + cleanupFuncs: []func(){ignoreRevsFileCleanup}, }, nil } -func tryCreateBlameIgnoreRevsFile(commit *Commit) *string { +func tryCreateBlameIgnoreRevsFile(commit *Commit) (string, func(), error) { entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs") if err != nil { - return nil + return "", nil, err } r, err := entry.Blob().DataAsync() if err != nil { - return nil + return "", nil, err } defer r.Close() - f, err := os.CreateTemp("", "gitea_git-blame-ignore-revs") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs") if err != nil { - return nil + return "", nil, err } - + filename := f.Name() _, err = io.Copy(f, r) _ = f.Close() if err != nil { - _ = util.Remove(f.Name()) - return nil + cleanup() + return "", nil, err } - return util.ToPointer(f.Name()) + return filename, cleanup, nil } diff --git a/modules/git/blame_sha256_test.go b/modules/git/blame_sha256_test.go index da451f22fc..c0a97bed3b 100644 --- a/modules/git/blame_sha256_test.go +++ b/modules/git/blame_sha256_test.go @@ -7,11 +7,14 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutputSha256(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + setting.AppDataPath = t.TempDir() + ctx, cancel := context.WithCancel(t.Context()) defer cancel() if isGogit { diff --git a/modules/git/blame_test.go b/modules/git/blame_test.go index 4220c85600..809d6fbcf7 100644 --- a/modules/git/blame_test.go +++ b/modules/git/blame_test.go @@ -7,11 +7,14 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutput(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + setting.AppDataPath = t.TempDir() + ctx, cancel := context.WithCancel(t.Context()) defer cancel() t.Run("Without .git-blame-ignore-revs", func(t *testing.T) { diff --git a/modules/git/blob.go b/modules/git/blob.go index bcecb42e16..40d8f44e79 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -7,7 +7,9 @@ package git import ( "bytes" "encoding/base64" + "errors" "io" + "strings" "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" @@ -20,22 +22,28 @@ func (b *Blob) Name() string { return b.name } -// GetBlobContent Gets the limited content of the blob as raw text -func (b *Blob) GetBlobContent(limit int64) (string, error) { +// GetBlobBytes Gets the limited content of the blob +func (b *Blob) GetBlobBytes(limit int64) ([]byte, error) { if limit <= 0 { - return "", nil + return nil, nil } dataRc, err := b.DataAsync() if err != nil { - return "", err + return nil, err } defer dataRc.Close() - buf, err := util.ReadWithLimit(dataRc, int(limit)) + return util.ReadWithLimit(dataRc, int(limit)) +} + +// GetBlobContent Gets the limited content of the blob as raw text +func (b *Blob) GetBlobContent(limit int64) (string, error) { + buf, err := b.GetBlobBytes(limit) return string(buf), err } -// GetBlobLineCount gets line count of the blob -func (b *Blob) GetBlobLineCount() (int, error) { +// GetBlobLineCount gets line count of the blob. +// It will also try to write the content to w if it's not nil, then we could pre-fetch the content without reading it again. +func (b *Blob) GetBlobLineCount(w io.Writer) (int, error) { reader, err := b.DataAsync() if err != nil { return 0, err @@ -44,59 +52,61 @@ func (b *Blob) GetBlobLineCount() (int, error) { buf := make([]byte, 32*1024) count := 1 lineSep := []byte{'\n'} - - c, err := reader.Read(buf) - if c == 0 && err == io.EOF { - return 0, nil - } for { + c, err := reader.Read(buf) + if w != nil { + if _, err := w.Write(buf[:c]); err != nil { + return count, err + } + } count += bytes.Count(buf[:c], lineSep) switch { - case err == io.EOF: + case errors.Is(err, io.EOF): return count, nil case err != nil: return count, err } - c, err = reader.Read(buf) } } -// GetBlobContentBase64 Reads the content of the blob with a base64 encode and returns the encoded string -func (b *Blob) GetBlobContentBase64() (string, error) { +// GetBlobContentBase64 Reads the content of the blob with a base64 encoding and returns the encoded string +func (b *Blob) GetBlobContentBase64(originContent *strings.Builder) (string, error) { dataRc, err := b.DataAsync() if err != nil { return "", err } defer dataRc.Close() - pr, pw := io.Pipe() - encoder := base64.NewEncoder(base64.StdEncoding, pw) - - go func() { - _, err := io.Copy(encoder, dataRc) - _ = encoder.Close() - - if err != nil { - _ = pw.CloseWithError(err) - } else { - _ = pw.Close() + base64buf := &strings.Builder{} + encoder := base64.NewEncoder(base64.StdEncoding, base64buf) + buf := make([]byte, 32*1024) +loop: + for { + n, err := dataRc.Read(buf) + if n > 0 { + if originContent != nil { + _, _ = originContent.Write(buf[:n]) + } + if _, err := encoder.Write(buf[:n]); err != nil { + return "", err + } + } + switch { + case errors.Is(err, io.EOF): + break loop + case err != nil: + return "", err } - }() - - out, err := io.ReadAll(pr) - if err != nil { - return "", err } - return string(out), nil + _ = encoder.Close() + return base64buf.String(), nil } // GuessContentType guesses the content type of the blob. func (b *Blob) GuessContentType() (typesniffer.SniffedType, error) { - r, err := b.DataAsync() + buf, err := b.GetBlobBytes(typesniffer.SniffContentSize) if err != nil { return typesniffer.SniffedType{}, err } - defer r.Close() - - return typesniffer.DetectContentTypeFromReader(r) + return typesniffer.DetectContentType(buf), nil } diff --git a/modules/git/blob_test.go b/modules/git/blob_test.go index 63374384f6..f21e8d146d 100644 --- a/modules/git/blob_test.go +++ b/modules/git/blob_test.go @@ -17,9 +17,7 @@ func TestBlob_Data(t *testing.T) { output := "file2\n" bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") repo, err := openRepositoryWithDefaultContext(bareRepo1Path) - if !assert.NoError(t, err) { - t.Fatal() - } + require.NoError(t, err) defer repo.Close() testBlob, err := repo.GetBlob("6c493ff740f9380390d5c9ddef4af18697ac9375") @@ -49,7 +47,7 @@ func Benchmark_Blob_Data(b *testing.B) { b.Fatal(err) } - for i := 0; i < b.N; i++ { + for b.Loop() { r, err := testBlob.DataAsync() if err != nil { b.Fatal(err) diff --git a/modules/git/cmdverb.go b/modules/git/cmdverb.go new file mode 100644 index 0000000000..3d6f4ae0c6 --- /dev/null +++ b/modules/git/cmdverb.go @@ -0,0 +1,36 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +const ( + CmdVerbUploadPack = "git-upload-pack" + CmdVerbUploadArchive = "git-upload-archive" + CmdVerbReceivePack = "git-receive-pack" + CmdVerbLfsAuthenticate = "git-lfs-authenticate" + CmdVerbLfsTransfer = "git-lfs-transfer" + + CmdSubVerbLfsUpload = "upload" + CmdSubVerbLfsDownload = "download" +) + +func IsAllowedVerbForServe(verb string) bool { + switch verb { + case CmdVerbUploadPack, + CmdVerbUploadArchive, + CmdVerbReceivePack, + CmdVerbLfsAuthenticate, + CmdVerbLfsTransfer: + return true + } + return false +} + +func IsAllowedVerbForServeLfs(verb string) bool { + switch verb { + case CmdVerbLfsAuthenticate, + CmdVerbLfsTransfer: + return true + } + return false +} diff --git a/modules/git/command.go b/modules/git/command.go index b231c3beea..22f1d02339 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -18,6 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/git/internal" //nolint:depguard // only this file can use the internal type CmdArg, other files and packages should use AddXxx functions + "code.gitea.io/gitea/modules/gtprof" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/util" @@ -43,9 +44,10 @@ const DefaultLocale = "C" type Command struct { prog string args []string - parentContext context.Context globalArgsLength int brokenArgs []string + cmd *exec.Cmd // for debug purpose only + configArgs []string } func logArgSanitize(arg string) string { @@ -54,7 +56,7 @@ func logArgSanitize(arg string) string { } else if filepath.IsAbs(arg) { base := filepath.Base(arg) dir := filepath.Dir(arg) - return filepath.Join(filepath.Base(dir), base) + return ".../" + filepath.Join(filepath.Base(dir), base) } return arg } @@ -79,9 +81,16 @@ func (c *Command) LogString() string { return strings.Join(a, " ") } +func (c *Command) ProcessState() string { + if c.cmd == nil { + return "" + } + return c.cmd.ProcessState.String() +} + // NewCommand creates and returns a new Git Command based on given command and arguments. // Each argument should be safe to be trusted. User-provided arguments should be passed to AddDynamicArguments instead. -func NewCommand(ctx context.Context, args ...internal.CmdArg) *Command { +func NewCommand(args ...internal.CmdArg) *Command { // Make an explicit copy of globalCommandArgs, otherwise append might overwrite it cargs := make([]string, 0, len(globalCommandArgs)+len(args)) for _, arg := range globalCommandArgs { @@ -93,31 +102,23 @@ func NewCommand(ctx context.Context, args ...internal.CmdArg) *Command { return &Command{ prog: GitExecutable, args: cargs, - parentContext: ctx, globalArgsLength: len(globalCommandArgs), } } -// NewCommandContextNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args +// NewCommandNoGlobals creates and returns a new Git Command based on given command and arguments only with the specified args and don't use global command args // Each argument should be safe to be trusted. User-provided arguments should be passed to AddDynamicArguments instead. -func NewCommandContextNoGlobals(ctx context.Context, args ...internal.CmdArg) *Command { +func NewCommandNoGlobals(args ...internal.CmdArg) *Command { cargs := make([]string, 0, len(args)) for _, arg := range args { cargs = append(cargs, string(arg)) } return &Command{ - prog: GitExecutable, - args: cargs, - parentContext: ctx, + prog: GitExecutable, + args: cargs, } } -// SetParentContext sets the parent context for this command -func (c *Command) SetParentContext(ctx context.Context) *Command { - c.parentContext = ctx - return c -} - // isSafeArgumentValue checks if the argument is safe to be used as a value (not an option) func isSafeArgumentValue(s string) bool { return s == "" || s[0] != '-' @@ -196,6 +197,16 @@ func (c *Command) AddDashesAndList(list ...string) *Command { return c } +func (c *Command) AddConfig(key, value string) *Command { + kv := key + "=" + value + if !isSafeArgumentValue(kv) { + c.brokenArgs = append(c.brokenArgs, key) + } else { + c.configArgs = append(c.configArgs, "-c", kv) + } + return c +} + // ToTrustedCmdArgs converts a list of strings (trusted as argument) to TrustedCmdArgs // In most cases, it shouldn't be used. Use NewCommand().AddXxx() function instead func ToTrustedCmdArgs(args []string) TrustedCmdArgs { @@ -236,10 +247,16 @@ type RunOpts struct { } func commonBaseEnvs() []string { - // at the moment, do not set "GIT_CONFIG_NOSYSTEM", users may have put some configs like "receive.certNonceSeed" in it envs := []string{ - "HOME=" + HomeDir(), // make Gitea use internal git config only, to prevent conflicts with user's git config - "GIT_NO_REPLACE_OBJECTS=1", // ignore replace references (https://git-scm.com/docs/git-replace) + // Make Gitea use internal git config only, to prevent conflicts with user's git config + // It's better to use GIT_CONFIG_GLOBAL, but it requires git >= 2.32, so we still use HOME at the moment. + "HOME=" + HomeDir(), + // Avoid using system git config, it would cause problems (eg: use macOS osxkeychain to show a modal dialog, auto installing lfs hooks) + // This might be a breaking change in 1.24, because some users said that they have put some configs like "receive.certNonceSeed" in "/etc/gitconfig" + // For these users, they need to migrate the necessary configs to Gitea's git config file manually. + "GIT_CONFIG_NOSYSTEM=1", + // Ignore replace references (https://git-scm.com/docs/git-replace) + "GIT_NO_REPLACE_OBJECTS=1", } // some environment variables should be passed to git command @@ -270,11 +287,11 @@ func CommonCmdServEnvs() []string { var ErrBrokenCommand = errors.New("git command is broken") // Run runs the command with the RunOpts -func (c *Command) Run(opts *RunOpts) error { - return c.run(1, opts) +func (c *Command) Run(ctx context.Context, opts *RunOpts) error { + return c.run(ctx, 1, opts) } -func (c *Command) run(skip int, opts *RunOpts) error { +func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { if len(c.brokenArgs) != 0 { log.Error("git command is broken: %s, broken args: %s", c.LogString(), strings.Join(c.brokenArgs, " ")) return ErrBrokenCommand @@ -289,29 +306,34 @@ func (c *Command) run(skip int, opts *RunOpts) error { timeout = defaultCommandExecutionTimeout } - var desc string + cmdLogString := c.LogString() callerInfo := util.CallerFuncName(1 /* util */ + 1 /* this */ + skip /* parent */) if pos := strings.LastIndex(callerInfo, "/"); pos >= 0 { callerInfo = callerInfo[pos+1:] } // these logs are for debugging purposes only, so no guarantee of correctness or stability - desc = fmt.Sprintf("git.Run(by:%s, repo:%s): %s", callerInfo, logArgSanitize(opts.Dir), c.LogString()) + desc := fmt.Sprintf("git.Run(by:%s, repo:%s): %s", callerInfo, logArgSanitize(opts.Dir), cmdLogString) log.Debug("git.Command: %s", desc) - var ctx context.Context + _, span := gtprof.GetTracer().Start(ctx, gtprof.TraceSpanGitRun) + defer span.End() + span.SetAttributeString(gtprof.TraceAttrFuncCaller, callerInfo) + span.SetAttributeString(gtprof.TraceAttrGitCommand, cmdLogString) + var cancel context.CancelFunc var finished context.CancelFunc if opts.UseContextTimeout { - ctx, cancel, finished = process.GetManager().AddContext(c.parentContext, desc) + ctx, cancel, finished = process.GetManager().AddContext(ctx, desc) } else { - ctx, cancel, finished = process.GetManager().AddContextTimeout(c.parentContext, timeout, desc) + ctx, cancel, finished = process.GetManager().AddContextTimeout(ctx, timeout, desc) } defer finished() startTime := time.Now() - cmd := exec.CommandContext(ctx, c.prog, c.args...) + cmd := exec.CommandContext(ctx, c.prog, append(c.configArgs, c.args...)...) + c.cmd = cmd // for debug purpose only if opts.Env == nil { cmd.Env = os.Environ() } else { @@ -346,9 +368,10 @@ func (c *Command) run(skip int, opts *RunOpts) error { // We need to check if the context is canceled by the program on Windows. // This is because Windows does not have signal checking when terminating the process. // It always returns exit code 1, unlike Linux, which has many exit codes for signals. + // `err.Error()` returns "exit status 1" when using the `git check-attr` command after the context is canceled. if runtime.GOOS == "windows" && err != nil && - err.Error() == "" && + (err.Error() == "" || err.Error() == "exit status 1") && cmd.ProcessState.ExitCode() == 1 && ctx.Err() == context.Canceled { return ctx.Err() @@ -398,8 +421,8 @@ func IsErrorExitCode(err error, code int) bool { } // RunStdString runs the command with options and returns stdout/stderr as string. and store stderr to returned error (err combined with stderr). -func (c *Command) RunStdString(opts *RunOpts) (stdout, stderr string, runErr RunStdError) { - stdoutBytes, stderrBytes, err := c.runStdBytes(opts) +func (c *Command) RunStdString(ctx context.Context, opts *RunOpts) (stdout, stderr string, runErr RunStdError) { + stdoutBytes, stderrBytes, err := c.runStdBytes(ctx, opts) stdout = util.UnsafeBytesToString(stdoutBytes) stderr = util.UnsafeBytesToString(stderrBytes) if err != nil { @@ -410,11 +433,11 @@ func (c *Command) RunStdString(opts *RunOpts) (stdout, stderr string, runErr Run } // RunStdBytes runs the command with options and returns stdout/stderr as bytes. and store stderr to returned error (err combined with stderr). -func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunStdError) { - return c.runStdBytes(opts) +func (c *Command) RunStdBytes(ctx context.Context, opts *RunOpts) (stdout, stderr []byte, runErr RunStdError) { + return c.runStdBytes(ctx, opts) } -func (c *Command) runStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunStdError) { +func (c *Command) runStdBytes(ctx context.Context, opts *RunOpts) (stdout, stderr []byte, runErr RunStdError) { if opts == nil { opts = &RunOpts{} } @@ -437,7 +460,7 @@ func (c *Command) runStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS PipelineFunc: opts.PipelineFunc, } - err := c.run(2, newOpts) + err := c.run(ctx, 2, newOpts) stderr = stderrBuf.Bytes() if err != nil { return nil, stderr, &runStdError{err: err, stderr: util.UnsafeBytesToString(stderr)} diff --git a/modules/git/command_race_test.go b/modules/git/command_race_test.go index f567406822..a6aa3a1580 100644 --- a/modules/git/command_race_test.go +++ b/modules/git/command_race_test.go @@ -15,9 +15,9 @@ func TestRunWithContextNoTimeout(t *testing.T) { maxLoops := 10 // 'git --version' does not block so it must be finished before the timeout triggered. - cmd := NewCommand(context.Background(), "--version") + cmd := NewCommand("--version") for i := 0; i < maxLoops; i++ { - if err := cmd.Run(&RunOpts{}); err != nil { + if err := cmd.Run(t.Context(), &RunOpts{}); err != nil { t.Fatal(err) } } @@ -27,9 +27,9 @@ func TestRunWithContextTimeout(t *testing.T) { maxLoops := 10 // 'git hash-object --stdin' blocks on stdin so we can have the timeout triggered. - cmd := NewCommand(context.Background(), "hash-object", "--stdin") + cmd := NewCommand("hash-object", "--stdin") for i := 0; i < maxLoops; i++ { - if err := cmd.Run(&RunOpts{Timeout: 1 * time.Millisecond}); err != nil { + if err := cmd.Run(t.Context(), &RunOpts{Timeout: 1 * time.Millisecond}); err != nil { if err != context.DeadlineExceeded { t.Fatalf("Testing %d/%d: %v", i, maxLoops, err) } diff --git a/modules/git/command_test.go b/modules/git/command_test.go index 0823afd7f7..eb112707e7 100644 --- a/modules/git/command_test.go +++ b/modules/git/command_test.go @@ -4,21 +4,20 @@ package git import ( - "context" "testing" "github.com/stretchr/testify/assert" ) func TestRunWithContextStd(t *testing.T) { - cmd := NewCommand(context.Background(), "--version") - stdout, stderr, err := cmd.RunStdString(&RunOpts{}) + cmd := NewCommand("--version") + stdout, stderr, err := cmd.RunStdString(t.Context(), &RunOpts{}) assert.NoError(t, err) assert.Empty(t, stderr) assert.Contains(t, stdout, "git version") - cmd = NewCommand(context.Background(), "--no-such-arg") - stdout, stderr, err = cmd.RunStdString(&RunOpts{}) + cmd = NewCommand("--no-such-arg") + stdout, stderr, err = cmd.RunStdString(t.Context(), &RunOpts{}) if assert.Error(t, err) { assert.Equal(t, stderr, err.Stderr()) assert.Contains(t, err.Stderr(), "unknown option:") @@ -26,17 +25,17 @@ func TestRunWithContextStd(t *testing.T) { assert.Empty(t, stdout) } - cmd = NewCommand(context.Background()) + cmd = NewCommand() cmd.AddDynamicArguments("-test") - assert.ErrorIs(t, cmd.Run(&RunOpts{}), ErrBrokenCommand) + assert.ErrorIs(t, cmd.Run(t.Context(), &RunOpts{}), ErrBrokenCommand) - cmd = NewCommand(context.Background()) + cmd = NewCommand() cmd.AddDynamicArguments("--test") - assert.ErrorIs(t, cmd.Run(&RunOpts{}), ErrBrokenCommand) + assert.ErrorIs(t, cmd.Run(t.Context(), &RunOpts{}), ErrBrokenCommand) subCmd := "version" - cmd = NewCommand(context.Background()).AddDynamicArguments(subCmd) // for test purpose only, the sub-command should never be dynamic for production - stdout, stderr, err = cmd.RunStdString(&RunOpts{}) + cmd = NewCommand().AddDynamicArguments(subCmd) // for test purpose only, the sub-command should never be dynamic for production + stdout, stderr, err = cmd.RunStdString(t.Context(), &RunOpts{}) assert.NoError(t, err) assert.Empty(t, stderr) assert.Contains(t, stdout, "git version") @@ -54,9 +53,9 @@ func TestGitArgument(t *testing.T) { } func TestCommandString(t *testing.T) { - cmd := NewCommandContextNoGlobals(context.Background(), "a", "-m msg", "it's a test", `say "hello"`) - assert.EqualValues(t, cmd.prog+` a "-m msg" "it's a test" "say \"hello\""`, cmd.LogString()) + cmd := NewCommandNoGlobals("a", "-m msg", "it's a test", `say "hello"`) + assert.Equal(t, cmd.prog+` a "-m msg" "it's a test" "say \"hello\""`, cmd.LogString()) - cmd = NewCommandContextNoGlobals(context.Background(), "url: https://a:b@c/", "/root/dir-a/dir-b") - assert.EqualValues(t, cmd.prog+` "url: https://sanitized-credential@c/" dir-a/dir-b`, cmd.LogString()) + cmd = NewCommandNoGlobals("url: https://a:b@c/", "/root/dir-a/dir-b") + assert.Equal(t, cmd.prog+` "url: https://sanitized-credential@c/" .../dir-a/dir-b`, cmd.LogString()) } diff --git a/modules/git/commit.go b/modules/git/commit.go index 0ed268e346..aae40c575b 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -20,10 +20,11 @@ import ( // Commit represents a git commit. type Commit struct { - Tree - ID ObjectID // The ID of this commit object - Author *Signature - Committer *Signature + Tree // FIXME: bad design, this field can be nil if the commit is from "last commit cache" + + ID ObjectID + Author *Signature // never nil + Committer *Signature // never nil CommitMessage string Signature *CommitSignature @@ -34,7 +35,7 @@ type Commit struct { // CommitSignature represents a git commit signature part. type CommitSignature struct { Signature string - Payload string // TODO check if can be reconstruct from the rest of commit information to not have duplicate data + Payload string } // Message returns the commit message. Same as retrieving CommitMessage directly. @@ -91,12 +92,12 @@ func AddChanges(repoPath string, all bool, files ...string) error { // AddChangesWithArgs marks local changes to be ready for commit. func AddChangesWithArgs(repoPath string, globalArgs TrustedCmdArgs, all bool, files ...string) error { - cmd := NewCommandContextNoGlobals(DefaultContext, globalArgs...).AddArguments("add") + cmd := NewCommandNoGlobals(globalArgs...).AddArguments("add") if all { cmd.AddArguments("--all") } cmd.AddDashesAndList(files...) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + _, _, err := cmd.RunStdString(DefaultContext, &RunOpts{Dir: repoPath}) return err } @@ -118,7 +119,7 @@ func CommitChanges(repoPath string, opts CommitChangesOptions) error { // CommitChangesWithArgs commits local changes with given committer, author and message. // If author is nil, it will be the same as committer. func CommitChangesWithArgs(repoPath string, args TrustedCmdArgs, opts CommitChangesOptions) error { - cmd := NewCommandContextNoGlobals(DefaultContext, args...) + cmd := NewCommandNoGlobals(args...) if opts.Committer != nil { cmd.AddOptionValues("-c", "user.name="+opts.Committer.Name) cmd.AddOptionValues("-c", "user.email="+opts.Committer.Email) @@ -133,7 +134,7 @@ func CommitChangesWithArgs(repoPath string, args TrustedCmdArgs, opts CommitChan } cmd.AddOptionFormat("--message=%s", opts.Message) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + _, _, err := cmd.RunStdString(DefaultContext, &RunOpts{Dir: repoPath}) // No stderr but exit status 1 means nothing to commit. if err != nil && err.Error() == "exit status 1" { return nil @@ -143,7 +144,7 @@ func CommitChangesWithArgs(repoPath string, args TrustedCmdArgs, opts CommitChan // AllCommitsCount returns count of all commits in repository func AllCommitsCount(ctx context.Context, repoPath string, hidePRRefs bool, files ...string) (int64, error) { - cmd := NewCommand(ctx, "rev-list") + cmd := NewCommand("rev-list") if hidePRRefs { cmd.AddArguments("--exclude=" + PullPrefix + "*") } @@ -152,7 +153,7 @@ func AllCommitsCount(ctx context.Context, repoPath string, hidePRRefs bool, file cmd.AddDashesAndList(files...) } - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return 0, err } @@ -166,11 +167,13 @@ type CommitsCountOptions struct { Not string Revision []string RelPath []string + Since string + Until string } // CommitsCount returns number of total commits of until given revision. func CommitsCount(ctx context.Context, opts CommitsCountOptions) (int64, error) { - cmd := NewCommand(ctx, "rev-list", "--count") + cmd := NewCommand("rev-list", "--count") cmd.AddDynamicArguments(opts.Revision...) @@ -182,7 +185,7 @@ func CommitsCount(ctx context.Context, opts CommitsCountOptions) (int64, error) cmd.AddDashesAndList(opts.RelPath...) } - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: opts.RepoPath}) + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: opts.RepoPath}) if err != nil { return 0, err } @@ -199,8 +202,8 @@ func (c *Commit) CommitsCount() (int64, error) { } // CommitsByRange returns the specific page commits before current revision, every page's number default by CommitsRangeSize -func (c *Commit) CommitsByRange(page, pageSize int, not string) ([]*Commit, error) { - return c.repo.commitsByRange(c.ID, page, pageSize, not) +func (c *Commit) CommitsByRange(page, pageSize int, not, since, until string) ([]*Commit, error) { + return c.repo.commitsByRangeWithTime(c.ID, page, pageSize, not, since, until) } // CommitsBefore returns all the commits before current revision @@ -217,7 +220,7 @@ func (c *Commit) HasPreviousCommit(objectID ObjectID) (bool, error) { return false, nil } - _, _, err := NewCommand(c.repo.Ctx, "merge-base", "--is-ancestor").AddDynamicArguments(that, this).RunStdString(&RunOpts{Dir: c.repo.Path}) + _, _, err := NewCommand("merge-base", "--is-ancestor").AddDynamicArguments(that, this).RunStdString(c.repo.Ctx, &RunOpts{Dir: c.repo.Path}) if err == nil { return true, nil } @@ -275,8 +278,8 @@ func NewSearchCommitsOptions(searchString string, forAllRefs bool) SearchCommits var keywords, authors, committers []string var after, before string - fields := strings.Fields(searchString) - for _, k := range fields { + fields := strings.FieldsSeq(searchString) + for k := range fields { switch { case strings.HasPrefix(k, "author:"): authors = append(authors, strings.TrimPrefix(k, "author:")) @@ -358,12 +361,12 @@ func (c *Commit) GetFileContent(filename string, limit int) (string, error) { // GetBranchName gets the closest branch name (as returned by 'git name-rev --name-only') func (c *Commit) GetBranchName() (string, error) { - cmd := NewCommand(c.repo.Ctx, "name-rev") + cmd := NewCommand("name-rev") if DefaultFeatures().CheckVersionAtLeast("2.13.0") { cmd.AddArguments("--exclude", "refs/tags/*") } cmd.AddArguments("--name-only", "--no-undefined").AddDynamicArguments(c.ID.String()) - data, _, err := cmd.RunStdString(&RunOpts{Dir: c.repo.Path}) + data, _, err := cmd.RunStdString(c.repo.Ctx, &RunOpts{Dir: c.repo.Path}) if err != nil { // handle special case where git can not describe commit if strings.Contains(err.Error(), "cannot describe") { @@ -441,7 +444,7 @@ func GetCommitFileStatus(ctx context.Context, repoPath, commitID string) (*Commi }() stderr := new(bytes.Buffer) - err := NewCommand(ctx, "log", "--name-status", "-m", "--pretty=format:", "--first-parent", "--no-renames", "-z", "-1").AddDynamicArguments(commitID).Run(&RunOpts{ + err := NewCommand("log", "--name-status", "-m", "--pretty=format:", "--first-parent", "--no-renames", "-z", "-1").AddDynamicArguments(commitID).Run(ctx, &RunOpts{ Dir: repoPath, Stdout: w, Stderr: stderr, @@ -457,7 +460,7 @@ func GetCommitFileStatus(ctx context.Context, repoPath, commitID string) (*Commi // GetFullCommitID returns full length (40) of commit ID by given short SHA in a repository. func GetFullCommitID(ctx context.Context, repoPath, shortID string) (string, error) { - commitID, _, err := NewCommand(ctx, "rev-parse").AddDynamicArguments(shortID).RunStdString(&RunOpts{Dir: repoPath}) + commitID, _, err := NewCommand("rev-parse").AddDynamicArguments(shortID).RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { if strings.Contains(err.Error(), "exit status 128") { return "", ErrNotExist{shortID, ""} @@ -476,8 +479,12 @@ func (c *Commit) GetRepositoryDefaultPublicGPGKey(forceUpdate bool) (*GPGSetting } func IsStringLikelyCommitID(objFmt ObjectFormat, s string, minLength ...int) bool { - minLen := util.OptionalArg(minLength, objFmt.FullLength()) - if len(s) < minLen || len(s) > objFmt.FullLength() { + maxLen := 64 // sha256 + if objFmt != nil { + maxLen = objFmt.FullLength() + } + minLen := util.OptionalArg(minLength, maxLen) + if len(s) < minLen || len(s) > maxLen { return false } for _, c := range s { diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go index 545081275b..4f76a28f31 100644 --- a/modules/git/commit_info.go +++ b/modules/git/commit_info.go @@ -7,5 +7,17 @@ package git type CommitInfo struct { Entry *TreeEntry Commit *Commit - SubModuleFile *CommitSubModuleFile + SubmoduleFile *CommitSubmoduleFile +} + +func GetCommitInfoSubmoduleFile(repoLink, fullPath string, commit *Commit, refCommitID ObjectID) (*CommitSubmoduleFile, error) { + submodule, err := commit.GetSubModule(fullPath) + if err != nil { + return nil, err + } + if submodule == nil { + // unable to find submodule from ".gitmodules" file + return NewCommitSubmoduleFile(repoLink, fullPath, "", refCommitID.String()), nil + } + return NewCommitSubmoduleFile(repoLink, fullPath, submodule.URL, refCommitID.String()), nil } diff --git a/modules/git/commit_info_gogit.go b/modules/git/commit_info_gogit.go index 11b44f7c35..73227347bc 100644 --- a/modules/git/commit_info_gogit.go +++ b/modules/git/commit_info_gogit.go @@ -16,7 +16,7 @@ import ( ) // GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) { +func (tes Entries) GetCommitsInfo(ctx context.Context, repoLink string, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) { entryPaths := make([]string, len(tes)+1) // Get the commit for the treePath itself entryPaths[0] = "" @@ -71,22 +71,12 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath commitsInfo[i].Commit = entryCommit } - // If the entry is a submodule add a submodule file for this + // If the entry is a submodule, add a submodule file for this if entry.IsSubModule() { - subModuleURL := "" - var fullPath string - if len(treePath) > 0 { - fullPath = treePath + "/" + entry.Name() - } else { - fullPath = entry.Name() - } - if subModule, err := commit.GetSubModule(fullPath); err != nil { + commitsInfo[i].SubmoduleFile, err = GetCommitInfoSubmoduleFile(repoLink, path.Join(treePath, entry.Name()), commit, entry.ID) + if err != nil { return nil, nil, err - } else if subModule != nil { - subModuleURL = subModule.URL } - subModuleFile := NewCommitSubModuleFile(subModuleURL, entry.ID.String()) - commitsInfo[i].SubModuleFile = subModuleFile } } diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index 20d586f0ff..ed775332a9 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -7,8 +7,7 @@ package git import ( "context" - "fmt" - "io" + "maps" "path" "sort" @@ -16,7 +15,7 @@ import ( ) // GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) { +func (tes Entries) GetCommitsInfo(ctx context.Context, repoLink string, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) { entryPaths := make([]string, len(tes)+1) // Get the commit for the treePath itself entryPaths[0] = "" @@ -40,9 +39,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath return nil, nil, err } - for pth, found := range commits { - revs[pth] = found - } + maps.Copy(revs, commits) } } else { sort.Strings(entryPaths) @@ -65,28 +62,18 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath log.Debug("missing commit for %s", entry.Name()) } - // If the entry if a submodule add a submodule file for this + // If the entry is a submodule, add a submodule file for this if entry.IsSubModule() { - subModuleURL := "" - var fullPath string - if len(treePath) > 0 { - fullPath = treePath + "/" + entry.Name() - } else { - fullPath = entry.Name() - } - if subModule, err := commit.GetSubModule(fullPath); err != nil { + commitsInfo[i].SubmoduleFile, err = GetCommitInfoSubmoduleFile(repoLink, path.Join(treePath, entry.Name()), commit, entry.ID) + if err != nil { return nil, nil, err - } else if subModule != nil { - subModuleURL = subModule.URL } - subModuleFile := NewCommitSubModuleFile(subModuleURL, entry.ID.String()) - commitsInfo[i].SubModuleFile = subModuleFile } } // Retrieve the commit for the treePath itself (see above). We basically - // get it for free during the tree traversal and it's used for listing - // pages to display information about newest commit for a given path. + // get it for free during the tree traversal, and it's used for listing + // pages to display information about the newest commit for a given path. var treeCommit *Commit var ok bool if treePath == "" { @@ -124,48 +111,25 @@ func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, return nil, err } - batchStdinWriter, batchReader, cancel, err := commit.repo.CatFileBatch(ctx) - if err != nil { - return nil, err - } - defer cancel() - commitsMap := map[string]*Commit{} commitsMap[commit.ID.String()] = commit commitCommits := map[string]*Commit{} for path, commitID := range revs { - c, ok := commitsMap[commitID] - if ok { - commitCommits[path] = c + if len(commitID) == 0 { continue } - if len(commitID) == 0 { + c, ok := commitsMap[commitID] + if ok { + commitCommits[path] = c continue } - _, err := batchStdinWriter.Write([]byte(commitID + "\n")) - if err != nil { - return nil, err - } - _, typ, size, err := ReadBatchLine(batchReader) - if err != nil { - return nil, err - } - if typ != "commit" { - if err := DiscardFull(batchReader, size+1); err != nil { - return nil, err - } - return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID) - } - c, err = CommitFromReader(commit.repo, MustIDFromString(commitID), io.LimitReader(batchReader, size)) + c, err := commit.repo.GetCommit(commitID) // Ensure the commit exists in the repository if err != nil { return nil, err } - if _, err := batchReader.Discard(1); err != nil { - return nil, err - } commitCommits[path] = c } diff --git a/modules/git/commit_info_test.go b/modules/git/commit_info_test.go index 1e331fac00..078b6815d2 100644 --- a/modules/git/commit_info_test.go +++ b/modules/git/commit_info_test.go @@ -4,12 +4,12 @@ package git import ( - "context" "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( @@ -83,7 +83,7 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) { } // FIXME: Context.TODO() - if graceful has started we should use its Shutdown context otherwise use install signals in TestMain. - commitsInfo, treeCommit, err := entries.GetCommitsInfo(context.TODO(), commit, testCase.Path) + commitsInfo, treeCommit, err := entries.GetCommitsInfo(t.Context(), "/any/repo-link", commit, testCase.Path) assert.NoError(t, err, "Unable to get commit information for entries of subtree: %s in commit: %s from testcase due to error: %v", testCase.Path, testCase.CommitID, err) if err != nil { t.FailNow() @@ -121,6 +121,23 @@ func TestEntries_GetCommitsInfo(t *testing.T) { defer clonedRepo1.Close() testGetCommitsInfo(t, clonedRepo1) + + t.Run("NonExistingSubmoduleAsNil", func(t *testing.T) { + commit, err := bareRepo1.GetCommit("HEAD") + require.NoError(t, err) + treeEntry, err := commit.GetTreeEntryByPath("file1.txt") + require.NoError(t, err) + cisf, err := GetCommitInfoSubmoduleFile("/any/repo-link", "file1.txt", commit, treeEntry.ID) + require.NoError(t, err) + assert.Equal(t, &CommitSubmoduleFile{ + repoLink: "/any/repo-link", + fullPath: "file1.txt", + refURL: "", + refID: "e2129701f1a4d54dc44f03c93bca0a2aec7c5449", + }, cisf) + // since there is no refURL, it means that the submodule info doesn't exist, so it won't have a web link + assert.Nil(t, cisf.SubmoduleWebLinkTree(t.Context())) + }) } func BenchmarkEntries_GetCommitsInfo(b *testing.B) { @@ -159,8 +176,8 @@ func BenchmarkEntries_GetCommitsInfo(b *testing.B) { entries.Sort() b.ResetTimer() b.Run(benchmark.name, func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _, err := entries.GetCommitsInfo(context.Background(), commit, "") + for b.Loop() { + _, _, err := entries.GetCommitsInfo(b.Context(), "/any/repo-link", commit, "") if err != nil { b.Fatal(err) } diff --git a/modules/git/commit_reader.go b/modules/git/commit_reader.go index 228bbaf314..eb8f4c6322 100644 --- a/modules/git/commit_reader.go +++ b/modules/git/commit_reader.go @@ -6,10 +6,44 @@ package git import ( "bufio" "bytes" + "fmt" "io" - "strings" ) +const ( + commitHeaderGpgsig = "gpgsig" + commitHeaderGpgsigSha256 = "gpgsig-sha256" +) + +func assignCommitFields(gitRepo *Repository, commit *Commit, headerKey string, headerValue []byte) error { + if len(headerValue) > 0 && headerValue[len(headerValue)-1] == '\n' { + headerValue = headerValue[:len(headerValue)-1] // remove trailing newline + } + switch headerKey { + case "tree": + objID, err := NewIDFromString(string(headerValue)) + if err != nil { + return fmt.Errorf("invalid tree ID %q: %w", string(headerValue), err) + } + commit.Tree = *NewTree(gitRepo, objID) + case "parent": + objID, err := NewIDFromString(string(headerValue)) + if err != nil { + return fmt.Errorf("invalid parent ID %q: %w", string(headerValue), err) + } + commit.Parents = append(commit.Parents, objID) + case "author": + commit.Author.Decode(headerValue) + case "committer": + commit.Committer.Decode(headerValue) + case commitHeaderGpgsig, commitHeaderGpgsigSha256: + // if there are duplicate "gpgsig" and "gpgsig-sha256" headers, then the signature must have already been invalid + // so we don't need to handle duplicate headers here + commit.Signature = &CommitSignature{Signature: string(headerValue)} + } + return nil +} + // CommitFromReader will generate a Commit from a provided reader // We need this to interpret commits from cat-file or cat-file --batch // @@ -21,90 +55,46 @@ func CommitFromReader(gitRepo *Repository, objectID ObjectID, reader io.Reader) Committer: &Signature{}, } - payloadSB := new(strings.Builder) - signatureSB := new(strings.Builder) - messageSB := new(strings.Builder) - message := false - pgpsig := false - - bufReader, ok := reader.(*bufio.Reader) - if !ok { - bufReader = bufio.NewReader(reader) - } - -readLoop: + bufReader := bufio.NewReader(reader) + inHeader := true + var payloadSB, messageSB bytes.Buffer + var headerKey string + var headerValue []byte for { line, err := bufReader.ReadBytes('\n') - if err != nil { - if err == io.EOF { - if message { - _, _ = messageSB.Write(line) - } - _, _ = payloadSB.Write(line) - break readLoop - } - return nil, err + if err != nil && err != io.EOF { + return nil, fmt.Errorf("unable to read commit %q: %w", objectID.String(), err) } - if pgpsig { - if len(line) > 0 && line[0] == ' ' { - _, _ = signatureSB.Write(line[1:]) - continue - } - pgpsig = false + if len(line) == 0 { + break } - if !message { - // This is probably not correct but is copied from go-gits interpretation... - trimmed := bytes.TrimSpace(line) - if len(trimmed) == 0 { - message = true - _, _ = payloadSB.Write(line) - continue - } - - split := bytes.SplitN(trimmed, []byte{' '}, 2) - var data []byte - if len(split) > 1 { - data = split[1] + if inHeader { + inHeader = !(len(line) == 1 && line[0] == '\n') // still in header if line is not just a newline + k, v, _ := bytes.Cut(line, []byte{' '}) + if len(k) != 0 || !inHeader { + if headerKey != "" { + if err = assignCommitFields(gitRepo, commit, headerKey, headerValue); err != nil { + return nil, fmt.Errorf("unable to parse commit %q: %w", objectID.String(), err) + } + } + headerKey = string(k) // it also resets the headerValue to empty string if not inHeader + headerValue = v + } else { + headerValue = append(headerValue, v...) } - - switch string(split[0]) { - case "tree": - commit.Tree = *NewTree(gitRepo, MustIDFromString(string(data))) + if headerKey != commitHeaderGpgsig && headerKey != commitHeaderGpgsigSha256 { _, _ = payloadSB.Write(line) - case "parent": - commit.Parents = append(commit.Parents, MustIDFromString(string(data))) - _, _ = payloadSB.Write(line) - case "author": - commit.Author = &Signature{} - commit.Author.Decode(data) - _, _ = payloadSB.Write(line) - case "committer": - commit.Committer = &Signature{} - commit.Committer.Decode(data) - _, _ = payloadSB.Write(line) - case "encoding": - _, _ = payloadSB.Write(line) - case "gpgsig": - fallthrough - case "gpgsig-sha256": // FIXME: no intertop, so only 1 exists at present. - _, _ = signatureSB.Write(data) - _ = signatureSB.WriteByte('\n') - pgpsig = true } } else { _, _ = messageSB.Write(line) _, _ = payloadSB.Write(line) } } + commit.CommitMessage = messageSB.String() - commit.Signature = &CommitSignature{ - Signature: signatureSB.String(), - Payload: payloadSB.String(), - } - if len(commit.Signature.Signature) == 0 { - commit.Signature = nil + if commit.Signature != nil { + commit.Signature.Payload = payloadSB.String() } - return commit, nil } diff --git a/modules/git/commit_sha256_test.go b/modules/git/commit_sha256_test.go index 2184a9c47c..97ccecdacc 100644 --- a/modules/git/commit_sha256_test.go +++ b/modules/git/commit_sha256_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCommitsCountSha256(t *testing.T) { @@ -59,8 +60,7 @@ func TestGetFullCommitIDErrorSha256(t *testing.T) { } func TestCommitFromReaderSha256(t *testing.T) { - commitString := `9433b2a62b964c17a4485ae180f45f595d3e69d31b786087775e28c6b6399df0 commit 1114 -tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e + commitString := `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e parent 26e9ccc29fad747e9c5d9f4c9ddeb7eff61cc45ef6a8dc258cbeb181afc055e8 author Adam Majer <amajer@suse.de> 1698676906 +0100 committer Adam Majer <amajer@suse.de> 1698676906 +0100 @@ -94,11 +94,9 @@ signed commit` commitFromReader, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString)) assert.NoError(t, err) - if !assert.NotNil(t, commitFromReader) { - return - } + require.NotNil(t, commitFromReader) assert.EqualValues(t, sha, commitFromReader.ID) - assert.EqualValues(t, `-----BEGIN PGP SIGNATURE----- + assert.Equal(t, `-----BEGIN PGP SIGNATURE----- iQIrBAABCgAtFiEES+fB08xlgTrzSdQvhkUIsBsmec8FAmU/wKoPHGFtYWplckBz dXNlLmRlAAoJEIZFCLAbJnnP4s4PQIJATa++WPzR6/H4etT7bsOGoMyguEJYyWOd @@ -113,21 +111,20 @@ VAEUo6ecdDxSpyt2naeg9pKus/BRi7P6g4B1hkk/zZstUX/QP4IQuAJbXjkvsC+X HKRr3NlRM/DygzTyj0gN74uoa0goCIbyAQhiT42nm0cuhM7uN/W0ayrlZjGF1cbR 8NCJUL2Nwj0ywKIavC99Ipkb8AsFwpVT6U6effs6 =xybZ ------END PGP SIGNATURE----- -`, commitFromReader.Signature.Signature) - assert.EqualValues(t, `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e +-----END PGP SIGNATURE-----`, commitFromReader.Signature.Signature) + assert.Equal(t, `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e parent 26e9ccc29fad747e9c5d9f4c9ddeb7eff61cc45ef6a8dc258cbeb181afc055e8 author Adam Majer <amajer@suse.de> 1698676906 +0100 committer Adam Majer <amajer@suse.de> 1698676906 +0100 signed commit`, commitFromReader.Signature.Payload) - assert.EqualValues(t, "Adam Majer <amajer@suse.de>", commitFromReader.Author.String()) + assert.Equal(t, "Adam Majer <amajer@suse.de>", commitFromReader.Author.String()) commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n")) assert.NoError(t, err) commitFromReader.CommitMessage += "\n\n" commitFromReader.Signature.Payload += "\n\n" - assert.EqualValues(t, commitFromReader, commitFromReader2) + assert.Equal(t, commitFromReader, commitFromReader2) } func TestHasPreviousCommitSha256(t *testing.T) { diff --git a/modules/git/commit_submodule.go b/modules/git/commit_submodule.go index 6603061da2..ff253b7eca 100644 --- a/modules/git/commit_submodule.go +++ b/modules/git/commit_submodule.go @@ -3,6 +3,10 @@ package git +type SubmoduleWebLink struct { + RepoWebLink, CommitWebLink string +} + // GetSubModules get all the submodules of current revision git tree func (c *Commit) GetSubModules() (*ObjectCache[*SubModule], error) { if c.submoduleCache != nil { @@ -31,7 +35,8 @@ func (c *Commit) GetSubModules() (*ObjectCache[*SubModule], error) { return c.submoduleCache, nil } -// GetSubModule get the submodule according entry name +// GetSubModule gets the submodule by the entry name. +// It returns "nil, nil" if the submodule does not exist, caller should always remember to check the "nil" func (c *Commit) GetSubModule(entryName string) (*SubModule, error) { modules, err := c.GetSubModules() if err != nil { diff --git a/modules/git/commit_submodule_file.go b/modules/git/commit_submodule_file.go index bdec35f682..efcf53b07c 100644 --- a/modules/git/commit_submodule_file.go +++ b/modules/git/commit_submodule_file.go @@ -5,107 +5,65 @@ package git import ( - "fmt" - "net" - "net/url" + "context" "path" - "regexp" "strings" + + giturl "code.gitea.io/gitea/modules/git/url" + "code.gitea.io/gitea/modules/util" ) -var scpSyntax = regexp.MustCompile(`^([a-zA-Z0-9_]+@)?([a-zA-Z0-9._-]+):(.*)$`) +// CommitSubmoduleFile represents a file with submodule type. +type CommitSubmoduleFile struct { + repoLink string + fullPath string + refURL string + refID string -// CommitSubModuleFile represents a file with submodule type. -type CommitSubModuleFile struct { - refURL string - refID string + parsed bool + parsedTargetLink string } -// NewCommitSubModuleFile create a new submodule file -func NewCommitSubModuleFile(refURL, refID string) *CommitSubModuleFile { - return &CommitSubModuleFile{ - refURL: refURL, - refID: refID, - } +// NewCommitSubmoduleFile create a new submodule file +func NewCommitSubmoduleFile(repoLink, fullPath, refURL, refID string) *CommitSubmoduleFile { + return &CommitSubmoduleFile{repoLink: repoLink, fullPath: fullPath, refURL: refURL, refID: refID} } -func getRefURL(refURL, urlPrefix, repoFullName, sshDomain string) string { - if refURL == "" { +// RefID returns the commit ID of the submodule, it returns empty string for nil receiver +func (sf *CommitSubmoduleFile) RefID() string { + if sf == nil { return "" } + return sf.refID +} - refURI := strings.TrimSuffix(refURL, ".git") - - prefixURL, _ := url.Parse(urlPrefix) - urlPrefixHostname, _, err := net.SplitHostPort(prefixURL.Host) - if err != nil { - urlPrefixHostname = prefixURL.Host - } - - urlPrefix = strings.TrimSuffix(urlPrefix, "/") - - // FIXME: Need to consider branch - which will require changes in modules/git/commit.go:GetSubModules - // Relative url prefix check (according to git submodule documentation) - if strings.HasPrefix(refURI, "./") || strings.HasPrefix(refURI, "../") { - return urlPrefix + path.Clean(path.Join("/", repoFullName, refURI)) - } - - if !strings.Contains(refURI, "://") { - // scp style syntax which contains *no* port number after the : (and is not parsed by net/url) - // ex: git@try.gitea.io:go-gitea/gitea - match := scpSyntax.FindAllStringSubmatch(refURI, -1) - if len(match) > 0 { - m := match[0] - refHostname := m[2] - pth := m[3] - - if !strings.HasPrefix(pth, "/") { - pth = "/" + pth - } - - if urlPrefixHostname == refHostname || refHostname == sshDomain { - return urlPrefix + path.Clean(path.Join("/", pth)) - } - return "http://" + refHostname + pth - } - } - - ref, err := url.Parse(refURI) - if err != nil { - return "" +func (sf *CommitSubmoduleFile) getWebLinkInTargetRepo(ctx context.Context, moreLinkPath string) *SubmoduleWebLink { + if sf == nil || sf.refURL == "" { + return nil } - - refHostname, _, err := net.SplitHostPort(ref.Host) - if err != nil { - refHostname = ref.Host + if strings.HasPrefix(sf.refURL, "../") { + targetLink := path.Join(sf.repoLink, sf.refURL) + return &SubmoduleWebLink{RepoWebLink: targetLink, CommitWebLink: targetLink + moreLinkPath} } - - supportedSchemes := []string{"http", "https", "git", "ssh", "git+ssh"} - - for _, scheme := range supportedSchemes { - if ref.Scheme == scheme { - if ref.Scheme == "http" || ref.Scheme == "https" { - if len(ref.User.Username()) > 0 { - return ref.Scheme + "://" + fmt.Sprintf("%v", ref.User) + "@" + ref.Host + ref.Path - } - return ref.Scheme + "://" + ref.Host + ref.Path - } else if urlPrefixHostname == refHostname || refHostname == sshDomain { - return urlPrefix + path.Clean(path.Join("/", ref.Path)) - } - return "http://" + refHostname + ref.Path + if !sf.parsed { + sf.parsed = true + parsedURL, err := giturl.ParseRepositoryURL(ctx, sf.refURL) + if err != nil { + return nil } + sf.parsedTargetLink = giturl.MakeRepositoryWebLink(parsedURL) } - - return "" + return &SubmoduleWebLink{RepoWebLink: sf.parsedTargetLink, CommitWebLink: sf.parsedTargetLink + moreLinkPath} } -// RefURL guesses and returns reference URL. -// FIXME: template passes AppURL as urlPrefix, it needs to figure out the correct approach (no hard-coded AppURL anymore) -func (sf *CommitSubModuleFile) RefURL(urlPrefix, repoFullName, sshDomain string) string { - return getRefURL(sf.refURL, urlPrefix, repoFullName, sshDomain) +// SubmoduleWebLinkTree tries to make the submodule's tree link in its own repo, it also works on "nil" receiver +// It returns nil if the submodule does not have a valid URL or is nil +func (sf *CommitSubmoduleFile) SubmoduleWebLinkTree(ctx context.Context, optCommitID ...string) *SubmoduleWebLink { + return sf.getWebLinkInTargetRepo(ctx, "/tree/"+util.OptionalArg(optCommitID, sf.RefID())) } -// RefID returns reference ID. -func (sf *CommitSubModuleFile) RefID() string { - return sf.refID +// SubmoduleWebLinkCompare tries to make the submodule's compare link in its own repo, it also works on "nil" receiver +// It returns nil if the submodule does not have a valid URL or is nil +func (sf *CommitSubmoduleFile) SubmoduleWebLinkCompare(ctx context.Context, commitID1, commitID2 string) *SubmoduleWebLink { + return sf.getWebLinkInTargetRepo(ctx, "/compare/"+commitID1+"..."+commitID2) } diff --git a/modules/git/commit_submodule_file_test.go b/modules/git/commit_submodule_file_test.go index 473b996b82..33fe146444 100644 --- a/modules/git/commit_submodule_file_test.go +++ b/modules/git/commit_submodule_file_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2024 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package git @@ -9,34 +9,32 @@ import ( "github.com/stretchr/testify/assert" ) -func TestCommitSubModuleFileGetRefURL(t *testing.T) { - kases := []struct { - refURL string - prefixURL string - parentPath string - SSHDomain string - expect string - }{ - {"git://github.com/user1/repo1", "/", "user1/repo2", "", "http://github.com/user1/repo1"}, - {"https://localhost/user1/repo1.git", "/", "user1/repo2", "", "https://localhost/user1/repo1"}, - {"http://localhost/user1/repo1.git", "/", "owner/reponame", "", "http://localhost/user1/repo1"}, - {"git@github.com:user1/repo1.git", "/", "owner/reponame", "", "http://github.com/user1/repo1"}, - {"ssh://git@git.zefie.net:2222/zefie/lge_g6_kernel_scripts.git", "/", "zefie/lge_g6_kernel", "", "http://git.zefie.net/zefie/lge_g6_kernel_scripts"}, - {"git@git.zefie.net:2222/zefie/lge_g6_kernel_scripts.git", "/", "zefie/lge_g6_kernel", "", "http://git.zefie.net/2222/zefie/lge_g6_kernel_scripts"}, - {"git@try.gitea.io:go-gitea/gitea", "https://try.gitea.io/", "go-gitea/sdk", "", "https://try.gitea.io/go-gitea/gitea"}, - {"ssh://git@try.gitea.io:9999/go-gitea/gitea", "https://try.gitea.io/", "go-gitea/sdk", "", "https://try.gitea.io/go-gitea/gitea"}, - {"git://git@try.gitea.io:9999/go-gitea/gitea", "https://try.gitea.io/", "go-gitea/sdk", "", "https://try.gitea.io/go-gitea/gitea"}, - {"ssh://git@127.0.0.1:9999/go-gitea/gitea", "https://127.0.0.1:3000/", "go-gitea/sdk", "", "https://127.0.0.1:3000/go-gitea/gitea"}, - {"https://gitea.com:3000/user1/repo1.git", "https://127.0.0.1:3000/", "user/repo2", "", "https://gitea.com:3000/user1/repo1"}, - {"https://example.gitea.com/gitea/user1/repo1.git", "https://example.gitea.com/gitea/", "", "user/repo2", "https://example.gitea.com/gitea/user1/repo1"}, - {"https://username:password@github.com/username/repository.git", "/", "username/repository2", "", "https://username:password@github.com/username/repository"}, - {"somethingbad", "https://127.0.0.1:3000/go-gitea/gitea", "/", "", ""}, - {"git@localhost:user/repo", "https://localhost/", "user2/repo1", "", "https://localhost/user/repo"}, - {"../path/to/repo.git/", "https://localhost/", "user/repo2", "", "https://localhost/user/path/to/repo.git"}, - {"ssh://git@ssh.gitea.io:2222/go-gitea/gitea", "https://try.gitea.io/", "go-gitea/sdk", "ssh.gitea.io", "https://try.gitea.io/go-gitea/gitea"}, - } - - for _, kase := range kases { - assert.EqualValues(t, kase.expect, getRefURL(kase.refURL, kase.prefixURL, kase.parentPath, kase.SSHDomain)) - } +func TestCommitSubmoduleLink(t *testing.T) { + assert.Nil(t, (*CommitSubmoduleFile)(nil).SubmoduleWebLinkTree(t.Context())) + assert.Nil(t, (*CommitSubmoduleFile)(nil).SubmoduleWebLinkCompare(t.Context(), "", "")) + assert.Nil(t, (&CommitSubmoduleFile{}).SubmoduleWebLinkTree(t.Context())) + assert.Nil(t, (&CommitSubmoduleFile{}).SubmoduleWebLinkCompare(t.Context(), "", "")) + + t.Run("GitHubRepo", func(t *testing.T) { + sf := NewCommitSubmoduleFile("/any/repo-link", "full-path", "git@github.com:user/repo.git", "aaaa") + wl := sf.SubmoduleWebLinkTree(t.Context()) + assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) + assert.Equal(t, "https://github.com/user/repo/tree/aaaa", wl.CommitWebLink) + + wl = sf.SubmoduleWebLinkCompare(t.Context(), "1111", "2222") + assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) + assert.Equal(t, "https://github.com/user/repo/compare/1111...2222", wl.CommitWebLink) + }) + + t.Run("RelativePath", func(t *testing.T) { + sf := NewCommitSubmoduleFile("/subpath/any/repo-home-link", "full-path", "../../user/repo", "aaaa") + wl := sf.SubmoduleWebLinkTree(t.Context()) + assert.Equal(t, "/subpath/user/repo", wl.RepoWebLink) + assert.Equal(t, "/subpath/user/repo/tree/aaaa", wl.CommitWebLink) + + sf = NewCommitSubmoduleFile("/subpath/any/repo-home-link", "dir/submodule", "../../user/repo", "aaaa") + wl = sf.SubmoduleWebLinkCompare(t.Context(), "1111", "2222") + assert.Equal(t, "/subpath/user/repo", wl.RepoWebLink) + assert.Equal(t, "/subpath/user/repo/compare/1111...2222", wl.CommitWebLink) + }) } diff --git a/modules/git/commit_test.go b/modules/git/commit_test.go index 6ac65564dc..81fb91dfc6 100644 --- a/modules/git/commit_test.go +++ b/modules/git/commit_test.go @@ -4,13 +4,13 @@ package git import ( - "context" "os" "path/filepath" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCommitsCount(t *testing.T) { @@ -59,8 +59,7 @@ func TestGetFullCommitIDError(t *testing.T) { } func TestCommitFromReader(t *testing.T) { - commitString := `feaf4ba6bc635fec442f46ddd4512416ec43c2c2 commit 1074 -tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 + commitString := `tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 parent 37991dec2c8e592043f47155ce4808d4580f9123 author silverwind <me@silverwind.io> 1563741793 +0200 committer silverwind <me@silverwind.io> 1563741793 +0200 @@ -91,11 +90,9 @@ empty commit` commitFromReader, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString)) assert.NoError(t, err) - if !assert.NotNil(t, commitFromReader) { - return - } + require.NotNil(t, commitFromReader) assert.EqualValues(t, sha, commitFromReader.ID) - assert.EqualValues(t, `-----BEGIN PGP SIGNATURE----- + assert.Equal(t, `-----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEEWPb2jX6FS2mqyJRQLmK0HJOGlEMFAl00zmEACgkQLmK0HJOG lEMDFBAAhQKKqLD1VICygJMEB8t1gBmNLgvziOLfpX4KPWdPtBk3v/QJ7OrfMrVK @@ -110,26 +107,24 @@ sD53z/f0J+We4VZjY+pidvA9BGZPFVdR3wd3xGs8/oH6UWaLJAMGkLG6dDb3qDLm mfeFhT57UbE4qukTDIQ0Y0WM40UYRTakRaDY7ubhXgLgx09Cnp9XTVMsHgT6j9/i 1pxsB104XLWjQHTjr1JtiaBQEwFh9r2OKTcpvaLcbNtYpo7CzOs= =FRsO ------END PGP SIGNATURE----- -`, commitFromReader.Signature.Signature) - assert.EqualValues(t, `tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 +-----END PGP SIGNATURE-----`, commitFromReader.Signature.Signature) + assert.Equal(t, `tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 parent 37991dec2c8e592043f47155ce4808d4580f9123 author silverwind <me@silverwind.io> 1563741793 +0200 committer silverwind <me@silverwind.io> 1563741793 +0200 empty commit`, commitFromReader.Signature.Payload) - assert.EqualValues(t, "silverwind <me@silverwind.io>", commitFromReader.Author.String()) + assert.Equal(t, "silverwind <me@silverwind.io>", commitFromReader.Author.String()) commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n")) assert.NoError(t, err) commitFromReader.CommitMessage += "\n\n" commitFromReader.Signature.Payload += "\n\n" - assert.EqualValues(t, commitFromReader, commitFromReader2) + assert.Equal(t, commitFromReader, commitFromReader2) } func TestCommitWithEncodingFromReader(t *testing.T) { - commitString := `feaf4ba6bc635fec442f46ddd4512416ec43c2c2 commit 1074 -tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 + commitString := `tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 parent 47b24e7ab977ed31c5a39989d570847d6d0052af author KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 committer KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 @@ -159,11 +154,9 @@ ISO-8859-1` commitFromReader, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString)) assert.NoError(t, err) - if !assert.NotNil(t, commitFromReader) { - return - } + require.NotNil(t, commitFromReader) assert.EqualValues(t, sha, commitFromReader.ID) - assert.EqualValues(t, `-----BEGIN PGP SIGNATURE----- + assert.Equal(t, `-----BEGIN PGP SIGNATURE----- iQGzBAABCgAdFiEE9HRrbqvYxPT8PXbefPSEkrowAa8FAmYGg7IACgkQfPSEkrow Aa9olwv+P0HhtCM6CRvlUmPaqswRsDPNR4i66xyXGiSxdI9V5oJL7HLiQIM7KrFR @@ -176,22 +169,21 @@ SONRzusmu5n3DgV956REL7x62h7JuqmBz/12HZkr0z0zgXkcZ04q08pSJATX5N1F yN+tWxTsWg+zhDk96d5Esdo9JMjcFvPv0eioo30GAERaz1hoD7zCMT4jgUFTQwgz jw4YcO5u =r3UU ------END PGP SIGNATURE----- -`, commitFromReader.Signature.Signature) - assert.EqualValues(t, `tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 +-----END PGP SIGNATURE-----`, commitFromReader.Signature.Signature) + assert.Equal(t, `tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 parent 47b24e7ab977ed31c5a39989d570847d6d0052af author KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 committer KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 encoding ISO-8859-1 ISO-8859-1`, commitFromReader.Signature.Payload) - assert.EqualValues(t, "KN4CK3R <admin@oldschoolhack.me>", commitFromReader.Author.String()) + assert.Equal(t, "KN4CK3R <admin@oldschoolhack.me>", commitFromReader.Author.String()) commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n")) assert.NoError(t, err) commitFromReader.CommitMessage += "\n\n" commitFromReader.Signature.Payload += "\n\n" - assert.EqualValues(t, commitFromReader, commitFromReader2) + assert.Equal(t, commitFromReader, commitFromReader2) } func TestHasPreviousCommit(t *testing.T) { @@ -350,15 +342,15 @@ func TestGetCommitFileStatusMerges(t *testing.T) { func Test_GetCommitBranchStart(t *testing.T) { bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") - repo, err := OpenRepository(context.Background(), bareRepo1Path) + repo, err := OpenRepository(t.Context(), bareRepo1Path) assert.NoError(t, err) defer repo.Close() commit, err := repo.GetBranchCommit("branch1") assert.NoError(t, err) - assert.EqualValues(t, "2839944139e0de9737a044f78b0e4b40d989a9e3", commit.ID.String()) + assert.Equal(t, "2839944139e0de9737a044f78b0e4b40d989a9e3", commit.ID.String()) startCommitID, err := repo.GetCommitBranchStart(os.Environ(), "branch1", commit.ID.String()) assert.NoError(t, err) assert.NotEmpty(t, startCommitID) - assert.EqualValues(t, "9c9aef8dd84e02bc7ec12641deb4c930a7c30185", startCommitID) + assert.Equal(t, "95bb4d39648ee7e325106df01a621c530863a653", startCommitID) } diff --git a/modules/git/config.go b/modules/git/config.go index 9c36cf1654..234be7b955 100644 --- a/modules/git/config.go +++ b/modules/git/config.go @@ -116,7 +116,7 @@ func syncGitConfig() (err error) { } func configSet(key, value string) error { - stdout, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil) + stdout, _, err := NewCommand("config", "--global", "--get").AddDynamicArguments(key).RunStdString(DefaultContext, nil) if err != nil && !IsErrorExitCode(err, 1) { return fmt.Errorf("failed to get git config %s, err: %w", key, err) } @@ -126,7 +126,7 @@ func configSet(key, value string) error { return nil } - _, _, err = NewCommand(DefaultContext, "config", "--global").AddDynamicArguments(key, value).RunStdString(nil) + _, _, err = NewCommand("config", "--global").AddDynamicArguments(key, value).RunStdString(DefaultContext, nil) if err != nil { return fmt.Errorf("failed to set git global config %s, err: %w", key, err) } @@ -135,14 +135,14 @@ func configSet(key, value string) error { } func configSetNonExist(key, value string) error { - _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil) + _, _, err := NewCommand("config", "--global", "--get").AddDynamicArguments(key).RunStdString(DefaultContext, nil) if err == nil { // already exist return nil } if IsErrorExitCode(err, 1) { // not exist, set new config - _, _, err = NewCommand(DefaultContext, "config", "--global").AddDynamicArguments(key, value).RunStdString(nil) + _, _, err = NewCommand("config", "--global").AddDynamicArguments(key, value).RunStdString(DefaultContext, nil) if err != nil { return fmt.Errorf("failed to set git global config %s, err: %w", key, err) } @@ -153,14 +153,14 @@ func configSetNonExist(key, value string) error { } func configAddNonExist(key, value string) error { - _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil) + _, _, err := NewCommand("config", "--global", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(DefaultContext, nil) if err == nil { // already exist return nil } if IsErrorExitCode(err, 1) { // not exist, add new config - _, _, err = NewCommand(DefaultContext, "config", "--global", "--add").AddDynamicArguments(key, value).RunStdString(nil) + _, _, err = NewCommand("config", "--global", "--add").AddDynamicArguments(key, value).RunStdString(DefaultContext, nil) if err != nil { return fmt.Errorf("failed to add git global config %s, err: %w", key, err) } @@ -170,10 +170,10 @@ func configAddNonExist(key, value string) error { } func configUnsetAll(key, value string) error { - _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil) + _, _, err := NewCommand("config", "--global", "--get").AddDynamicArguments(key).RunStdString(DefaultContext, nil) if err == nil { // exist, need to remove - _, _, err = NewCommand(DefaultContext, "config", "--global", "--unset-all").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil) + _, _, err = NewCommand("config", "--global", "--unset-all").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(DefaultContext, nil) if err != nil { return fmt.Errorf("failed to unset git global config %s, err: %w", key, err) } diff --git a/modules/git/diff.go b/modules/git/diff.go index 833f6220f9..35d115be0e 100644 --- a/modules/git/diff.go +++ b/modules/git/diff.go @@ -34,8 +34,8 @@ func GetRawDiff(repo *Repository, commitID string, diffType RawDiffType, writer // GetReverseRawDiff dumps the reverse diff results of repository in given commit ID to io.Writer. func GetReverseRawDiff(ctx context.Context, repoPath, commitID string, writer io.Writer) error { stderr := new(bytes.Buffer) - cmd := NewCommand(ctx, "show", "--pretty=format:revert %H%n", "-R").AddDynamicArguments(commitID) - if err := cmd.Run(&RunOpts{ + cmd := NewCommand("show", "--pretty=format:revert %H%n", "-R").AddDynamicArguments(commitID) + if err := cmd.Run(ctx, &RunOpts{ Dir: repoPath, Stdout: writer, Stderr: stderr, @@ -56,7 +56,7 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff files = append(files, file) } - cmd := NewCommand(repo.Ctx) + cmd := NewCommand() switch diffType { case RawDiffNormal: if len(startCommit) != 0 { @@ -64,7 +64,10 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff } else if commit.ParentCount() == 0 { cmd.AddArguments("show").AddDynamicArguments(endCommit).AddDashesAndList(files...) } else { - c, _ := commit.Parent(0) + c, err := commit.Parent(0) + if err != nil { + return err + } cmd.AddArguments("diff", "-M").AddDynamicArguments(c.ID.String(), endCommit).AddDashesAndList(files...) } case RawDiffPatch: @@ -74,7 +77,10 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff } else if commit.ParentCount() == 0 { cmd.AddArguments("format-patch", "--no-signature", "--stdout", "--root").AddDynamicArguments(endCommit).AddDashesAndList(files...) } else { - c, _ := commit.Parent(0) + c, err := commit.Parent(0) + if err != nil { + return err + } query := fmt.Sprintf("%s...%s", endCommit, c.ID.String()) cmd.AddArguments("format-patch", "--no-signature", "--stdout").AddDynamicArguments(query).AddDashesAndList(files...) } @@ -83,7 +89,7 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff } stderr := new(bytes.Buffer) - if err = cmd.Run(&RunOpts{ + if err = cmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: writer, Stderr: stderr, @@ -93,9 +99,9 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff return nil } -// ParseDiffHunkString parse the diffhunk content and return -func ParseDiffHunkString(diffhunk string) (leftLine, leftHunk, rightLine, righHunk int) { - ss := strings.Split(diffhunk, "@@") +// ParseDiffHunkString parse the diff hunk content and return +func ParseDiffHunkString(diffHunk string) (leftLine, leftHunk, rightLine, rightHunk int) { + ss := strings.Split(diffHunk, "@@") ranges := strings.Split(ss[1][1:], " ") leftRange := strings.Split(ranges[0], ",") leftLine, _ = strconv.Atoi(leftRange[0][1:]) @@ -106,14 +112,19 @@ func ParseDiffHunkString(diffhunk string) (leftLine, leftHunk, rightLine, righHu rightRange := strings.Split(ranges[1], ",") rightLine, _ = strconv.Atoi(rightRange[0]) if len(rightRange) > 1 { - righHunk, _ = strconv.Atoi(rightRange[1]) + rightHunk, _ = strconv.Atoi(rightRange[1]) } } else { - log.Debug("Parse line number failed: %v", diffhunk) + log.Debug("Parse line number failed: %v", diffHunk) rightLine = leftLine - righHunk = leftHunk + rightHunk = leftHunk + } + if rightLine == 0 { + // FIXME: GIT-DIFF-CUT-BUG search this tag to see details + // this is only a hacky patch, the rightLine&rightHunk might still be incorrect in some cases. + rightLine++ } - return leftLine, leftHunk, rightLine, righHunk + return leftLine, leftHunk, rightLine, rightHunk } // Example: @@ -1,8 +1,9 @@ => [..., 1, 8, 1, 9] @@ -264,6 +275,12 @@ func CutDiffAroundLine(originalDiff io.Reader, line int64, old bool, numbersOfLi oldNumOfLines++ } } + + // "git diff" outputs "@@ -1 +1,3 @@" for "OLD" => "A\nB\nC" + // FIXME: GIT-DIFF-CUT-BUG But there is a bug in CutDiffAroundLine, then the "Patch" stored in the comment model becomes "@@ -1,1 +0,4 @@" + // It may generate incorrect results for difference cases, for example: delete 2 line add 1 line, delete 2 line add 2 line etc, need to double check. + // For example: "L1\nL2" => "A\nB", then the patch shows "L2" as line 1 on the left (deleted part) + // construct the new hunk header newHunk[headerLines] = fmt.Sprintf("@@ -%d,%d +%d,%d @@", oldBegin, oldNumOfLines, newBegin, newNumOfLines) @@ -295,8 +312,8 @@ func GetAffectedFiles(repo *Repository, branchName, oldCommitID, newCommitID str affectedFiles := make([]string, 0, 32) // Run `git diff --name-only` to get the names of the changed files - err = NewCommand(repo.Ctx, "diff", "--name-only").AddDynamicArguments(oldCommitID, newCommitID). - Run(&RunOpts{ + err = NewCommand("diff", "--name-only").AddDynamicArguments(oldCommitID, newCommitID). + Run(repo.Ctx, &RunOpts{ Env: env, Dir: repo.Path, Stdout: stdoutWriter, diff --git a/modules/git/diff_test.go b/modules/git/diff_test.go index 0f865c52a8..7671fffcc1 100644 --- a/modules/git/diff_test.go +++ b/modules/git/diff_test.go @@ -154,7 +154,7 @@ func TestCutDiffAroundLine(t *testing.T) { } func BenchmarkCutDiffAroundLine(b *testing.B) { - for n := 0; n < b.N; n++ { + for b.Loop() { CutDiffAroundLine(strings.NewReader(exampleDiff), 3, true, 3) } } @@ -177,8 +177,8 @@ func ExampleCutDiffAroundLine() { func TestParseDiffHunkString(t *testing.T) { leftLine, leftHunk, rightLine, rightHunk := ParseDiffHunkString("@@ -19,3 +19,5 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER") - assert.EqualValues(t, 19, leftLine) - assert.EqualValues(t, 3, leftHunk) - assert.EqualValues(t, 19, rightLine) - assert.EqualValues(t, 5, rightHunk) + assert.Equal(t, 19, leftLine) + assert.Equal(t, 3, leftHunk) + assert.Equal(t, 19, rightLine) + assert.Equal(t, 5, rightHunk) } diff --git a/modules/git/error.go b/modules/git/error.go index 10fb37be07..7d131345d0 100644 --- a/modules/git/error.go +++ b/modules/git/error.go @@ -32,22 +32,6 @@ func (err ErrNotExist) Unwrap() error { return util.ErrNotExist } -// ErrBadLink entry.FollowLink error -type ErrBadLink struct { - Name string - Message string -} - -func (err ErrBadLink) Error() string { - return fmt.Sprintf("%s: %s", err.Name, err.Message) -} - -// IsErrBadLink if some error is ErrBadLink -func IsErrBadLink(err error) bool { - _, ok := err.(ErrBadLink) - return ok -} - // ErrBranchNotExist represents a "BranchNotExist" kind of error. type ErrBranchNotExist struct { Name string diff --git a/modules/git/foreachref/format.go b/modules/git/foreachref/format.go index 97e8ee4724..d9573a55d6 100644 --- a/modules/git/foreachref/format.go +++ b/modules/git/foreachref/format.go @@ -76,7 +76,7 @@ func (f Format) Parser(r io.Reader) *Parser { // would turn into "%0a%00". func (f Format) hexEscaped(delim []byte) string { escaped := "" - for i := 0; i < len(delim); i++ { + for i := range delim { escaped += "%" + hex.EncodeToString([]byte{delim[i]}) } return escaped diff --git a/modules/git/fsck.go b/modules/git/fsck.go index cec27f165b..a52684c84f 100644 --- a/modules/git/fsck.go +++ b/modules/git/fsck.go @@ -10,5 +10,5 @@ import ( // Fsck verifies the connectivity and validity of the objects in the database func Fsck(ctx context.Context, repoPath string, timeout time.Duration, args TrustedCmdArgs) error { - return NewCommand(ctx, "fsck").AddArguments(args...).Run(&RunOpts{Timeout: timeout, Dir: repoPath}) + return NewCommand("fsck").AddArguments(args...).Run(ctx, &RunOpts{Timeout: timeout, Dir: repoPath}) } diff --git a/modules/git/git.go b/modules/git/git.go index e3e5b83274..a2ffd6d289 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -30,6 +30,7 @@ type Features struct { SupportProcReceive bool // >= 2.29 SupportHashSha256 bool // >= 2.42, SHA-256 repositories no longer an ‘experimental curiosity’ SupportedObjectFormats []ObjectFormat // sha1, sha256 + SupportCheckAttrOnBare bool // >= 2.40 } var ( @@ -60,7 +61,7 @@ func DefaultFeatures() *Features { } func loadGitVersionFeatures() (*Features, error) { - stdout, _, runErr := NewCommand(DefaultContext, "version").RunStdString(nil) + stdout, _, runErr := NewCommand("version").RunStdString(DefaultContext, nil) if runErr != nil { return nil, runErr } @@ -77,6 +78,7 @@ func loadGitVersionFeatures() (*Features, error) { if features.SupportHashSha256 { features.SupportedObjectFormats = append(features.SupportedObjectFormats, Sha256ObjectFormat) } + features.SupportCheckAttrOnBare = features.CheckVersionAtLeast("2.40") return features, nil } diff --git a/modules/git/git_test.go b/modules/git/git_test.go index 5472842b76..58ba01cabc 100644 --- a/modules/git/git_test.go +++ b/modules/git/git_test.go @@ -10,18 +10,19 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/tempdir" "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" ) func testRun(m *testing.M) error { - gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + gitHomePath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("git-home") if err != nil { return fmt.Errorf("unable to create temp dir: %w", err) } - defer util.RemoveAll(gitHomePath) + defer cleanup() + setting.Git.HomePath = gitHomePath if err = InitFull(context.Background()); err != nil { diff --git a/modules/git/grep.go b/modules/git/grep.go index bf6b41a886..66711650c9 100644 --- a/modules/git/grep.go +++ b/modules/git/grep.go @@ -23,11 +23,19 @@ type GrepResult struct { LineCodes []string } +type GrepModeType string + +const ( + GrepModeExact GrepModeType = "exact" + GrepModeWords GrepModeType = "words" + GrepModeRegexp GrepModeType = "regexp" +) + type GrepOptions struct { RefName string MaxResultLimit int ContextLineNumber int - IsFuzzy bool + GrepMode GrepModeType MaxLineLength int // the maximum length of a line to parse, exceeding chars will be truncated PathspecList []string } @@ -52,21 +60,30 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO 2^@repo: go-gitea/gitea */ var results []*GrepResult - cmd := NewCommand(ctx, "grep", "--null", "--break", "--heading", "--fixed-strings", "--line-number", "--ignore-case", "--full-name") - cmd.AddOptionValues("--context", fmt.Sprint(opts.ContextLineNumber)) - if opts.IsFuzzy { + cmd := NewCommand("grep", "--null", "--break", "--heading", "--line-number", "--full-name") + cmd.AddOptionValues("--context", strconv.Itoa(opts.ContextLineNumber)) + switch opts.GrepMode { + case GrepModeExact: + cmd.AddArguments("--fixed-strings") + cmd.AddOptionValues("-e", strings.TrimLeft(search, "-")) + case GrepModeRegexp: + cmd.AddArguments("--perl-regexp") + cmd.AddOptionValues("-e", strings.TrimLeft(search, "-")) + default: /* words */ words := strings.Fields(search) - for _, word := range words { + cmd.AddArguments("--fixed-strings", "--ignore-case") + for i, word := range words { cmd.AddOptionValues("-e", strings.TrimLeft(word, "-")) + if i < len(words)-1 { + cmd.AddOptionValues("--and") + } } - } else { - cmd.AddOptionValues("-e", strings.TrimLeft(search, "-")) } cmd.AddDynamicArguments(util.IfZero(opts.RefName, "HEAD")) cmd.AddDashesAndList(opts.PathspecList...) opts.MaxResultLimit = util.IfZero(opts.MaxResultLimit, 50) stderr := bytes.Buffer{} - err = cmd.Run(&RunOpts{ + err = cmd.Run(ctx, &RunOpts{ Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr, diff --git a/modules/git/grep_test.go b/modules/git/grep_test.go index 005d539726..0dce464b7c 100644 --- a/modules/git/grep_test.go +++ b/modules/git/grep_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "path/filepath" "testing" @@ -16,7 +15,7 @@ func TestGrepSearch(t *testing.T) { assert.NoError(t, err) defer repo.Close() - res, err := GrepSearch(context.Background(), repo, "void", GrepOptions{}) + res, err := GrepSearch(t.Context(), repo, "void", GrepOptions{}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -31,7 +30,7 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "void", GrepOptions{PathspecList: []string{":(glob)java-hello/*"}}) + res, err = GrepSearch(t.Context(), repo, "void", GrepOptions{PathspecList: []string{":(glob)java-hello/*"}}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -41,7 +40,7 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "void", GrepOptions{PathspecList: []string{":(glob,exclude)java-hello/*"}}) + res, err = GrepSearch(t.Context(), repo, "void", GrepOptions{PathspecList: []string{":(glob,exclude)java-hello/*"}}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -51,7 +50,7 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "void", GrepOptions{MaxResultLimit: 1}) + res, err = GrepSearch(t.Context(), repo, "void", GrepOptions{MaxResultLimit: 1}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -61,7 +60,7 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "void", GrepOptions{MaxResultLimit: 1, MaxLineLength: 39}) + res, err = GrepSearch(t.Context(), repo, "void", GrepOptions{MaxResultLimit: 1, MaxLineLength: 39}) assert.NoError(t, err) assert.Equal(t, []*GrepResult{ { @@ -71,11 +70,11 @@ func TestGrepSearch(t *testing.T) { }, }, res) - res, err = GrepSearch(context.Background(), repo, "no-such-content", GrepOptions{}) + res, err = GrepSearch(t.Context(), repo, "no-such-content", GrepOptions{}) assert.NoError(t, err) assert.Empty(t, res) - res, err = GrepSearch(context.Background(), &Repository{Path: "no-such-git-repo"}, "no-such-content", GrepOptions{}) + res, err = GrepSearch(t.Context(), &Repository{Path: "no-such-git-repo"}, "no-such-content", GrepOptions{}) assert.Error(t, err) assert.Empty(t, res) } diff --git a/modules/git/hook.go b/modules/git/hook.go index 46f93ce13e..548a59971d 100644 --- a/modules/git/hook.go +++ b/modules/git/hook.go @@ -7,11 +7,10 @@ package git import ( "errors" "os" - "path" "path/filepath" + "slices" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -27,12 +26,7 @@ var ErrNotValidHook = errors.New("not a valid Git hook") // IsValidHookName returns true if given name is a valid Git hook. func IsValidHookName(name string) bool { - for _, hn := range hookNames { - if hn == name { - return true - } - } - return false + return slices.Contains(hookNames, name) } // Hook represents a Git hook. @@ -51,17 +45,28 @@ func GetHook(repoPath, name string) (*Hook, error) { } h := &Hook{ name: name, - path: path.Join(repoPath, "hooks", name+".d", name), + path: filepath.Join(repoPath, "hooks", name+".d", name), } - samplePath := filepath.Join(repoPath, "hooks", name+".sample") - if isFile(h.path) { + isFile, err := util.IsFile(h.path) + if err != nil { + return nil, err + } + if isFile { data, err := os.ReadFile(h.path) if err != nil { return nil, err } h.IsActive = true h.Content = string(data) - } else if isFile(samplePath) { + return h, nil + } + + samplePath := filepath.Join(repoPath, "hooks", name+".sample") + isFile, err = util.IsFile(samplePath) + if err != nil { + return nil, err + } + if isFile { data, err := os.ReadFile(samplePath) if err != nil { return nil, err @@ -79,7 +84,11 @@ func (h *Hook) Name() string { // Update updates hook settings. func (h *Hook) Update() error { if len(strings.TrimSpace(h.Content)) == 0 { - if isExist(h.path) { + exist, err := util.IsExist(h.path) + if err != nil { + return err + } + if exist { err := util.Remove(h.path) if err != nil { return err @@ -103,7 +112,10 @@ func (h *Hook) Update() error { // ListHooks returns a list of Git hooks of given repository. func ListHooks(repoPath string) (_ []*Hook, err error) { - if !isDir(path.Join(repoPath, "hooks")) { + exist, err := util.IsDir(filepath.Join(repoPath, "hooks")) + if err != nil { + return nil, err + } else if !exist { return nil, errors.New("hooks path does not exist") } @@ -116,28 +128,3 @@ func ListHooks(repoPath string) (_ []*Hook, err error) { } return hooks, nil } - -const ( - // HookPathUpdate hook update path - HookPathUpdate = "hooks/update" -) - -// SetUpdateHook writes given content to update hook of the repository. -func SetUpdateHook(repoPath, content string) (err error) { - log.Debug("Setting update hook: %s", repoPath) - hookPath := path.Join(repoPath, HookPathUpdate) - isExist, err := util.IsExist(hookPath) - if err != nil { - log.Debug("Unable to check if %s exists. Error: %v", hookPath, err) - return err - } - if isExist { - err = util.Remove(hookPath) - } else { - err = os.MkdirAll(path.Dir(hookPath), os.ModePerm) - } - if err != nil { - return err - } - return os.WriteFile(hookPath, []byte(content), 0o777) -} diff --git a/modules/git/key.go b/modules/git/key.go new file mode 100644 index 0000000000..2513c048b7 --- /dev/null +++ b/modules/git/key.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +// Based on https://git-scm.com/docs/git-config#Documentation/git-config.txt-gpgformat +const ( + SigningKeyFormatOpenPGP = "openpgp" // for GPG keys, the expected default of git cli + SigningKeyFormatSSH = "ssh" +) + +type SigningKey struct { + KeyID string + Format string +} diff --git a/modules/git/repo_language_stats.go b/modules/git/languagestats/language_stats.go index 8551ea9d24..a71284c3e4 100644 --- a/modules/git/repo_language_stats.go +++ b/modules/git/languagestats/language_stats.go @@ -1,13 +1,15 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package git +package languagestats import ( + "context" "strings" "unicode" - "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" ) const ( @@ -49,19 +51,15 @@ func mergeLanguageStats(stats map[string]int64) map[string]int64 { return res } -func TryReadLanguageAttribute(attrs map[string]string) optional.Option[string] { - language := AttributeToString(attrs, AttributeLinguistLanguage) - if language.Value() == "" { - language = AttributeToString(attrs, AttributeGitlabLanguage) - if language.Has() { - raw := language.Value() - // gitlab-language may have additional parameters after the language - // ignore them and just use the main language - // https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type - if idx := strings.IndexByte(raw, '?'); idx >= 0 { - language = optional.Some(raw[:idx]) - } - } +// GetFileLanguage tries to get the (linguist) language of the file content +func GetFileLanguage(ctx context.Context, gitRepo *git.Repository, treeish, treePath string) (string, error) { + attributesMap, err := attribute.CheckAttributes(ctx, gitRepo, treeish, attribute.CheckAttributeOpts{ + Attributes: []string{attribute.LinguistLanguage, attribute.GitlabLanguage}, + Filenames: []string{treePath}, + }) + if err != nil { + return "", err } - return language + + return attributesMap[treePath].GetLanguage().Value(), nil } diff --git a/modules/git/repo_language_stats_gogit.go b/modules/git/languagestats/language_stats_gogit.go index a34c03c781..418c05b157 100644 --- a/modules/git/repo_language_stats_gogit.go +++ b/modules/git/languagestats/language_stats_gogit.go @@ -3,13 +3,15 @@ //go:build gogit -package git +package languagestats import ( "bytes" "io" "code.gitea.io/gitea/modules/analyze" + git_module "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/optional" "github.com/go-enry/go-enry/v2" @@ -19,7 +21,7 @@ import ( ) // GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { +func GetLanguageStats(repo *git_module.Repository, commitID string) (map[string]int64, error) { r, err := git.PlainOpen(repo.Path) if err != nil { return nil, err @@ -40,8 +42,11 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } - checker, deferable := repo.CheckAttributeReader(commitID) - defer deferable() + checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes) + if err != nil { + return nil, err + } + defer checker.Close() // sizes contains the current calculated size of all files by language sizes := make(map[string]int64) @@ -62,43 +67,41 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err isDocumentation := optional.None[bool]() isDetectable := optional.None[bool]() - if checker != nil { - attrs, err := checker.CheckPath(f.Name) - if err == nil { - isVendored = AttributeToBool(attrs, AttributeLinguistVendored) - if isVendored.ValueOrDefault(false) { - return nil - } - - isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) - if isGenerated.ValueOrDefault(false) { - return nil - } + attrs, err := checker.CheckPath(f.Name) + if err == nil { + isVendored = attrs.GetVendored() + if isVendored.ValueOrDefault(false) { + return nil + } - isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) - if isDocumentation.ValueOrDefault(false) { - return nil - } + isGenerated = attrs.GetGenerated() + if isGenerated.ValueOrDefault(false) { + return nil + } - isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) - if !isDetectable.ValueOrDefault(true) { - return nil - } + isDocumentation = attrs.GetDocumentation() + if isDocumentation.ValueOrDefault(false) { + return nil + } - hasLanguage := TryReadLanguageAttribute(attrs) - if hasLanguage.Value() != "" { - language := hasLanguage.Value() + isDetectable = attrs.GetDetectable() + if !isDetectable.ValueOrDefault(true) { + return nil + } - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } + hasLanguage := attrs.GetLanguage() + if hasLanguage.Value() != "" { + language := hasLanguage.Value() - // this language will always be added to the size - sizes[language] += f.Size - return nil + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if len(group) != 0 { + language = group } + + // this language will always be added to the size + sizes[language] += f.Size + return nil } } diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/languagestats/language_stats_nogogit.go index de7707bd6c..94cf9fff8c 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/languagestats/language_stats_nogogit.go @@ -3,13 +3,15 @@ //go:build !gogit -package git +package languagestats import ( "bytes" "io" "code.gitea.io/gitea/modules/analyze" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" @@ -17,7 +19,7 @@ import ( ) // GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { +func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64, error) { // We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. // so let's create a batch stdin and stdout batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx) @@ -34,19 +36,19 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if err := writeID(commitID); err != nil { return nil, err } - shaBytes, typ, size, err := ReadBatchLine(batchReader) + shaBytes, typ, size, err := git.ReadBatchLine(batchReader) if typ != "commit" { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} + return nil, git.ErrNotExist{ID: commitID} } - sha, err := NewIDFromString(string(shaBytes)) + sha, err := git.NewIDFromString(string(shaBytes)) if err != nil { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} + return nil, git.ErrNotExist{ID: commitID} } - commit, err := CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) + commit, err := git.CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) if err != nil { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) return nil, err @@ -62,8 +64,11 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } - checker, deferable := repo.CheckAttributeReader(commitID) - defer deferable() + checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes) + if err != nil { + return nil, err + } + defer checker.Close() contentBuf := bytes.Buffer{} var content []byte @@ -92,47 +97,40 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err } isVendored := optional.None[bool]() - isGenerated := optional.None[bool]() isDocumentation := optional.None[bool]() isDetectable := optional.None[bool]() - if checker != nil { - attrs, err := checker.CheckPath(f.Name()) - if err == nil { - isVendored = AttributeToBool(attrs, AttributeLinguistVendored) - if isVendored.ValueOrDefault(false) { - continue - } - - isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) - if isGenerated.ValueOrDefault(false) { - continue - } + attrs, err := checker.CheckPath(f.Name()) + attrLinguistGenerated := optional.None[bool]() + if err == nil { + if isVendored = attrs.GetVendored(); isVendored.ValueOrDefault(false) { + continue + } - isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) - if isDocumentation.ValueOrDefault(false) { - continue - } + if attrLinguistGenerated = attrs.GetGenerated(); attrLinguistGenerated.ValueOrDefault(false) { + continue + } - isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) - if !isDetectable.ValueOrDefault(true) { - continue - } + if isDocumentation = attrs.GetDocumentation(); isDocumentation.ValueOrDefault(false) { + continue + } - hasLanguage := TryReadLanguageAttribute(attrs) - if hasLanguage.Value() != "" { - language := hasLanguage.Value() + if isDetectable = attrs.GetDetectable(); !isDetectable.ValueOrDefault(true) { + continue + } - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } + if hasLanguage := attrs.GetLanguage(); hasLanguage.Value() != "" { + language := hasLanguage.Value() - // this language will always be added to the size - sizes[language] += f.Size() - continue + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if len(group) != 0 { + language = group } + + // this language will always be added to the size + sizes[language] += f.Size() + continue } } @@ -149,7 +147,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if err := writeID(f.ID.String()); err != nil { return nil, err } - _, _, size, err := ReadBatchLine(batchReader) + _, _, size, err := git.ReadBatchLine(batchReader) if err != nil { log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err) return nil, err @@ -167,11 +165,19 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } content = contentBuf.Bytes() - if err := DiscardFull(batchReader, discard); err != nil { + if err := git.DiscardFull(batchReader, discard); err != nil { return nil, err } } - if !isGenerated.Has() && enry.IsGenerated(f.Name(), content) { + + // if "generated" attribute is set, use it, otherwise use enry.IsGenerated to guess + var isGenerated bool + if attrLinguistGenerated.Has() { + isGenerated = attrLinguistGenerated.Value() + } else { + isGenerated = enry.IsGenerated(f.Name(), content) + } + if isGenerated { continue } diff --git a/modules/git/repo_language_stats_test.go b/modules/git/languagestats/language_stats_test.go index da3871e909..b908ae6413 100644 --- a/modules/git/repo_language_stats_test.go +++ b/modules/git/languagestats/language_stats_test.go @@ -3,36 +3,36 @@ //go:build !gogit -package git +package languagestats import ( - "path/filepath" "testing" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRepository_GetLanguageStats(t *testing.T) { - repoPath := filepath.Join(testReposDir, "language_stats_repo") - gitRepo, err := openRepositoryWithDefaultContext(repoPath) - if !assert.NoError(t, err) { - t.Fatal() - } + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + require.NoError(t, err) defer gitRepo.Close() - stats, err := gitRepo.GetLanguageStats("8fee858da5796dfb37704761701bb8e800ad9ef3") - if !assert.NoError(t, err) { - t.Fatal() - } + stats, err := GetLanguageStats(gitRepo, "8fee858da5796dfb37704761701bb8e800ad9ef3") + require.NoError(t, err) - assert.EqualValues(t, map[string]int64{ + assert.Equal(t, map[string]int64{ "Python": 134, "Java": 112, }, stats) } func TestMergeLanguageStats(t *testing.T) { - assert.EqualValues(t, map[string]int64{ + assert.Equal(t, map[string]int64{ "PHP": 1, "python": 10, "JAVA": 700, diff --git a/modules/git/languagestats/main_test.go b/modules/git/languagestats/main_test.go new file mode 100644 index 0000000000..707d268c81 --- /dev/null +++ b/modules/git/languagestats/main_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package languagestats + +import ( + "context" + "fmt" + "os" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func testRun(m *testing.M) error { + gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + if err != nil { + return fmt.Errorf("unable to create temp dir: %w", err) + } + defer util.RemoveAll(gitHomePath) + setting.Git.HomePath = gitHomePath + + if err = git.InitFull(context.Background()); err != nil { + return fmt.Errorf("failed to call Init: %w", err) + } + + exitCode := m.Run() + if exitCode != 0 { + return fmt.Errorf("run test failed, ExitCode=%d", exitCode) + } + return nil +} + +func TestMain(m *testing.M) { + if err := testRun(m); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Test failed: %v", err) + os.Exit(1) + } +} diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index cf9c10d7b4..cff2556083 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -13,7 +13,7 @@ import ( ) func getCacheKey(repoPath, commitID, entryPath string) string { - hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, commitID, entryPath))) + hashBytes := sha256.Sum256(fmt.Appendf(nil, "%s:%s:%s", repoPath, commitID, entryPath)) return fmt.Sprintf("last_commit:%x", hashBytes) } diff --git a/modules/git/log_name_status.go b/modules/git/log_name_status.go index 1fd58abfcd..dfdef38ef9 100644 --- a/modules/git/log_name_status.go +++ b/modules/git/log_name_status.go @@ -34,7 +34,7 @@ func LogNameStatusRepo(ctx context.Context, repository, head, treepath string, p _ = stdoutWriter.Close() } - cmd := NewCommand(ctx) + cmd := NewCommand() cmd.AddArguments("log", "--name-status", "-c", "--format=commit%x00%H %P%x00", "--parents", "--no-renames", "-t", "-z").AddDynamicArguments(head) var files []string @@ -64,7 +64,7 @@ func LogNameStatusRepo(ctx context.Context, repository, head, treepath string, p go func() { stderr := strings.Builder{} - err := cmd.Run(&RunOpts{ + err := cmd.Run(ctx, &RunOpts{ Dir: repository, Stdout: stdoutWriter, Stderr: &stderr, @@ -118,11 +118,12 @@ func (g *LogNameStatusRepoParser) Next(treepath string, paths2ids map[string]int g.buffull = false g.next, err = g.rd.ReadSlice('\x00') if err != nil { - if err == bufio.ErrBufferFull { + switch err { + case bufio.ErrBufferFull: g.buffull = true - } else if err == io.EOF { + case io.EOF: return nil, nil - } else { + default: return nil, err } } @@ -132,11 +133,12 @@ func (g *LogNameStatusRepoParser) Next(treepath string, paths2ids map[string]int if bytes.Equal(g.next, []byte("commit\000")) { g.next, err = g.rd.ReadSlice('\x00') if err != nil { - if err == bufio.ErrBufferFull { + switch err { + case bufio.ErrBufferFull: g.buffull = true - } else if err == io.EOF { + case io.EOF: return nil, nil - } else { + default: return nil, err } } @@ -214,11 +216,12 @@ diffloop: } g.next, err = g.rd.ReadSlice('\x00') if err != nil { - if err == bufio.ErrBufferFull { + switch err { + case bufio.ErrBufferFull: g.buffull = true - } else if err == io.EOF { + case io.EOF: return &ret, nil - } else { + default: return nil, err } } @@ -343,10 +346,7 @@ func WalkGitLog(ctx context.Context, repo *Repository, head *Commit, treepath st results := make([]string, len(paths)) remaining := len(paths) - nextRestart := (len(paths) * 3) / 4 - if nextRestart > 70 { - nextRestart = 70 - } + nextRestart := min((len(paths)*3)/4, 70) lastEmptyParent := head.ID.String() commitSinceLastEmptyParent := uint64(0) commitSinceNextRestart := uint64(0) diff --git a/modules/git/notes_test.go b/modules/git/notes_test.go index 267671d8fa..ca05a9e525 100644 --- a/modules/git/notes_test.go +++ b/modules/git/notes_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "path/filepath" "testing" @@ -18,7 +17,7 @@ func TestGetNotes(t *testing.T) { defer bareRepo1.Close() note := Note{} - err = GetNote(context.Background(), bareRepo1, "95bb4d39648ee7e325106df01a621c530863a653", ¬e) + err = GetNote(t.Context(), bareRepo1, "95bb4d39648ee7e325106df01a621c530863a653", ¬e) assert.NoError(t, err) assert.Equal(t, []byte("Note contents\n"), note.Message) assert.Equal(t, "Vladimir Panteleev", note.Commit.Author.Name) @@ -31,10 +30,10 @@ func TestGetNestedNotes(t *testing.T) { defer repo.Close() note := Note{} - err = GetNote(context.Background(), repo, "3e668dbfac39cbc80a9ff9c61eb565d944453ba4", ¬e) + err = GetNote(t.Context(), repo, "3e668dbfac39cbc80a9ff9c61eb565d944453ba4", ¬e) assert.NoError(t, err) assert.Equal(t, []byte("Note 2"), note.Message) - err = GetNote(context.Background(), repo, "ba0a96fa63532d6c5087ecef070b0250ed72fa47", ¬e) + err = GetNote(t.Context(), repo, "ba0a96fa63532d6c5087ecef070b0250ed72fa47", ¬e) assert.NoError(t, err) assert.Equal(t, []byte("Note 1"), note.Message) } @@ -46,7 +45,7 @@ func TestGetNonExistentNotes(t *testing.T) { defer bareRepo1.Close() note := Note{} - err = GetNote(context.Background(), bareRepo1, "non_existent_sha", ¬e) + err = GetNote(t.Context(), bareRepo1, "non_existent_sha", ¬e) assert.Error(t, err) assert.IsType(t, ErrNotExist{}, err) } diff --git a/modules/git/object_id.go b/modules/git/object_id.go index 82d30184df..25dfef3ec5 100644 --- a/modules/git/object_id.go +++ b/modules/git/object_id.go @@ -99,5 +99,5 @@ type ErrInvalidSHA struct { } func (err ErrInvalidSHA) Error() string { - return fmt.Sprintf("invalid sha: %s", err.SHA) + return "invalid sha: " + err.SHA } diff --git a/modules/git/parse.go b/modules/git/parse.go new file mode 100644 index 0000000000..a7f5c58e89 --- /dev/null +++ b/modules/git/parse.go @@ -0,0 +1,68 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "bytes" + "fmt" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/optional" +) + +var sepSpace = []byte{' '} + +type LsTreeEntry struct { + ID ObjectID + EntryMode EntryMode + Name string + Size optional.Option[int64] +} + +func parseLsTreeLine(line []byte) (*LsTreeEntry, error) { + // expect line to be of the form: + // <mode> <type> <sha> <space-padded-size>\t<filename> + // <mode> <type> <sha>\t<filename> + + var err error + posTab := bytes.IndexByte(line, '\t') + if posTab == -1 { + return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line) + } + + entry := new(LsTreeEntry) + + entryAttrs := line[:posTab] + entryName := line[posTab+1:] + + entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) + _ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type + entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) + if len(entryAttrs) > 0 { + entrySize := entryAttrs // the last field is the space-padded-size + size, _ := strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64) + entry.Size = optional.Some(size) + } + + entry.EntryMode, err = ParseEntryMode(string(entryMode)) + if err != nil || entry.EntryMode == EntryModeNoEntry { + return nil, fmt.Errorf("invalid ls-tree output (invalid mode): %q, err: %w", line, err) + } + + entry.ID, err = NewIDFromString(string(entryObjectID)) + if err != nil { + return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err) + } + + if len(entryName) > 0 && entryName[0] == '"' { + entry.Name, err = strconv.Unquote(string(entryName)) + if err != nil { + return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err) + } + } else { + entry.Name = string(entryName) + } + return entry, nil +} diff --git a/modules/git/parse_nogogit.go b/modules/git/parse_nogogit.go index 546b38be37..78a0162889 100644 --- a/modules/git/parse_nogogit.go +++ b/modules/git/parse_nogogit.go @@ -10,8 +10,6 @@ import ( "bytes" "fmt" "io" - "strconv" - "strings" "code.gitea.io/gitea/modules/log" ) @@ -21,71 +19,30 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) { return parseTreeEntries(data, nil) } -var sepSpace = []byte{' '} - +// parseTreeEntries FIXME this function's design is not right, it should not make the caller read all data into memory func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { - var err error entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1) for pos := 0; pos < len(data); { - // expect line to be of the form: - // <mode> <type> <sha> <space-padded-size>\t<filename> - // <mode> <type> <sha>\t<filename> posEnd := bytes.IndexByte(data[pos:], '\n') if posEnd == -1 { posEnd = len(data) } else { posEnd += pos } - line := data[pos:posEnd] - posTab := bytes.IndexByte(line, '\t') - if posTab == -1 { - return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line) - } - - entry := new(TreeEntry) - entry.ptree = ptree - - entryAttrs := line[:posTab] - entryName := line[posTab+1:] - - entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) - _ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type - entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace) - if len(entryAttrs) > 0 { - entrySize := entryAttrs // the last field is the space-padded-size - entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64) - entry.sized = true - } - switch string(entryMode) { - case "100644": - entry.entryMode = EntryModeBlob - case "100755": - entry.entryMode = EntryModeExec - case "120000": - entry.entryMode = EntryModeSymlink - case "160000": - entry.entryMode = EntryModeCommit - case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons - entry.entryMode = EntryModeTree - default: - return nil, fmt.Errorf("unknown type: %v", string(entryMode)) - } - - entry.ID, err = NewIDFromString(string(entryObjectID)) + line := data[pos:posEnd] + lsTreeLine, err := parseLsTreeLine(line) if err != nil { - return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err) + return nil, err } - - if len(entryName) > 0 && entryName[0] == '"' { - entry.name, err = strconv.Unquote(string(entryName)) - if err != nil { - return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err) - } - } else { - entry.name = string(entryName) + entry := &TreeEntry{ + ptree: ptree, + ID: lsTreeLine.ID, + entryMode: lsTreeLine.EntryMode, + name: lsTreeLine.Name, + size: lsTreeLine.Size.Value(), + sized: lsTreeLine.Size.Has(), } - pos = posEnd + 1 entries = append(entries, entry) } @@ -100,7 +57,7 @@ func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio. loop: for sz > 0 { - mode, fname, sha, count, err := ParseTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf) + mode, fname, sha, count, err := ParseCatFileTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf) if err != nil { if err == io.EOF { break loop diff --git a/modules/git/parse_nogogit_test.go b/modules/git/parse_nogogit_test.go index a4436ce499..6594c84269 100644 --- a/modules/git/parse_nogogit_test.go +++ b/modules/git/parse_nogogit_test.go @@ -58,7 +58,7 @@ func TestParseTreeEntriesLong(t *testing.T) { assert.NoError(t, err) assert.Len(t, entries, len(testCase.Expected)) for i, entry := range entries { - assert.EqualValues(t, testCase.Expected[i], entry) + assert.Equal(t, testCase.Expected[i], entry) } } } @@ -91,7 +91,7 @@ func TestParseTreeEntriesShort(t *testing.T) { assert.NoError(t, err) assert.Len(t, entries, len(testCase.Expected)) for i, entry := range entries { - assert.EqualValues(t, testCase.Expected[i], entry) + assert.Equal(t, testCase.Expected[i], entry) } } } diff --git a/modules/git/pipeline/catfile.go b/modules/git/pipeline/catfile.go index 4677218150..5ddc36cc01 100644 --- a/modules/git/pipeline/catfile.go +++ b/modules/git/pipeline/catfile.go @@ -25,8 +25,8 @@ func CatFileBatchCheck(ctx context.Context, shasToCheckReader *io.PipeReader, ca stderr := new(bytes.Buffer) var errbuf strings.Builder - cmd := git.NewCommand(ctx, "cat-file", "--batch-check") - if err := cmd.Run(&git.RunOpts{ + cmd := git.NewCommand("cat-file", "--batch-check") + if err := cmd.Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdin: shasToCheckReader, Stdout: catFileCheckWriter, @@ -43,8 +43,8 @@ func CatFileBatchCheckAllObjects(ctx context.Context, catFileCheckWriter *io.Pip stderr := new(bytes.Buffer) var errbuf strings.Builder - cmd := git.NewCommand(ctx, "cat-file", "--batch-check", "--batch-all-objects") - if err := cmd.Run(&git.RunOpts{ + cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects") + if err := cmd.Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdout: catFileCheckWriter, Stderr: stderr, @@ -64,7 +64,7 @@ func CatFileBatch(ctx context.Context, shasToBatchReader *io.PipeReader, catFile stderr := new(bytes.Buffer) var errbuf strings.Builder - if err := git.NewCommand(ctx, "cat-file", "--batch").Run(&git.RunOpts{ + if err := git.NewCommand("cat-file", "--batch").Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdout: catFileBatchWriter, Stdin: shasToBatchReader, diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index b22805c132..c5eed73701 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -32,7 +32,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err go func() { stderr := strings.Builder{} - err := git.NewCommand(repo.Ctx, "rev-list", "--all").Run(&git.RunOpts{ + err := git.NewCommand("rev-list", "--all").Run(repo.Ctx, &git.RunOpts{ Dir: repo.Path, Stdout: revListWriter, Stderr: &stderr, @@ -114,7 +114,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err case "tree": var n int64 for n < size { - mode, fname, binObjectID, count, err := git.ParseTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf) + mode, fname, binObjectID, count, err := git.ParseCatFileTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf) if err != nil { return nil, err } diff --git a/modules/git/pipeline/namerev.go b/modules/git/pipeline/namerev.go index ad583a7479..06731c5051 100644 --- a/modules/git/pipeline/namerev.go +++ b/modules/git/pipeline/namerev.go @@ -22,7 +22,7 @@ func NameRevStdin(ctx context.Context, shasToNameReader *io.PipeReader, nameRevS stderr := new(bytes.Buffer) var errbuf strings.Builder - if err := git.NewCommand(ctx, "name-rev", "--stdin", "--name-only", "--always").Run(&git.RunOpts{ + if err := git.NewCommand("name-rev", "--stdin", "--name-only", "--always").Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdout: nameRevStdinWriter, Stdin: shasToNameReader, diff --git a/modules/git/pipeline/revlist.go b/modules/git/pipeline/revlist.go index d88ebe78ef..31627a0f3a 100644 --- a/modules/git/pipeline/revlist.go +++ b/modules/git/pipeline/revlist.go @@ -23,8 +23,8 @@ func RevListAllObjects(ctx context.Context, revListWriter *io.PipeWriter, wg *sy stderr := new(bytes.Buffer) var errbuf strings.Builder - cmd := git.NewCommand(ctx, "rev-list", "--objects", "--all") - if err := cmd.Run(&git.RunOpts{ + cmd := git.NewCommand("rev-list", "--objects", "--all") + if err := cmd.Run(ctx, &git.RunOpts{ Dir: basePath, Stdout: revListWriter, Stderr: stderr, @@ -42,11 +42,11 @@ func RevListObjects(ctx context.Context, revListWriter *io.PipeWriter, wg *sync. defer revListWriter.Close() stderr := new(bytes.Buffer) var errbuf strings.Builder - cmd := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(headSHA) + cmd := git.NewCommand("rev-list", "--objects").AddDynamicArguments(headSHA) if baseSHA != "" { cmd = cmd.AddArguments("--not").AddDynamicArguments(baseSHA) } - if err := cmd.Run(&git.RunOpts{ + if err := cmd.Run(ctx, &git.RunOpts{ Dir: tmpBasePath, Stdout: revListWriter, Stderr: stderr, diff --git a/modules/git/ref.go b/modules/git/ref.go index aab4c5d77d..56b2db858a 100644 --- a/modules/git/ref.go +++ b/modules/git/ref.go @@ -80,6 +80,10 @@ func RefNameFromTag(shortName string) RefName { return RefName(TagPrefix + shortName) } +func RefNameFromCommit(shortName string) RefName { + return RefName(shortName) +} + func (ref RefName) String() string { return string(ref) } @@ -105,8 +109,8 @@ func (ref RefName) IsFor() bool { } func (ref RefName) nameWithoutPrefix(prefix string) string { - if strings.HasPrefix(string(ref), prefix) { - return strings.TrimPrefix(string(ref), prefix) + if after, ok := strings.CutPrefix(string(ref), prefix); ok { + return after } return "" } @@ -181,32 +185,38 @@ func (ref RefName) RefGroup() string { return "" } +// RefType is a simple ref type of the reference, it is used for UI and webhooks +type RefType string + +const ( + RefTypeBranch RefType = "branch" + RefTypeTag RefType = "tag" + RefTypeCommit RefType = "commit" +) + // RefType returns the simple ref type of the reference, e.g. branch, tag // It's different from RefGroup, which is using the name of the directory under .git/refs -// Here we using branch but not heads, using tag but not tags -func (ref RefName) RefType() string { - var refType string - if ref.IsBranch() { - refType = "branch" - } else if ref.IsTag() { - refType = "tag" +func (ref RefName) RefType() RefType { + switch { + case ref.IsBranch(): + return RefTypeBranch + case ref.IsTag(): + return RefTypeTag + case IsStringLikelyCommitID(nil, string(ref), 6): + return RefTypeCommit } - return refType + return "" } -// RefURL returns the absolute URL for a ref in a repository -func RefURL(repoURL, ref string) string { - refFullName := RefName(ref) - refName := util.PathEscapeSegments(refFullName.ShortName()) - switch { - case refFullName.IsBranch(): - return repoURL + "/src/branch/" + refName - case refFullName.IsTag(): - return repoURL + "/src/tag/" + refName - case !Sha1ObjectFormat.IsValid(ref): - // assume they mean a branch - return repoURL + "/src/branch/" + refName - default: - return repoURL + "/src/commit/" + refName +// RefWebLinkPath returns a path for the reference that can be used in a web link: +// * "branch/<branch_name>" +// * "tag/<tag_name>" +// * "commit/<commit_id>" +// It returns an empty string if the reference is not a branch, tag or commit. +func (ref RefName) RefWebLinkPath() string { + refType := ref.RefType() + if refType == "" { + return "" } + return string(refType) + "/" + util.PathEscapeSegments(ref.ShortName()) } diff --git a/modules/git/ref_test.go b/modules/git/ref_test.go index 58f679b7d6..5397191561 100644 --- a/modules/git/ref_test.go +++ b/modules/git/ref_test.go @@ -20,6 +20,8 @@ func TestRefName(t *testing.T) { // Test pull names assert.Equal(t, "1", RefName("refs/pull/1/head").PullName()) + assert.True(t, RefName("refs/pull/1/head").IsPull()) + assert.True(t, RefName("refs/pull/1/merge").IsPull()) assert.Equal(t, "my/pull", RefName("refs/pull/my/pull/head").PullName()) // Test for branch names @@ -30,9 +32,8 @@ func TestRefName(t *testing.T) { assert.Equal(t, "c0ffee", RefName("c0ffee").ShortName()) } -func TestRefURL(t *testing.T) { - repoURL := "/user/repo" - assert.Equal(t, repoURL+"/src/branch/foo", RefURL(repoURL, "refs/heads/foo")) - assert.Equal(t, repoURL+"/src/tag/foo", RefURL(repoURL, "refs/tags/foo")) - assert.Equal(t, repoURL+"/src/commit/c0ffee", RefURL(repoURL, "c0ffee")) +func TestRefWebLinkPath(t *testing.T) { + assert.Equal(t, "branch/foo", RefName("refs/heads/foo").RefWebLinkPath()) + assert.Equal(t, "tag/foo", RefName("refs/tags/foo").RefWebLinkPath()) + assert.Equal(t, "commit/c0ffee", RefName("c0ffee").RefWebLinkPath()) } diff --git a/modules/git/remote.go b/modules/git/remote.go index de8d74eded..876c3d6acb 100644 --- a/modules/git/remote.go +++ b/modules/git/remote.go @@ -17,12 +17,12 @@ import ( func GetRemoteAddress(ctx context.Context, repoPath, remoteName string) (string, error) { var cmd *Command if DefaultFeatures().CheckVersionAtLeast("2.7") { - cmd = NewCommand(ctx, "remote", "get-url").AddDynamicArguments(remoteName) + cmd = NewCommand("remote", "get-url").AddDynamicArguments(remoteName) } else { - cmd = NewCommand(ctx, "config", "--get").AddDynamicArguments("remote." + remoteName + ".url") + cmd = NewCommand("config", "--get").AddDynamicArguments("remote." + remoteName + ".url") } - result, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + result, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return "", err } @@ -39,7 +39,7 @@ func GetRemoteURL(ctx context.Context, repoPath, remoteName string) (*giturl.Git if err != nil { return nil, err } - return giturl.Parse(addr) + return giturl.ParseGitURL(addr) } // ErrInvalidCloneAddr represents a "InvalidCloneAddr" kind of error. @@ -79,6 +79,15 @@ func (err *ErrInvalidCloneAddr) Unwrap() error { return util.ErrInvalidArgument } +// IsRemoteNotExistError checks the prefix of the error message to see whether a remote does not exist. +func IsRemoteNotExistError(err error) bool { + // see: https://github.com/go-gitea/gitea/issues/32889#issuecomment-2571848216 + // Should not add space in the end, sometimes git will add a `:` + prefix1 := "exit status 128 - fatal: No such remote" // git < 2.30 + prefix2 := "exit status 2 - error: No such remote" // git >= 2.30 + return strings.HasPrefix(err.Error(), prefix1) || strings.HasPrefix(err.Error(), prefix2) +} + // ParseRemoteAddr checks if given remote address is valid, // and returns composed URL with needed username and password. func ParseRemoteAddr(remoteAddr, authUsername, authPassword string) (string, error) { diff --git a/modules/git/repo.go b/modules/git/repo.go index 0993a4ac37..f1f6902773 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -18,6 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" ) // GPGSettings represents the default GPG settings for this repository @@ -27,6 +28,7 @@ type GPGSettings struct { Email string Name string PublicKeyContent string + Format string } const prettyLogFormat = `--pretty=format:%H` @@ -42,9 +44,9 @@ func (repo *Repository) parsePrettyFormatLogToList(logs []byte) ([]*Commit, erro return commits, nil } - parts := bytes.Split(logs, []byte{'\n'}) + parts := bytes.SplitSeq(logs, []byte{'\n'}) - for _, commitID := range parts { + for commitID := range parts { commit, err := repo.GetCommit(string(commitID)) if err != nil { return nil, err @@ -57,7 +59,7 @@ func (repo *Repository) parsePrettyFormatLogToList(logs []byte) ([]*Commit, erro // IsRepoURLAccessible checks if given repository URL is accessible. func IsRepoURLAccessible(ctx context.Context, url string) bool { - _, _, err := NewCommand(ctx, "ls-remote", "-q", "-h").AddDynamicArguments(url, "HEAD").RunStdString(nil) + _, _, err := NewCommand("ls-remote", "-q", "-h").AddDynamicArguments(url, "HEAD").RunStdString(ctx, nil) return err == nil } @@ -68,7 +70,7 @@ func InitRepository(ctx context.Context, repoPath string, bare bool, objectForma return err } - cmd := NewCommand(ctx, "init") + cmd := NewCommand("init") if !IsValidObjectFormat(objectFormatName) { return fmt.Errorf("invalid object format: %s", objectFormatName) @@ -80,15 +82,15 @@ func InitRepository(ctx context.Context, repoPath string, bare bool, objectForma if bare { cmd.AddArguments("--bare") } - _, _, err = cmd.RunStdString(&RunOpts{Dir: repoPath}) + _, _, err = cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) return err } // IsEmpty Check if repository is empty. func (repo *Repository) IsEmpty() (bool, error) { var errbuf, output strings.Builder - if err := NewCommand(repo.Ctx).AddOptionFormat("--git-dir=%s", repo.Path).AddArguments("rev-list", "-n", "1", "--all"). - Run(&RunOpts{ + if err := NewCommand().AddOptionFormat("--git-dir=%s", repo.Path).AddArguments("rev-list", "-n", "1", "--all"). + Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: &output, Stderr: &errbuf, @@ -129,7 +131,7 @@ func CloneWithArgs(ctx context.Context, args TrustedCmdArgs, from, to string, op return err } - cmd := NewCommandContextNoGlobals(ctx, args...).AddArguments("clone") + cmd := NewCommandNoGlobals(args...).AddArguments("clone") if opts.SkipTLSVerify { cmd.AddArguments("-c", "http.sslVerify=false") } @@ -170,7 +172,7 @@ func CloneWithArgs(ctx context.Context, args TrustedCmdArgs, from, to string, op } stderr := new(bytes.Buffer) - if err = cmd.Run(&RunOpts{ + if err = cmd.Run(ctx, &RunOpts{ Timeout: opts.Timeout, Env: envs, Stdout: io.Discard, @@ -193,7 +195,7 @@ type PushOptions struct { // Push pushs local commits to given remote branch. func Push(ctx context.Context, repoPath string, opts PushOptions) error { - cmd := NewCommand(ctx, "push") + cmd := NewCommand("push") if opts.Force { cmd.AddArguments("-f") } @@ -206,7 +208,7 @@ func Push(ctx context.Context, repoPath string, opts PushOptions) error { } cmd.AddDashesAndList(remoteBranchArgs...) - stdout, stderr, err := cmd.RunStdString(&RunOpts{Env: opts.Env, Timeout: opts.Timeout, Dir: repoPath}) + stdout, stderr, err := cmd.RunStdString(ctx, &RunOpts{Env: opts.Env, Timeout: opts.Timeout, Dir: repoPath}) if err != nil { if strings.Contains(stderr, "non-fast-forward") { return &ErrPushOutOfDate{StdOut: stdout, StdErr: stderr, Err: err} @@ -225,8 +227,8 @@ func Push(ctx context.Context, repoPath string, opts PushOptions) error { // GetLatestCommitTime returns time for latest commit in repository (across all branches) func GetLatestCommitTime(ctx context.Context, repoPath string) (time.Time, error) { - cmd := NewCommand(ctx, "for-each-ref", "--sort=-committerdate", BranchPrefix, "--count", "1", "--format=%(committerdate)") - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + cmd := NewCommand("for-each-ref", "--sort=-committerdate", BranchPrefix, "--count", "1", "--format=%(committerdate)") + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return time.Time{}, err } @@ -242,9 +244,9 @@ type DivergeObject struct { // GetDivergingCommits returns the number of commits a targetBranch is ahead or behind a baseBranch func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch string) (do DivergeObject, err error) { - cmd := NewCommand(ctx, "rev-list", "--count", "--left-right"). + cmd := NewCommand("rev-list", "--count", "--left-right"). AddDynamicArguments(baseBranch + "..." + targetBranch).AddArguments("--") - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return do, err } @@ -266,30 +268,30 @@ func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch // CreateBundle create bundle content to the target path func (repo *Repository) CreateBundle(ctx context.Context, commit string, out io.Writer) error { - tmp, err := os.MkdirTemp(os.TempDir(), "gitea-bundle") + tmp, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-bundle") if err != nil { return err } - defer os.RemoveAll(tmp) + defer cleanup() env := append(os.Environ(), "GIT_OBJECT_DIRECTORY="+filepath.Join(repo.Path, "objects")) - _, _, err = NewCommand(ctx, "init", "--bare").RunStdString(&RunOpts{Dir: tmp, Env: env}) + _, _, err = NewCommand("init", "--bare").RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) if err != nil { return err } - _, _, err = NewCommand(ctx, "reset", "--soft").AddDynamicArguments(commit).RunStdString(&RunOpts{Dir: tmp, Env: env}) + _, _, err = NewCommand("reset", "--soft").AddDynamicArguments(commit).RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) if err != nil { return err } - _, _, err = NewCommand(ctx, "branch", "-m", "bundle").RunStdString(&RunOpts{Dir: tmp, Env: env}) + _, _, err = NewCommand("branch", "-m", "bundle").RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) if err != nil { return err } tmpFile := filepath.Join(tmp, "bundle") - _, _, err = NewCommand(ctx, "bundle", "create").AddDynamicArguments(tmpFile, "bundle", "HEAD").RunStdString(&RunOpts{Dir: tmp, Env: env}) + _, _, err = NewCommand("bundle", "create").AddDynamicArguments(tmpFile, "bundle", "HEAD").RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) if err != nil { return err } diff --git a/modules/git/repo_archive.go b/modules/git/repo_archive.go index 1bf1aa41b9..0b2f6f2a45 100644 --- a/modules/git/repo_archive.go +++ b/modules/git/repo_archive.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "io" - "os" "path/filepath" "strings" ) @@ -17,37 +16,35 @@ import ( type ArchiveType int const ( - // ZIP zip archive type - ZIP ArchiveType = iota + 1 - // TARGZ tar gz archive type - TARGZ - // BUNDLE bundle archive type - BUNDLE + ArchiveUnknown ArchiveType = iota + ArchiveZip // 1 + ArchiveTarGz // 2 + ArchiveBundle // 3 ) -// String converts an ArchiveType to string +// String converts an ArchiveType to string: the extension of the archive file without prefix dot func (a ArchiveType) String() string { switch a { - case ZIP: + case ArchiveZip: return "zip" - case TARGZ: + case ArchiveTarGz: return "tar.gz" - case BUNDLE: + case ArchiveBundle: return "bundle" } return "unknown" } -func ToArchiveType(s string) ArchiveType { - switch s { - case "zip": - return ZIP - case "tar.gz": - return TARGZ - case "bundle": - return BUNDLE +func SplitArchiveNameType(s string) (string, ArchiveType) { + switch { + case strings.HasSuffix(s, ".zip"): + return strings.TrimSuffix(s, ".zip"), ArchiveZip + case strings.HasSuffix(s, ".tar.gz"): + return strings.TrimSuffix(s, ".tar.gz"), ArchiveTarGz + case strings.HasSuffix(s, ".bundle"): + return strings.TrimSuffix(s, ".bundle"), ArchiveBundle } - return 0 + return s, ArchiveUnknown } // CreateArchive create archive content to the target path @@ -56,22 +53,18 @@ func (repo *Repository) CreateArchive(ctx context.Context, format ArchiveType, t return fmt.Errorf("unknown format: %v", format) } - cmd := NewCommand(ctx, "archive") + cmd := NewCommand("archive") if usePrefix { cmd.AddOptionFormat("--prefix=%s", filepath.Base(strings.TrimSuffix(repo.Path, ".git"))+"/") } cmd.AddOptionFormat("--format=%s", format.String()) cmd.AddDynamicArguments(commitID) - // Avoid LFS hooks getting installed because of /etc/gitconfig, which can break pull requests. - env := append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1") - var stderr strings.Builder - err := cmd.Run(&RunOpts{ + err := cmd.Run(ctx, &RunOpts{ Dir: repo.Path, Stdout: target, Stderr: &stderr, - Env: env, }) if err != nil { return ConcatenateError(err, stderr.String()) diff --git a/modules/git/repo_archive_test.go b/modules/git/repo_archive_test.go new file mode 100644 index 0000000000..ff7e2dfce1 --- /dev/null +++ b/modules/git/repo_archive_test.go @@ -0,0 +1,32 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestArchiveType(t *testing.T) { + name, archiveType := SplitArchiveNameType("test.tar.gz") + assert.Equal(t, "test", name) + assert.Equal(t, "tar.gz", archiveType.String()) + + name, archiveType = SplitArchiveNameType("a/b/test.zip") + assert.Equal(t, "a/b/test", name) + assert.Equal(t, "zip", archiveType.String()) + + name, archiveType = SplitArchiveNameType("1234.bundle") + assert.Equal(t, "1234", name) + assert.Equal(t, "bundle", archiveType.String()) + + name, archiveType = SplitArchiveNameType("test") + assert.Equal(t, "test", name) + assert.Equal(t, "unknown", archiveType.String()) + + name, archiveType = SplitArchiveNameType("test.xz") + assert.Equal(t, "test.xz", name) + assert.Equal(t, "unknown", archiveType.String()) +} diff --git a/modules/git/repo_attribute.go b/modules/git/repo_attribute.go deleted file mode 100644 index 90eb783fe8..0000000000 --- a/modules/git/repo_attribute.go +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - - "code.gitea.io/gitea/modules/log" -) - -// CheckAttributeOpts represents the possible options to CheckAttribute -type CheckAttributeOpts struct { - CachedOnly bool - AllAttributes bool - Attributes []string - Filenames []string - IndexFile string - WorkTree string -} - -// CheckAttribute return the Blame object of file -func (repo *Repository) CheckAttribute(opts CheckAttributeOpts) (map[string]map[string]string, error) { - env := []string{} - - if len(opts.IndexFile) > 0 { - env = append(env, "GIT_INDEX_FILE="+opts.IndexFile) - } - if len(opts.WorkTree) > 0 { - env = append(env, "GIT_WORK_TREE="+opts.WorkTree) - } - - if len(env) > 0 { - env = append(os.Environ(), env...) - } - - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - - cmd := NewCommand(repo.Ctx, "check-attr", "-z") - - if opts.AllAttributes { - cmd.AddArguments("-a") - } else { - for _, attribute := range opts.Attributes { - if attribute != "" { - cmd.AddDynamicArguments(attribute) - } - } - } - - if opts.CachedOnly { - cmd.AddArguments("--cached") - } - - cmd.AddDashesAndList(opts.Filenames...) - - if err := cmd.Run(&RunOpts{ - Env: env, - Dir: repo.Path, - Stdout: stdOut, - Stderr: stdErr, - }); err != nil { - return nil, fmt.Errorf("failed to run check-attr: %w\n%s\n%s", err, stdOut.String(), stdErr.String()) - } - - // FIXME: This is incorrect on versions < 1.8.5 - fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) - - if len(fields)%3 != 1 { - return nil, fmt.Errorf("wrong number of fields in return from check-attr") - } - - name2attribute2info := make(map[string]map[string]string) - - for i := 0; i < (len(fields) / 3); i++ { - filename := string(fields[3*i]) - attribute := string(fields[3*i+1]) - info := string(fields[3*i+2]) - attribute2info := name2attribute2info[filename] - if attribute2info == nil { - attribute2info = make(map[string]string) - } - attribute2info[attribute] = info - name2attribute2info[filename] = attribute2info - } - - return name2attribute2info, nil -} - -// CheckAttributeReader provides a reader for check-attribute content that can be long running -type CheckAttributeReader struct { - // params - Attributes []string - Repo *Repository - IndexFile string - WorkTree string - - stdinReader io.ReadCloser - stdinWriter *os.File - stdOut attributeWriter - cmd *Command - env []string - ctx context.Context - cancel context.CancelFunc -} - -// Init initializes the CheckAttributeReader -func (c *CheckAttributeReader) Init(ctx context.Context) error { - if len(c.Attributes) == 0 { - lw := new(nulSeparatedAttributeWriter) - lw.attributes = make(chan attributeTriple) - lw.closed = make(chan struct{}) - - c.stdOut = lw - c.stdOut.Close() - return fmt.Errorf("no provided Attributes to check") - } - - c.ctx, c.cancel = context.WithCancel(ctx) - c.cmd = NewCommand(c.ctx, "check-attr", "--stdin", "-z") - - if len(c.IndexFile) > 0 { - c.cmd.AddArguments("--cached") - c.env = append(c.env, "GIT_INDEX_FILE="+c.IndexFile) - } - - if len(c.WorkTree) > 0 { - c.env = append(c.env, "GIT_WORK_TREE="+c.WorkTree) - } - - c.env = append(c.env, "GIT_FLUSH=1") - - c.cmd.AddDynamicArguments(c.Attributes...) - - var err error - - c.stdinReader, c.stdinWriter, err = os.Pipe() - if err != nil { - c.cancel() - return err - } - - lw := new(nulSeparatedAttributeWriter) - lw.attributes = make(chan attributeTriple, 5) - lw.closed = make(chan struct{}) - c.stdOut = lw - return nil -} - -// Run run cmd -func (c *CheckAttributeReader) Run() error { - defer func() { - _ = c.stdinReader.Close() - _ = c.stdOut.Close() - }() - stdErr := new(bytes.Buffer) - err := c.cmd.Run(&RunOpts{ - Env: c.env, - Dir: c.Repo.Path, - Stdin: c.stdinReader, - Stdout: c.stdOut, - Stderr: stdErr, - }) - if err != nil && !IsErrCanceledOrKilled(err) { - return fmt.Errorf("failed to run attr-check. Error: %w\nStderr: %s", err, stdErr.String()) - } - return nil -} - -// CheckPath check attr for given path -func (c *CheckAttributeReader) CheckPath(path string) (rs map[string]string, err error) { - defer func() { - if err != nil && err != c.ctx.Err() { - log.Error("Unexpected error when checking path %s in %s. Error: %v", path, c.Repo.Path, err) - } - }() - - select { - case <-c.ctx.Done(): - return nil, c.ctx.Err() - default: - } - - if _, err = c.stdinWriter.Write([]byte(path + "\x00")); err != nil { - defer c.Close() - return nil, err - } - - rs = make(map[string]string) - for range c.Attributes { - select { - case attr, ok := <-c.stdOut.ReadAttribute(): - if !ok { - return nil, c.ctx.Err() - } - rs[attr.Attribute] = attr.Value - case <-c.ctx.Done(): - return nil, c.ctx.Err() - } - } - return rs, nil -} - -// Close close pip after use -func (c *CheckAttributeReader) Close() error { - c.cancel() - err := c.stdinWriter.Close() - return err -} - -type attributeWriter interface { - io.WriteCloser - ReadAttribute() <-chan attributeTriple -} - -type attributeTriple struct { - Filename string - Attribute string - Value string -} - -type nulSeparatedAttributeWriter struct { - tmp []byte - attributes chan attributeTriple - closed chan struct{} - working attributeTriple - pos int -} - -func (wr *nulSeparatedAttributeWriter) Write(p []byte) (n int, err error) { - l, read := len(p), 0 - - nulIdx := bytes.IndexByte(p, '\x00') - for nulIdx >= 0 { - wr.tmp = append(wr.tmp, p[:nulIdx]...) - switch wr.pos { - case 0: - wr.working = attributeTriple{ - Filename: string(wr.tmp), - } - case 1: - wr.working.Attribute = string(wr.tmp) - case 2: - wr.working.Value = string(wr.tmp) - } - wr.tmp = wr.tmp[:0] - wr.pos++ - if wr.pos > 2 { - wr.attributes <- wr.working - wr.pos = 0 - } - read += nulIdx + 1 - if l > read { - p = p[nulIdx+1:] - nulIdx = bytes.IndexByte(p, '\x00') - } else { - return l, nil - } - } - wr.tmp = append(wr.tmp, p...) - return len(p), nil -} - -func (wr *nulSeparatedAttributeWriter) ReadAttribute() <-chan attributeTriple { - return wr.attributes -} - -func (wr *nulSeparatedAttributeWriter) Close() error { - select { - case <-wr.closed: - return nil - default: - } - close(wr.attributes) - close(wr.closed) - return nil -} - -// Create a check attribute reader for the current repository and provided commit ID -func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeReader, context.CancelFunc) { - indexFilename, worktree, deleteTemporaryFile, err := repo.ReadTreeToTemporaryIndex(commitID) - if err != nil { - return nil, func() {} - } - - checker := &CheckAttributeReader{ - Attributes: []string{ - AttributeLinguistVendored, - AttributeLinguistGenerated, - AttributeLinguistDocumentation, - AttributeLinguistDetectable, - AttributeLinguistLanguage, - AttributeGitlabLanguage, - }, - Repo: repo, - IndexFile: indexFilename, - WorkTree: worktree, - } - ctx, cancel := context.WithCancel(repo.Ctx) - if err := checker.Init(ctx); err != nil { - log.Error("Unable to open checker for %s. Error: %v", commitID, err) - } else { - go func() { - err := checker.Run() - if err != nil && err != ctx.Err() { - log.Error("Unable to open checker for %s. Error: %v", commitID, err) - } - cancel() - }() - } - deferable := func() { - _ = checker.Close() - cancel() - deleteTemporaryFile() - } - - return checker, deferable -} diff --git a/modules/git/repo_attribute_test.go b/modules/git/repo_attribute_test.go deleted file mode 100644 index 0fcd94b4c7..0000000000 --- a/modules/git/repo_attribute_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { - wr := &nulSeparatedAttributeWriter{ - attributes: make(chan attributeTriple, 5), - } - - testStr := ".gitignore\"\n\x00linguist-vendored\x00unspecified\x00" - - n, err := wr.Write([]byte(testStr)) - - assert.Len(t, testStr, n) - assert.NoError(t, err) - select { - case attr := <-wr.ReadAttribute(): - assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, AttributeLinguistVendored, attr.Attribute) - assert.Equal(t, "unspecified", attr.Value) - case <-time.After(100 * time.Millisecond): - assert.FailNow(t, "took too long to read an attribute from the list") - } - // Write a second attribute again - n, err = wr.Write([]byte(testStr)) - - assert.Len(t, testStr, n) - assert.NoError(t, err) - - select { - case attr := <-wr.ReadAttribute(): - assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, AttributeLinguistVendored, attr.Attribute) - assert.Equal(t, "unspecified", attr.Value) - case <-time.After(100 * time.Millisecond): - assert.FailNow(t, "took too long to read an attribute from the list") - } - - // Write a partial attribute - _, err = wr.Write([]byte("incomplete-file")) - assert.NoError(t, err) - _, err = wr.Write([]byte("name\x00")) - assert.NoError(t, err) - - select { - case <-wr.ReadAttribute(): - assert.FailNow(t, "There should not be an attribute ready to read") - case <-time.After(100 * time.Millisecond): - } - _, err = wr.Write([]byte("attribute\x00")) - assert.NoError(t, err) - select { - case <-wr.ReadAttribute(): - assert.FailNow(t, "There should not be an attribute ready to read") - case <-time.After(100 * time.Millisecond): - } - - _, err = wr.Write([]byte("value\x00")) - assert.NoError(t, err) - - attr := <-wr.ReadAttribute() - assert.Equal(t, "incomplete-filename", attr.Filename) - assert.Equal(t, "attribute", attr.Attribute) - assert.Equal(t, "value", attr.Value) - - _, err = wr.Write([]byte("shouldbe.vendor\x00linguist-vendored\x00set\x00shouldbe.vendor\x00linguist-generated\x00unspecified\x00shouldbe.vendor\x00linguist-language\x00unspecified\x00")) - assert.NoError(t, err) - attr = <-wr.ReadAttribute() - assert.NoError(t, err) - assert.EqualValues(t, attributeTriple{ - Filename: "shouldbe.vendor", - Attribute: AttributeLinguistVendored, - Value: "set", - }, attr) - attr = <-wr.ReadAttribute() - assert.NoError(t, err) - assert.EqualValues(t, attributeTriple{ - Filename: "shouldbe.vendor", - Attribute: AttributeLinguistGenerated, - Value: "unspecified", - }, attr) - attr = <-wr.ReadAttribute() - assert.NoError(t, err) - assert.EqualValues(t, attributeTriple{ - Filename: "shouldbe.vendor", - Attribute: AttributeLinguistLanguage, - Value: "unspecified", - }, attr) -} diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go index 0ca1ea79c2..293aca159c 100644 --- a/modules/git/repo_base_gogit.go +++ b/modules/git/repo_base_gogit.go @@ -49,7 +49,12 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { repoPath, err := filepath.Abs(repoPath) if err != nil { return nil, err - } else if !isDir(repoPath) { + } + exist, err := util.IsDir(repoPath) + if err != nil { + return nil, err + } + if !exist { return nil, util.NewNotExistErrorf("no such file or directory") } diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go index 477e3b8742..6f9bfd4b43 100644 --- a/modules/git/repo_base_nogogit.go +++ b/modules/git/repo_base_nogogit.go @@ -47,7 +47,12 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { repoPath, err := filepath.Abs(repoPath) if err != nil { return nil, err - } else if !isDir(repoPath) { + } + exist, err := util.IsDir(repoPath) + if err != nil { + return nil, err + } + if !exist { return nil, util.NewNotExistErrorf("no such file or directory") } @@ -62,7 +67,7 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { if repo.batch == nil { var err error - repo.batch, err = repo.NewBatch(ctx) + repo.batch, err = NewBatch(ctx, repo.Path) if err != nil { return nil, nil, nil, err } @@ -76,7 +81,7 @@ func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bu } log.Debug("Opening temporary cat file batch for: %s", repo.Path) - tempBatch, err := repo.NewBatch(ctx) + tempBatch, err := NewBatch(ctx, repo.Path) if err != nil { return nil, nil, nil, err } @@ -87,7 +92,7 @@ func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bu func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { if repo.check == nil { var err error - repo.check, err = repo.NewBatchCheck(ctx) + repo.check, err = NewBatchCheck(ctx, repo.Path) if err != nil { return nil, nil, nil, err } @@ -101,7 +106,7 @@ func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError } log.Debug("Opening temporary cat file batch-check for: %s", repo.Path) - tempBatchCheck, err := repo.NewBatchCheck(ctx) + tempBatchCheck, err := NewBatchCheck(ctx, repo.Path) if err != nil { return nil, nil, nil, err } diff --git a/modules/git/repo_blame.go b/modules/git/repo_blame.go index 139cdd7be9..6941a76c42 100644 --- a/modules/git/repo_blame.go +++ b/modules/git/repo_blame.go @@ -9,10 +9,10 @@ import ( // LineBlame returns the latest commit at the given line func (repo *Repository) LineBlame(revision, path, file string, line uint) (*Commit, error) { - res, _, err := NewCommand(repo.Ctx, "blame"). + res, _, err := NewCommand("blame"). AddOptionFormat("-L %d,%d", line, line). AddOptionValues("-p", revision). - AddDashesAndList(file).RunStdString(&RunOpts{Dir: path}) + AddDashesAndList(file).RunStdString(repo.Ctx, &RunOpts{Dir: path}) if err != nil { return nil, err } diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go index 552ae2bb8c..e7ecf53f51 100644 --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -7,7 +7,6 @@ package git import ( "context" "errors" - "fmt" "strings" ) @@ -16,7 +15,7 @@ const BranchPrefix = "refs/heads/" // IsReferenceExist returns true if given reference exists in the repository. func IsReferenceExist(ctx context.Context, repoPath, name string) bool { - _, _, err := NewCommand(ctx, "show-ref", "--verify").AddDashesAndList(name).RunStdString(&RunOpts{Dir: repoPath}) + _, _, err := NewCommand("show-ref", "--verify").AddDashesAndList(name).RunStdString(ctx, &RunOpts{Dir: repoPath}) return err == nil } @@ -25,38 +24,8 @@ func IsBranchExist(ctx context.Context, repoPath, name string) bool { return IsReferenceExist(ctx, repoPath, BranchPrefix+name) } -// Branch represents a Git branch. -type Branch struct { - Name string - Path string - - gitRepo *Repository -} - -// GetHEADBranch returns corresponding branch of HEAD. -func (repo *Repository) GetHEADBranch() (*Branch, error) { - if repo == nil { - return nil, fmt.Errorf("nil repo") - } - stdout, _, err := NewCommand(repo.Ctx, "symbolic-ref", "HEAD").RunStdString(&RunOpts{Dir: repo.Path}) - if err != nil { - return nil, err - } - stdout = strings.TrimSpace(stdout) - - if !strings.HasPrefix(stdout, BranchPrefix) { - return nil, fmt.Errorf("invalid HEAD branch: %v", stdout) - } - - return &Branch{ - Name: stdout[len(BranchPrefix):], - Path: stdout, - gitRepo: repo, - }, nil -} - func GetDefaultBranch(ctx context.Context, repoPath string) (string, error) { - stdout, _, err := NewCommand(ctx, "symbolic-ref", "HEAD").RunStdString(&RunOpts{Dir: repoPath}) + stdout, _, err := NewCommand("symbolic-ref", "HEAD").RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return "", err } @@ -67,37 +36,6 @@ func GetDefaultBranch(ctx context.Context, repoPath string) (string, error) { return strings.TrimPrefix(stdout, BranchPrefix), nil } -// GetBranch returns a branch by it's name -func (repo *Repository) GetBranch(branch string) (*Branch, error) { - if !repo.IsBranchExist(branch) { - return nil, ErrBranchNotExist{branch} - } - return &Branch{ - Path: repo.Path, - Name: branch, - gitRepo: repo, - }, nil -} - -// GetBranches returns a slice of *git.Branch -func (repo *Repository) GetBranches(skip, limit int) ([]*Branch, int, error) { - brs, countAll, err := repo.GetBranchNames(skip, limit) - if err != nil { - return nil, 0, err - } - - branches := make([]*Branch, len(brs)) - for i := range brs { - branches[i] = &Branch{ - Path: repo.Path, - Name: brs[i], - gitRepo: repo, - } - } - - return branches, countAll, nil -} - // DeleteBranchOptions Option(s) for delete branch type DeleteBranchOptions struct { Force bool @@ -105,7 +43,7 @@ type DeleteBranchOptions struct { // DeleteBranch delete a branch by name on repository. func (repo *Repository) DeleteBranch(name string, opts DeleteBranchOptions) error { - cmd := NewCommand(repo.Ctx, "branch") + cmd := NewCommand("branch") if opts.Force { cmd.AddArguments("-D") @@ -114,46 +52,41 @@ func (repo *Repository) DeleteBranch(name string, opts DeleteBranchOptions) erro } cmd.AddDashesAndList(name) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := cmd.RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // CreateBranch create a new branch func (repo *Repository) CreateBranch(branch, oldbranchOrCommit string) error { - cmd := NewCommand(repo.Ctx, "branch") + cmd := NewCommand("branch") cmd.AddDashesAndList(branch, oldbranchOrCommit) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := cmd.RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // AddRemote adds a new remote to repository. func (repo *Repository) AddRemote(name, url string, fetch bool) error { - cmd := NewCommand(repo.Ctx, "remote", "add") + cmd := NewCommand("remote", "add") if fetch { cmd.AddArguments("-f") } cmd.AddDynamicArguments(name, url) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := cmd.RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // RemoveRemote removes a remote from repository. func (repo *Repository) RemoveRemote(name string) error { - _, _, err := NewCommand(repo.Ctx, "remote", "rm").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("remote", "rm").AddDynamicArguments(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } -// GetCommit returns the head commit of a branch -func (branch *Branch) GetCommit() (*Commit, error) { - return branch.gitRepo.GetBranchCommit(branch.Name) -} - // RenameBranch rename a branch func (repo *Repository) RenameBranch(from, to string) error { - _, _, err := NewCommand(repo.Ctx, "branch", "-m").AddDynamicArguments(from, to).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("branch", "-m").AddDynamicArguments(from, to).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go index dbc4a5fedc..77aecb21eb 100644 --- a/modules/git/repo_branch_gogit.go +++ b/modules/git/repo_branch_gogit.go @@ -57,7 +57,7 @@ func (repo *Repository) IsBranchExist(name string) bool { // GetBranches returns branches from the repository, skipping "skip" initial branches and // returning at most "limit" branches, or all branches if "limit" is 0. -// Branches are returned with sort of `-commiterdate` as the nogogit +// Branches are returned with sort of `-committerdate` as the nogogit // implementation. This requires full fetch, sort and then the // skip/limit applies later as gogit returns in undefined order. func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index 0d2efd4a6b..0d11198523 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -109,7 +109,7 @@ func WalkShowRef(ctx context.Context, repoPath string, extraArgs TrustedCmdArgs, stderrBuilder := &strings.Builder{} args := TrustedCmdArgs{"for-each-ref", "--format=%(objectname) %(refname)"} args = append(args, extraArgs...) - err := NewCommand(ctx, args...).Run(&RunOpts{ + err := NewCommand(args...).Run(ctx, &RunOpts{ Dir: repoPath, Stdout: stdoutWriter, Stderr: stderrBuilder, diff --git a/modules/git/repo_branch_test.go b/modules/git/repo_branch_test.go index 5d3b8abb3a..8e8ea16fcd 100644 --- a/modules/git/repo_branch_test.go +++ b/modules/git/repo_branch_test.go @@ -21,21 +21,21 @@ func TestRepository_GetBranches(t *testing.T) { assert.NoError(t, err) assert.Len(t, branches, 2) - assert.EqualValues(t, 3, countAll) + assert.Equal(t, 3, countAll) assert.ElementsMatch(t, []string{"master", "branch2"}, branches) branches, countAll, err = bareRepo1.GetBranchNames(0, 0) assert.NoError(t, err) assert.Len(t, branches, 3) - assert.EqualValues(t, 3, countAll) + assert.Equal(t, 3, countAll) assert.ElementsMatch(t, []string{"master", "branch2", "branch1"}, branches) branches, countAll, err = bareRepo1.GetBranchNames(5, 1) assert.NoError(t, err) assert.Empty(t, branches) - assert.EqualValues(t, 3, countAll) + assert.Equal(t, 3, countAll) assert.ElementsMatch(t, []string{}, branches) } @@ -47,7 +47,7 @@ func BenchmarkRepository_GetBranches(b *testing.B) { } defer bareRepo1.Close() - for i := 0; i < b.N; i++ { + for b.Loop() { _, _, err := bareRepo1.GetBranchNames(0, 0) if err != nil { b.Fatal(err) @@ -71,15 +71,15 @@ func TestGetRefsBySha(t *testing.T) { // refs/pull/1/head branches, err = bareRepo5.GetRefsBySha("c83380d7056593c51a699d12b9c00627bd5743e9", PullPrefix) assert.NoError(t, err) - assert.EqualValues(t, []string{"refs/pull/1/head"}, branches) + assert.Equal(t, []string{"refs/pull/1/head"}, branches) branches, err = bareRepo5.GetRefsBySha("d8e0bbb45f200e67d9a784ce55bd90821af45ebd", BranchPrefix) assert.NoError(t, err) - assert.EqualValues(t, []string{"refs/heads/master", "refs/heads/master-clone"}, branches) + assert.Equal(t, []string{"refs/heads/master", "refs/heads/master-clone"}, branches) branches, err = bareRepo5.GetRefsBySha("58a4bcc53ac13e7ff76127e0fb518b5262bf09af", BranchPrefix) assert.NoError(t, err) - assert.EqualValues(t, []string{"refs/heads/test-patch-1"}, branches) + assert.Equal(t, []string{"refs/heads/test-patch-1"}, branches) } func BenchmarkGetRefsBySha(b *testing.B) { diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index 9ffadb833d..4066a1ca7b 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -59,7 +59,7 @@ func (repo *Repository) getCommitByPathWithID(id ObjectID, relpath string) (*Com relpath = `\` + relpath } - stdout, _, runErr := NewCommand(repo.Ctx, "log", "-1", prettyLogFormat).AddDynamicArguments(id.String()).AddDashesAndList(relpath).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand("log", "-1", prettyLogFormat).AddDynamicArguments(id.String()).AddDashesAndList(relpath).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -74,7 +74,7 @@ func (repo *Repository) getCommitByPathWithID(id ObjectID, relpath string) (*Com // GetCommitByPath returns the last commit of relative path. func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) { - stdout, _, runErr := NewCommand(repo.Ctx, "log", "-1", prettyLogFormat).AddDashesAndList(relpath).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand("log", "-1", prettyLogFormat).AddDashesAndList(relpath).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -89,8 +89,9 @@ func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) { return commits[0], nil } -func (repo *Repository) commitsByRange(id ObjectID, page, pageSize int, not string) ([]*Commit, error) { - cmd := NewCommand(repo.Ctx, "log"). +// commitsByRangeWithTime returns the specific page commits before current revision, with not, since, until support +func (repo *Repository) commitsByRangeWithTime(id ObjectID, page, pageSize int, not, since, until string) ([]*Commit, error) { + cmd := NewCommand("log"). AddOptionFormat("--skip=%d", (page-1)*pageSize). AddOptionFormat("--max-count=%d", pageSize). AddArguments(prettyLogFormat). @@ -99,8 +100,14 @@ func (repo *Repository) commitsByRange(id ObjectID, page, pageSize int, not stri if not != "" { cmd.AddOptionValues("--not", not) } + if since != "" { + cmd.AddOptionFormat("--since=%s", since) + } + if until != "" { + cmd.AddOptionFormat("--until=%s", until) + } - stdout, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } @@ -134,7 +141,7 @@ func (repo *Repository) searchCommits(id ObjectID, opts SearchCommitsOptions) ([ } // create new git log command with limit of 100 commits - cmd := NewCommand(repo.Ctx, "log", "-100", prettyLogFormat).AddDynamicArguments(id.String()) + cmd := NewCommand("log", "-100", prettyLogFormat).AddDynamicArguments(id.String()) // pretend that all refs along with HEAD were listed on command line as <commis> // https://git-scm.com/docs/git-log#Documentation/git-log.txt---all @@ -154,7 +161,7 @@ func (repo *Repository) searchCommits(id ObjectID, opts SearchCommitsOptions) ([ // search for commits matching given constraints and keywords in commit msg addCommonSearchArgs(cmd) - stdout, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } @@ -168,14 +175,14 @@ func (repo *Repository) searchCommits(id ObjectID, opts SearchCommitsOptions) ([ // ignore anything not matching a valid sha pattern if id.Type().IsValid(v) { // create new git log command with 1 commit limit - hashCmd := NewCommand(repo.Ctx, "log", "-1", prettyLogFormat) + hashCmd := NewCommand("log", "-1", prettyLogFormat) // add previous arguments except for --grep and --all addCommonSearchArgs(hashCmd) // add keyword as <commit> hashCmd.AddDynamicArguments(v) // search with given constraints for commit matching sha hash of v - hashMatching, _, err := hashCmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + hashMatching, _, err := hashCmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil || bytes.Contains(stdout, hashMatching) { continue } @@ -190,7 +197,7 @@ func (repo *Repository) searchCommits(id ObjectID, opts SearchCommitsOptions) ([ // FileChangedBetweenCommits Returns true if the file changed between commit IDs id1 and id2 // You must ensure that id1 and id2 are valid commit ids. func (repo *Repository) FileChangedBetweenCommits(filename, id1, id2 string) (bool, error) { - stdout, _, err := NewCommand(repo.Ctx, "diff", "--name-only", "-z").AddDynamicArguments(id1, id2).AddDashesAndList(filename).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("diff", "--name-only", "-z").AddDynamicArguments(id1, id2).AddDashesAndList(filename).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return false, err } @@ -212,12 +219,12 @@ type CommitsByFileAndRangeOptions struct { File string Not string Page int + Since string + Until string } // CommitsByFileAndRange return the commits according revision file and the page func (repo *Repository) CommitsByFileAndRange(opts CommitsByFileAndRangeOptions) ([]*Commit, error) { - skip := (opts.Page - 1) * setting.Git.CommitsRangeSize - stdoutReader, stdoutWriter := io.Pipe() defer func() { _ = stdoutReader.Close() @@ -225,17 +232,23 @@ func (repo *Repository) CommitsByFileAndRange(opts CommitsByFileAndRangeOptions) }() go func() { stderr := strings.Builder{} - gitCmd := NewCommand(repo.Ctx, "rev-list"). - AddOptionFormat("--max-count=%d", setting.Git.CommitsRangeSize*opts.Page). - AddOptionFormat("--skip=%d", skip) + gitCmd := NewCommand("rev-list"). + AddOptionFormat("--max-count=%d", setting.Git.CommitsRangeSize). + AddOptionFormat("--skip=%d", (opts.Page-1)*setting.Git.CommitsRangeSize) gitCmd.AddDynamicArguments(opts.Revision) if opts.Not != "" { gitCmd.AddOptionValues("--not", opts.Not) } + if opts.Since != "" { + gitCmd.AddOptionFormat("--since=%s", opts.Since) + } + if opts.Until != "" { + gitCmd.AddOptionFormat("--until=%s", opts.Until) + } gitCmd.AddDashesAndList(opts.File) - err := gitCmd.Run(&RunOpts{ + err := gitCmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr, @@ -277,11 +290,11 @@ func (repo *Repository) CommitsByFileAndRange(opts CommitsByFileAndRangeOptions) // FilesCountBetween return the number of files changed between two commits func (repo *Repository) FilesCountBetween(startCommitID, endCommitID string) (int, error) { - stdout, _, err := NewCommand(repo.Ctx, "diff", "--name-only").AddDynamicArguments(startCommitID + "..." + endCommitID).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("diff", "--name-only").AddDynamicArguments(startCommitID+"..."+endCommitID).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil && strings.Contains(err.Error(), "no merge base") { // git >= 2.28 now returns an error if startCommitID and endCommitID have become unrelated. // previously it would return the results of git diff --name-only startCommitID endCommitID so let's try that... - stdout, _, err = NewCommand(repo.Ctx, "diff", "--name-only").AddDynamicArguments(startCommitID, endCommitID).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("diff", "--name-only").AddDynamicArguments(startCommitID, endCommitID).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) } if err != nil { return 0, err @@ -295,13 +308,13 @@ func (repo *Repository) CommitsBetween(last, before *Commit) ([]*Commit, error) var stdout []byte var err error if before == nil { - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } else { - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String() + ".." + last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(before.ID.String()+".."+last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil && strings.Contains(err.Error(), "no merge base") { // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. // previously it would return the results of git rev-list before last so let's try that... - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } } if err != nil { @@ -315,22 +328,22 @@ func (repo *Repository) CommitsBetweenLimit(last, before *Commit, limit, skip in var stdout []byte var err error if before == nil { - stdout, _, err = NewCommand(repo.Ctx, "rev-list"). + stdout, _, err = NewCommand("rev-list"). AddOptionValues("--max-count", strconv.Itoa(limit)). AddOptionValues("--skip", strconv.Itoa(skip)). - AddDynamicArguments(last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + AddDynamicArguments(last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } else { - stdout, _, err = NewCommand(repo.Ctx, "rev-list"). + stdout, _, err = NewCommand("rev-list"). AddOptionValues("--max-count", strconv.Itoa(limit)). AddOptionValues("--skip", strconv.Itoa(skip)). - AddDynamicArguments(before.ID.String() + ".." + last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + AddDynamicArguments(before.ID.String()+".."+last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil && strings.Contains(err.Error(), "no merge base") { // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. // previously it would return the results of git rev-list --max-count n before last so let's try that... - stdout, _, err = NewCommand(repo.Ctx, "rev-list"). + stdout, _, err = NewCommand("rev-list"). AddOptionValues("--max-count", strconv.Itoa(limit)). AddOptionValues("--skip", strconv.Itoa(skip)). - AddDynamicArguments(before.ID.String(), last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) + AddDynamicArguments(before.ID.String(), last.ID.String()).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } } if err != nil { @@ -345,13 +358,13 @@ func (repo *Repository) CommitsBetweenNotBase(last, before *Commit, baseBranch s var stdout []byte var err error if before == nil { - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } else { - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String()+".."+last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(before.ID.String()+".."+last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil && strings.Contains(err.Error(), "no merge base") { // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. // previously it would return the results of git rev-list before last so let's try that... - stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, err = NewCommand("rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) } } if err != nil { @@ -397,13 +410,13 @@ func (repo *Repository) CommitsCountBetween(start, end string) (int64, error) { // commitsBefore the limit is depth, not total number of returned commits. func (repo *Repository) commitsBefore(id ObjectID, limit int) ([]*Commit, error) { - cmd := NewCommand(repo.Ctx, "log", prettyLogFormat) + cmd := NewCommand("log", prettyLogFormat) if limit > 0 { cmd.AddOptionFormat("-%d", limit) } cmd.AddDynamicArguments(id.String()) - stdout, _, runErr := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -440,10 +453,10 @@ func (repo *Repository) getCommitsBeforeLimit(id ObjectID, num int) ([]*Commit, func (repo *Repository) getBranches(env []string, commitID string, limit int) ([]string, error) { if DefaultFeatures().CheckVersionAtLeast("2.7.0") { - stdout, _, err := NewCommand(repo.Ctx, "for-each-ref", "--format=%(refname:strip=2)"). + stdout, _, err := NewCommand("for-each-ref", "--format=%(refname:strip=2)"). AddOptionFormat("--count=%d", limit). AddOptionValues("--contains", commitID, BranchPrefix). - RunStdString(&RunOpts{ + RunStdString(repo.Ctx, &RunOpts{ Dir: repo.Path, Env: env, }) @@ -455,7 +468,7 @@ func (repo *Repository) getBranches(env []string, commitID string, limit int) ([ return branches, nil } - stdout, _, err := NewCommand(repo.Ctx, "branch").AddOptionValues("--contains", commitID).RunStdString(&RunOpts{ + stdout, _, err := NewCommand("branch").AddOptionValues("--contains", commitID).RunStdString(repo.Ctx, &RunOpts{ Dir: repo.Path, Env: env, }) @@ -497,7 +510,7 @@ func (repo *Repository) GetCommitsFromIDs(commitIDs []string) []*Commit { // IsCommitInBranch check if the commit is on the branch func (repo *Repository) IsCommitInBranch(commitID, branch string) (r bool, err error) { - stdout, _, err := NewCommand(repo.Ctx, "branch", "--contains").AddDynamicArguments(commitID, branch).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("branch", "--contains").AddDynamicArguments(commitID, branch).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return false, err } @@ -521,11 +534,12 @@ func (repo *Repository) AddLastCommitCache(cacheKey, fullName, sha string) error return nil } +// GetCommitBranchStart returns the commit where the branch diverged func (repo *Repository) GetCommitBranchStart(env []string, branch, endCommitID string) (string, error) { - cmd := NewCommand(repo.Ctx, "log", prettyLogFormat) + cmd := NewCommand("log", prettyLogFormat) cmd.AddDynamicArguments(endCommitID) - stdout, _, runErr := cmd.RunStdBytes(&RunOpts{ + stdout, _, runErr := cmd.RunStdBytes(repo.Ctx, &RunOpts{ Dir: repo.Path, Env: env, }) @@ -533,21 +547,20 @@ func (repo *Repository) GetCommitBranchStart(env []string, branch, endCommitID s return "", runErr } - parts := bytes.Split(bytes.TrimSpace(stdout), []byte{'\n'}) + parts := bytes.SplitSeq(bytes.TrimSpace(stdout), []byte{'\n'}) - var startCommitID string - for _, commitID := range parts { + // check the commits one by one until we find a commit contained by another branch + // and we think this commit is the divergence point + for commitID := range parts { branches, err := repo.getBranches(env, string(commitID), 2) if err != nil { return "", err } for _, b := range branches { if b != branch { - return startCommitID, nil + return string(commitID), nil } } - - startCommitID = string(commitID) } return "", nil diff --git a/modules/git/repo_commit_gogit.go b/modules/git/repo_commit_gogit.go index 993013eef7..a88902e209 100644 --- a/modules/git/repo_commit_gogit.go +++ b/modules/git/repo_commit_gogit.go @@ -59,7 +59,7 @@ func (repo *Repository) ConvertToGitID(commitID string) (ObjectID, error) { } } - actualCommitID, _, err := NewCommand(repo.Ctx, "rev-parse", "--verify").AddDynamicArguments(commitID).RunStdString(&RunOpts{Dir: repo.Path}) + actualCommitID, _, err := NewCommand("rev-parse", "--verify").AddDynamicArguments(commitID).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) actualCommitID = strings.TrimSpace(actualCommitID) if err != nil { if strings.Contains(err.Error(), "unknown revision or path") || diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index f5ed282a45..3ead3e2216 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -16,7 +16,7 @@ import ( // ResolveReference resolves a name to a reference func (repo *Repository) ResolveReference(name string) (string, error) { - stdout, _, err := NewCommand(repo.Ctx, "show-ref", "--hash").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("show-ref", "--hash").AddDynamicArguments(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { if strings.Contains(err.Error(), "not a valid ref") { return "", ErrNotExist{name, ""} @@ -52,13 +52,13 @@ func (repo *Repository) GetRefCommitID(name string) (string, error) { // SetReference sets the commit ID string of given reference (e.g. branch or tag). func (repo *Repository) SetReference(name, commitID string) error { - _, _, err := NewCommand(repo.Ctx, "update-ref").AddDynamicArguments(name, commitID).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("update-ref").AddDynamicArguments(name, commitID).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // RemoveReference removes the given reference (e.g. branch or tag). func (repo *Repository) RemoveReference(name string) error { - _, _, err := NewCommand(repo.Ctx, "update-ref", "--no-deref", "-d").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("update-ref", "--no-deref", "-d").AddDynamicArguments(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } @@ -68,7 +68,7 @@ func (repo *Repository) IsCommitExist(name string) bool { log.Error("IsCommitExist: %v", err) return false } - _, _, err := NewCommand(repo.Ctx, "cat-file", "-e").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("cat-file", "-e").AddDynamicArguments(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err == nil } @@ -81,10 +81,10 @@ func (repo *Repository) getCommit(id ObjectID) (*Commit, error) { _, _ = wr.Write([]byte(id.String() + "\n")) - return repo.getCommitFromBatchReader(rd, id) + return repo.getCommitFromBatchReader(wr, rd, id) } -func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id ObjectID) (*Commit, error) { +func (repo *Repository) getCommitFromBatchReader(wr WriteCloserError, rd *bufio.Reader, id ObjectID) (*Commit, error) { _, typ, size, err := ReadBatchLine(rd) if err != nil { if errors.Is(err, io.EOF) || IsErrNotExist(err) { @@ -112,7 +112,11 @@ func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id ObjectID) return nil, err } - commit, err := tag.Commit(repo) + if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { + return nil, err + } + + commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) if err != nil { return nil, err } diff --git a/modules/git/repo_commit_test.go b/modules/git/repo_commit_test.go index 4c26fa2a48..e9f469accd 100644 --- a/modules/git/repo_commit_test.go +++ b/modules/git/repo_commit_test.go @@ -8,7 +8,11 @@ import ( "path/filepath" "testing" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRepository_GetCommitBranches(t *testing.T) { @@ -126,3 +130,21 @@ func TestGetRefCommitID(t *testing.T) { } } } + +func TestCommitsByFileAndRange(t *testing.T) { + defer test.MockVariableValue(&setting.Git.CommitsRangeSize, 2)() + + bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") + bareRepo1, err := openRepositoryWithDefaultContext(bareRepo1Path) + require.NoError(t, err) + defer bareRepo1.Close() + + // "foo" has 3 commits in "master" branch + commits, err := bareRepo1.CommitsByFileAndRange(CommitsByFileAndRangeOptions{Revision: "master", File: "foo", Page: 1}) + require.NoError(t, err) + assert.Len(t, commits, 2) + + commits, err = bareRepo1.CommitsByFileAndRange(CommitsByFileAndRangeOptions{Revision: "master", File: "foo", Page: 2}) + require.NoError(t, err) + assert.Len(t, commits, 1) +} diff --git a/modules/git/repo_commitgraph.go b/modules/git/repo_commitgraph.go index 087d5bcec4..62c6378054 100644 --- a/modules/git/repo_commitgraph.go +++ b/modules/git/repo_commitgraph.go @@ -12,7 +12,7 @@ import ( // this requires git v2.18 to be installed func WriteCommitGraph(ctx context.Context, repoPath string) error { if DefaultFeatures().CheckVersionAtLeast("2.18") { - if _, _, err := NewCommand(ctx, "commit-graph", "write").RunStdString(&RunOpts{Dir: repoPath}); err != nil { + if _, _, err := NewCommand("commit-graph", "write").RunStdString(ctx, &RunOpts{Dir: repoPath}); err != nil { return fmt.Errorf("unable to write commit-graph for '%s' : %w", repoPath, err) } } diff --git a/modules/git/repo_commitgraph_gogit.go b/modules/git/repo_commitgraph_gogit.go index d3182f15c6..c0082b62c8 100644 --- a/modules/git/repo_commitgraph_gogit.go +++ b/modules/git/repo_commitgraph_gogit.go @@ -8,7 +8,7 @@ package git import ( "os" - "path" + "path/filepath" gitealog "code.gitea.io/gitea/modules/log" @@ -18,7 +18,7 @@ import ( // CommitNodeIndex returns the index for walking commit graph func (r *Repository) CommitNodeIndex() (cgobject.CommitNodeIndex, *os.File) { - indexPath := path.Join(r.Path, "objects", "info", "commit-graph") + indexPath := filepath.Join(r.Path, "objects", "info", "commit-graph") file, err := os.Open(indexPath) if err == nil { diff --git a/modules/git/repo_compare.go b/modules/git/repo_compare.go index 16fcdcf4c8..ff44506e13 100644 --- a/modules/git/repo_compare.go +++ b/modules/git/repo_compare.go @@ -39,13 +39,13 @@ func (repo *Repository) GetMergeBase(tmpRemote, base, head string) (string, stri if tmpRemote != "origin" { tmpBaseName := RemotePrefix + tmpRemote + "/tmp_" + base // Fetch commit into a temporary branch in order to be able to handle commits and tags - _, _, err := NewCommand(repo.Ctx, "fetch", "--no-tags").AddDynamicArguments(tmpRemote).AddDashesAndList(base + ":" + tmpBaseName).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("fetch", "--no-tags").AddDynamicArguments(tmpRemote).AddDashesAndList(base+":"+tmpBaseName).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err == nil { base = tmpBaseName } } - stdout, _, err := NewCommand(repo.Ctx, "merge-base").AddDashesAndList(base, head).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("merge-base").AddDashesAndList(base, head).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return strings.TrimSpace(stdout), base, err } @@ -94,9 +94,9 @@ func (repo *Repository) GetCompareInfo(basePath, baseBranch, headBranch string, if !fileOnly { // avoid: ambiguous argument 'refs/a...refs/b': unknown revision or path not in the working tree. Use '--': 'git <command> [<revision>...] -- [<file>...]' var logs []byte - logs, _, err = NewCommand(repo.Ctx, "log").AddArguments(prettyLogFormat). - AddDynamicArguments(baseCommitID + separator + headBranch).AddArguments("--"). - RunStdBytes(&RunOpts{Dir: repo.Path}) + logs, _, err = NewCommand("log").AddArguments(prettyLogFormat). + AddDynamicArguments(baseCommitID+separator+headBranch).AddArguments("--"). + RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } @@ -150,8 +150,8 @@ func (repo *Repository) GetDiffNumChangedFiles(base, head string, directComparis } // avoid: ambiguous argument 'refs/a...refs/b': unknown revision or path not in the working tree. Use '--': 'git <command> [<revision>...] -- [<file>...]' - if err := NewCommand(repo.Ctx, "diff", "-z", "--name-only").AddDynamicArguments(base + separator + head).AddArguments("--"). - Run(&RunOpts{ + if err := NewCommand("diff", "-z", "--name-only").AddDynamicArguments(base+separator+head).AddArguments("--"). + Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: w, Stderr: stderr, @@ -161,7 +161,7 @@ func (repo *Repository) GetDiffNumChangedFiles(base, head string, directComparis // previously it would return the results of git diff -z --name-only base head so let's try that... w = &lineCountWriter{} stderr.Reset() - if err = NewCommand(repo.Ctx, "diff", "-z", "--name-only").AddDynamicArguments(base, head).AddArguments("--").Run(&RunOpts{ + if err = NewCommand("diff", "-z", "--name-only").AddDynamicArguments(base, head).AddArguments("--").Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: w, Stderr: stderr, @@ -174,23 +174,15 @@ func (repo *Repository) GetDiffNumChangedFiles(base, head string, directComparis return w.numLines, nil } -// GetDiffShortStat counts number of changed files, number of additions and deletions -func (repo *Repository) GetDiffShortStat(base, head string) (numFiles, totalAdditions, totalDeletions int, err error) { - numFiles, totalAdditions, totalDeletions, err = GetDiffShortStat(repo.Ctx, repo.Path, nil, base+"..."+head) - if err != nil && strings.Contains(err.Error(), "no merge base") { - return GetDiffShortStat(repo.Ctx, repo.Path, nil, base, head) - } - return numFiles, totalAdditions, totalDeletions, err -} - -// GetDiffShortStat counts number of changed files, number of additions and deletions -func GetDiffShortStat(ctx context.Context, repoPath string, trustedArgs TrustedCmdArgs, dynamicArgs ...string) (numFiles, totalAdditions, totalDeletions int, err error) { +// GetDiffShortStatByCmdArgs counts number of changed files, number of additions and deletions +// TODO: it can be merged with another "GetDiffShortStat" in the future +func GetDiffShortStatByCmdArgs(ctx context.Context, repoPath string, trustedArgs TrustedCmdArgs, dynamicArgs ...string) (numFiles, totalAdditions, totalDeletions int, err error) { // Now if we call: // $ git diff --shortstat 1ebb35b98889ff77299f24d82da426b434b0cca0...788b8b1440462d477f45b0088875 // we get: // " 9902 files changed, 2034198 insertions(+), 298800 deletions(-)\n" - cmd := NewCommand(ctx, "diff", "--shortstat").AddArguments(trustedArgs...).AddDynamicArguments(dynamicArgs...) - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + cmd := NewCommand("diff", "--shortstat").AddArguments(trustedArgs...).AddDynamicArguments(dynamicArgs...) + stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { return 0, 0, 0, err } @@ -233,72 +225,34 @@ func parseDiffStat(stdout string) (numFiles, totalAdditions, totalDeletions int, return numFiles, totalAdditions, totalDeletions, err } -// GetDiffOrPatch generates either diff or formatted patch data between given revisions -func (repo *Repository) GetDiffOrPatch(base, head string, w io.Writer, patch, binary bool) error { - if patch { - return repo.GetPatch(base, head, w) - } - if binary { - return repo.GetDiffBinary(base, head, w) - } - return repo.GetDiff(base, head, w) -} - // GetDiff generates and returns patch data between given revisions, optimized for human readability -func (repo *Repository) GetDiff(base, head string, w io.Writer) error { +func (repo *Repository) GetDiff(compareArg string, w io.Writer) error { stderr := new(bytes.Buffer) - err := NewCommand(repo.Ctx, "diff", "-p").AddDynamicArguments(base + "..." + head). - Run(&RunOpts{ + return NewCommand("diff", "-p").AddDynamicArguments(compareArg). + Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: w, Stderr: stderr, }) - if err != nil && bytes.Contains(stderr.Bytes(), []byte("no merge base")) { - return NewCommand(repo.Ctx, "diff", "-p").AddDynamicArguments(base, head). - Run(&RunOpts{ - Dir: repo.Path, - Stdout: w, - }) - } - return err } // GetDiffBinary generates and returns patch data between given revisions, including binary diffs. -func (repo *Repository) GetDiffBinary(base, head string, w io.Writer) error { - stderr := new(bytes.Buffer) - err := NewCommand(repo.Ctx, "diff", "-p", "--binary", "--histogram").AddDynamicArguments(base + "..." + head). - Run(&RunOpts{ - Dir: repo.Path, - Stdout: w, - Stderr: stderr, - }) - if err != nil && bytes.Contains(stderr.Bytes(), []byte("no merge base")) { - return NewCommand(repo.Ctx, "diff", "-p", "--binary", "--histogram").AddDynamicArguments(base, head). - Run(&RunOpts{ - Dir: repo.Path, - Stdout: w, - }) - } - return err +func (repo *Repository) GetDiffBinary(compareArg string, w io.Writer) error { + return NewCommand("diff", "-p", "--binary", "--histogram").AddDynamicArguments(compareArg).Run(repo.Ctx, &RunOpts{ + Dir: repo.Path, + Stdout: w, + }) } // GetPatch generates and returns format-patch data between given revisions, able to be used with `git apply` -func (repo *Repository) GetPatch(base, head string, w io.Writer) error { +func (repo *Repository) GetPatch(compareArg string, w io.Writer) error { stderr := new(bytes.Buffer) - err := NewCommand(repo.Ctx, "format-patch", "--binary", "--stdout").AddDynamicArguments(base + "..." + head). - Run(&RunOpts{ + return NewCommand("format-patch", "--binary", "--stdout").AddDynamicArguments(compareArg). + Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: w, Stderr: stderr, }) - if err != nil && bytes.Contains(stderr.Bytes(), []byte("no merge base")) { - return NewCommand(repo.Ctx, "format-patch", "--binary", "--stdout").AddDynamicArguments(base, head). - Run(&RunOpts{ - Dir: repo.Path, - Stdout: w, - }) - } - return err } // GetFilesChangedBetween returns a list of all files that have been changed between the given commits @@ -309,13 +263,13 @@ func (repo *Repository) GetFilesChangedBetween(base, head string) ([]string, err if err != nil { return nil, err } - cmd := NewCommand(repo.Ctx, "diff-tree", "--name-only", "--root", "--no-commit-id", "-r", "-z") + cmd := NewCommand("diff-tree", "--name-only", "--root", "--no-commit-id", "-r", "-z") if base == objectFormat.EmptyObjectID().String() { cmd.AddDynamicArguments(head) } else { cmd.AddDynamicArguments(base, head) } - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := cmd.RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } @@ -329,21 +283,6 @@ func (repo *Repository) GetFilesChangedBetween(base, head string) ([]string, err return split, err } -// GetDiffFromMergeBase generates and return patch data from merge base to head -func (repo *Repository) GetDiffFromMergeBase(base, head string, w io.Writer) error { - stderr := new(bytes.Buffer) - err := NewCommand(repo.Ctx, "diff", "-p", "--binary").AddDynamicArguments(base + "..." + head). - Run(&RunOpts{ - Dir: repo.Path, - Stdout: w, - Stderr: stderr, - }) - if err != nil && bytes.Contains(stderr.Bytes(), []byte("no merge base")) { - return repo.GetDiffBinary(base, head, w) - } - return err -} - // ReadPatchCommit will check if a diff patch exists and return stats func (repo *Repository) ReadPatchCommit(prID int64) (commitSHA string, err error) { // Migrated repositories download patches to "pulls" location diff --git a/modules/git/repo_compare_test.go b/modules/git/repo_compare_test.go index 454ed6b9f8..25ee4c5198 100644 --- a/modules/git/repo_compare_test.go +++ b/modules/git/repo_compare_test.go @@ -28,7 +28,7 @@ func TestGetFormatPatch(t *testing.T) { defer repo.Close() rd := &bytes.Buffer{} - err = repo.GetPatch("8d92fc95^", "8d92fc95", rd) + err = repo.GetPatch("8d92fc95^...8d92fc95", rd) if err != nil { assert.NoError(t, err) return diff --git a/modules/git/repo_gpg.go b/modules/git/repo_gpg.go index e2b45064fd..0021a7bda7 100644 --- a/modules/git/repo_gpg.go +++ b/modules/git/repo_gpg.go @@ -6,6 +6,7 @@ package git import ( "fmt" + "os" "strings" "code.gitea.io/gitea/modules/process" @@ -13,6 +14,14 @@ import ( // LoadPublicKeyContent will load the key from gpg func (gpgSettings *GPGSettings) LoadPublicKeyContent() error { + if gpgSettings.Format == SigningKeyFormatSSH { + content, err := os.ReadFile(gpgSettings.KeyID) + if err != nil { + return fmt.Errorf("unable to read SSH public key file: %s, %w", gpgSettings.KeyID, err) + } + gpgSettings.PublicKeyContent = string(content) + return nil + } content, stderr, err := process.GetManager().Exec( "gpg -a --export", "gpg", "-a", "--export", gpgSettings.KeyID) @@ -33,7 +42,7 @@ func (repo *Repository) GetDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, Sign: true, } - value, _, _ := NewCommand(repo.Ctx, "config", "--get", "commit.gpgsign").RunStdString(&RunOpts{Dir: repo.Path}) + value, _, _ := NewCommand("config", "--get", "commit.gpgsign").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) sign, valid := ParseBool(strings.TrimSpace(value)) if !sign || !valid { gpgSettings.Sign = false @@ -41,13 +50,16 @@ func (repo *Repository) GetDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, return gpgSettings, nil } - signingKey, _, _ := NewCommand(repo.Ctx, "config", "--get", "user.signingkey").RunStdString(&RunOpts{Dir: repo.Path}) + signingKey, _, _ := NewCommand("config", "--get", "user.signingkey").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) gpgSettings.KeyID = strings.TrimSpace(signingKey) - defaultEmail, _, _ := NewCommand(repo.Ctx, "config", "--get", "user.email").RunStdString(&RunOpts{Dir: repo.Path}) + format, _, _ := NewCommand("config", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) + gpgSettings.Format = strings.TrimSpace(format) + + defaultEmail, _, _ := NewCommand("config", "--get", "user.email").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) gpgSettings.Email = strings.TrimSpace(defaultEmail) - defaultName, _, _ := NewCommand(repo.Ctx, "config", "--get", "user.name").RunStdString(&RunOpts{Dir: repo.Path}) + defaultName, _, _ := NewCommand("config", "--get", "user.name").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) gpgSettings.Name = strings.TrimSpace(defaultName) if err := gpgSettings.LoadPublicKeyContent(); err != nil { diff --git a/modules/git/repo_index.go b/modules/git/repo_index.go index f45b6e6191..4879121a41 100644 --- a/modules/git/repo_index.go +++ b/modules/git/repo_index.go @@ -10,8 +10,7 @@ import ( "path/filepath" "strings" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/setting" ) // ReadTreeToIndex reads a treeish to the index @@ -22,7 +21,7 @@ func (repo *Repository) ReadTreeToIndex(treeish string, indexFilename ...string) } if len(treeish) != objectFormat.FullLength() { - res, _, err := NewCommand(repo.Ctx, "rev-parse", "--verify").AddDynamicArguments(treeish).RunStdString(&RunOpts{Dir: repo.Path}) + res, _, err := NewCommand("rev-parse", "--verify").AddDynamicArguments(treeish).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return err } @@ -42,7 +41,7 @@ func (repo *Repository) readTreeToIndex(id ObjectID, indexFilename ...string) er if len(indexFilename) > 0 { env = append(os.Environ(), "GIT_INDEX_FILE="+indexFilename[0]) } - _, _, err := NewCommand(repo.Ctx, "read-tree").AddDynamicArguments(id.String()).RunStdString(&RunOpts{Dir: repo.Path, Env: env}) + _, _, err := NewCommand("read-tree").AddDynamicArguments(id.String()).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path, Env: env}) if err != nil { return err } @@ -59,43 +58,35 @@ func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (tmpIndexFilena } }() - removeDirFn := func(dir string) func() { // it can't use the return value "tmpDir" directly because it is empty when error occurs - return func() { - if err := util.RemoveAll(dir); err != nil { - log.Error("failed to remove tmp index dir: %v", err) - } - } - } - - tmpDir, err = os.MkdirTemp("", "index") + tmpDir, cancel, err = setting.AppDataTempDir("git-repo-content").MkdirTempRandom("index") if err != nil { return "", "", nil, err } tmpIndexFilename = filepath.Join(tmpDir, ".tmp-index") - cancel = removeDirFn(tmpDir) + err = repo.ReadTreeToIndex(treeish, tmpIndexFilename) if err != nil { return "", "", cancel, err } - return tmpIndexFilename, tmpDir, cancel, err + return tmpIndexFilename, tmpDir, cancel, nil } // EmptyIndex empties the index func (repo *Repository) EmptyIndex() error { - _, _, err := NewCommand(repo.Ctx, "read-tree", "--empty").RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("read-tree", "--empty").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // LsFiles checks if the given filenames are in the index func (repo *Repository) LsFiles(filenames ...string) ([]string, error) { - cmd := NewCommand(repo.Ctx, "ls-files", "-z").AddDashesAndList(filenames...) - res, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + cmd := NewCommand("ls-files", "-z").AddDashesAndList(filenames...) + res, _, err := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } filelist := make([]string, 0, len(filenames)) - for _, line := range bytes.Split(res, []byte{'\000'}) { + for line := range bytes.SplitSeq(res, []byte{'\000'}) { filelist = append(filelist, string(line)) } @@ -108,7 +99,7 @@ func (repo *Repository) RemoveFilesFromIndex(filenames ...string) error { if err != nil { return err } - cmd := NewCommand(repo.Ctx, "update-index", "--remove", "-z", "--index-info") + cmd := NewCommand("update-index", "--remove", "-z", "--index-info") stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) buffer := new(bytes.Buffer) @@ -118,7 +109,7 @@ func (repo *Repository) RemoveFilesFromIndex(filenames ...string) error { buffer.WriteString("0 blob " + objectFormat.EmptyObjectID().String() + "\t" + file + "\000") } } - return cmd.Run(&RunOpts{ + return cmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdin: bytes.NewReader(buffer.Bytes()), Stdout: stdout, @@ -134,7 +125,7 @@ type IndexObjectInfo struct { // AddObjectsToIndex adds the provided object hashes to the index at the provided filenames func (repo *Repository) AddObjectsToIndex(objects ...IndexObjectInfo) error { - cmd := NewCommand(repo.Ctx, "update-index", "--add", "--replace", "-z", "--index-info") + cmd := NewCommand("update-index", "--add", "--replace", "-z", "--index-info") stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) buffer := new(bytes.Buffer) @@ -142,7 +133,7 @@ func (repo *Repository) AddObjectsToIndex(objects ...IndexObjectInfo) error { // using format: mode SP type SP sha1 TAB path buffer.WriteString(object.Mode + " blob " + object.Object.String() + "\t" + object.Filename + "\000") } - return cmd.Run(&RunOpts{ + return cmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdin: bytes.NewReader(buffer.Bytes()), Stdout: stdout, @@ -157,7 +148,7 @@ func (repo *Repository) AddObjectToIndex(mode string, object ObjectID, filename // WriteTree writes the current index as a tree to the object db and returns its hash func (repo *Repository) WriteTree() (*Tree, error) { - stdout, _, runErr := NewCommand(repo.Ctx, "write-tree").RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand("write-tree").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } diff --git a/modules/git/repo_object.go b/modules/git/repo_object.go index 3d48b91c6d..08e0413311 100644 --- a/modules/git/repo_object.go +++ b/modules/git/repo_object.go @@ -68,13 +68,13 @@ func (repo *Repository) HashObject(reader io.Reader) (ObjectID, error) { func (repo *Repository) hashObject(reader io.Reader, save bool) (string, error) { var cmd *Command if save { - cmd = NewCommand(repo.Ctx, "hash-object", "-w", "--stdin") + cmd = NewCommand("hash-object", "-w", "--stdin") } else { - cmd = NewCommand(repo.Ctx, "hash-object", "--stdin") + cmd = NewCommand("hash-object", "--stdin") } stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) - err := cmd.Run(&RunOpts{ + err := cmd.Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdin: reader, Stdout: stdout, @@ -85,17 +85,3 @@ func (repo *Repository) hashObject(reader io.Reader, save bool) (string, error) } return strings.TrimSpace(stdout.String()), nil } - -// GetRefType gets the type of the ref based on the string -func (repo *Repository) GetRefType(ref string) ObjectType { - if repo.IsTagExist(ref) { - return ObjectTag - } else if repo.IsBranchExist(ref) { - return ObjectBranch - } else if repo.IsCommitExist(ref) { - return ObjectCommit - } else if _, err := repo.GetBlob(ref); err == nil { - return ObjectBlob - } - return ObjectType("invalid") -} diff --git a/modules/git/repo_ref.go b/modules/git/repo_ref.go index 850ec65502..554f9f73e1 100644 --- a/modules/git/repo_ref.go +++ b/modules/git/repo_ref.go @@ -18,15 +18,16 @@ func (repo *Repository) GetRefs() ([]*Reference, error) { // ListOccurrences lists all refs of the given refType the given commit appears in sorted by creation date DESC // refType should only be a literal "branch" or "tag" and nothing else func (repo *Repository) ListOccurrences(ctx context.Context, refType, commitSHA string) ([]string, error) { - cmd := NewCommand(ctx) - if refType == "branch" { + cmd := NewCommand() + switch refType { + case "branch": cmd.AddArguments("branch") - } else if refType == "tag" { + case "tag": cmd.AddArguments("tag") - } else { + default: return nil, util.NewInvalidArgumentErrorf(`can only use "branch" or "tag" for refType, but got %q`, refType) } - stdout, _, err := cmd.AddArguments("--no-color", "--sort=-creatordate", "--contains").AddDynamicArguments(commitSHA).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := cmd.AddArguments("--no-color", "--sort=-creatordate", "--contains").AddDynamicArguments(commitSHA).RunStdString(ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } diff --git a/modules/git/repo_ref_nogogit.go b/modules/git/repo_ref_nogogit.go index ac53d661b5..8d34713eaf 100644 --- a/modules/git/repo_ref_nogogit.go +++ b/modules/git/repo_ref_nogogit.go @@ -21,7 +21,7 @@ func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { go func() { stderrBuilder := &strings.Builder{} - err := NewCommand(repo.Ctx, "for-each-ref").Run(&RunOpts{ + err := NewCommand("for-each-ref").Run(repo.Ctx, &RunOpts{ Dir: repo.Path, Stdout: stdoutWriter, Stderr: stderrBuilder, diff --git a/modules/git/repo_stats.go b/modules/git/repo_stats.go index 83220104bd..8c6f31c38c 100644 --- a/modules/git/repo_stats.go +++ b/modules/git/repo_stats.go @@ -40,7 +40,9 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) since := fromTime.Format(time.RFC3339) - stdout, _, runErr := NewCommand(repo.Ctx, "rev-list", "--count", "--no-merges", "--branches=*", "--date=iso").AddOptionFormat("--since='%s'", since).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand("rev-list", "--count", "--no-merges", "--branches=*", "--date=iso"). + AddOptionFormat("--since=%s", since). + RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -60,7 +62,8 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) _ = stdoutWriter.Close() }() - gitCmd := NewCommand(repo.Ctx, "log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso").AddOptionFormat("--since='%s'", since) + gitCmd := NewCommand("log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso"). + AddOptionFormat("--since=%s", since) if len(branch) == 0 { gitCmd.AddArguments("--branches=*") } else { @@ -68,7 +71,7 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) } stderr := new(strings.Builder) - err = gitCmd.Run(&RunOpts{ + err = gitCmd.Run(repo.Ctx, &RunOpts{ Env: []string{}, Dir: repo.Path, Stdout: stdoutWriter, diff --git a/modules/git/repo_stats_test.go b/modules/git/repo_stats_test.go index 3d032385ee..85d8807a6e 100644 --- a/modules/git/repo_stats_test.go +++ b/modules/git/repo_stats_test.go @@ -30,7 +30,7 @@ func TestRepository_GetCodeActivityStats(t *testing.T) { assert.EqualValues(t, 10, code.Additions) assert.EqualValues(t, 1, code.Deletions) assert.Len(t, code.Authors, 3) - assert.EqualValues(t, "tris.git@shoddynet.org", code.Authors[1].Email) + assert.Equal(t, "tris.git@shoddynet.org", code.Authors[1].Email) assert.EqualValues(t, 3, code.Authors[1].Commits) assert.EqualValues(t, 5, code.Authors[0].Commits) } diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index 2026a4c9f5..c8d72eee02 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -5,7 +5,6 @@ package git import ( - "context" "fmt" "io" "strings" @@ -17,20 +16,15 @@ import ( // TagPrefix tags prefix path on the repository const TagPrefix = "refs/tags/" -// IsTagExist returns true if given tag exists in the repository. -func IsTagExist(ctx context.Context, repoPath, name string) bool { - return IsReferenceExist(ctx, repoPath, TagPrefix+name) -} - // CreateTag create one tag in the repository func (repo *Repository) CreateTag(name, revision string) error { - _, _, err := NewCommand(repo.Ctx, "tag").AddDashesAndList(name, revision).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("tag").AddDashesAndList(name, revision).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } // CreateAnnotatedTag create one annotated tag in the repository func (repo *Repository) CreateAnnotatedTag(name, message, revision string) error { - _, _, err := NewCommand(repo.Ctx, "tag", "-a", "-m").AddDynamicArguments(message).AddDashesAndList(name, revision).RunStdString(&RunOpts{Dir: repo.Path}) + _, _, err := NewCommand("tag", "-a", "-m").AddDynamicArguments(message).AddDashesAndList(name, revision).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) return err } @@ -40,13 +34,13 @@ func (repo *Repository) GetTagNameBySHA(sha string) (string, error) { return "", fmt.Errorf("SHA is too short: %s", sha) } - stdout, _, err := NewCommand(repo.Ctx, "show-ref", "--tags", "-d").RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("show-ref", "--tags", "-d").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return "", err } - tagRefs := strings.Split(stdout, "\n") - for _, tagRef := range tagRefs { + tagRefs := strings.SplitSeq(stdout, "\n") + for tagRef := range tagRefs { if len(strings.TrimSpace(tagRef)) > 0 { fields := strings.Fields(tagRef) if strings.HasPrefix(fields[0], sha) && strings.HasPrefix(fields[1], TagPrefix) { @@ -63,12 +57,12 @@ func (repo *Repository) GetTagNameBySHA(sha string) (string, error) { // GetTagID returns the object ID for a tag (annotated tags have both an object SHA AND a commit SHA) func (repo *Repository) GetTagID(name string) (string, error) { - stdout, _, err := NewCommand(repo.Ctx, "show-ref", "--tags").AddDashesAndList(name).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand("show-ref", "--tags").AddDashesAndList(name).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return "", err } // Make sure exact match is used: "v1" != "release/v1" - for _, line := range strings.Split(stdout, "\n") { + for line := range strings.SplitSeq(stdout, "\n") { fields := strings.Fields(line) if len(fields) == 2 && fields[1] == "refs/tags/"+name { return fields[0], nil @@ -123,9 +117,9 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) { rc := &RunOpts{Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr} go func() { - err := NewCommand(repo.Ctx, "for-each-ref"). + err := NewCommand("for-each-ref"). AddOptionFormat("--format=%s", forEachRefFmt.Flag()). - AddArguments("--sort", "-*creatordate", "refs/tags").Run(rc) + AddArguments("--sort", "-*creatordate", "refs/tags").Run(repo.Ctx, rc) if err != nil { _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String())) } else { diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index e0a3104249..3d2b4f52bd 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -41,8 +41,11 @@ func (repo *Repository) GetTagType(id ObjectID) (string, error) { return "", err } _, typ, _, err := ReadBatchLine(rd) - if IsErrNotExist(err) { - return "", ErrNotExist{ID: id.String()} + if err != nil { + if IsErrNotExist(err) { + return "", ErrNotExist{ID: id.String()} + } + return "", err } return typ, nil } diff --git a/modules/git/repo_tag_test.go b/modules/git/repo_tag_test.go index 0117cb902d..f1f5ff6664 100644 --- a/modules/git/repo_tag_test.go +++ b/modules/git/repo_tag_test.go @@ -27,12 +27,12 @@ func TestRepository_GetTags(t *testing.T) { } assert.Len(t, tags, 2) assert.Len(t, tags, total) - assert.EqualValues(t, "signed-tag", tags[0].Name) - assert.EqualValues(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", tags[0].ID.String()) - assert.EqualValues(t, "tag", tags[0].Type) - assert.EqualValues(t, "test", tags[1].Name) - assert.EqualValues(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[1].ID.String()) - assert.EqualValues(t, "tag", tags[1].Type) + assert.Equal(t, "signed-tag", tags[0].Name) + assert.Equal(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", tags[0].ID.String()) + assert.Equal(t, "tag", tags[0].Type) + assert.Equal(t, "test", tags[1].Name) + assert.Equal(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[1].ID.String()) + assert.Equal(t, "tag", tags[1].Type) } func TestRepository_GetTag(t *testing.T) { @@ -64,18 +64,13 @@ func TestRepository_GetTag(t *testing.T) { // and try to get the Tag for lightweight tag lTag, err := bareRepo1.GetTag(lTagName) - if err != nil { - assert.NoError(t, err) - return - } - if lTag == nil { - assert.NotNil(t, lTag) - assert.FailNow(t, "nil lTag: %s", lTagName) - } - assert.EqualValues(t, lTagName, lTag.Name) - assert.EqualValues(t, lTagCommitID, lTag.ID.String()) - assert.EqualValues(t, lTagCommitID, lTag.Object.String()) - assert.EqualValues(t, "commit", lTag.Type) + require.NoError(t, err) + require.NotNil(t, lTag, "nil lTag: %s", lTagName) + + assert.Equal(t, lTagName, lTag.Name) + assert.Equal(t, lTagCommitID, lTag.ID.String()) + assert.Equal(t, lTagCommitID, lTag.Object.String()) + assert.Equal(t, "commit", lTag.Type) // ANNOTATED TAGS aTagCommitID := "8006ff9adbf0cb94da7dad9e537e53817f9fa5c0" @@ -97,19 +92,14 @@ func TestRepository_GetTag(t *testing.T) { } aTag, err := bareRepo1.GetTag(aTagName) - if err != nil { - assert.NoError(t, err) - return - } - if aTag == nil { - assert.NotNil(t, aTag) - assert.FailNow(t, "nil aTag: %s", aTagName) - } - assert.EqualValues(t, aTagName, aTag.Name) - assert.EqualValues(t, aTagID, aTag.ID.String()) + require.NoError(t, err) + require.NotNil(t, aTag, "nil aTag: %s", aTagName) + + assert.Equal(t, aTagName, aTag.Name) + assert.Equal(t, aTagID, aTag.ID.String()) assert.NotEqual(t, aTagID, aTag.Object.String()) - assert.EqualValues(t, aTagCommitID, aTag.Object.String()) - assert.EqualValues(t, "tag", aTag.Type) + assert.Equal(t, aTagCommitID, aTag.Object.String()) + assert.Equal(t, "tag", aTag.Type) // RELEASE TAGS @@ -127,14 +117,14 @@ func TestRepository_GetTag(t *testing.T) { assert.NoError(t, err) return } - assert.EqualValues(t, rTagCommitID, rTagID) + assert.Equal(t, rTagCommitID, rTagID) oTagID, err := bareRepo1.GetTagID(lTagName) if err != nil { assert.NoError(t, err) return } - assert.EqualValues(t, lTagCommitID, oTagID) + assert.Equal(t, lTagCommitID, oTagID) } func TestRepository_GetAnnotatedTag(t *testing.T) { @@ -170,9 +160,9 @@ func TestRepository_GetAnnotatedTag(t *testing.T) { return } assert.NotNil(t, tag) - assert.EqualValues(t, aTagName, tag.Name) - assert.EqualValues(t, aTagID, tag.ID.String()) - assert.EqualValues(t, "tag", tag.Type) + assert.Equal(t, aTagName, tag.Name) + assert.Equal(t, aTagID, tag.ID.String()) + assert.Equal(t, "tag", tag.Type) // Annotated tag's Commit ID should fail tag2, err := bareRepo1.GetAnnotatedTag(aTagCommitID) @@ -182,7 +172,6 @@ func TestRepository_GetAnnotatedTag(t *testing.T) { // Annotated tag's name should fail tag3, err := bareRepo1.GetAnnotatedTag(aTagName) - assert.Error(t, err) assert.Errorf(t, err, "Length must be 40: %d", len(aTagName)) assert.Nil(t, tag3) diff --git a/modules/git/repo_test.go b/modules/git/repo_test.go index 9db78153a1..4638bdac1f 100644 --- a/modules/git/repo_test.go +++ b/modules/git/repo_test.go @@ -4,7 +4,6 @@ package git import ( - "context" "path/filepath" "testing" @@ -33,21 +32,21 @@ func TestRepoIsEmpty(t *testing.T) { func TestRepoGetDivergingCommits(t *testing.T) { bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") - do, err := GetDivergingCommits(context.Background(), bareRepo1Path, "master", "branch2") + do, err := GetDivergingCommits(t.Context(), bareRepo1Path, "master", "branch2") assert.NoError(t, err) assert.Equal(t, DivergeObject{ Ahead: 1, Behind: 5, }, do) - do, err = GetDivergingCommits(context.Background(), bareRepo1Path, "master", "master") + do, err = GetDivergingCommits(t.Context(), bareRepo1Path, "master", "master") assert.NoError(t, err) assert.Equal(t, DivergeObject{ Ahead: 0, Behind: 0, }, do) - do, err = GetDivergingCommits(context.Background(), bareRepo1Path, "master", "test") + do, err = GetDivergingCommits(t.Context(), bareRepo1Path, "master", "test") assert.NoError(t, err) assert.Equal(t, DivergeObject{ Ahead: 0, diff --git a/modules/git/repo_tree.go b/modules/git/repo_tree.go index ab48d47d13..309a73d759 100644 --- a/modules/git/repo_tree.go +++ b/modules/git/repo_tree.go @@ -15,7 +15,7 @@ import ( type CommitTreeOpts struct { Parents []string Message string - KeyID string + Key *SigningKey NoGPGSign bool AlwaysSign bool } @@ -33,7 +33,7 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt "GIT_COMMITTER_EMAIL="+committer.Email, "GIT_COMMITTER_DATE="+commitTimeStr, ) - cmd := NewCommand(repo.Ctx, "commit-tree").AddDynamicArguments(tree.ID.String()) + cmd := NewCommand("commit-tree").AddDynamicArguments(tree.ID.String()) for _, parent := range opts.Parents { cmd.AddArguments("-p").AddDynamicArguments(parent) @@ -43,8 +43,13 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt _, _ = messageBytes.WriteString(opts.Message) _, _ = messageBytes.WriteString("\n") - if opts.KeyID != "" || opts.AlwaysSign { - cmd.AddOptionFormat("-S%s", opts.KeyID) + if opts.Key != nil { + if opts.Key.Format != "" { + cmd.AddConfig("gpg.format", opts.Key.Format) + } + cmd.AddOptionFormat("-S%s", opts.Key.KeyID) + } else if opts.AlwaysSign { + cmd.AddOptionFormat("-S") } if opts.NoGPGSign { @@ -53,7 +58,7 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) - err := cmd.Run(&RunOpts{ + err := cmd.Run(repo.Ctx, &RunOpts{ Env: env, Dir: repo.Path, Stdin: messageBytes, diff --git a/modules/git/repo_tree_gogit.go b/modules/git/repo_tree_gogit.go index 651794a5aa..f77cd83612 100644 --- a/modules/git/repo_tree_gogit.go +++ b/modules/git/repo_tree_gogit.go @@ -36,7 +36,7 @@ func (repo *Repository) GetTree(idStr string) (*Tree, error) { } if len(idStr) != objectFormat.FullLength() { - res, _, err := NewCommand(repo.Ctx, "rev-parse", "--verify").AddDynamicArguments(idStr).RunStdString(&RunOpts{Dir: repo.Path}) + res, _, err := NewCommand("rev-parse", "--verify").AddDynamicArguments(idStr).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go index d74769ccb2..1954f85162 100644 --- a/modules/git/repo_tree_nogogit.go +++ b/modules/git/repo_tree_nogogit.go @@ -35,7 +35,11 @@ func (repo *Repository) getTree(id ObjectID) (*Tree, error) { if err != nil { return nil, err } - commit, err := tag.Commit(repo) + + if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { + return nil, err + } + commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) if err != nil { return nil, err } diff --git a/modules/git/signature_test.go b/modules/git/signature_test.go index 92681feea9..b9b5aff61b 100644 --- a/modules/git/signature_test.go +++ b/modules/git/signature_test.go @@ -42,6 +42,6 @@ func TestParseSignatureFromCommitLine(t *testing.T) { } for _, test := range tests { got := parseSignatureFromCommitLine(test.line) - assert.EqualValues(t, test.want, got) + assert.Equal(t, test.want, got) } } diff --git a/modules/git/submodule.go b/modules/git/submodule.go new file mode 100644 index 0000000000..31a32f1a9e --- /dev/null +++ b/modules/git/submodule.go @@ -0,0 +1,66 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "bufio" + "context" + "fmt" + "os" + + "code.gitea.io/gitea/modules/log" +) + +type TemplateSubmoduleCommit struct { + Path string + Commit string +} + +// GetTemplateSubmoduleCommits returns a list of submodules paths and their commits from a repository +// This function is only for generating new repos based on existing template, the template couldn't be too large. +func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []TemplateSubmoduleCommit, _ error) { + stdoutReader, stdoutWriter, err := os.Pipe() + if err != nil { + return nil, err + } + opts := &RunOpts{ + Dir: repoPath, + Stdout: stdoutWriter, + PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { + _ = stdoutWriter.Close() + defer stdoutReader.Close() + + scanner := bufio.NewScanner(stdoutReader) + for scanner.Scan() { + entry, err := parseLsTreeLine(scanner.Bytes()) + if err != nil { + cancel() + return err + } + if entry.EntryMode == EntryModeCommit { + submoduleCommits = append(submoduleCommits, TemplateSubmoduleCommit{Path: entry.Name, Commit: entry.ID.String()}) + } + } + return scanner.Err() + }, + } + err = NewCommand("ls-tree", "-r", "--", "HEAD").Run(ctx, opts) + if err != nil { + return nil, fmt.Errorf("GetTemplateSubmoduleCommits: error running git ls-tree: %v", err) + } + return submoduleCommits, nil +} + +// AddTemplateSubmoduleIndexes Adds the given submodules to the git index. +// It is only for generating new repos based on existing template, requires the .gitmodules file to be already present in the work dir. +func AddTemplateSubmoduleIndexes(ctx context.Context, repoPath string, submodules []TemplateSubmoduleCommit) error { + for _, submodule := range submodules { + cmd := NewCommand("update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path) + if stdout, _, err := cmd.RunStdString(ctx, &RunOpts{Dir: repoPath}); err != nil { + log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Path, repoPath, stdout, err) + return err + } + } + return nil +} diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go new file mode 100644 index 0000000000..7893b95e3a --- /dev/null +++ b/modules/git/submodule_test.go @@ -0,0 +1,47 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetTemplateSubmoduleCommits(t *testing.T) { + testRepoPath := filepath.Join(testReposDir, "repo4_submodules") + submodules, err := GetTemplateSubmoduleCommits(DefaultContext, testRepoPath) + require.NoError(t, err) + + assert.Len(t, submodules, 2) + + assert.Equal(t, "<°)))><", submodules[0].Path) + assert.Equal(t, "d2932de67963f23d43e1c7ecf20173e92ee6c43c", submodules[0].Commit) + + assert.Equal(t, "libtest", submodules[1].Path) + assert.Equal(t, "1234567890123456789012345678901234567890", submodules[1].Commit) +} + +func TestAddTemplateSubmoduleIndexes(t *testing.T) { + ctx := t.Context() + tmpDir := t.TempDir() + var err error + _, _, err = NewCommand("init").RunStdString(ctx, &RunOpts{Dir: tmpDir}) + require.NoError(t, err) + _ = os.Mkdir(filepath.Join(tmpDir, "new-dir"), 0o755) + err = AddTemplateSubmoduleIndexes(ctx, tmpDir, []TemplateSubmoduleCommit{{Path: "new-dir", Commit: "1234567890123456789012345678901234567890"}}) + require.NoError(t, err) + _, _, err = NewCommand("add", "--all").RunStdString(ctx, &RunOpts{Dir: tmpDir}) + require.NoError(t, err) + _, _, err = NewCommand("-c", "user.name=a", "-c", "user.email=b", "commit", "-m=test").RunStdString(ctx, &RunOpts{Dir: tmpDir}) + require.NoError(t, err) + submodules, err := GetTemplateSubmoduleCommits(DefaultContext, tmpDir) + require.NoError(t, err) + assert.Len(t, submodules, 1) + assert.Equal(t, "new-dir", submodules[0].Path) + assert.Equal(t, "1234567890123456789012345678901234567890", submodules[0].Commit) +} diff --git a/modules/git/tag.go b/modules/git/tag.go index f7666aa89b..8bf3658d62 100644 --- a/modules/git/tag.go +++ b/modules/git/tag.go @@ -21,11 +21,6 @@ type Tag struct { Signature *CommitSignature } -// Commit return the commit of the tag reference -func (tag *Tag) Commit(gitRepo *Repository) (*Commit, error) { - return gitRepo.getCommit(tag.Object) -} - func parsePayloadSignature(data []byte, messageStart int) (payload, msg, sign string) { pos := messageStart signStart, signEnd := -1, -1 diff --git a/modules/git/tests/repos/language_stats_repo/config b/modules/git/tests/repos/language_stats_repo/config index 515f483629..a4ef456cbc 100644 --- a/modules/git/tests/repos/language_stats_repo/config +++ b/modules/git/tests/repos/language_stats_repo/config @@ -1,5 +1,5 @@ [core] repositoryformatversion = 0 filemode = true - bare = false + bare = true logallrefupdates = true diff --git a/modules/git/tests/repos/repo3_notes/config b/modules/git/tests/repos/repo3_notes/config index d545cdabdb..5ed22e23d1 100644 --- a/modules/git/tests/repos/repo3_notes/config +++ b/modules/git/tests/repos/repo3_notes/config @@ -1,7 +1,7 @@ [core] repositoryformatversion = 0 filemode = false - bare = false + bare = true logallrefupdates = true symlinks = false ignorecase = true diff --git a/modules/git/tests/repos/repo4_commitsbetween/config b/modules/git/tests/repos/repo4_commitsbetween/config index d545cdabdb..5ed22e23d1 100644 --- a/modules/git/tests/repos/repo4_commitsbetween/config +++ b/modules/git/tests/repos/repo4_commitsbetween/config @@ -1,7 +1,7 @@ [core] repositoryformatversion = 0 filemode = false - bare = false + bare = true logallrefupdates = true symlinks = false ignorecase = true diff --git a/modules/git/tests/repos/repo4_submodules/HEAD b/modules/git/tests/repos/repo4_submodules/HEAD new file mode 100644 index 0000000000..cb089cd89a --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/modules/git/tests/repos/repo4_submodules/config b/modules/git/tests/repos/repo4_submodules/config new file mode 100644 index 0000000000..07d359d07c --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/config @@ -0,0 +1,4 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true diff --git a/modules/git/tests/repos/repo4_submodules/objects/97/c3d30df0e6492348292600920a6482feaebb74 b/modules/git/tests/repos/repo4_submodules/objects/97/c3d30df0e6492348292600920a6482feaebb74 Binary files differnew file mode 100644 index 0000000000..7596090b49 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/objects/97/c3d30df0e6492348292600920a6482feaebb74 diff --git a/modules/git/tests/repos/repo4_submodules/objects/c7/e064ed49b44523cba8a5dfbc37d2ce1bb41d34 b/modules/git/tests/repos/repo4_submodules/objects/c7/e064ed49b44523cba8a5dfbc37d2ce1bb41d34 Binary files differnew file mode 100644 index 0000000000..e3a13c156d --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/objects/c7/e064ed49b44523cba8a5dfbc37d2ce1bb41d34 diff --git a/modules/git/tests/repos/repo4_submodules/objects/e1/e59caba97193d48862d6809912043871f37437 b/modules/git/tests/repos/repo4_submodules/objects/e1/e59caba97193d48862d6809912043871f37437 new file mode 100644 index 0000000000..a8d6e5c17c --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/objects/e1/e59caba97193d48862d6809912043871f37437 @@ -0,0 +1,2 @@ +x[ +Â0EýÎ*æ_é$MÑ5tifBk IÅ•¹7æk~ÞÃ9ܘ—åÜ ü¦ð.jÖÈ ÅOÚ äÉ"zÂ`ß#IirF…µÍ¹ÀØ$%¹Âçò|4)°¯?t¼É=” Ë:K¦ï#[$D¿¯ û¿^˜…¡®Ó’y½HU/f?G
\ No newline at end of file diff --git a/modules/git/tests/repos/repo4_submodules/refs/heads/master b/modules/git/tests/repos/repo4_submodules/refs/heads/master new file mode 100644 index 0000000000..102bc34da8 --- /dev/null +++ b/modules/git/tests/repos/repo4_submodules/refs/heads/master @@ -0,0 +1 @@ +e1e59caba97193d48862d6809912043871f37437 diff --git a/modules/git/tree.go b/modules/git/tree.go index 1da4a9fa5d..38fb45f3b1 100644 --- a/modules/git/tree.go +++ b/modules/git/tree.go @@ -17,7 +17,7 @@ func NewTree(repo *Repository, id ObjectID) *Tree { } } -// SubTree get a sub tree by the sub dir path +// SubTree get a subtree by the sub dir path func (t *Tree) SubTree(rpath string) (*Tree, error) { if len(rpath) == 0 { return t, nil @@ -48,17 +48,28 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) { // LsTree checks if the given filenames are in the tree func (repo *Repository) LsTree(ref string, filenames ...string) ([]string, error) { - cmd := NewCommand(repo.Ctx, "ls-tree", "-z", "--name-only"). + cmd := NewCommand("ls-tree", "-z", "--name-only"). AddDashesAndList(append([]string{ref}, filenames...)...) - res, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) + res, _, err := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { return nil, err } filelist := make([]string, 0, len(filenames)) - for _, line := range bytes.Split(res, []byte{'\000'}) { + for line := range bytes.SplitSeq(res, []byte{'\000'}) { filelist = append(filelist, string(line)) } return filelist, err } + +// GetTreePathLatestCommit returns the latest commit of a tree path +func (repo *Repository) GetTreePathLatestCommit(refName, treePath string) (*Commit, error) { + stdout, _, err := NewCommand("rev-list", "-1"). + AddDynamicArguments(refName).AddDashesAndList(treePath). + RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) + if err != nil { + return nil, err + } + return repo.GetCommit(strings.TrimSpace(stdout)) +} diff --git a/modules/git/tree_blob_gogit.go b/modules/git/tree_blob_gogit.go index 92c25cb92c..f29e8f8b9e 100644 --- a/modules/git/tree_blob_gogit.go +++ b/modules/git/tree_blob_gogit.go @@ -21,6 +21,7 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { return &TreeEntry{ ID: t.ID, // Type: ObjectTree, + ptree: t, gogitTreeEntry: &object.TreeEntry{ Name: "", Mode: filemode.Dir, diff --git a/modules/git/tree_blob_nogogit.go b/modules/git/tree_blob_nogogit.go index 92d3d107a7..b18d0fa05e 100644 --- a/modules/git/tree_blob_nogogit.go +++ b/modules/git/tree_blob_nogogit.go @@ -11,38 +11,35 @@ import ( ) // GetTreeEntryByPath get the tree entries according the sub dir -func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { +func (t *Tree) GetTreeEntryByPath(relpath string) (_ *TreeEntry, err error) { if len(relpath) == 0 { return &TreeEntry{ ptree: t, ID: t.ID, name: "", - fullName: "", entryMode: EntryModeTree, }, nil } - // FIXME: This should probably use git cat-file --batch to be a bit more efficient relpath = path.Clean(relpath) parts := strings.Split(relpath, "/") - var err error + tree := t - for i, name := range parts { - if i == len(parts)-1 { - entries, err := tree.ListEntries() - if err != nil { - return nil, err - } - for _, v := range entries { - if v.Name() == name { - return v, nil - } - } - } else { - tree, err = tree.SubTree(name) - if err != nil { - return nil, err - } + for _, name := range parts[:len(parts)-1] { + tree, err = tree.SubTree(name) + if err != nil { + return nil, err + } + } + + name := parts[len(parts)-1] + entries, err := tree.ListEntries() + if err != nil { + return nil, err + } + for _, v := range entries { + if v.Name() == name { + return v, nil } } return nil, ErrNotExist{"", relpath} diff --git a/modules/git/tree_entry.go b/modules/git/tree_entry.go index 9513121487..5099d8ee79 100644 --- a/modules/git/tree_entry.go +++ b/modules/git/tree_entry.go @@ -5,9 +5,11 @@ package git import ( - "io" + "path" "sort" "strings" + + "code.gitea.io/gitea/modules/util" ) // Type returns the type of the entry (commit, tree, blob) @@ -22,83 +24,57 @@ func (te *TreeEntry) Type() string { } } -// FollowLink returns the entry pointed to by a symlink -func (te *TreeEntry) FollowLink() (*TreeEntry, error) { +type EntryFollowResult struct { + SymlinkContent string + TargetFullPath string + TargetEntry *TreeEntry +} + +func EntryFollowLink(commit *Commit, fullPath string, te *TreeEntry) (*EntryFollowResult, error) { if !te.IsLink() { - return nil, ErrBadLink{te.Name(), "not a symlink"} + return nil, util.ErrorWrap(util.ErrUnprocessableContent, "%q is not a symlink", fullPath) } - // read the link - r, err := te.Blob().DataAsync() - if err != nil { - return nil, err + // git's filename max length is 4096, hopefully a link won't be longer than multiple of that + const maxSymlinkSize = 20 * 4096 + if te.Blob().Size() > maxSymlinkSize { + return nil, util.ErrorWrap(util.ErrUnprocessableContent, "%q content exceeds symlink limit", fullPath) } - closed := false - defer func() { - if !closed { - _ = r.Close() - } - }() - buf := make([]byte, te.Size()) - _, err = io.ReadFull(r, buf) + + link, err := te.Blob().GetBlobContent(maxSymlinkSize) if err != nil { return nil, err } - _ = r.Close() - closed = true - - lnk := string(buf) - t := te.ptree - - // traverse up directories - for ; t != nil && strings.HasPrefix(lnk, "../"); lnk = lnk[3:] { - t = t.ptree + if strings.HasPrefix(link, "/") { + // It's said that absolute path will be stored as is in Git + return &EntryFollowResult{SymlinkContent: link}, util.ErrorWrap(util.ErrUnprocessableContent, "%q is an absolute symlink", fullPath) } - if t == nil { - return nil, ErrBadLink{te.Name(), "points outside of repo"} - } - - target, err := t.GetTreeEntryByPath(lnk) + targetFullPath := path.Join(path.Dir(fullPath), link) + targetEntry, err := commit.GetTreeEntryByPath(targetFullPath) if err != nil { - if IsErrNotExist(err) { - return nil, ErrBadLink{te.Name(), "broken link"} - } - return nil, err + return &EntryFollowResult{SymlinkContent: link}, err } - return target, nil + return &EntryFollowResult{SymlinkContent: link, TargetFullPath: targetFullPath, TargetEntry: targetEntry}, nil } -// FollowLinks returns the entry ultimately pointed to by a symlink -func (te *TreeEntry) FollowLinks() (*TreeEntry, error) { - if !te.IsLink() { - return nil, ErrBadLink{te.Name(), "not a symlink"} - } - entry := te - for i := 0; i < 999; i++ { - if entry.IsLink() { - next, err := entry.FollowLink() - if err != nil { - return nil, err - } - if next.ID == entry.ID { - return nil, ErrBadLink{ - entry.Name(), - "recursive link", - } - } - entry = next - } else { +func EntryFollowLinks(commit *Commit, firstFullPath string, firstTreeEntry *TreeEntry, optLimit ...int) (res *EntryFollowResult, err error) { + limit := util.OptionalArg(optLimit, 10) + treeEntry, fullPath := firstTreeEntry, firstFullPath + for range limit { + res, err = EntryFollowLink(commit, fullPath, treeEntry) + if err != nil { + return res, err + } + treeEntry, fullPath = res.TargetEntry, res.TargetFullPath + if !treeEntry.IsLink() { break } } - if entry.IsLink() { - return nil, ErrBadLink{ - te.Name(), - "too many levels of symbolic links", - } + if treeEntry.IsLink() { + return res, util.ErrorWrap(util.ErrUnprocessableContent, "%q has too many links", firstFullPath) } - return entry, nil + return res, nil } // returns the Tree pointed to by this TreeEntry, or nil if this is not a tree diff --git a/modules/git/tree_entry_common_test.go b/modules/git/tree_entry_common_test.go new file mode 100644 index 0000000000..8b63bbb993 --- /dev/null +++ b/modules/git/tree_entry_common_test.go @@ -0,0 +1,76 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "testing" + + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFollowLink(t *testing.T) { + r, err := openRepositoryWithDefaultContext("tests/repos/repo1_bare") + require.NoError(t, err) + defer r.Close() + + commit, err := r.GetCommit("37991dec2c8e592043f47155ce4808d4580f9123") + require.NoError(t, err) + + // get the symlink + { + lnkFullPath := "foo/bar/link_to_hello" + lnk, err := commit.Tree.GetTreeEntryByPath("foo/bar/link_to_hello") + require.NoError(t, err) + assert.True(t, lnk.IsLink()) + + // should be able to dereference to target + res, err := EntryFollowLink(commit, lnkFullPath, lnk) + require.NoError(t, err) + assert.Equal(t, "hello", res.TargetEntry.Name()) + assert.Equal(t, "foo/nar/hello", res.TargetFullPath) + assert.False(t, res.TargetEntry.IsLink()) + assert.Equal(t, "b14df6442ea5a1b382985a6549b85d435376c351", res.TargetEntry.ID.String()) + } + + { + // should error when called on a normal file + entry, err := commit.Tree.GetTreeEntryByPath("file1.txt") + require.NoError(t, err) + res, err := EntryFollowLink(commit, "file1.txt", entry) + assert.ErrorIs(t, err, util.ErrUnprocessableContent) + assert.Nil(t, res) + } + + { + // should error for broken links + entry, err := commit.Tree.GetTreeEntryByPath("foo/broken_link") + require.NoError(t, err) + assert.True(t, entry.IsLink()) + res, err := EntryFollowLink(commit, "foo/broken_link", entry) + assert.ErrorIs(t, err, util.ErrNotExist) + assert.Equal(t, "nar/broken_link", res.SymlinkContent) + } + + { + // should error for external links + entry, err := commit.Tree.GetTreeEntryByPath("foo/outside_repo") + require.NoError(t, err) + assert.True(t, entry.IsLink()) + res, err := EntryFollowLink(commit, "foo/outside_repo", entry) + assert.ErrorIs(t, err, util.ErrNotExist) + assert.Equal(t, "../../outside_repo", res.SymlinkContent) + } + + { + // testing fix for short link bug + entry, err := commit.Tree.GetTreeEntryByPath("foo/link_short") + require.NoError(t, err) + res, err := EntryFollowLink(commit, "foo/link_short", entry) + assert.ErrorIs(t, err, util.ErrNotExist) + assert.Equal(t, "a", res.SymlinkContent) + } +} diff --git a/modules/git/tree_entry_gogit.go b/modules/git/tree_entry_gogit.go index eb9b012681..e6845f1c77 100644 --- a/modules/git/tree_entry_gogit.go +++ b/modules/git/tree_entry_gogit.go @@ -19,16 +19,12 @@ type TreeEntry struct { gogitTreeEntry *object.TreeEntry ptree *Tree - size int64 - sized bool - fullName string + size int64 + sized bool } // Name returns the name of the entry func (te *TreeEntry) Name() string { - if te.fullName != "" { - return te.fullName - } return te.gogitTreeEntry.Name } @@ -55,7 +51,7 @@ func (te *TreeEntry) Size() int64 { return te.size } -// IsSubModule if the entry is a sub module +// IsSubModule if the entry is a submodule func (te *TreeEntry) IsSubModule() bool { return te.gogitTreeEntry.Mode == filemode.Submodule } diff --git a/modules/git/tree_entry_mode.go b/modules/git/tree_entry_mode.go index a399118cf8..f36c07bc2a 100644 --- a/modules/git/tree_entry_mode.go +++ b/modules/git/tree_entry_mode.go @@ -3,7 +3,10 @@ package git -import "strconv" +import ( + "fmt" + "strconv" +) // EntryMode the type of the object in the git tree type EntryMode int @@ -11,16 +14,15 @@ type EntryMode int // There are only a few file modes in Git. They look like unix file modes, but they can only be // one of these. const ( - // EntryModeBlob - EntryModeBlob EntryMode = 0o100644 - // EntryModeExec - EntryModeExec EntryMode = 0o100755 - // EntryModeSymlink + // EntryModeNoEntry is possible if the file was added or removed in a commit. In the case of + // when adding the base commit doesn't have the file in its tree, a mode of 0o000000 is used. + EntryModeNoEntry EntryMode = 0o000000 + + EntryModeBlob EntryMode = 0o100644 + EntryModeExec EntryMode = 0o100755 EntryModeSymlink EntryMode = 0o120000 - // EntryModeCommit - EntryModeCommit EntryMode = 0o160000 - // EntryModeTree - EntryModeTree EntryMode = 0o040000 + EntryModeCommit EntryMode = 0o160000 + EntryModeTree EntryMode = 0o040000 ) // String converts an EntryMode to a string @@ -28,8 +30,46 @@ func (e EntryMode) String() string { return strconv.FormatInt(int64(e), 8) } -// ToEntryMode converts a string to an EntryMode -func ToEntryMode(value string) EntryMode { - v, _ := strconv.ParseInt(value, 8, 32) - return EntryMode(v) +// IsSubModule if the entry is a submodule +func (e EntryMode) IsSubModule() bool { + return e == EntryModeCommit +} + +// IsDir if the entry is a sub dir +func (e EntryMode) IsDir() bool { + return e == EntryModeTree +} + +// IsLink if the entry is a symlink +func (e EntryMode) IsLink() bool { + return e == EntryModeSymlink +} + +// IsRegular if the entry is a regular file +func (e EntryMode) IsRegular() bool { + return e == EntryModeBlob +} + +// IsExecutable if the entry is an executable file (not necessarily binary) +func (e EntryMode) IsExecutable() bool { + return e == EntryModeExec +} + +func ParseEntryMode(mode string) (EntryMode, error) { + switch mode { + case "000000": + return EntryModeNoEntry, nil + case "100644": + return EntryModeBlob, nil + case "100755": + return EntryModeExec, nil + case "120000": + return EntryModeSymlink, nil + case "160000": + return EntryModeCommit, nil + case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons + return EntryModeTree, nil + default: + return 0, fmt.Errorf("unparsable entry mode: %s", mode) + } } diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go index 1c3bcd197a..8fad96cdf8 100644 --- a/modules/git/tree_entry_nogogit.go +++ b/modules/git/tree_entry_nogogit.go @@ -9,23 +9,17 @@ import "code.gitea.io/gitea/modules/log" // TreeEntry the leaf in the git tree type TreeEntry struct { - ID ObjectID - + ID ObjectID ptree *Tree entryMode EntryMode name string - - size int64 - sized bool - fullName string + size int64 + sized bool } -// Name returns the name of the entry +// Name returns the name of the entry (base name) func (te *TreeEntry) Name() string { - if te.fullName != "" { - return te.fullName - } return te.name } @@ -63,29 +57,29 @@ func (te *TreeEntry) Size() int64 { return te.size } -// IsSubModule if the entry is a sub module +// IsSubModule if the entry is a submodule func (te *TreeEntry) IsSubModule() bool { - return te.entryMode == EntryModeCommit + return te.entryMode.IsSubModule() } // IsDir if the entry is a sub dir func (te *TreeEntry) IsDir() bool { - return te.entryMode == EntryModeTree + return te.entryMode.IsDir() } // IsLink if the entry is a symlink func (te *TreeEntry) IsLink() bool { - return te.entryMode == EntryModeSymlink + return te.entryMode.IsLink() } // IsRegular if the entry is a regular file func (te *TreeEntry) IsRegular() bool { - return te.entryMode == EntryModeBlob + return te.entryMode.IsRegular() } // IsExecutable if the entry is an executable file (not necessarily binary) func (te *TreeEntry) IsExecutable() bool { - return te.entryMode == EntryModeExec + return te.entryMode.IsExecutable() } // Blob returns the blob object the entry diff --git a/modules/git/tree_entry_test.go b/modules/git/tree_entry_test.go index 30eee13669..9ca82675e0 100644 --- a/modules/git/tree_entry_test.go +++ b/modules/git/tree_entry_test.go @@ -53,50 +53,3 @@ func TestEntriesCustomSort(t *testing.T) { assert.Equal(t, "bcd", entries[6].Name()) assert.Equal(t, "abc", entries[7].Name()) } - -func TestFollowLink(t *testing.T) { - r, err := openRepositoryWithDefaultContext("tests/repos/repo1_bare") - assert.NoError(t, err) - defer r.Close() - - commit, err := r.GetCommit("37991dec2c8e592043f47155ce4808d4580f9123") - assert.NoError(t, err) - - // get the symlink - lnk, err := commit.Tree.GetTreeEntryByPath("foo/bar/link_to_hello") - assert.NoError(t, err) - assert.True(t, lnk.IsLink()) - - // should be able to dereference to target - target, err := lnk.FollowLink() - assert.NoError(t, err) - assert.Equal(t, "hello", target.Name()) - assert.False(t, target.IsLink()) - assert.Equal(t, "b14df6442ea5a1b382985a6549b85d435376c351", target.ID.String()) - - // should error when called on normal file - target, err = commit.Tree.GetTreeEntryByPath("file1.txt") - assert.NoError(t, err) - _, err = target.FollowLink() - assert.EqualError(t, err, "file1.txt: not a symlink") - - // should error for broken links - target, err = commit.Tree.GetTreeEntryByPath("foo/broken_link") - assert.NoError(t, err) - assert.True(t, target.IsLink()) - _, err = target.FollowLink() - assert.EqualError(t, err, "broken_link: broken link") - - // should error for external links - target, err = commit.Tree.GetTreeEntryByPath("foo/outside_repo") - assert.NoError(t, err) - assert.True(t, target.IsLink()) - _, err = target.FollowLink() - assert.EqualError(t, err, "outside_repo: points outside of repo") - - // testing fix for short link bug - target, err = commit.Tree.GetTreeEntryByPath("foo/link_short") - assert.NoError(t, err) - _, err = target.FollowLink() - assert.EqualError(t, err, "link_short: broken link") -} diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go index 421b0ecb0f..272b018ffd 100644 --- a/modules/git/tree_gogit.go +++ b/modules/git/tree_gogit.go @@ -69,7 +69,7 @@ func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { seen := map[plumbing.Hash]bool{} walker := object.NewTreeWalker(t.gogitTree, true, seen) for { - fullName, entry, err := walker.Next() + _, entry, err := walker.Next() if err == io.EOF { break } @@ -84,7 +84,6 @@ func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { ID: ParseGogitHash(entry.Hash), gogitTreeEntry: &entry, ptree: t, - fullName: fullName, } entries = append(entries, convertedEntry) } diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 993b98edc2..f88788418e 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -70,7 +70,7 @@ func (t *Tree) ListEntries() (Entries, error) { } } - stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-l").AddDynamicArguments(t.ID.String()).RunStdBytes(&RunOpts{Dir: t.repo.Path}) + stdout, _, runErr := NewCommand("ls-tree", "-l").AddDynamicArguments(t.ID.String()).RunStdBytes(t.repo.Ctx, &RunOpts{Dir: t.repo.Path}) if runErr != nil { if strings.Contains(runErr.Error(), "fatal: Not a valid object name") || strings.Contains(runErr.Error(), "fatal: not a tree object") { return nil, ErrNotExist{ @@ -96,10 +96,10 @@ func (t *Tree) listEntriesRecursive(extraArgs TrustedCmdArgs) (Entries, error) { return t.entriesRecursive, nil } - stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r"). + stdout, _, runErr := NewCommand("ls-tree", "-t", "-r"). AddArguments(extraArgs...). AddDynamicArguments(t.ID.String()). - RunStdBytes(&RunOpts{Dir: t.repo.Path}) + RunStdBytes(t.repo.Ctx, &RunOpts{Dir: t.repo.Path}) if runErr != nil { return nil, runErr } diff --git a/modules/git/tree_test.go b/modules/git/tree_test.go index 6d2b5c84d5..cae11c4b1b 100644 --- a/modules/git/tree_test.go +++ b/modules/git/tree_test.go @@ -19,9 +19,24 @@ func TestSubTree_Issue29101(t *testing.T) { assert.NoError(t, err) // old code could produce a different error if called multiple times - for i := 0; i < 10; i++ { + for range 10 { _, err = commit.SubTree("file1.txt") assert.Error(t, err) assert.True(t, IsErrNotExist(err)) } } + +func Test_GetTreePathLatestCommit(t *testing.T) { + repo, err := openRepositoryWithDefaultContext(filepath.Join(testReposDir, "repo6_blame")) + assert.NoError(t, err) + defer repo.Close() + + commitID, err := repo.GetBranchCommitID("master") + assert.NoError(t, err) + assert.Equal(t, "544d8f7a3b15927cddf2299b4b562d6ebd71b6a7", commitID) + + commit, err := repo.GetTreePathLatestCommit("master", "blame.txt") + assert.NoError(t, err) + assert.NotNil(t, commit) + assert.Equal(t, "45fb6cbc12f970b04eacd5cd4165edd11c8d7376", commit.ID.String()) +} diff --git a/modules/git/url/url.go b/modules/git/url/url.go index 637685183e..aa6fa31c5e 100644 --- a/modules/git/url/url.go +++ b/modules/git/url/url.go @@ -4,9 +4,15 @@ package url import ( + "context" "fmt" + "net" stdurl "net/url" "strings" + + "code.gitea.io/gitea/modules/httplib" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) // ErrWrongURLFormat represents an error with wrong url format @@ -21,7 +27,7 @@ func (err ErrWrongURLFormat) Error() string { // GitURL represents a git URL type GitURL struct { *stdurl.URL - extraMark int // 0 no extra 1 scp 2 file path with no prefix + extraMark int // 0: standard URL with scheme, 1: scp short syntax (no scheme), 2: file path with no prefix } // String returns the URL's string @@ -38,8 +44,11 @@ func (u *GitURL) String() string { } } -// Parse parse all kinds of git URL -func Parse(remote string) (*GitURL, error) { +// ParseGitURL parse all kinds of git URL: +// * Full URL: http://git@host/path, http://git@host:port/path +// * SCP short syntax: git@host:/path +// * File path: /dir/repo/path +func ParseGitURL(remote string) (*GitURL, error) { if strings.Contains(remote, "://") { u, err := stdurl.Parse(remote) if err != nil { @@ -87,3 +96,88 @@ func Parse(remote string) (*GitURL, error) { extraMark: 2, }, nil } + +type RepositoryURL struct { + GitURL *GitURL + + // if the URL belongs to current Gitea instance, then the below fields have values + OwnerName string + RepoName string + RemainingPath string +} + +// ParseRepositoryURL tries to parse a Git URL and extract the owner/repository name if it belongs to current Gitea instance. +func ParseRepositoryURL(ctx context.Context, repoURL string) (*RepositoryURL, error) { + // possible urls for git: + // https://my.domain/sub-path/<owner>/<repo>[.git] + // git+ssh://user@my.domain/<owner>/<repo>[.git] + // ssh://user@my.domain/<owner>/<repo>[.git] + // user@my.domain:<owner>/<repo>[.git] + parsed, err := ParseGitURL(repoURL) + if err != nil { + return nil, err + } + + ret := &RepositoryURL{} + ret.GitURL = parsed + + fillPathParts := func(s string) { + s = strings.TrimPrefix(s, "/") + fields := strings.SplitN(s, "/", 3) + if len(fields) >= 2 { + ret.OwnerName = fields[0] + ret.RepoName = strings.TrimSuffix(fields[1], ".git") + if len(fields) == 3 { + ret.RemainingPath = "/" + fields[2] + } + } + } + + switch parsed.URL.Scheme { + case "http", "https": + if !httplib.IsCurrentGiteaSiteURL(ctx, repoURL) { + return ret, nil + } + fillPathParts(strings.TrimPrefix(parsed.URL.Path, setting.AppSubURL)) + case "ssh", "git+ssh": + domainSSH := setting.SSH.Domain + domainCur := httplib.GuessCurrentHostDomain(ctx) + urlDomain, _, _ := net.SplitHostPort(parsed.URL.Host) + urlDomain = util.IfZero(urlDomain, parsed.URL.Host) + if urlDomain == "" { + return ret, nil + } + // check whether URL domain is the App domain + domainMatches := domainSSH == urlDomain + // check whether URL domain is current domain from context + domainMatches = domainMatches || (domainCur != "" && domainCur == urlDomain) + if domainMatches { + fillPathParts(parsed.URL.Path) + } + } + return ret, nil +} + +// MakeRepositoryWebLink generates a web link (http/https) for a git repository (by guessing sometimes) +func MakeRepositoryWebLink(repoURL *RepositoryURL) string { + if repoURL.OwnerName != "" { + return setting.AppSubURL + "/" + repoURL.OwnerName + "/" + repoURL.RepoName + } + + // now, let's guess, for example: + // * git@github.com:owner/submodule.git + // * https://github.com/example/submodule1.git + switch repoURL.GitURL.Scheme { + case "http", "https": + return strings.TrimSuffix(repoURL.GitURL.String(), ".git") + case "ssh", "git+ssh": + hostname, _, _ := net.SplitHostPort(repoURL.GitURL.Host) + hostname = util.IfZero(hostname, repoURL.GitURL.Host) + urlPath := strings.TrimSuffix(repoURL.GitURL.Path, ".git") + urlPath = strings.TrimPrefix(urlPath, "/") + urlFull := fmt.Sprintf("https://%s/%s", hostname, urlPath) + urlFull = strings.TrimSuffix(urlFull, "/") + return urlFull + } + return "" +} diff --git a/modules/git/url/url_test.go b/modules/git/url/url_test.go index da820ed889..6655c20be3 100644 --- a/modules/git/url/url_test.go +++ b/modules/git/url/url_test.go @@ -4,9 +4,15 @@ package url import ( + "context" + "net/http" "net/url" "testing" + "code.gitea.io/gitea/modules/httplib" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" ) @@ -157,10 +163,105 @@ func TestParseGitURLs(t *testing.T) { for _, kase := range kases { t.Run(kase.kase, func(t *testing.T) { - u, err := Parse(kase.kase) + u, err := ParseGitURL(kase.kase) assert.NoError(t, err) - assert.EqualValues(t, kase.expected.extraMark, u.extraMark) - assert.EqualValues(t, *kase.expected, *u) + assert.Equal(t, kase.expected.extraMark, u.extraMark) + assert.Equal(t, *kase.expected, *u) + }) + } +} + +func TestParseRepositoryURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "https://localhost:3000")() + defer test.MockVariableValue(&setting.SSH.Domain, "try.gitea.io")() + + ctxURL, _ := url.Parse("https://gitea") + ctxReq := &http.Request{URL: ctxURL, Header: http.Header{}} + ctxReq.Host = ctxURL.Host + ctxReq.Header.Add("X-Forwarded-Proto", ctxURL.Scheme) + ctx := context.WithValue(t.Context(), httplib.RequestContextKey, ctxReq) + cases := []struct { + input string + ownerName, repoName, remaining string + }{ + {input: "/user/repo"}, + + {input: "https://localhost:3000/user/repo", ownerName: "user", repoName: "repo"}, + {input: "https://external:3000/user/repo"}, + + {input: "https://localhost:3000/user/repo.git/other", ownerName: "user", repoName: "repo", remaining: "/other"}, + + {input: "https://gitea/user/repo", ownerName: "user", repoName: "repo"}, + {input: "https://gitea:3333/user/repo"}, + + {input: "ssh://try.gitea.io:2222/user/repo", ownerName: "user", repoName: "repo"}, + {input: "ssh://external:2222/user/repo"}, + + {input: "git+ssh://user@try.gitea.io/user/repo.git", ownerName: "user", repoName: "repo"}, + {input: "git+ssh://user@external/user/repo.git"}, + + {input: "root@try.gitea.io:user/repo.git", ownerName: "user", repoName: "repo"}, + {input: "root@gitea:user/repo.git", ownerName: "user", repoName: "repo"}, + {input: "root@external:user/repo.git"}, + } + + for _, c := range cases { + t.Run(c.input, func(t *testing.T) { + ret, _ := ParseRepositoryURL(ctx, c.input) + assert.Equal(t, c.ownerName, ret.OwnerName) + assert.Equal(t, c.repoName, ret.RepoName) + assert.Equal(t, c.remaining, ret.RemainingPath) }) } + + t.Run("WithSubpath", func(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "https://localhost:3000/subpath")() + defer test.MockVariableValue(&setting.AppSubURL, "/subpath")() + cases = []struct { + input string + ownerName, repoName, remaining string + }{ + {input: "https://localhost:3000/user/repo"}, + {input: "https://localhost:3000/subpath/user/repo.git/other", ownerName: "user", repoName: "repo", remaining: "/other"}, + + {input: "ssh://try.gitea.io:2222/user/repo", ownerName: "user", repoName: "repo"}, + {input: "ssh://external:2222/user/repo"}, + + {input: "git+ssh://user@try.gitea.io/user/repo.git", ownerName: "user", repoName: "repo"}, + {input: "git+ssh://user@external/user/repo.git"}, + + {input: "root@try.gitea.io:user/repo.git", ownerName: "user", repoName: "repo"}, + {input: "root@external:user/repo.git"}, + } + + for _, c := range cases { + t.Run(c.input, func(t *testing.T) { + ret, _ := ParseRepositoryURL(ctx, c.input) + assert.Equal(t, c.ownerName, ret.OwnerName) + assert.Equal(t, c.repoName, ret.RepoName) + assert.Equal(t, c.remaining, ret.RemainingPath) + }) + } + }) +} + +func TestMakeRepositoryBaseLink(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "https://localhost:3000/subpath")() + defer test.MockVariableValue(&setting.AppSubURL, "/subpath")() + + u, err := ParseRepositoryURL(t.Context(), "https://localhost:3000/subpath/user/repo.git") + assert.NoError(t, err) + assert.Equal(t, "/subpath/user/repo", MakeRepositoryWebLink(u)) + + u, err = ParseRepositoryURL(t.Context(), "https://github.com/owner/repo.git") + assert.NoError(t, err) + assert.Equal(t, "https://github.com/owner/repo", MakeRepositoryWebLink(u)) + + u, err = ParseRepositoryURL(t.Context(), "git@github.com:owner/repo.git") + assert.NoError(t, err) + assert.Equal(t, "https://github.com/owner/repo", MakeRepositoryWebLink(u)) + + u, err = ParseRepositoryURL(t.Context(), "git+ssh://other:123/owner/repo.git") + assert.NoError(t, err) + assert.Equal(t, "https://other/owner/repo", MakeRepositoryWebLink(u)) } diff --git a/modules/git/utils.go b/modules/git/utils.go index 56cba9087a..e2bdf7866b 100644 --- a/modules/git/utils.go +++ b/modules/git/utils.go @@ -8,10 +8,11 @@ import ( "encoding/hex" "fmt" "io" - "os" "strconv" "strings" "sync" + + "code.gitea.io/gitea/modules/util" ) // ObjectCache provides thread-safe cache operations. @@ -41,33 +42,6 @@ func (oc *ObjectCache[T]) Get(id string) (T, bool) { return obj, has } -// isDir returns true if given path is a directory, -// or returns false when it's a file or does not exist. -func isDir(dir string) bool { - f, e := os.Stat(dir) - if e != nil { - return false - } - return f.IsDir() -} - -// isFile returns true if given path is a file, -// or returns false when it's a directory or does not exist. -func isFile(filePath string) bool { - f, e := os.Stat(filePath) - if e != nil { - return false - } - return !f.IsDir() -} - -// isExist checks whether a file or directory exists. -// It returns false when the file or directory does not exist. -func isExist(path string) bool { - _, err := os.Stat(path) - return err == nil || os.IsExist(err) -} - // ConcatenateError concatenats an error with stderr string func ConcatenateError(err error, stderr string) error { if len(stderr) == 0 { @@ -134,3 +108,16 @@ func HashFilePathForWebUI(s string) string { _, _ = h.Write([]byte(s)) return hex.EncodeToString(h.Sum(nil)) } + +func SplitCommitTitleBody(commitMessage string, titleRuneLimit int) (title, body string) { + title, body, _ = strings.Cut(commitMessage, "\n") + title, title2 := util.EllipsisTruncateRunes(title, titleRuneLimit) + if title2 != "" { + if body == "" { + body = title2 + } else { + body = title2 + "\n" + body + } + } + return title, body +} diff --git a/modules/git/utils_test.go b/modules/git/utils_test.go index 1291cee637..f09a047136 100644 --- a/modules/git/utils_test.go +++ b/modules/git/utils_test.go @@ -15,3 +15,17 @@ func TestHashFilePathForWebUI(t *testing.T) { HashFilePathForWebUI("foobar"), ) } + +func TestSplitCommitTitleBody(t *testing.T) { + title, body := SplitCommitTitleBody("å•Šbcdefg", 4) + assert.Equal(t, "啊…", title) + assert.Equal(t, "…bcdefg", body) + + title, body = SplitCommitTitleBody("abcdefg\n1234567", 4) + assert.Equal(t, "a…", title) + assert.Equal(t, "…bcdefg\n1234567", body) + + title, body = SplitCommitTitleBody("abcdefg\n1234567", 100) + assert.Equal(t, "abcdefg", title) + assert.Equal(t, "1234567", body) +} diff --git a/modules/gitgraph/graph.go b/modules/gitgraph/graph.go deleted file mode 100644 index 7e12be030f..0000000000 --- a/modules/gitgraph/graph.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package gitgraph - -import ( - "bufio" - "bytes" - "context" - "os" - "strings" - - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/setting" -) - -// GetCommitGraph return a list of commit (GraphItems) from all branches -func GetCommitGraph(r *git.Repository, page, maxAllowedColors int, hidePRRefs bool, branches, files []string) (*Graph, error) { - format := "DATA:%D|%H|%ad|%h|%s" - - if page == 0 { - page = 1 - } - - graphCmd := git.NewCommand(r.Ctx, "log", "--graph", "--date-order", "--decorate=full") - - if hidePRRefs { - graphCmd.AddArguments("--exclude=" + git.PullPrefix + "*") - } - - if len(branches) == 0 { - graphCmd.AddArguments("--all") - } - - graphCmd.AddArguments("-C", "-M", "--date=iso-strict"). - AddOptionFormat("-n %d", setting.UI.GraphMaxCommitNum*page). - AddOptionFormat("--pretty=format:%s", format) - - if len(branches) > 0 { - graphCmd.AddDynamicArguments(branches...) - } - if len(files) > 0 { - graphCmd.AddDashesAndList(files...) - } - graph := NewGraph() - - stderr := new(strings.Builder) - stdoutReader, stdoutWriter, err := os.Pipe() - if err != nil { - return nil, err - } - commitsToSkip := setting.UI.GraphMaxCommitNum * (page - 1) - - scanner := bufio.NewScanner(stdoutReader) - - if err := graphCmd.Run(&git.RunOpts{ - Dir: r.Path, - Stdout: stdoutWriter, - Stderr: stderr, - PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { - _ = stdoutWriter.Close() - defer stdoutReader.Close() - parser := &Parser{} - parser.firstInUse = -1 - parser.maxAllowedColors = maxAllowedColors - if maxAllowedColors > 0 { - parser.availableColors = make([]int, maxAllowedColors) - for i := range parser.availableColors { - parser.availableColors[i] = i + 1 - } - } else { - parser.availableColors = []int{1, 2} - } - for commitsToSkip > 0 && scanner.Scan() { - line := scanner.Bytes() - dataIdx := bytes.Index(line, []byte("DATA:")) - if dataIdx < 0 { - dataIdx = len(line) - } - starIdx := bytes.IndexByte(line, '*') - if starIdx >= 0 && starIdx < dataIdx { - commitsToSkip-- - } - parser.ParseGlyphs(line[:dataIdx]) - } - - row := 0 - - // Skip initial non-commit lines - for scanner.Scan() { - line := scanner.Bytes() - if bytes.IndexByte(line, '*') >= 0 { - if err := parser.AddLineToGraph(graph, row, line); err != nil { - cancel() - return err - } - break - } - parser.ParseGlyphs(line) - } - - for scanner.Scan() { - row++ - line := scanner.Bytes() - if err := parser.AddLineToGraph(graph, row, line); err != nil { - cancel() - return err - } - } - return scanner.Err() - }, - }); err != nil { - return graph, err - } - return graph, nil -} diff --git a/modules/gitgraph/graph_models.go b/modules/gitgraph/graph_models.go deleted file mode 100644 index 191b0b3afc..0000000000 --- a/modules/gitgraph/graph_models.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package gitgraph - -import ( - "bytes" - "context" - "fmt" - "strings" - "time" - - asymkey_model "code.gitea.io/gitea/models/asymkey" - "code.gitea.io/gitea/models/db" - git_model "code.gitea.io/gitea/models/git" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" -) - -// NewGraph creates a basic graph -func NewGraph() *Graph { - graph := &Graph{} - graph.relationCommit = &Commit{ - Row: -1, - Column: -1, - } - graph.Flows = map[int64]*Flow{} - return graph -} - -// Graph represents a collection of flows -type Graph struct { - Flows map[int64]*Flow - Commits []*Commit - MinRow int - MinColumn int - MaxRow int - MaxColumn int - relationCommit *Commit -} - -// Width returns the width of the graph -func (graph *Graph) Width() int { - return graph.MaxColumn - graph.MinColumn + 1 -} - -// Height returns the height of the graph -func (graph *Graph) Height() int { - return graph.MaxRow - graph.MinRow + 1 -} - -// AddGlyph adds glyph to flows -func (graph *Graph) AddGlyph(row, column int, flowID int64, color int, glyph byte) { - flow, ok := graph.Flows[flowID] - if !ok { - flow = NewFlow(flowID, color, row, column) - graph.Flows[flowID] = flow - } - flow.AddGlyph(row, column, glyph) - - if row < graph.MinRow { - graph.MinRow = row - } - if row > graph.MaxRow { - graph.MaxRow = row - } - if column < graph.MinColumn { - graph.MinColumn = column - } - if column > graph.MaxColumn { - graph.MaxColumn = column - } -} - -// AddCommit adds a commit at row, column on flowID with the provided data -func (graph *Graph) AddCommit(row, column int, flowID int64, data []byte) error { - commit, err := NewCommit(row, column, data) - if err != nil { - return err - } - commit.Flow = flowID - graph.Commits = append(graph.Commits, commit) - - graph.Flows[flowID].Commits = append(graph.Flows[flowID].Commits, commit) - return nil -} - -// LoadAndProcessCommits will load the git.Commits for each commit in the graph, -// the associate the commit with the user author, and check the commit verification -// before finally retrieving the latest status -func (graph *Graph) LoadAndProcessCommits(ctx context.Context, repository *repo_model.Repository, gitRepo *git.Repository) error { - var err error - var ok bool - - emails := map[string]*user_model.User{} - keyMap := map[string]bool{} - - for _, c := range graph.Commits { - if len(c.Rev) == 0 { - continue - } - c.Commit, err = gitRepo.GetCommit(c.Rev) - if err != nil { - return fmt.Errorf("GetCommit: %s Error: %w", c.Rev, err) - } - - if c.Commit.Author != nil { - email := c.Commit.Author.Email - if c.User, ok = emails[email]; !ok { - c.User, _ = user_model.GetUserByEmail(ctx, email) - emails[email] = c.User - } - } - - c.Verification = asymkey_model.ParseCommitWithSignature(ctx, c.Commit) - - _ = asymkey_model.CalculateTrustStatus(c.Verification, repository.GetTrustModel(), func(user *user_model.User) (bool, error) { - return repo_model.IsOwnerMemberCollaborator(ctx, repository, user.ID) - }, &keyMap) - - statuses, _, err := git_model.GetLatestCommitStatus(ctx, repository.ID, c.Commit.ID.String(), db.ListOptions{}) - if err != nil { - log.Error("GetLatestCommitStatus: %v", err) - } else { - c.Status = git_model.CalcCommitStatus(statuses) - } - } - return nil -} - -// NewFlow creates a new flow -func NewFlow(flowID int64, color, row, column int) *Flow { - return &Flow{ - ID: flowID, - ColorNumber: color, - MinRow: row, - MinColumn: column, - MaxRow: row, - MaxColumn: column, - } -} - -// Flow represents a series of glyphs -type Flow struct { - ID int64 - ColorNumber int - Glyphs []Glyph - Commits []*Commit - MinRow int - MinColumn int - MaxRow int - MaxColumn int -} - -// Color16 wraps the color numbers around mod 16 -func (flow *Flow) Color16() int { - return flow.ColorNumber % 16 -} - -// AddGlyph adds glyph at row and column -func (flow *Flow) AddGlyph(row, column int, glyph byte) { - if row < flow.MinRow { - flow.MinRow = row - } - if row > flow.MaxRow { - flow.MaxRow = row - } - if column < flow.MinColumn { - flow.MinColumn = column - } - if column > flow.MaxColumn { - flow.MaxColumn = column - } - - flow.Glyphs = append(flow.Glyphs, Glyph{ - row, - column, - glyph, - }) -} - -// Glyph represents a co-ordinate and glyph -type Glyph struct { - Row int - Column int - Glyph byte -} - -// RelationCommit represents an empty relation commit -var RelationCommit = &Commit{ - Row: -1, -} - -func parseGitTime(timeStr string) time.Time { - t, err := time.Parse(time.RFC3339, timeStr) - if err != nil { - return time.Unix(0, 0) - } - return t -} - -// NewCommit creates a new commit from a provided line -func NewCommit(row, column int, line []byte) (*Commit, error) { - data := bytes.SplitN(line, []byte("|"), 5) - if len(data) < 5 { - return nil, fmt.Errorf("malformed data section on line %d with commit: %s", row, string(line)) - } - return &Commit{ - Row: row, - Column: column, - // 0 matches git log --pretty=format:%d => ref names, like the --decorate option of git-log(1) - Refs: newRefsFromRefNames(data[0]), - // 1 matches git log --pretty=format:%H => commit hash - Rev: string(data[1]), - // 2 matches git log --pretty=format:%ad => author date (format respects --date= option) - Date: parseGitTime(string(data[2])), - // 3 matches git log --pretty=format:%h => abbreviated commit hash - ShortRev: string(data[3]), - // 4 matches git log --pretty=format:%s => subject - Subject: string(data[4]), - }, nil -} - -func newRefsFromRefNames(refNames []byte) []git.Reference { - refBytes := bytes.Split(refNames, []byte{',', ' '}) - refs := make([]git.Reference, 0, len(refBytes)) - for _, refNameBytes := range refBytes { - if len(refNameBytes) == 0 { - continue - } - refName := string(refNameBytes) - if strings.HasPrefix(refName, "tag: ") { - refName = strings.TrimPrefix(refName, "tag: ") - } else { - refName = strings.TrimPrefix(refName, "HEAD -> ") - } - refs = append(refs, git.Reference{ - Name: refName, - }) - } - return refs -} - -// Commit represents a commit at co-ordinate X, Y with the data -type Commit struct { - Commit *git.Commit - User *user_model.User - Verification *asymkey_model.CommitVerification - Status *git_model.CommitStatus - Flow int64 - Row int - Column int - Refs []git.Reference - Rev string - Date time.Time - ShortRev string - Subject string -} - -// OnlyRelation returns whether this a relation only commit -func (c *Commit) OnlyRelation() bool { - return c.Row == -1 -} diff --git a/modules/gitgraph/graph_test.go b/modules/gitgraph/graph_test.go deleted file mode 100644 index 18d427acd9..0000000000 --- a/modules/gitgraph/graph_test.go +++ /dev/null @@ -1,714 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package gitgraph - -import ( - "bytes" - "fmt" - "strings" - "testing" - - "code.gitea.io/gitea/modules/git" -) - -func BenchmarkGetCommitGraph(b *testing.B) { - currentRepo, err := git.OpenRepository(git.DefaultContext, ".") - if err != nil || currentRepo == nil { - b.Error("Could not open repository") - } - defer currentRepo.Close() - - for i := 0; i < b.N; i++ { - graph, err := GetCommitGraph(currentRepo, 1, 0, false, nil, nil) - if err != nil { - b.Error("Could get commit graph") - } - - if len(graph.Commits) < 100 { - b.Error("Should get 100 log lines.") - } - } -} - -func BenchmarkParseCommitString(b *testing.B) { - testString := "* DATA:|4e61bacab44e9b4730e44a6615d04098dd3a8eaf|2016-12-20 21:10:41 +0100|4e61bac|Add route for graph" - - parser := &Parser{} - parser.Reset() - for i := 0; i < b.N; i++ { - parser.Reset() - graph := NewGraph() - if err := parser.AddLineToGraph(graph, 0, []byte(testString)); err != nil { - b.Error("could not parse teststring") - } - if graph.Flows[1].Commits[0].Rev != "4e61bacab44e9b4730e44a6615d04098dd3a8eaf" { - b.Error("Did not get expected data") - } - } -} - -func BenchmarkParseGlyphs(b *testing.B) { - parser := &Parser{} - parser.Reset() - tgBytes := []byte(testglyphs) - var tg []byte - for i := 0; i < b.N; i++ { - parser.Reset() - tg = tgBytes - idx := bytes.Index(tg, []byte("\n")) - for idx > 0 { - parser.ParseGlyphs(tg[:idx]) - tg = tg[idx+1:] - idx = bytes.Index(tg, []byte("\n")) - } - } -} - -func TestReleaseUnusedColors(t *testing.T) { - testcases := []struct { - availableColors []int - oldColors []int - firstInUse int // these values have to be either be correct or suggest less is - firstAvailable int // available than possibly is - i.e. you cannot say 10 is available when it - }{ - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - oldColors: []int{1, 1, 1, 1, 1}, - firstAvailable: -1, - firstInUse: 1, - }, - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - oldColors: []int{1, 2, 3, 4}, - firstAvailable: 6, - firstInUse: 0, - }, - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - oldColors: []int{6, 0, 3, 5, 3, 4, 0, 0}, - firstAvailable: 6, - firstInUse: 0, - }, - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7}, - oldColors: []int{6, 1, 3, 5, 3, 4, 2, 7}, - firstAvailable: -1, - firstInUse: 0, - }, - { - availableColors: []int{1, 2, 3, 4, 5, 6, 7}, - oldColors: []int{6, 0, 3, 5, 3, 4, 2, 7}, - firstAvailable: -1, - firstInUse: 0, - }, - } - for _, testcase := range testcases { - parser := &Parser{} - parser.Reset() - parser.availableColors = append([]int{}, testcase.availableColors...) - parser.oldColors = append(parser.oldColors, testcase.oldColors...) - parser.firstAvailable = testcase.firstAvailable - parser.firstInUse = testcase.firstInUse - parser.releaseUnusedColors() - - if parser.firstAvailable == -1 { - // All in use - for _, color := range parser.availableColors { - found := false - for _, oldColor := range parser.oldColors { - if oldColor == color { - found = true - break - } - } - if !found { - t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", - testcase.availableColors, - testcase.oldColors, - testcase.firstAvailable, - testcase.firstInUse, - parser.availableColors, - parser.oldColors, - parser.firstAvailable, - parser.firstInUse, - color) - } - } - } else if parser.firstInUse != -1 { - // Some in use - for i := parser.firstInUse; i != parser.firstAvailable; i = (i + 1) % len(parser.availableColors) { - color := parser.availableColors[i] - found := false - for _, oldColor := range parser.oldColors { - if oldColor == color { - found = true - break - } - } - if !found { - t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", - testcase.availableColors, - testcase.oldColors, - testcase.firstAvailable, - testcase.firstInUse, - parser.availableColors, - parser.oldColors, - parser.firstAvailable, - parser.firstInUse, - color) - } - } - for i := parser.firstAvailable; i != parser.firstInUse; i = (i + 1) % len(parser.availableColors) { - color := parser.availableColors[i] - found := false - for _, oldColor := range parser.oldColors { - if oldColor == color { - found = true - break - } - } - if found { - t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should not be available but is", - testcase.availableColors, - testcase.oldColors, - testcase.firstAvailable, - testcase.firstInUse, - parser.availableColors, - parser.oldColors, - parser.firstAvailable, - parser.firstInUse, - color) - } - } - } else { - // None in use - for _, color := range parser.oldColors { - if color != 0 { - t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should not be available but is", - testcase.availableColors, - testcase.oldColors, - testcase.firstAvailable, - testcase.firstInUse, - parser.availableColors, - parser.oldColors, - parser.firstAvailable, - parser.firstInUse, - color) - } - } - } - } -} - -func TestParseGlyphs(t *testing.T) { - parser := &Parser{} - parser.Reset() - tgBytes := []byte(testglyphs) - tg := tgBytes - idx := bytes.Index(tg, []byte("\n")) - row := 0 - for idx > 0 { - parser.ParseGlyphs(tg[:idx]) - tg = tg[idx+1:] - idx = bytes.Index(tg, []byte("\n")) - if parser.flows[0] != 1 { - t.Errorf("First column flow should be 1 but was %d", parser.flows[0]) - } - colorToFlow := map[int]int64{} - flowToColor := map[int64]int{} - - for i, flow := range parser.flows { - if flow == 0 { - continue - } - color := parser.colors[i] - - if fColor, in := flowToColor[flow]; in && fColor != color { - t.Errorf("Row %d column %d flow %d has color %d but should be %d", row, i, flow, color, fColor) - } - flowToColor[flow] = color - if cFlow, in := colorToFlow[color]; in && cFlow != flow { - t.Errorf("Row %d column %d flow %d has color %d but conflicts with flow %d", row, i, flow, color, cFlow) - } - colorToFlow[color] = flow - } - row++ - } - if len(parser.availableColors) != 9 { - t.Errorf("Expected 9 colors but have %d", len(parser.availableColors)) - } -} - -func TestCommitStringParsing(t *testing.T) { - dataFirstPart := "* DATA:|4e61bacab44e9b4730e44a6615d04098dd3a8eaf|2016-12-20 21:10:41 +0100|4e61bac|" - tests := []struct { - shouldPass bool - testName string - commitMessage string - }{ - {true, "normal", "not a fancy message"}, - {true, "extra pipe", "An extra pipe: |"}, - {true, "extra 'Data:'", "DATA: might be trouble"}, - } - - for _, test := range tests { - t.Run(test.testName, func(t *testing.T) { - testString := fmt.Sprintf("%s%s", dataFirstPart, test.commitMessage) - idx := strings.Index(testString, "DATA:") - commit, err := NewCommit(0, 0, []byte(testString[idx+5:])) - if err != nil && test.shouldPass { - t.Errorf("Could not parse %s", testString) - return - } - - if test.commitMessage != commit.Subject { - t.Errorf("%s does not match %s", test.commitMessage, commit.Subject) - } - }) - } -} - -var testglyphs = `* -* -* -* -* -* -* -* -|\ -* | -* | -* | -* | -* | -| * -* | -| * -| |\ -* | | -| | * -| | |\ -* | | \ -|\ \ \ \ -| * | | | -| |\| | | -* | | | | -|/ / / / -| | | * -| * | | -| * | | -| * | | -* | | | -* | | | -* | | | -* | | | -* | | | -|\ \ \ \ -| | * | | -| | |\| | -| | | * | -| | | | * -* | | | | -* | | | | -* | | | | -* | | | | -* | | | | -|\ \ \ \ \ -| * | | | | -|/| | | | | -| | |/ / / -| |/| | | -| | | | * -| * | | | -|/| | | | -| * | | | -|/| | | | -| | |/ / -| |/| | -| * | | -| * | | -| |\ \ \ -| | * | | -| |/| | | -| | | |/ -| | |/| -| * | | -| * | | -| * | | -| | * | -| | |\ \ -| | | * | -| | |/| | -| | | * | -| | | |\ \ -| | | | * | -| | | |/| | -| | * | | | -| | * | | | -| | |\ \ \ \ -| | | * | | | -| | |/| | | | -| | | | | * | -| | | | |/ / -* | | | / / -|/ / / / / -* | | | | -|\ \ \ \ \ -| * | | | | -|/| | | | | -| * | | | | -| * | | | | -| |\ \ \ \ \ -| | | * \ \ \ -| | | |\ \ \ \ -| | | | * | | | -| | | |/| | | | -| | | | | |/ / -| | | | |/| | -* | | | | | | -* | | | | | | -* | | | | | | -| | | | * | | -* | | | | | | -| | * | | | | -| |/| | | | | -* | | | | | | -| |/ / / / / -|/| | | | | -| | | | * | -| | | |/ / -| | |/| | -| * | | | -| | | | * -| | * | | -| | |\ \ \ -| | | * | | -| | |/| | | -| | | |/ / -| | | * | -| | * | | -| | |\ \ \ -| | | * | | -| | |/| | | -| | | |/ / -| | | * | -* | | | | -|\ \ \ \ \ -| * \ \ \ \ -| |\ \ \ \ \ -| | | |/ / / -| | |/| | | -| | | | * | -| | | | * | -* | | | | | -* | | | | | -|/ / / / / -| | | * | -* | | | | -* | | | | -* | | | | -* | | | | -|\ \ \ \ \ -| * | | | | -|/| | | | | -| | * | | | -| | |\ \ \ \ -| | | * | | | -| | |/| | | | -| |/| | |/ / -| | | |/| | -| | | | | * -| |_|_|_|/ -|/| | | | -| | * | | -| |/ / / -* | | | -* | | | -| | * | -* | | | -* | | | -| * | | -| | * | -| * | | -* | | | -|\ \ \ \ -| * | | | -|/| | | | -| |/ / / -| * | | -| |\ \ \ -| | * | | -| |/| | | -| | |/ / -| | * | -| | |\ \ -| | | * | -| | |/| | -* | | | | -* | | | | -|\ \ \ \ \ -| * | | | | -|/| | | | | -| | * | | | -| | * | | | -| | * | | | -| |/ / / / -| * | | | -| |\ \ \ \ -| | * | | | -| |/| | | | -* | | | | | -* | | | | | -* | | | | | -* | | | | | -* | | | | | -| | | | * | -* | | | | | -|\ \ \ \ \ \ -| * | | | | | -|/| | | | | | -| | | | | * | -| | | | |/ / -* | | | | | -|\ \ \ \ \ \ -* | | | | | | -* | | | | | | -| | | | * | | -* | | | | | | -* | | | | | | -|\ \ \ \ \ \ \ -| | |_|_|/ / / -| |/| | | | | -| | | | * | | -| | | | * | | -| | | | * | | -| | | | * | | -| | | | * | | -| | | | * | | -| | | |/ / / -| | | * | | -| | | * | | -| | | * | | -| | |/| | | -| | | * | | -| | |/| | | -| | | |/ / -| | * | | -| |/| | | -| | | * | -| | |/ / -| | * | -| * | | -| |\ \ \ -| * | | | -| | * | | -| |/| | | -| | |/ / -| | * | -| | |\ \ -| | * | | -* | | | | -|\| | | | -| * | | | -| * | | | -| * | | | -| | * | | -| * | | | -| |\| | | -| * | | | -| | * | | -| | * | | -| * | | | -| * | | | -| * | | | -| * | | | -| * | | | -| * | | | -| * | | | -| * | | | -| | * | | -| * | | | -| * | | | -| * | | | -| * | | | -| | * | | -* | | | | -|\| | | | -| | * | | -| * | | | -| |\| | | -| | * | | -| | * | | -| | * | | -| | | * | -* | | | | -|\| | | | -| | * | | -| | |/ / -| * | | -| * | | -| |\| | -* | | | -|\| | | -| | * | -| | * | -| | * | -| * | | -| | * | -| * | | -| | * | -| | * | -| | * | -| * | | -| * | | -| * | | -| * | | -| * | | -| * | | -| * | | -* | | | -|\| | | -| * | | -| |\| | -| | * | -| | |\ \ -* | | | | -|\| | | | -| * | | | -| |\| | | -| | * | | -| | | * | -| | |/ / -* | | | -* | | | -|\| | | -| * | | -| |\| | -| | * | -| | * | -| | * | -| | | * -* | | | -|\| | | -| * | | -| * | | -| | | * -| | | |\ -* | | | | -| |_|_|/ -|/| | | -| * | | -| |\| | -| | * | -| | * | -| | * | -| | * | -| | * | -| * | | -* | | | -|\| | | -| * | | -|/| | | -| |/ / -| * | -| |\ \ -| * | | -| * | | -* | | | -|\| | | -| | * | -| * | | -| * | | -| * | | -* | | | -|\| | | -| * | | -| * | | -| | * | -| | |\ \ -| | |/ / -| |/| | -| * | | -* | | | -|\| | | -| * | | -* | | | -|\| | | -| * | | -| |\ \ \ -| * | | | -| * | | | -| | | * | -| * | | | -| * | | | -| | |/ / -| |/| | -| | * | -* | | | -|\| | | -| * | | -| * | | -| * | | -| * | | -| * | | -| |\ \ \ -* | | | | -|\| | | | -| * | | | -| * | | | -* | | | | -* | | | | -|\| | | | -| | | | * -| | | | |\ -| |_|_|_|/ -|/| | | | -| * | | | -* | | | | -* | | | | -|\| | | | -| * | | | -| |\ \ \ \ -| | | |/ / -| | |/| | -| * | | | -| * | | | -| * | | | -| * | | | -| | * | | -| | | * | -| | |/ / -| |/| | -* | | | -|\| | | -| * | | -| * | | -| * | | -| * | | -| * | | -* | | | -|\| | | -| * | | -| * | | -* | | | -| * | | -| * | | -| * | | -* | | | -* | | | -* | | | -|\| | | -| * | | -* | | | -* | | | -* | | | -* | | | -| | | * -* | | | -|\| | | -| * | | -| * | | -| * | | -` diff --git a/modules/gitgraph/parser.go b/modules/gitgraph/parser.go deleted file mode 100644 index f6bf9b0b90..0000000000 --- a/modules/gitgraph/parser.go +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package gitgraph - -import ( - "bytes" - "fmt" -) - -// Parser represents a git graph parser. It is stateful containing the previous -// glyphs, detected flows and color assignments. -type Parser struct { - glyphs []byte - oldGlyphs []byte - flows []int64 - oldFlows []int64 - maxFlow int64 - colors []int - oldColors []int - availableColors []int - nextAvailable int - firstInUse int - firstAvailable int - maxAllowedColors int -} - -// Reset resets the internal parser state. -func (parser *Parser) Reset() { - parser.glyphs = parser.glyphs[0:0] - parser.oldGlyphs = parser.oldGlyphs[0:0] - parser.flows = parser.flows[0:0] - parser.oldFlows = parser.oldFlows[0:0] - parser.maxFlow = 0 - parser.colors = parser.colors[0:0] - parser.oldColors = parser.oldColors[0:0] - parser.availableColors = parser.availableColors[0:0] - parser.availableColors = append(parser.availableColors, 1, 2) - parser.nextAvailable = 0 - parser.firstInUse = -1 - parser.firstAvailable = 0 - parser.maxAllowedColors = 0 -} - -// AddLineToGraph adds the line as a row to the graph -func (parser *Parser) AddLineToGraph(graph *Graph, row int, line []byte) error { - idx := bytes.Index(line, []byte("DATA:")) - if idx < 0 { - parser.ParseGlyphs(line) - } else { - parser.ParseGlyphs(line[:idx]) - } - - var err error - commitDone := false - - for column, glyph := range parser.glyphs { - if glyph == ' ' { - continue - } - - flowID := parser.flows[column] - - graph.AddGlyph(row, column, flowID, parser.colors[column], glyph) - - if glyph == '*' { - if commitDone { - if err != nil { - err = fmt.Errorf("double commit on line %d: %s. %w", row, string(line), err) - } else { - err = fmt.Errorf("double commit on line %d: %s", row, string(line)) - } - } - commitDone = true - if idx < 0 { - if err != nil { - err = fmt.Errorf("missing data section on line %d with commit: %s. %w", row, string(line), err) - } else { - err = fmt.Errorf("missing data section on line %d with commit: %s", row, string(line)) - } - continue - } - err2 := graph.AddCommit(row, column, flowID, line[idx+5:]) - if err != nil && err2 != nil { - err = fmt.Errorf("%v %w", err2, err) - continue - } else if err2 != nil { - err = err2 - continue - } - } - } - if !commitDone { - graph.Commits = append(graph.Commits, RelationCommit) - } - return err -} - -func (parser *Parser) releaseUnusedColors() { - if parser.firstInUse > -1 { - // Here we step through the old colors, searching for them in the - // "in-use" section of availableColors (that is, the colors between - // firstInUse and firstAvailable) - // Ensure that the benchmarks are not worsened with proposed changes - stepstaken := 0 - position := parser.firstInUse - for _, color := range parser.oldColors { - if color == 0 { - continue - } - found := false - i := position - for j := stepstaken; i != parser.firstAvailable && j < len(parser.availableColors); j++ { - colorToCheck := parser.availableColors[i] - if colorToCheck == color { - found = true - break - } - i = (i + 1) % len(parser.availableColors) - } - if !found { - // Duplicate color - continue - } - // Swap them around - parser.availableColors[position], parser.availableColors[i] = parser.availableColors[i], parser.availableColors[position] - stepstaken++ - position = (parser.firstInUse + stepstaken) % len(parser.availableColors) - if position == parser.firstAvailable || stepstaken == len(parser.availableColors) { - break - } - } - if stepstaken == len(parser.availableColors) { - parser.firstAvailable = -1 - } else { - parser.firstAvailable = position - if parser.nextAvailable == -1 { - parser.nextAvailable = parser.firstAvailable - } - } - } -} - -// ParseGlyphs parses the provided glyphs and sets the internal state -func (parser *Parser) ParseGlyphs(glyphs []byte) { - // Clean state for parsing this row - parser.glyphs, parser.oldGlyphs = parser.oldGlyphs, parser.glyphs - parser.glyphs = parser.glyphs[0:0] - parser.flows, parser.oldFlows = parser.oldFlows, parser.flows - parser.flows = parser.flows[0:0] - parser.colors, parser.oldColors = parser.oldColors, parser.colors - - // Ensure we have enough flows and colors - parser.colors = parser.colors[0:0] - for range glyphs { - parser.flows = append(parser.flows, 0) - parser.colors = append(parser.colors, 0) - } - - // Copy the provided glyphs in to state.glyphs for safekeeping - parser.glyphs = append(parser.glyphs, glyphs...) - - // release unused colors - parser.releaseUnusedColors() - - for i := len(glyphs) - 1; i >= 0; i-- { - glyph := glyphs[i] - switch glyph { - case '|': - fallthrough - case '*': - parser.setUpFlow(i) - case '/': - parser.setOutFlow(i) - case '\\': - parser.setInFlow(i) - case '_': - parser.setRightFlow(i) - case '.': - fallthrough - case '-': - parser.setLeftFlow(i) - case ' ': - // no-op - default: - parser.newFlow(i) - } - } -} - -func (parser *Parser) takePreviousFlow(i, j int) { - if j < len(parser.oldFlows) && parser.oldFlows[j] > 0 { - parser.flows[i] = parser.oldFlows[j] - parser.oldFlows[j] = 0 - parser.colors[i] = parser.oldColors[j] - parser.oldColors[j] = 0 - } else { - parser.newFlow(i) - } -} - -func (parser *Parser) takeCurrentFlow(i, j int) { - if j < len(parser.flows) && parser.flows[j] > 0 { - parser.flows[i] = parser.flows[j] - parser.colors[i] = parser.colors[j] - } else { - parser.newFlow(i) - } -} - -func (parser *Parser) newFlow(i int) { - parser.maxFlow++ - parser.flows[i] = parser.maxFlow - - // Now give this flow a color - if parser.nextAvailable == -1 { - next := len(parser.availableColors) - if parser.maxAllowedColors < 1 || next < parser.maxAllowedColors { - parser.nextAvailable = next - parser.firstAvailable = next - parser.availableColors = append(parser.availableColors, next+1) - } - } - parser.colors[i] = parser.availableColors[parser.nextAvailable] - if parser.firstInUse == -1 { - parser.firstInUse = parser.nextAvailable - } - parser.availableColors[parser.firstAvailable], parser.availableColors[parser.nextAvailable] = parser.availableColors[parser.nextAvailable], parser.availableColors[parser.firstAvailable] - - parser.nextAvailable = (parser.nextAvailable + 1) % len(parser.availableColors) - parser.firstAvailable = (parser.firstAvailable + 1) % len(parser.availableColors) - - if parser.nextAvailable == parser.firstInUse { - parser.nextAvailable = parser.firstAvailable - } - if parser.nextAvailable == parser.firstInUse { - parser.nextAvailable = -1 - parser.firstAvailable = -1 - } -} - -// setUpFlow handles '|' or '*' -func (parser *Parser) setUpFlow(i int) { - // In preference order: - // - // Previous Row: '\? ' ' |' ' /' - // Current Row: ' | ' ' |' ' | ' - if i > 0 && i-1 < len(parser.oldGlyphs) && parser.oldGlyphs[i-1] == '\\' { - parser.takePreviousFlow(i, i-1) - } else if i < len(parser.oldGlyphs) && (parser.oldGlyphs[i] == '|' || parser.oldGlyphs[i] == '*') { - parser.takePreviousFlow(i, i) - } else if i+1 < len(parser.oldGlyphs) && parser.oldGlyphs[i+1] == '/' { - parser.takePreviousFlow(i, i+1) - } else { - parser.newFlow(i) - } -} - -// setOutFlow handles '/' -func (parser *Parser) setOutFlow(i int) { - // In preference order: - // - // Previous Row: ' |/' ' |_' ' |' ' /' ' _' '\' - // Current Row: '/| ' '/| ' '/ ' '/ ' '/ ' '/' - if i+2 < len(parser.oldGlyphs) && - (parser.oldGlyphs[i+1] == '|' || parser.oldGlyphs[i+1] == '*') && - (parser.oldGlyphs[i+2] == '/' || parser.oldGlyphs[i+2] == '_') && - i+1 < len(parser.glyphs) && - (parser.glyphs[i+1] == '|' || parser.glyphs[i+1] == '*') { - parser.takePreviousFlow(i, i+2) - } else if i+1 < len(parser.oldGlyphs) && - (parser.oldGlyphs[i+1] == '|' || parser.oldGlyphs[i+1] == '*' || - parser.oldGlyphs[i+1] == '/' || parser.oldGlyphs[i+1] == '_') { - parser.takePreviousFlow(i, i+1) - if parser.oldGlyphs[i+1] == '/' { - parser.glyphs[i] = '|' - } - } else if i < len(parser.oldGlyphs) && parser.oldGlyphs[i] == '\\' { - parser.takePreviousFlow(i, i) - } else { - parser.newFlow(i) - } -} - -// setInFlow handles '\' -func (parser *Parser) setInFlow(i int) { - // In preference order: - // - // Previous Row: '| ' '-. ' '| ' '\ ' '/' '---' - // Current Row: '|\' ' \' ' \' ' \' '\' ' \ ' - if i > 0 && i-1 < len(parser.oldGlyphs) && - (parser.oldGlyphs[i-1] == '|' || parser.oldGlyphs[i-1] == '*') && - (parser.glyphs[i-1] == '|' || parser.glyphs[i-1] == '*') { - parser.newFlow(i) - } else if i > 0 && i-1 < len(parser.oldGlyphs) && - (parser.oldGlyphs[i-1] == '|' || parser.oldGlyphs[i-1] == '*' || - parser.oldGlyphs[i-1] == '.' || parser.oldGlyphs[i-1] == '\\') { - parser.takePreviousFlow(i, i-1) - if parser.oldGlyphs[i-1] == '\\' { - parser.glyphs[i] = '|' - } - } else if i < len(parser.oldGlyphs) && parser.oldGlyphs[i] == '/' { - parser.takePreviousFlow(i, i) - } else { - parser.newFlow(i) - } -} - -// setRightFlow handles '_' -func (parser *Parser) setRightFlow(i int) { - // In preference order: - // - // Current Row: '__' '_/' '_|_' '_|/' - if i+1 < len(parser.glyphs) && - (parser.glyphs[i+1] == '_' || parser.glyphs[i+1] == '/') { - parser.takeCurrentFlow(i, i+1) - } else if i+2 < len(parser.glyphs) && - (parser.glyphs[i+1] == '|' || parser.glyphs[i+1] == '*') && - (parser.glyphs[i+2] == '_' || parser.glyphs[i+2] == '/') { - parser.takeCurrentFlow(i, i+2) - } else { - parser.newFlow(i) - } -} - -// setLeftFlow handles '----.' -func (parser *Parser) setLeftFlow(i int) { - if parser.glyphs[i] == '.' { - parser.newFlow(i) - } else if i+1 < len(parser.glyphs) && - (parser.glyphs[i+1] == '-' || parser.glyphs[i+1] == '.') { - parser.takeCurrentFlow(i, i+1) - } else { - parser.newFlow(i) - } -} diff --git a/modules/gitrepo/branch.go b/modules/gitrepo/branch.go index e13a4c82e1..d7857819e4 100644 --- a/modules/gitrepo/branch.go +++ b/modules/gitrepo/branch.go @@ -11,14 +11,14 @@ import ( // GetBranchesByPath returns a branch by its path // if limit = 0 it will not limit -func GetBranchesByPath(ctx context.Context, repo Repository, skip, limit int) ([]*git.Branch, int, error) { +func GetBranchesByPath(ctx context.Context, repo Repository, skip, limit int) ([]string, int, error) { gitRepo, err := OpenRepository(ctx, repo) if err != nil { return nil, 0, err } defer gitRepo.Close() - return gitRepo.GetBranches(skip, limit) + return gitRepo.GetBranchNames(skip, limit) } func GetBranchCommitID(ctx context.Context, repo Repository, branch string) (string, error) { @@ -33,9 +33,9 @@ func GetBranchCommitID(ctx context.Context, repo Repository, branch string) (str // SetDefaultBranch sets default branch of repository. func SetDefaultBranch(ctx context.Context, repo Repository, name string) error { - _, _, err := git.NewCommand(ctx, "symbolic-ref", "HEAD"). - AddDynamicArguments(git.BranchPrefix + name). - RunStdString(&git.RunOpts{Dir: repoPath(repo)}) + _, _, err := git.NewCommand("symbolic-ref", "HEAD"). + AddDynamicArguments(git.BranchPrefix+name). + RunStdString(ctx, &git.RunOpts{Dir: repoPath(repo)}) return err } @@ -44,6 +44,12 @@ func GetDefaultBranch(ctx context.Context, repo Repository) (string, error) { return git.GetDefaultBranch(ctx, repoPath(repo)) } -func GetWikiDefaultBranch(ctx context.Context, repo Repository) (string, error) { - return git.GetDefaultBranch(ctx, wikiPath(repo)) +// IsReferenceExist returns true if given reference exists in the repository. +func IsReferenceExist(ctx context.Context, repo Repository, name string) bool { + return git.IsReferenceExist(ctx, repoPath(repo), name) +} + +// IsBranchExist returns true if given branch exists in the repository. +func IsBranchExist(ctx context.Context, repo Repository, name string) bool { + return IsReferenceExist(ctx, repo, git.BranchPrefix+name) } diff --git a/modules/gitrepo/gitrepo.go b/modules/gitrepo/gitrepo.go index 14d809aedb..5da65e2452 100644 --- a/modules/gitrepo/gitrepo.go +++ b/modules/gitrepo/gitrepo.go @@ -5,26 +5,25 @@ package gitrepo import ( "context" + "fmt" "io" "path/filepath" - "strings" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) +// Repository represents a git repository which stored in a disk type Repository interface { - GetName() string - GetOwnerName() string + RelativePath() string // We don't assume how the directory structure of the repository is, so we only need the relative path } +// RelativePath should be an unix style path like username/reponame.git +// This method should change it according to the current OS. func repoPath(repo Repository) string { - return filepath.Join(setting.RepoRootPath, strings.ToLower(repo.GetOwnerName()), strings.ToLower(repo.GetName())+".git") -} - -func wikiPath(repo Repository) string { - return filepath.Join(setting.RepoRootPath, strings.ToLower(repo.GetOwnerName()), strings.ToLower(repo.GetName())+".wiki.git") + return filepath.Join(setting.RepoRootPath, filepath.FromSlash(repo.RelativePath())) } // OpenRepository opens the repository at the given relative path with the provided context. @@ -32,69 +31,53 @@ func OpenRepository(ctx context.Context, repo Repository) (*git.Repository, erro return git.OpenRepository(ctx, repoPath(repo)) } -func OpenWikiRepository(ctx context.Context, repo Repository) (*git.Repository, error) { - return git.OpenRepository(ctx, wikiPath(repo)) -} - // contextKey is a value for use with context.WithValue. type contextKey struct { - name string -} - -// RepositoryContextKey is a context key. It is used with context.Value() to get the current Repository for the context -var RepositoryContextKey = &contextKey{"repository"} - -// RepositoryFromContext attempts to get the repository from the context -func repositoryFromContext(ctx context.Context, repo Repository) *git.Repository { - value := ctx.Value(RepositoryContextKey) - if value == nil { - return nil - } - - if gitRepo, ok := value.(*git.Repository); ok && gitRepo != nil { - if gitRepo.Path == repoPath(repo) { - return gitRepo - } - } - - return nil + repoPath string } // RepositoryFromContextOrOpen attempts to get the repository from the context or just opens it +// The caller must call "defer gitRepo.Close()" func RepositoryFromContextOrOpen(ctx context.Context, repo Repository) (*git.Repository, io.Closer, error) { - gitRepo := repositoryFromContext(ctx, repo) - if gitRepo != nil { - return gitRepo, util.NopCloser{}, nil + reqCtx := reqctx.FromContext(ctx) + if reqCtx != nil { + gitRepo, err := RepositoryFromRequestContextOrOpen(reqCtx, repo) + return gitRepo, util.NopCloser{}, err } - gitRepo, err := OpenRepository(ctx, repo) return gitRepo, gitRepo, err } -// repositoryFromContextPath attempts to get the repository from the context -func repositoryFromContextPath(ctx context.Context, path string) *git.Repository { - value := ctx.Value(RepositoryContextKey) - if value == nil { - return nil +// RepositoryFromRequestContextOrOpen opens the repository at the given relative path in the provided request context. +// Caller shouldn't close the git repo manually, the git repo will be automatically closed when the request context is done. +func RepositoryFromRequestContextOrOpen(ctx reqctx.RequestContext, repo Repository) (*git.Repository, error) { + ck := contextKey{repoPath: repoPath(repo)} + if gitRepo, ok := ctx.Value(ck).(*git.Repository); ok { + return gitRepo, nil } - - if repo, ok := value.(*git.Repository); ok && repo != nil { - if repo.Path == path { - return repo - } + gitRepo, err := git.OpenRepository(ctx, ck.repoPath) + if err != nil { + return nil, err } + ctx.AddCloser(gitRepo) + ctx.SetContextValue(ck, gitRepo) + return gitRepo, nil +} - return nil +// IsRepositoryExist returns true if the repository directory exists in the disk +func IsRepositoryExist(ctx context.Context, repo Repository) (bool, error) { + return util.IsExist(repoPath(repo)) } -// RepositoryFromContextOrOpenPath attempts to get the repository from the context or just opens it -// Deprecated: Use RepositoryFromContextOrOpen instead -func RepositoryFromContextOrOpenPath(ctx context.Context, path string) (*git.Repository, io.Closer, error) { - gitRepo := repositoryFromContextPath(ctx, path) - if gitRepo != nil { - return gitRepo, util.NopCloser{}, nil - } +// DeleteRepository deletes the repository directory from the disk +func DeleteRepository(ctx context.Context, repo Repository) error { + return util.RemoveAll(repoPath(repo)) +} - gitRepo, err := git.OpenRepository(ctx, path) - return gitRepo, gitRepo, err +// RenameRepository renames a repository's name on disk +func RenameRepository(ctx context.Context, repo, newRepo Repository) error { + if err := util.Rename(repoPath(repo), repoPath(newRepo)); err != nil { + return fmt.Errorf("rename repository directory: %w", err) + } + return nil } diff --git a/modules/repository/hooks.go b/modules/gitrepo/hooks.go index 95849789ab..d9d4a88ff1 100644 --- a/modules/repository/hooks.go +++ b/modules/gitrepo/hooks.go @@ -1,9 +1,10 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package repository +package gitrepo import ( + "context" "fmt" "os" "path/filepath" @@ -106,9 +107,12 @@ done } // CreateDelegateHooks creates all the hooks scripts for the repo -func CreateDelegateHooks(repoPath string) (err error) { +func CreateDelegateHooks(_ context.Context, repo Repository) (err error) { + return createDelegateHooks(filepath.Join(repoPath(repo), "hooks")) +} + +func createDelegateHooks(hookDir string) (err error) { hookNames, hookTpls, giteaHookTpls := getHookTemplates() - hookDir := filepath.Join(repoPath, "hooks") for i, hookName := range hookNames { oldHookPath := filepath.Join(hookDir, hookName) @@ -170,10 +174,13 @@ func ensureExecutable(filename string) error { } // CheckDelegateHooks checks the hooks scripts for the repo -func CheckDelegateHooks(repoPath string) ([]string, error) { +func CheckDelegateHooks(_ context.Context, repo Repository) ([]string, error) { + return checkDelegateHooks(filepath.Join(repoPath(repo), "hooks")) +} + +func checkDelegateHooks(hookDir string) ([]string, error) { hookNames, hookTpls, giteaHookTpls := getHookTemplates() - hookDir := filepath.Join(repoPath, "hooks") results := make([]string, 0, 10) for i, hookName := range hookNames { diff --git a/modules/gitrepo/tag.go b/modules/gitrepo/tag.go new file mode 100644 index 0000000000..58ed204a99 --- /dev/null +++ b/modules/gitrepo/tag.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitrepo + +import ( + "context" + + "code.gitea.io/gitea/modules/git" +) + +// IsTagExist returns true if given tag exists in the repository. +func IsTagExist(ctx context.Context, repo Repository, name string) bool { + return IsReferenceExist(ctx, repo, git.TagPrefix+name) +} diff --git a/modules/gitrepo/walk_gogit.go b/modules/gitrepo/walk_gogit.go index 6370faf08e..709897ba0c 100644 --- a/modules/gitrepo/walk_gogit.go +++ b/modules/gitrepo/walk_gogit.go @@ -14,15 +14,11 @@ import ( // WalkReferences walks all the references from the repository // refname is empty, ObjectTag or ObjectBranch. All other values should be treated as equivalent to empty. func WalkReferences(ctx context.Context, repo Repository, walkfn func(sha1, refname string) error) (int, error) { - gitRepo := repositoryFromContext(ctx, repo) - if gitRepo == nil { - var err error - gitRepo, err = OpenRepository(ctx, repo) - if err != nil { - return 0, err - } - defer gitRepo.Close() + gitRepo, closer, err := RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return 0, err } + defer closer.Close() i := 0 iter, err := gitRepo.GoGitRepo().References() diff --git a/modules/globallock/globallock_test.go b/modules/globallock/globallock_test.go index f14c7d656b..8d55d9f699 100644 --- a/modules/globallock/globallock_test.go +++ b/modules/globallock/globallock_test.go @@ -66,11 +66,11 @@ func TestLockAndDo(t *testing.T) { func testLockAndDo(t *testing.T) { const concurrency = 50 - ctx := context.Background() + ctx := t.Context() count := 0 wg := sync.WaitGroup{} wg.Add(concurrency) - for i := 0; i < concurrency; i++ { + for range concurrency { go func() { defer wg.Done() err := LockAndDo(ctx, "test", func(ctx context.Context) error { diff --git a/modules/globallock/locker_test.go b/modules/globallock/locker_test.go index bee4d34b34..c9e73c25d2 100644 --- a/modules/globallock/locker_test.go +++ b/modules/globallock/locker_test.go @@ -46,14 +46,14 @@ func TestLocker(t *testing.T) { func testLocker(t *testing.T, locker Locker) { t.Run("lock", func(t *testing.T) { - parentCtx := context.Background() + parentCtx := t.Context() release, err := locker.Lock(parentCtx, "test") defer release() assert.NoError(t, err) func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() release, err := locker.Lock(ctx, "test") defer release() @@ -64,7 +64,7 @@ func testLocker(t *testing.T, locker Locker) { release() func() { - release, err := locker.Lock(context.Background(), "test") + release, err := locker.Lock(t.Context(), "test") defer release() assert.NoError(t, err) @@ -72,7 +72,7 @@ func testLocker(t *testing.T, locker Locker) { }) t.Run("try lock", func(t *testing.T) { - parentCtx := context.Background() + parentCtx := t.Context() ok, release, err := locker.TryLock(parentCtx, "test") defer release() @@ -80,7 +80,7 @@ func testLocker(t *testing.T, locker Locker) { assert.NoError(t, err) func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() ok, release, err := locker.TryLock(ctx, "test") defer release() @@ -92,7 +92,7 @@ func testLocker(t *testing.T, locker Locker) { release() func() { - ok, release, _ := locker.TryLock(context.Background(), "test") + ok, release, _ := locker.TryLock(t.Context(), "test") defer release() assert.True(t, ok) @@ -100,7 +100,7 @@ func testLocker(t *testing.T, locker Locker) { }) t.Run("wait and acquired", func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() release, err := locker.Lock(ctx, "test") require.NoError(t, err) @@ -109,7 +109,7 @@ func testLocker(t *testing.T, locker Locker) { go func() { defer wg.Done() started := time.Now() - release, err := locker.Lock(context.Background(), "test") // should be blocked for seconds + release, err := locker.Lock(t.Context(), "test") // should be blocked for seconds defer release() assert.Greater(t, time.Since(started), time.Second) assert.NoError(t, err) @@ -122,7 +122,7 @@ func testLocker(t *testing.T, locker Locker) { }) t.Run("multiple release", func(t *testing.T) { - ctx := context.Background() + ctx := t.Context() release1, err := locker.Lock(ctx, "test") require.NoError(t, err) @@ -159,13 +159,13 @@ func testRedisLocker(t *testing.T, locker *redisLocker) { // Otherwise, it will affect other tests. t.Run("close", func(t *testing.T) { assert.NoError(t, locker.Close()) - _, err := locker.Lock(context.Background(), "test") + _, err := locker.Lock(t.Context(), "test") assert.Error(t, err) }) }() t.Run("failed extend", func(t *testing.T) { - release, err := locker.Lock(context.Background(), "test") + release, err := locker.Lock(t.Context(), "test") defer release() require.NoError(t, err) diff --git a/modules/globallock/redis_locker.go b/modules/globallock/redis_locker.go index 34ed9e389b..45dc769fd4 100644 --- a/modules/globallock/redis_locker.go +++ b/modules/globallock/redis_locker.go @@ -6,7 +6,6 @@ package globallock import ( "context" "errors" - "fmt" "sync" "sync/atomic" "time" @@ -78,7 +77,7 @@ func (l *redisLocker) Close() error { func (l *redisLocker) lock(ctx context.Context, key string, tries int) (ReleaseFunc, error) { if l.closed.Load() { - return func() {}, fmt.Errorf("locker is closed") + return func() {}, errors.New("locker is closed") } options := []redsync.Option{ diff --git a/modules/graceful/manager_windows.go b/modules/graceful/manager_windows.go index d776e0e9f9..457768d6ca 100644 --- a/modules/graceful/manager_windows.go +++ b/modules/graceful/manager_windows.go @@ -41,8 +41,7 @@ func (g *Manager) start() { // Make SVC process run := svc.Run - //lint:ignore SA1019 We use IsAnInteractiveSession because IsWindowsService has a different permissions profile - isAnInteractiveSession, err := svc.IsAnInteractiveSession() //nolint:staticcheck + isAnInteractiveSession, err := svc.IsAnInteractiveSession() //nolint:staticcheck // must use IsAnInteractiveSession because IsWindowsService has a different permissions profile if err != nil { log.Error("Unable to ascertain if running as an Windows Service: %v", err) return diff --git a/modules/graceful/releasereopen/releasereopen_test.go b/modules/graceful/releasereopen/releasereopen_test.go index 0e8b48257d..46e67c2046 100644 --- a/modules/graceful/releasereopen/releasereopen_test.go +++ b/modules/graceful/releasereopen/releasereopen_test.go @@ -30,14 +30,14 @@ func TestManager(t *testing.T) { _ = m.Register(t3) assert.NoError(t, m.ReleaseReopen()) - assert.EqualValues(t, 1, t1.count) - assert.EqualValues(t, 1, t2.count) - assert.EqualValues(t, 1, t3.count) + assert.Equal(t, 1, t1.count) + assert.Equal(t, 1, t2.count) + assert.Equal(t, 1, t3.count) c2() assert.NoError(t, m.ReleaseReopen()) - assert.EqualValues(t, 2, t1.count) - assert.EqualValues(t, 1, t2.count) - assert.EqualValues(t, 2, t3.count) + assert.Equal(t, 2, t1.count) + assert.Equal(t, 1, t2.count) + assert.Equal(t, 2, t3.count) } diff --git a/modules/gtprof/event.go b/modules/gtprof/event.go new file mode 100644 index 0000000000..da4a0faff9 --- /dev/null +++ b/modules/gtprof/event.go @@ -0,0 +1,32 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +type EventConfig struct { + attributes []*TraceAttribute +} + +type EventOption interface { + applyEvent(*EventConfig) +} + +type applyEventFunc func(*EventConfig) + +func (f applyEventFunc) applyEvent(cfg *EventConfig) { + f(cfg) +} + +func WithAttributes(attrs ...*TraceAttribute) EventOption { + return applyEventFunc(func(cfg *EventConfig) { + cfg.attributes = append(cfg.attributes, attrs...) + }) +} + +func eventConfigFromOptions(options ...EventOption) *EventConfig { + cfg := &EventConfig{} + for _, opt := range options { + opt.applyEvent(cfg) + } + return cfg +} diff --git a/modules/gtprof/trace.go b/modules/gtprof/trace.go new file mode 100644 index 0000000000..ad67c226dc --- /dev/null +++ b/modules/gtprof/trace.go @@ -0,0 +1,175 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +import ( + "context" + "fmt" + "sync" + "time" + + "code.gitea.io/gitea/modules/util" +) + +type contextKey struct { + name string +} + +var contextKeySpan = &contextKey{"span"} + +type traceStarter interface { + start(ctx context.Context, traceSpan *TraceSpan, internalSpanIdx int) (context.Context, traceSpanInternal) +} + +type traceSpanInternal interface { + addEvent(name string, cfg *EventConfig) + recordError(err error, cfg *EventConfig) + end() +} + +type TraceSpan struct { + // immutable + parent *TraceSpan + internalSpans []traceSpanInternal + internalContexts []context.Context + + // mutable, must be protected by mutex + mu sync.RWMutex + name string + statusCode uint32 + statusDesc string + startTime time.Time + endTime time.Time + attributes []*TraceAttribute + children []*TraceSpan +} + +type TraceAttribute struct { + Key string + Value TraceValue +} + +type TraceValue struct { + v any +} + +func (t *TraceValue) AsString() string { + return fmt.Sprint(t.v) +} + +func (t *TraceValue) AsInt64() int64 { + v, _ := util.ToInt64(t.v) + return v +} + +func (t *TraceValue) AsFloat64() float64 { + v, _ := util.ToFloat64(t.v) + return v +} + +var globalTraceStarters []traceStarter + +type Tracer struct { + starters []traceStarter +} + +func (s *TraceSpan) SetName(name string) { + s.mu.Lock() + defer s.mu.Unlock() + s.name = name +} + +func (s *TraceSpan) SetStatus(code uint32, desc string) { + s.mu.Lock() + defer s.mu.Unlock() + s.statusCode, s.statusDesc = code, desc +} + +func (s *TraceSpan) AddEvent(name string, options ...EventOption) { + cfg := eventConfigFromOptions(options...) + for _, tsp := range s.internalSpans { + tsp.addEvent(name, cfg) + } +} + +func (s *TraceSpan) RecordError(err error, options ...EventOption) { + cfg := eventConfigFromOptions(options...) + for _, tsp := range s.internalSpans { + tsp.recordError(err, cfg) + } +} + +func (s *TraceSpan) SetAttributeString(key, value string) *TraceSpan { + s.mu.Lock() + defer s.mu.Unlock() + + s.attributes = append(s.attributes, &TraceAttribute{Key: key, Value: TraceValue{v: value}}) + return s +} + +func (t *Tracer) Start(ctx context.Context, spanName string) (context.Context, *TraceSpan) { + starters := t.starters + if starters == nil { + starters = globalTraceStarters + } + ts := &TraceSpan{name: spanName, startTime: time.Now()} + parentSpan := GetContextSpan(ctx) + if parentSpan != nil { + parentSpan.mu.Lock() + parentSpan.children = append(parentSpan.children, ts) + parentSpan.mu.Unlock() + ts.parent = parentSpan + } + + parentCtx := ctx + for internalSpanIdx, tsp := range starters { + var internalSpan traceSpanInternal + if parentSpan != nil { + parentCtx = parentSpan.internalContexts[internalSpanIdx] + } + ctx, internalSpan = tsp.start(parentCtx, ts, internalSpanIdx) + ts.internalContexts = append(ts.internalContexts, ctx) + ts.internalSpans = append(ts.internalSpans, internalSpan) + } + ctx = context.WithValue(ctx, contextKeySpan, ts) + return ctx, ts +} + +type mutableContext interface { + context.Context + SetContextValue(key, value any) + GetContextValue(key any) any +} + +// StartInContext starts a trace span in Gitea's mutable context (usually the web request context). +// Due to the design limitation of Gitea's web framework, it can't use `context.WithValue` to bind a new span into a new context. +// So here we use our "reqctx" framework to achieve the same result: web request context could always see the latest "span". +func (t *Tracer) StartInContext(ctx mutableContext, spanName string) (*TraceSpan, func()) { + curTraceSpan := GetContextSpan(ctx) + _, newTraceSpan := GetTracer().Start(ctx, spanName) + ctx.SetContextValue(contextKeySpan, newTraceSpan) + return newTraceSpan, func() { + newTraceSpan.End() + ctx.SetContextValue(contextKeySpan, curTraceSpan) + } +} + +func (s *TraceSpan) End() { + s.mu.Lock() + s.endTime = time.Now() + s.mu.Unlock() + + for _, tsp := range s.internalSpans { + tsp.end() + } +} + +func GetTracer() *Tracer { + return &Tracer{} +} + +func GetContextSpan(ctx context.Context) *TraceSpan { + ts, _ := ctx.Value(contextKeySpan).(*TraceSpan) + return ts +} diff --git a/modules/gtprof/trace_builtin.go b/modules/gtprof/trace_builtin.go new file mode 100644 index 0000000000..2590ed3a13 --- /dev/null +++ b/modules/gtprof/trace_builtin.go @@ -0,0 +1,96 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +import ( + "context" + "fmt" + "strings" + "sync/atomic" + "time" + + "code.gitea.io/gitea/modules/tailmsg" +) + +type traceBuiltinStarter struct{} + +type traceBuiltinSpan struct { + ts *TraceSpan + + internalSpanIdx int +} + +func (t *traceBuiltinSpan) addEvent(name string, cfg *EventConfig) { + // No-op because builtin tracer doesn't need it. + // In the future we might use it to mark the time point between backend logic and network response. +} + +func (t *traceBuiltinSpan) recordError(err error, cfg *EventConfig) { + // No-op because builtin tracer doesn't need it. + // Actually Gitea doesn't handle err this way in most cases +} + +func (t *traceBuiltinSpan) toString(out *strings.Builder, indent int) { + t.ts.mu.RLock() + defer t.ts.mu.RUnlock() + + out.WriteString(strings.Repeat(" ", indent)) + out.WriteString(t.ts.name) + if t.ts.endTime.IsZero() { + out.WriteString(" duration: (not ended)") + } else { + fmt.Fprintf(out, " duration=%.4fs", t.ts.endTime.Sub(t.ts.startTime).Seconds()) + } + for _, a := range t.ts.attributes { + out.WriteString(" ") + out.WriteString(a.Key) + out.WriteString("=") + value := a.Value.AsString() + if strings.ContainsAny(value, " \t\r\n") { + quoted := false + for _, c := range "\"'`" { + if quoted = !strings.Contains(value, string(c)); quoted { + value = string(c) + value + string(c) + break + } + } + if !quoted { + value = fmt.Sprintf("%q", value) + } + } + out.WriteString(value) + } + out.WriteString("\n") + for _, c := range t.ts.children { + span := c.internalSpans[t.internalSpanIdx].(*traceBuiltinSpan) + span.toString(out, indent+2) + } +} + +func (t *traceBuiltinSpan) end() { + if t.ts.parent == nil { + // TODO: debug purpose only + // TODO: it should distinguish between http response network lag and actual processing time + threshold := time.Duration(traceBuiltinThreshold.Load()) + if threshold != 0 && t.ts.endTime.Sub(t.ts.startTime) > threshold { + sb := &strings.Builder{} + t.toString(sb, 0) + tailmsg.GetManager().GetTraceRecorder().Record(sb.String()) + } + } +} + +func (t *traceBuiltinStarter) start(ctx context.Context, traceSpan *TraceSpan, internalSpanIdx int) (context.Context, traceSpanInternal) { + return ctx, &traceBuiltinSpan{ts: traceSpan, internalSpanIdx: internalSpanIdx} +} + +func init() { + globalTraceStarters = append(globalTraceStarters, &traceBuiltinStarter{}) +} + +var traceBuiltinThreshold atomic.Int64 + +func EnableBuiltinTracer(threshold time.Duration) { + traceBuiltinThreshold.Store(int64(threshold)) +} diff --git a/modules/gtprof/trace_const.go b/modules/gtprof/trace_const.go new file mode 100644 index 0000000000..af9ce9223f --- /dev/null +++ b/modules/gtprof/trace_const.go @@ -0,0 +1,19 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +// Some interesting names could be found in https://github.com/open-telemetry/opentelemetry-go/tree/main/semconv + +const ( + TraceSpanHTTP = "http" + TraceSpanGitRun = "git-run" + TraceSpanDatabase = "database" +) + +const ( + TraceAttrFuncCaller = "func.caller" + TraceAttrDbSQL = "db.sql" + TraceAttrGitCommand = "git.command" + TraceAttrHTTPRoute = "http.route" +) diff --git a/modules/gtprof/trace_test.go b/modules/gtprof/trace_test.go new file mode 100644 index 0000000000..0f4e3facba --- /dev/null +++ b/modules/gtprof/trace_test.go @@ -0,0 +1,93 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gtprof + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +// "vendor span" is a simple demo for a span from a vendor library + +var vendorContextKey any = "vendorContextKey" + +type vendorSpan struct { + name string + children []*vendorSpan +} + +func vendorTraceStart(ctx context.Context, name string) (context.Context, *vendorSpan) { + span := &vendorSpan{name: name} + parentSpan, ok := ctx.Value(vendorContextKey).(*vendorSpan) + if ok { + parentSpan.children = append(parentSpan.children, span) + } + ctx = context.WithValue(ctx, vendorContextKey, span) + return ctx, span +} + +// below "testTrace*" integrate the vendor span into our trace system + +type testTraceSpan struct { + vendorSpan *vendorSpan +} + +func (t *testTraceSpan) addEvent(name string, cfg *EventConfig) {} + +func (t *testTraceSpan) recordError(err error, cfg *EventConfig) {} + +func (t *testTraceSpan) end() {} + +type testTraceStarter struct{} + +func (t *testTraceStarter) start(ctx context.Context, traceSpan *TraceSpan, internalSpanIdx int) (context.Context, traceSpanInternal) { + ctx, span := vendorTraceStart(ctx, traceSpan.name) + return ctx, &testTraceSpan{span} +} + +func TestTraceStarter(t *testing.T) { + globalTraceStarters = []traceStarter{&testTraceStarter{}} + + ctx := t.Context() + ctx, span := GetTracer().Start(ctx, "root") + defer span.End() + + func(ctx context.Context) { + ctx, span := GetTracer().Start(ctx, "span1") + defer span.End() + func(ctx context.Context) { + _, span := GetTracer().Start(ctx, "spanA") + defer span.End() + }(ctx) + func(ctx context.Context) { + _, span := GetTracer().Start(ctx, "spanB") + defer span.End() + }(ctx) + }(ctx) + + func(ctx context.Context) { + _, span := GetTracer().Start(ctx, "span2") + defer span.End() + }(ctx) + + var spanFullNames []string + var collectSpanNames func(parentFullName string, s *vendorSpan) + collectSpanNames = func(parentFullName string, s *vendorSpan) { + fullName := parentFullName + "/" + s.name + spanFullNames = append(spanFullNames, fullName) + for _, c := range s.children { + collectSpanNames(fullName, c) + } + } + collectSpanNames("", span.internalSpans[0].(*testTraceSpan).vendorSpan) + assert.Equal(t, []string{ + "/root", + "/root/span1", + "/root/span1/spanA", + "/root/span1/spanB", + "/root/span2", + }, spanFullNames) +} diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index d7ab3f7afd..77f24fa3f3 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -11,6 +11,7 @@ import ( gohtml "html" "html/template" "io" + "path" "path/filepath" "strings" "sync" @@ -83,7 +84,7 @@ func Code(fileName, language, code string) (output template.HTML, lexerName stri } if lexer == nil { - if val, ok := highlightMapping[filepath.Ext(fileName)]; ok { + if val, ok := highlightMapping[path.Ext(fileName)]; ok { // use mapped value to find lexer lexer = lexers.Get(val) } diff --git a/modules/highlight/highlight_test.go b/modules/highlight/highlight_test.go index 659688bd0f..b36de98c5c 100644 --- a/modules/highlight/highlight_test.go +++ b/modules/highlight/highlight_test.go @@ -114,7 +114,7 @@ c=2 t.Run(tt.name, func(t *testing.T) { out, lexerName, err := File(tt.name, "", []byte(tt.code)) assert.NoError(t, err) - assert.EqualValues(t, tt.want, out) + assert.Equal(t, tt.want, out) assert.Equal(t, tt.lexerName, lexerName) }) } @@ -177,7 +177,7 @@ c=2`), for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { out := PlainText([]byte(tt.code)) - assert.EqualValues(t, tt.want, out) + assert.Equal(t, tt.want, out) }) } } diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go index 1069310316..15c6371422 100644 --- a/modules/hostmatcher/hostmatcher.go +++ b/modules/hostmatcher/hostmatcher.go @@ -6,6 +6,7 @@ package hostmatcher import ( "net" "path/filepath" + "slices" "strings" ) @@ -38,7 +39,7 @@ func isBuiltin(s string) bool { // ParseHostMatchList parses the host list HostMatchList func ParseHostMatchList(settingKeyHint, hostList string) *HostMatchList { hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList} - for _, s := range strings.Split(hostList, ",") { + for s := range strings.SplitSeq(hostList, ",") { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { continue @@ -61,7 +62,7 @@ func ParseSimpleMatchList(settingKeyHint, matchList string) *HostMatchList { SettingKeyHint: settingKeyHint, SettingValue: matchList, } - for _, s := range strings.Split(matchList, ",") { + for s := range strings.SplitSeq(matchList, ",") { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { continue @@ -98,10 +99,8 @@ func (hl *HostMatchList) checkPattern(host string) bool { } func (hl *HostMatchList) checkIP(ip net.IP) bool { - for _, pattern := range hl.patterns { - if pattern == "*" { - return true - } + if slices.Contains(hl.patterns, "*") { + return true } for _, builtin := range hl.builtins { switch builtin { diff --git a/modules/htmlutil/html.go b/modules/htmlutil/html.go index 9b5f5a92d8..efbc174b2e 100644 --- a/modules/htmlutil/html.go +++ b/modules/htmlutil/html.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "slices" + "strings" ) // ParseSizeAndClass get size and class from string with default values @@ -30,7 +31,10 @@ func ParseSizeAndClass(defaultSize int, defaultClass string, others ...any) (int return size, class } -func HTMLFormat(s string, rawArgs ...any) template.HTML { +func HTMLFormat(s template.HTML, rawArgs ...any) template.HTML { + if !strings.Contains(string(s), "%") || len(rawArgs) == 0 { + panic("HTMLFormat requires one or more arguments") + } args := slices.Clone(rawArgs) for i, v := range args { switch v := v.(type) { @@ -38,11 +42,13 @@ func HTMLFormat(s string, rawArgs ...any) template.HTML { // for most basic types (including template.HTML which is safe), just do nothing and use it case string: args[i] = template.HTMLEscapeString(v) + case template.URL: + args[i] = template.HTMLEscapeString(string(v)) case fmt.Stringer: args[i] = template.HTMLEscapeString(v.String()) default: args[i] = template.HTMLEscapeString(fmt.Sprint(v)) } } - return template.HTML(fmt.Sprintf(s, args...)) + return template.HTML(fmt.Sprintf(string(s), args...)) } diff --git a/modules/htmlutil/html_test.go b/modules/htmlutil/html_test.go index 5ff05d75b3..22258ce59d 100644 --- a/modules/htmlutil/html_test.go +++ b/modules/htmlutil/html_test.go @@ -10,6 +10,15 @@ import ( "github.com/stretchr/testify/assert" ) +type testStringer struct{} + +func (t testStringer) String() string { + return "&StringMethod" +} + func TestHTMLFormat(t *testing.T) { assert.Equal(t, template.HTML("<a>< < 1</a>"), HTMLFormat("<a>%s %s %d</a>", "<", template.HTML("<"), 1)) + assert.Equal(t, template.HTML("%!s(<nil>)"), HTMLFormat("%s", nil)) + assert.Equal(t, template.HTML("<>"), HTMLFormat("%s", template.URL("<>"))) + assert.Equal(t, template.HTML("&StringMethod &StringMethod"), HTMLFormat("%s %s", testStringer{}, &testStringer{})) } diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go index 2c9af94405..dd3efab7a5 100644 --- a/modules/httpcache/httpcache.go +++ b/modules/httpcache/httpcache.go @@ -4,40 +4,60 @@ package httpcache import ( - "io" + "fmt" "net/http" "strconv" "strings" "time" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) +type CacheControlOptions struct { + IsPublic bool + MaxAge time.Duration + NoTransform bool +} + // SetCacheControlInHeader sets suitable cache-control headers in the response -func SetCacheControlInHeader(h http.Header, maxAge time.Duration, additionalDirectives ...string) { - directives := make([]string, 0, 2+len(additionalDirectives)) +func SetCacheControlInHeader(h http.Header, opts *CacheControlOptions) { + directives := make([]string, 0, 4) // "max-age=0 + must-revalidate" (aka "no-cache") is preferred instead of "no-store" // because browsers may restore some input fields after navigate-back / reload a page. + publicPrivate := util.Iif(opts.IsPublic, "public", "private") if setting.IsProd { - if maxAge == 0 { + if opts.MaxAge == 0 { directives = append(directives, "max-age=0", "private", "must-revalidate") } else { - directives = append(directives, "private", "max-age="+strconv.Itoa(int(maxAge.Seconds()))) + directives = append(directives, publicPrivate, "max-age="+strconv.Itoa(int(opts.MaxAge.Seconds()))) } } else { - directives = append(directives, "max-age=0", "private", "must-revalidate") + // use dev-related controls, and remind users they are using non-prod setting. + directives = append(directives, "max-age=0", publicPrivate, "must-revalidate") + h.Set("X-Gitea-Debug", fmt.Sprintf("RUN_MODE=%v, MaxAge=%s", setting.RunMode, opts.MaxAge)) + } - // to remind users they are using non-prod setting. - h.Set("X-Gitea-Debug", "RUN_MODE="+setting.RunMode) + if opts.NoTransform { + directives = append(directives, "no-transform") } + h.Set("Cache-Control", strings.Join(directives, ", ")) +} - h.Set("Cache-Control", strings.Join(append(directives, additionalDirectives...), ", ")) +func CacheControlForPublicStatic() *CacheControlOptions { + return &CacheControlOptions{ + IsPublic: true, + MaxAge: setting.StaticCacheTime, + NoTransform: true, + } } -func ServeContentWithCacheControl(w http.ResponseWriter, req *http.Request, name string, modTime time.Time, content io.ReadSeeker) { - SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) - http.ServeContent(w, req, name, modTime, content) +func CacheControlForPrivateStatic() *CacheControlOptions { + return &CacheControlOptions{ + MaxAge: setting.StaticCacheTime, + NoTransform: true, + } } // HandleGenericETagCache handles ETag-based caching for a HTTP request. @@ -50,7 +70,8 @@ func HandleGenericETagCache(req *http.Request, w http.ResponseWriter, etag strin return true } } - SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) + // not sure whether it is a public content, so just use "private" (old behavior) + SetCacheControlInHeader(w.Header(), CacheControlForPrivateStatic()) return false } @@ -58,7 +79,7 @@ func HandleGenericETagCache(req *http.Request, w http.ResponseWriter, etag strin func checkIfNoneMatchIsValid(req *http.Request, etag string) bool { ifNoneMatch := req.Header.Get("If-None-Match") if len(ifNoneMatch) > 0 { - for _, item := range strings.Split(ifNoneMatch, ",") { + for item := range strings.SplitSeq(ifNoneMatch, ",") { item = strings.TrimPrefix(strings.TrimSpace(item), "W/") // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#directives if item == etag { return true @@ -95,6 +116,8 @@ func HandleGenericETagTimeCache(req *http.Request, w http.ResponseWriter, etag s } } } - SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) + + // not sure whether it is a public content, so just use "private" (old behavior) + SetCacheControlInHeader(w.Header(), CacheControlForPrivateStatic()) return false } diff --git a/modules/httplib/request.go b/modules/httplib/request.go index 880d7ad3cb..49ea6f4b73 100644 --- a/modules/httplib/request.go +++ b/modules/httplib/request.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "crypto/tls" + "errors" "fmt" "io" "net" @@ -99,18 +100,27 @@ func (r *Request) Param(key, value string) *Request { return r } -// Body adds request raw body. -// it supports string and []byte. +// Body adds request raw body. It supports string, []byte and io.Reader as body. func (r *Request) Body(data any) *Request { + if r == nil { + return nil + } switch t := data.(type) { + case nil: // do nothing case string: - bf := bytes.NewBufferString(t) + bf := strings.NewReader(t) r.req.Body = io.NopCloser(bf) r.req.ContentLength = int64(len(t)) case []byte: bf := bytes.NewBuffer(t) r.req.Body = io.NopCloser(bf) r.req.ContentLength = int64(len(t)) + case io.ReadCloser: + r.req.Body = t + case io.Reader: + r.req.Body = io.NopCloser(t) + default: + panic(fmt.Sprintf("unsupported request body type %T", t)) } return r } @@ -133,15 +143,15 @@ func (r *Request) getResponse() (*http.Response, error) { paramBody = paramBody[0 : len(paramBody)-1] } - if r.req.Method == "GET" && len(paramBody) > 0 { + if r.req.Method == http.MethodGet && len(paramBody) > 0 { if strings.Contains(r.url, "?") { r.url += "&" + paramBody } else { r.url = r.url + "?" + paramBody } - } else if r.req.Method == "POST" && r.req.Body == nil && len(paramBody) > 0 { + } else if r.req.Method == http.MethodPost && r.req.Body == nil && len(paramBody) > 0 { r.Header("Content-Type", "application/x-www-form-urlencoded") - r.Body(paramBody) + r.Body(paramBody) // string } var err error @@ -185,7 +195,11 @@ func (r *Request) getResponse() (*http.Response, error) { } // Response executes request client gets response manually. +// Caller MUST close the response body if no error occurs func (r *Request) Response() (*http.Response, error) { + if r == nil { + return nil, errors.New("invalid request") + } return r.getResponse() } diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go index 8fb667876e..7c1edf432d 100644 --- a/modules/httplib/serve.go +++ b/modules/httplib/serve.go @@ -33,6 +33,7 @@ type ServeHeaderOptions struct { ContentLength *int64 Disposition string // defaults to "attachment" Filename string + CacheIsPublic bool CacheDuration time.Duration // defaults to 5 minutes LastModified time.Time } @@ -72,11 +73,11 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) { header.Set("Access-Control-Expose-Headers", "Content-Disposition") } - duration := opts.CacheDuration - if duration == 0 { - duration = 5 * time.Minute - } - httpcache.SetCacheControlInHeader(header, duration) + httpcache.SetCacheControlInHeader(header, &httpcache.CacheControlOptions{ + IsPublic: opts.CacheIsPublic, + MaxAge: opts.CacheDuration, + NoTransform: true, + }) if !opts.LastModified.IsZero() { // http.TimeFormat required a UTC time, refer to https://pkg.go.dev/net/http#TimeFormat @@ -85,19 +86,15 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) { } // ServeData download file from io.Reader -func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath string, mineBuf []byte) { +func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byte, opts *ServeHeaderOptions) { // do not set "Content-Length", because the length could only be set by callers, and it needs to support range requests - opts := &ServeHeaderOptions{ - Filename: path.Base(filePath), - } - sniffedType := typesniffer.DetectContentType(mineBuf) // the "render" parameter came from year 2016: 638dd24c, it doesn't have clear meaning, so I think it could be removed later isPlain := sniffedType.IsText() || r.FormValue("render") != "" if setting.MimeTypeMap.Enabled { - fileExtension := strings.ToLower(filepath.Ext(filePath)) + fileExtension := strings.ToLower(filepath.Ext(opts.Filename)) opts.ContentType = setting.MimeTypeMap.Map[fileExtension] } @@ -114,7 +111,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri if isPlain { charset, err := charsetModule.DetectEncoding(mineBuf) if err != nil { - log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err) + log.Error("Detect raw file %s charset failed: %v, using by default utf-8", opts.Filename, err) charset = "utf-8" } opts.ContentTypeCharset = strings.ToLower(charset) @@ -142,7 +139,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri const mimeDetectionBufferLen = 1024 -func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath string, size int64, reader io.Reader) { +func ServeContentByReader(r *http.Request, w http.ResponseWriter, size int64, reader io.Reader, opts *ServeHeaderOptions) { buf := make([]byte, mimeDetectionBufferLen) n, err := util.ReadAtMost(reader, buf) if err != nil { @@ -152,7 +149,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin if n >= 0 { buf = buf[:n] } - setServeHeadersByFile(r, w, filePath, buf) + setServeHeadersByFile(r, w, buf, opts) // reset the reader to the beginning reader = io.MultiReader(bytes.NewReader(buf), reader) @@ -215,7 +212,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin _, _ = io.CopyN(w, reader, partialLength) // just like http.ServeContent, not necessary to handle the error } -func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath string, modTime *time.Time, reader io.ReadSeeker) { +func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, modTime *time.Time, reader io.ReadSeeker, opts *ServeHeaderOptions) { buf := make([]byte, mimeDetectionBufferLen) n, err := util.ReadAtMost(reader, buf) if err != nil { @@ -229,9 +226,9 @@ func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath s if n >= 0 { buf = buf[:n] } - setServeHeadersByFile(r, w, filePath, buf) + setServeHeadersByFile(r, w, buf, opts) if modTime == nil { modTime = &time.Time{} } - http.ServeContent(w, r, path.Base(filePath), *modTime, reader) + http.ServeContent(w, r, opts.Filename, *modTime, reader) } diff --git a/modules/httplib/serve_test.go b/modules/httplib/serve_test.go index c2229dffe9..78b88c9b5f 100644 --- a/modules/httplib/serve_test.go +++ b/modules/httplib/serve_test.go @@ -4,15 +4,16 @@ package httplib import ( - "fmt" "net/http" "net/http/httptest" "net/url" "os" + "strconv" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestServeContentByReader(t *testing.T) { @@ -22,14 +23,14 @@ func TestServeContentByReader(t *testing.T) { _, rangeStr, _ := strings.Cut(t.Name(), "_range_") r := &http.Request{Header: http.Header{}, Form: url.Values{}} if rangeStr != "" { - r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + r.Header.Set("Range", "bytes="+rangeStr) } reader := strings.NewReader(data) w := httptest.NewRecorder() - ServeContentByReader(r, w, "test", int64(len(data)), reader) + ServeContentByReader(r, w, int64(len(data)), reader, &ServeHeaderOptions{}) assert.Equal(t, expectedStatusCode, w.Code) if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { - assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, strconv.Itoa(len(expectedContent)), w.Header().Get("Content-Length")) assert.Equal(t, expectedContent, w.Body.String()) } } @@ -67,20 +68,18 @@ func TestServeContentByReadSeeker(t *testing.T) { _, rangeStr, _ := strings.Cut(t.Name(), "_range_") r := &http.Request{Header: http.Header{}, Form: url.Values{}} if rangeStr != "" { - r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + r.Header.Set("Range", "bytes="+rangeStr) } seekReader, err := os.OpenFile(tmpFile, os.O_RDONLY, 0o644) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) defer seekReader.Close() w := httptest.NewRecorder() - ServeContentByReadSeeker(r, w, "test", nil, seekReader) + ServeContentByReadSeeker(r, w, nil, seekReader, &ServeHeaderOptions{}) assert.Equal(t, expectedStatusCode, w.Code) if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { - assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, strconv.Itoa(len(expectedContent)), w.Header().Get("Content-Length")) assert.Equal(t, expectedContent, w.Body.String()) } } diff --git a/modules/httplib/url.go b/modules/httplib/url.go index e3bad1e5fb..f51506ac3b 100644 --- a/modules/httplib/url.go +++ b/modules/httplib/url.go @@ -5,6 +5,7 @@ package httplib import ( "context" + "net" "net/http" "net/url" "strings" @@ -52,28 +53,34 @@ func getRequestScheme(req *http.Request) string { return "" } -// GuessCurrentAppURL tries to guess the current full app URL (with sub-path) by http headers. It always has a '/' suffix, exactly the same as setting.AppURL +// GuessCurrentAppURL tries to guess the current full public URL (with sub-path) by http headers. It always has a '/' suffix, exactly the same as setting.AppURL +// TODO: should rename it to GuessCurrentPublicURL in the future func GuessCurrentAppURL(ctx context.Context) string { return GuessCurrentHostURL(ctx) + setting.AppSubURL + "/" } // GuessCurrentHostURL tries to guess the current full host URL (no sub-path) by http headers, there is no trailing slash. func GuessCurrentHostURL(ctx context.Context) string { - req, ok := ctx.Value(RequestContextKey).(*http.Request) - if !ok { - return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") - } - // If no scheme provided by reverse proxy, then do not guess the AppURL, use the configured one. + // Try the best guess to get the current host URL (will be used for public URL) by http headers. // At the moment, if site admin doesn't configure the proxy headers correctly, then Gitea would guess wrong. // There are some cases: // 1. The reverse proxy is configured correctly, it passes "X-Forwarded-Proto/Host" headers. Perfect, Gitea can handle it correctly. // 2. The reverse proxy is not configured correctly, doesn't pass "X-Forwarded-Proto/Host" headers, eg: only one "proxy_pass http://gitea:3000" in Nginx. // 3. There is no reverse proxy. - // Without an extra config option, Gitea is impossible to distinguish between case 2 and case 3, - // then case 2 would result in wrong guess like guessed AppURL becomes "http://gitea:3000/", which is not accessible by end users. - // So in the future maybe it should introduce a new config option, to let site admin decide how to guess the AppURL. + // Without more information, Gitea is impossible to distinguish between case 2 and case 3, then case 2 would result in + // wrong guess like guessed public URL becomes "http://gitea:3000/" behind a "https" reverse proxy, which is not accessible by end users. + // So we introduced "PUBLIC_URL_DETECTION" option, to control the guessing behavior to satisfy different use cases. + req, ok := ctx.Value(RequestContextKey).(*http.Request) + if !ok { + return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") + } reqScheme := getRequestScheme(req) if reqScheme == "" { + // if no reverse proxy header, try to use "Host" header for absolute URL + if setting.PublicURLDetection == setting.PublicURLAuto && req.Host != "" { + return util.Iif(req.TLS == nil, "http://", "https://") + req.Host + } + // fall back to default AppURL return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") } // X-Forwarded-Host has many problems: non-standard, not well-defined (X-Forwarded-Port or not), conflicts with Host header. @@ -81,8 +88,14 @@ func GuessCurrentHostURL(ctx context.Context) string { return reqScheme + "://" + req.Host } -// MakeAbsoluteURL tries to make a link to an absolute URL: -// * If link is empty, it returns the current app URL. +func GuessCurrentHostDomain(ctx context.Context) string { + _, host, _ := strings.Cut(GuessCurrentHostURL(ctx), "://") + domain, _, _ := net.SplitHostPort(host) + return util.IfZero(domain, host) +} + +// MakeAbsoluteURL tries to make a link to an absolute public URL: +// * If link is empty, it returns the current public URL. // * If link is absolute, it returns the link. // * Otherwise, it returns the current host URL + link, the link itself should have correct sub-path (AppSubURL) if needed. func MakeAbsoluteURL(ctx context.Context, link string) string { @@ -95,25 +108,77 @@ func MakeAbsoluteURL(ctx context.Context, link string) string { return GuessCurrentHostURL(ctx) + "/" + strings.TrimPrefix(link, "/") } -func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool { +type urlType int + +const ( + urlTypeGiteaAbsolute urlType = iota + 1 // "http://gitea/subpath" + urlTypeGiteaPageRelative // "/subpath" + urlTypeGiteaSiteRelative // "?key=val" + urlTypeUnknown // "http://other" +) + +func detectURLRoutePath(ctx context.Context, s string) (routePath string, ut urlType) { u, err := url.Parse(s) if err != nil { - return false + return "", urlTypeUnknown } + cleanedPath := "" if u.Path != "" { - cleanedPath := util.PathJoinRelX(u.Path) - if cleanedPath == "" || cleanedPath == "." { - u.Path = "/" - } else { - u.Path += "/" + cleanedPath + "/" - } + cleanedPath = util.PathJoinRelX(u.Path) + cleanedPath = util.Iif(cleanedPath == ".", "", "/"+cleanedPath) } if urlIsRelative(s, u) { - return u.Path == "" || strings.HasPrefix(strings.ToLower(u.Path), strings.ToLower(setting.AppSubURL+"/")) - } - if u.Path == "" { - u.Path = "/" + if u.Path == "" { + return "", urlTypeGiteaPageRelative + } + if strings.HasPrefix(strings.ToLower(cleanedPath+"/"), strings.ToLower(setting.AppSubURL+"/")) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaSiteRelative + } + return "", urlTypeUnknown } + u.Path = cleanedPath + "/" urlLower := strings.ToLower(u.String()) - return strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) || strings.HasPrefix(urlLower, strings.ToLower(GuessCurrentAppURL(ctx))) + if strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute + } + guessedCurURL := GuessCurrentAppURL(ctx) + if strings.HasPrefix(urlLower, strings.ToLower(guessedCurURL)) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute + } + return "", urlTypeUnknown +} + +func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool { + _, ut := detectURLRoutePath(ctx, s) + return ut != urlTypeUnknown +} + +type GiteaSiteURL struct { + RoutePath string + OwnerName string + RepoName string + RepoSubPath string +} + +func ParseGiteaSiteURL(ctx context.Context, s string) *GiteaSiteURL { + routePath, ut := detectURLRoutePath(ctx, s) + if ut == urlTypeUnknown || ut == urlTypeGiteaPageRelative { + return nil + } + ret := &GiteaSiteURL{RoutePath: routePath} + fields := strings.SplitN(strings.TrimPrefix(ret.RoutePath, "/"), "/", 3) + + // TODO: now it only does a quick check for some known reserved paths, should do more strict checks in the future + if fields[0] == "attachments" { + return ret + } + if len(fields) < 2 { + return ret + } + ret.OwnerName = fields[0] + ret.RepoName = fields[1] + if len(fields) == 3 { + ret.RepoSubPath = "/" + fields[2] + } + return ret } diff --git a/modules/httplib/url_test.go b/modules/httplib/url_test.go index fc6c91cd3a..0ffb0cac05 100644 --- a/modules/httplib/url_test.go +++ b/modules/httplib/url_test.go @@ -5,6 +5,7 @@ package httplib import ( "context" + "crypto/tls" "net/http" "testing" @@ -39,12 +40,48 @@ func TestIsRelativeURL(t *testing.T) { } } +func TestGuessCurrentHostURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + headersWithProto := http.Header{"X-Forwarded-Proto": {"https"}} + + t.Run("Legacy", func(t *testing.T) { + defer test.MockVariableValue(&setting.PublicURLDetection, setting.PublicURLLegacy)() + + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(t.Context())) + + // legacy: "Host" is not used when there is no "X-Forwarded-Proto" header + ctx := context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000"}) + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(ctx)) + + // if "X-Forwarded-Proto" exists, then use it and "Host" header + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000", Header: headersWithProto}) + assert.Equal(t, "https://req-host:3000", GuessCurrentHostURL(ctx)) + }) + + t.Run("Auto", func(t *testing.T) { + defer test.MockVariableValue(&setting.PublicURLDetection, setting.PublicURLAuto)() + + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(t.Context())) + + // auto: always use "Host" header, the scheme is determined by "X-Forwarded-Proto" header, or TLS config if no "X-Forwarded-Proto" header + ctx := context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000"}) + assert.Equal(t, "http://req-host:3000", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host", TLS: &tls.ConnectionState{}}) + assert.Equal(t, "https://req-host", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000", Header: headersWithProto}) + assert.Equal(t, "https://req-host:3000", GuessCurrentHostURL(ctx)) + }) +} + func TestMakeAbsoluteURL(t *testing.T) { defer test.MockVariableValue(&setting.Protocol, "http")() defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() defer test.MockVariableValue(&setting.AppSubURL, "/sub")() - ctx := context.Background() + ctx := t.Context() assert.Equal(t, "http://cfg-host/sub/", MakeAbsoluteURL(ctx, "")) assert.Equal(t, "http://cfg-host/foo", MakeAbsoluteURL(ctx, "foo")) assert.Equal(t, "http://cfg-host/foo", MakeAbsoluteURL(ctx, "/foo")) @@ -76,7 +113,7 @@ func TestMakeAbsoluteURL(t *testing.T) { func TestIsCurrentGiteaSiteURL(t *testing.T) { defer test.MockVariableValue(&setting.AppURL, "http://localhost:3000/sub/")() defer test.MockVariableValue(&setting.AppSubURL, "/sub")() - ctx := context.Background() + ctx := t.Context() good := []string{ "?key=val", "/sub", @@ -122,3 +159,26 @@ func TestIsCurrentGiteaSiteURL(t *testing.T) { assert.True(t, IsCurrentGiteaSiteURL(ctx, "https://user-host")) assert.False(t, IsCurrentGiteaSiteURL(ctx, "https://forwarded-host")) } + +func TestParseGiteaSiteURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://localhost:3000/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + ctx := t.Context() + tests := []struct { + url string + exp *GiteaSiteURL + }{ + {"http://localhost:3000/sub?k=v", &GiteaSiteURL{RoutePath: ""}}, + {"http://localhost:3000/sub/", &GiteaSiteURL{RoutePath: ""}}, + {"http://localhost:3000/sub/foo", &GiteaSiteURL{RoutePath: "/foo"}}, + {"http://localhost:3000/sub/foo/bar", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}}, + {"http://localhost:3000/sub/foo/bar/", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}}, + {"http://localhost:3000/sub/attachments/bar", &GiteaSiteURL{RoutePath: "/attachments/bar"}}, + {"http://localhost:3000/other", nil}, + {"http://other/", nil}, + } + for _, test := range tests { + su := ParseGiteaSiteURL(ctx, test.url) + assert.Equal(t, test.exp, su, "URL = %s", test.url) + } +} diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index 772317fa59..70f0995a01 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -16,7 +16,7 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/indexer" path_filter "code.gitea.io/gitea/modules/indexer/code/bleve/token/path" "code.gitea.io/gitea/modules/indexer/code/internal" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" @@ -24,11 +24,11 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" "github.com/blevesearch/bleve/v2" analyzer_custom "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - "github.com/blevesearch/bleve/v2/analysis/token/camelcase" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter" @@ -70,7 +70,7 @@ const ( filenameIndexerAnalyzer = "filenameIndexerAnalyzer" filenameIndexerTokenizer = "filenameIndexerTokenizer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 8 + repoIndexerLatestVersion = 9 ) // generateBleveIndexMapping generates a bleve index mapping for the repo indexer @@ -107,7 +107,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { "type": analyzer_custom.Name, "char_filters": []string{}, "tokenizer": letter.Name, - "token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name}, + "token_filters": []string{unicodeNormalizeName, lowercase.Name}, }); err != nil { return nil, err } @@ -136,6 +136,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new bleve local indexer func NewIndexer(indexDir string) *Indexer { inner := inner_bleve.NewIndexer(indexDir, repoIndexerLatestVersion, generateBleveIndexMapping) @@ -158,7 +162,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro var err error if !update.Sized { var stdout string - stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, err = git.NewCommand("cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil { return err } @@ -185,7 +189,8 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro return err } else if !typesniffer.DetectContentType(fileContents).IsText() { // FIXME: UTF-16 files will probably fail here - return nil + // Even if the file is not recognized as a "text file", we could still put its name into the indexers to make the filename become searchable, while leave the content to empty. + fileContents = nil } if _, err = batchReader.Discard(1); err != nil { @@ -211,12 +216,7 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize) if len(changes.Updates) > 0 { - r, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - return err - } - defer r.Close() - gitBatch, err := r.NewBatch(ctx) + gitBatch, err := git.NewBatch(ctx, repo.RepoPath()) if err != nil { return err } @@ -260,17 +260,31 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int var ( indexerQuery query.Query keywordQuery query.Query + contentQuery query.Query ) pathQuery := bleve.NewPrefixQuery(strings.ToLower(opts.Keyword)) pathQuery.FieldVal = "Filename" pathQuery.SetBoost(10) - contentQuery := bleve.NewMatchQuery(opts.Keyword) - contentQuery.FieldVal = "Content" - - if opts.IsKeywordFuzzy { - contentQuery.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword) + searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeExact { + // 1.21 used NewPrefixQuery, but it seems not working well, and later releases changed to NewMatchPhraseQuery + q := bleve.NewMatchPhraseQuery(opts.Keyword) + q.Analyzer = repoIndexerAnalyzer + q.FieldVal = "Content" + contentQuery = q + } else /* words */ { + q := bleve.NewMatchQuery(opts.Keyword) + q.FieldVal = "Content" + q.Analyzer = repoIndexerAnalyzer + if searchMode == indexer.SearchModeFuzzy { + // this logic doesn't seem right, it is only used to pass the test-case `Keyword: "dESCRIPTION"`, which doesn't seem to be a real-life use-case. + q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword) + } else { + q.Operator = query.MatchQueryOperatorAnd + } + contentQuery = q } keywordQuery = bleve.NewDisjunctionQuery(contentQuery, pathQuery) diff --git a/modules/indexer/code/bleve/token/path/path.go b/modules/indexer/code/bleve/token/path/path.go index 107e0da109..6dfc12f146 100644 --- a/modules/indexer/code/bleve/token/path/path.go +++ b/modules/indexer/code/bleve/token/path/path.go @@ -51,13 +51,13 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke slices.Reverse(input) } - for i := 0; i < len(input); i++ { + for i := range input { var sb strings.Builder - sb.WriteString(string(input[0].Term)) + sb.Write(input[0].Term) for j := 1; j < i; j++ { sb.WriteString("/") - sb.WriteString(string(input[j].Term)) + sb.Write(input[j].Term) } term := sb.String() @@ -97,5 +97,9 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke } func init() { - registry.RegisterTokenFilter(Name, TokenFilterConstructor) + // FIXME: move it to the bleve's init function, but do not call it in global init + err := registry.RegisterTokenFilter(Name, TokenFilterConstructor) + if err != nil { + panic(err) + } } diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index 1c4dd39eff..f925ce396a 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -15,7 +15,7 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/code/internal" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" "github.com/go-enry/go-enry/v2" "github.com/olivere/elastic/v7" @@ -45,6 +46,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new elasticsearch indexer func NewIndexer(url, indexerName string) *Indexer { inner := inner_elasticsearch.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping) @@ -142,7 +147,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro var err error if !update.Sized { var stdout string - stdout, _, err = git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, err = git.NewCommand("cat-file", "-s").AddDynamicArguments(update.BlobSha).RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil { return nil, err } @@ -203,12 +208,7 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elasti func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { reqs := make([]elastic.BulkableRequest, 0) if len(changes.Updates) > 0 { - r, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - return err - } - defer r.Close() - batch, err := r.NewBatch(ctx) + batch, err := git.NewBatch(ctx, repo.RepoPath()) if err != nil { return err } @@ -359,13 +359,16 @@ func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLan // Search searches for codes and language stats by given conditions. func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { - searchType := esMultiMatchTypePhrasePrefix - if opts.IsKeywordFuzzy { - searchType = esMultiMatchTypeBestFields + var contentQuery elastic.Query + searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeExact { + // 1.21 used NewMultiMatchQuery().Type(esMultiMatchTypePhrasePrefix), but later releases changed to NewMatchPhraseQuery + contentQuery = elastic.NewMatchPhraseQuery("content", opts.Keyword) + } else /* words */ { + contentQuery = elastic.NewMultiMatchQuery("content", opts.Keyword).Type(esMultiMatchTypeBestFields).Operator("and") } - kwQuery := elastic.NewBoolQuery().Should( - elastic.NewMultiMatchQuery(opts.Keyword, "content").Type(searchType), + contentQuery, elastic.NewMultiMatchQuery(opts.Keyword, "filename^10").Type(esMultiMatchTypePhrasePrefix), ) query := elastic.NewBoolQuery() diff --git a/modules/indexer/code/elasticsearch/elasticsearch_test.go b/modules/indexer/code/elasticsearch/elasticsearch_test.go index a6d2af92b2..e8f1f202ce 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/code/elasticsearch/elasticsearch_test.go @@ -11,6 +11,6 @@ import ( func TestIndexPos(t *testing.T) { startIdx, endIdx := contentMatchIndexPos("test index start and end", "start", "end") - assert.EqualValues(t, 11, startIdx) - assert.EqualValues(t, 15, endIdx) + assert.Equal(t, 11, startIdx) + assert.Equal(t, 15, endIdx) } diff --git a/modules/indexer/code/git.go b/modules/indexer/code/git.go index df9783288b..41bc74e6ec 100644 --- a/modules/indexer/code/git.go +++ b/modules/indexer/code/git.go @@ -16,7 +16,7 @@ import ( ) func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (string, error) { - stdout, _, err := git.NewCommand(ctx, "show-ref", "-s").AddDynamicArguments(git.BranchPrefix + repo.DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, err := git.NewCommand("show-ref", "-s").AddDynamicArguments(git.BranchPrefix+repo.DefaultBranch).RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil { return "", err } @@ -32,8 +32,8 @@ func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision s needGenesis := len(status.CommitSha) == 0 if !needGenesis { - hasAncestorCmd := git.NewCommand(ctx, "merge-base").AddDynamicArguments(status.CommitSha, revision) - stdout, _, _ := hasAncestorCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + hasAncestorCmd := git.NewCommand("merge-base").AddDynamicArguments(status.CommitSha, revision) + stdout, _, _ := hasAncestorCmd.RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) needGenesis = len(stdout) == 0 } @@ -86,7 +86,7 @@ func parseGitLsTreeOutput(stdout []byte) ([]internal.FileUpdate, error) { // genesisChanges get changes to add repo to the indexer for the first time func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) { var changes internal.RepoChanges - stdout, _, runErr := git.NewCommand(ctx, "ls-tree", "--full-tree", "-l", "-r").AddDynamicArguments(revision).RunStdBytes(&git.RunOpts{Dir: repo.RepoPath()}) + stdout, _, runErr := git.NewCommand("ls-tree", "--full-tree", "-l", "-r").AddDynamicArguments(revision).RunStdBytes(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if runErr != nil { return nil, runErr } @@ -98,8 +98,8 @@ func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision s // nonGenesisChanges get changes since the previous indexer update func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) { - diffCmd := git.NewCommand(ctx, "diff", "--name-status").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision) - stdout, _, runErr := diffCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) + diffCmd := git.NewCommand("diff", "--name-status").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision) + stdout, _, runErr := diffCmd.RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if runErr != nil { // previous commit sha may have been removed by a force push, so // try rebuilding from scratch @@ -115,9 +115,9 @@ func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revisio updatedFilenames := make([]string, 0, 10) updateChanges := func() error { - cmd := git.NewCommand(ctx, "ls-tree", "--full-tree", "-l").AddDynamicArguments(revision). + cmd := git.NewCommand("ls-tree", "--full-tree", "-l").AddDynamicArguments(revision). AddDashesAndList(updatedFilenames...) - lsTreeStdout, _, err := cmd.RunStdBytes(&git.RunOpts{Dir: repo.RepoPath()}) + lsTreeStdout, _, err := cmd.RunStdBytes(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil { return err } @@ -129,8 +129,8 @@ func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revisio changes.Updates = append(changes.Updates, updates...) return nil } - lines := strings.Split(stdout, "\n") - for _, line := range lines { + lines := strings.SplitSeq(stdout, "\n") + for line := range lines { line = strings.TrimSpace(line) if len(line) == 0 { continue diff --git a/modules/indexer/code/gitgrep/gitgrep.go b/modules/indexer/code/gitgrep/gitgrep.go new file mode 100644 index 0000000000..6f6e0b47b9 --- /dev/null +++ b/modules/indexer/code/gitgrep/gitgrep.go @@ -0,0 +1,66 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgrep + +import ( + "context" + "fmt" + "strings" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/indexer" + code_indexer "code.gitea.io/gitea/modules/indexer/code" + "code.gitea.io/gitea/modules/setting" +) + +func indexSettingToGitGrepPathspecList() (list []string) { + for _, expr := range setting.Indexer.IncludePatterns { + list = append(list, ":(glob)"+expr.PatternString()) + } + for _, expr := range setting.Indexer.ExcludePatterns { + list = append(list, ":(glob,exclude)"+expr.PatternString()) + } + return list +} + +func PerformSearch(ctx context.Context, page int, repoID int64, gitRepo *git.Repository, ref git.RefName, keyword string, searchMode indexer.SearchModeType) (searchResults []*code_indexer.Result, total int, err error) { + grepMode := git.GrepModeWords + switch searchMode { + case indexer.SearchModeExact: + grepMode = git.GrepModeExact + case indexer.SearchModeRegexp: + grepMode = git.GrepModeRegexp + } + res, err := git.GrepSearch(ctx, gitRepo, keyword, git.GrepOptions{ + ContextLineNumber: 1, + GrepMode: grepMode, + RefName: ref.String(), + PathspecList: indexSettingToGitGrepPathspecList(), + }) + if err != nil { + // TODO: if no branch exists, it reports: exit status 128, fatal: this operation must be run in a work tree. + return nil, 0, fmt.Errorf("git.GrepSearch: %w", err) + } + commitID, err := gitRepo.GetRefCommitID(ref.String()) + if err != nil { + return nil, 0, fmt.Errorf("gitRepo.GetRefCommitID: %w", err) + } + + total = len(res) + pageStart := min((page-1)*setting.UI.RepoSearchPagingNum, len(res)) + pageEnd := min(page*setting.UI.RepoSearchPagingNum, len(res)) + res = res[pageStart:pageEnd] + for _, r := range res { + searchResults = append(searchResults, &code_indexer.Result{ + RepoID: repoID, + Filename: r.Filename, + CommitID: commitID, + // UpdatedUnix: not supported yet + // Language: not supported yet + // Color: not supported yet + Lines: code_indexer.HighlightSearchResultCode(r.Filename, "", r.LineNumbers, strings.Join(r.LineCodes, "\n")), + }) + } + return searchResults, total, nil +} diff --git a/modules/indexer/code/gitgrep/gitgrep_test.go b/modules/indexer/code/gitgrep/gitgrep_test.go new file mode 100644 index 0000000000..97dda9d966 --- /dev/null +++ b/modules/indexer/code/gitgrep/gitgrep_test.go @@ -0,0 +1,19 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgrep + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" +) + +func TestIndexSettingToGitGrepPathspecList(t *testing.T) { + defer test.MockVariableValue(&setting.Indexer.IncludePatterns, setting.IndexerGlobFromString("a"))() + defer test.MockVariableValue(&setting.Indexer.ExcludePatterns, setting.IndexerGlobFromString("b"))() + assert.Equal(t, []string{":(glob)a", ":(glob,exclude)b"}, indexSettingToGitGrepPathspecList()) +} diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index c1ab26569c..6035ddfe95 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/code/bleve" "code.gitea.io/gitea/modules/indexer/code/elasticsearch" "code.gitea.io/gitea/modules/indexer/code/internal" @@ -29,13 +30,11 @@ var ( // When the real indexer is not ready, it will be a dummy indexer which will return error to explain it's not ready. // So it's always safe use it as *globalIndexer.Load() and call its methods. globalIndexer atomic.Pointer[internal.Indexer] - dummyIndexer *internal.Indexer ) func init() { - i := internal.NewDummyIndexer() - dummyIndexer = &i - globalIndexer.Store(dummyIndexer) + dummyIndexer := internal.NewDummyIndexer() + globalIndexer.Store(&dummyIndexer) } func index(ctx context.Context, indexer internal.Indexer, repoID int64) error { @@ -123,13 +122,12 @@ func Init() { for _, indexerData := range items { log.Trace("IndexerData Process Repo: %d", indexerData.RepoID) if err := index(ctx, indexer, indexerData.RepoID); err != nil { - unhandled = append(unhandled, indexerData) if !setting.IsInTesting { log.Error("Codes indexer handler: index error for repo %v: %v", indexerData.RepoID, err) } } } - return unhandled + return nil // do not re-queue the failed items, otherwise some broken repo will block the queue } indexerQueue = queue.CreateUniqueQueue(ctx, "code_indexer", handler) @@ -305,3 +303,11 @@ func populateRepoIndexer(ctx context.Context) { } log.Info("Done (re)populating the repo indexer with existing repositories") } + +func SupportedSearchModes() []indexer.SearchMode { + gi := globalIndexer.Load() + if gi == nil { + return nil + } + return (*gi).SupportedSearchModes() +} diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index d04088531a..78fea22f10 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -11,10 +11,13 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/git" + indexer_module "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/code/bleve" "code.gitea.io/gitea/modules/indexer/code/elasticsearch" "code.gitea.io/gitea/modules/indexer/code/internal" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/util" _ "code.gitea.io/gitea/models" _ "code.gitea.io/gitea/models/actions" @@ -35,13 +38,14 @@ func TestMain(m *testing.M) { func testIndexer(name string, t *testing.T, indexer internal.Indexer) { t.Run(name, func(t *testing.T) { - assert.NoError(t, setupRepositoryIndexes(git.DefaultContext, indexer)) + assert.NoError(t, setupRepositoryIndexes(t.Context(), indexer)) keywords := []struct { - RepoIDs []int64 - Keyword string - Langs int - Results []codeSearchResult + RepoIDs []int64 + Keyword string + Langs int + SearchMode indexer_module.SearchModeType + Results []codeSearchResult }{ // Search for an exact match on the contents of a file // This scenario yields a single result (the file README.md on the repo '1') @@ -182,9 +186,10 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, // Search for matches on the contents of files regardless of case. { - RepoIDs: nil, - Keyword: "dESCRIPTION", - Langs: 1, + RepoIDs: nil, + Keyword: "dESCRIPTION", + Langs: 1, + SearchMode: indexer_module.SearchModeFuzzy, Results: []codeSearchResult{ { Filename: "README.md", @@ -192,7 +197,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, }, }, - // Search for an exact match on the filename within the repo '62' (case insenstive). + // Search for an exact match on the filename within the repo '62' (case-insensitive). // This scenario yields a single result (the file avocado.md on the repo '62') { RepoIDs: []int64{62}, @@ -205,7 +210,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, }, }, - // Search for matches on the contents of files when the criteria is a expression. + // Search for matches on the contents of files when the criteria are an expression. { RepoIDs: []int64{62}, Keyword: "console.log", @@ -217,7 +222,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, }, }, - // Search for matches on the contents of files when the criteria is part of a expression. + // Search for matches on the contents of files when the criteria are parts of an expression. { RepoIDs: []int64{62}, Keyword: "log", @@ -233,17 +238,17 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { for _, kw := range keywords { t.Run(kw.Keyword, func(t *testing.T) { - total, res, langs, err := indexer.Search(context.TODO(), &internal.SearchOptions{ - RepoIDs: kw.RepoIDs, - Keyword: kw.Keyword, + total, res, langs, err := indexer.Search(t.Context(), &internal.SearchOptions{ + RepoIDs: kw.RepoIDs, + Keyword: kw.Keyword, + SearchMode: util.IfZero(kw.SearchMode, indexer_module.SearchModeWords), Paginator: &db.ListOptions{ Page: 1, PageSize: 10, }, - IsKeywordFuzzy: true, }) - assert.NoError(t, err) - assert.Len(t, langs, kw.Langs) + require.NoError(t, err) + require.Len(t, langs, kw.Langs) hits := make([]codeSearchResult, 0, len(res)) @@ -273,22 +278,22 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }) } - assert.NoError(t, tearDownRepositoryIndexes(indexer)) + assert.NoError(t, tearDownRepositoryIndexes(t.Context(), indexer)) }) } func TestBleveIndexAndSearch(t *testing.T) { unittest.PrepareTestEnv(t) - + defer test.MockVariableValue(&setting.Indexer.TypeBleveMaxFuzzniess, 2)() dir := t.TempDir() idx := bleve.NewIndexer(dir) defer idx.Close() - _, err := idx.Init(context.Background()) + _, err := idx.Init(t.Context()) require.NoError(t, err) - testIndexer("beleve", t, idx) + testIndexer("bleve", t, idx) } func TestESIndexAndSearch(t *testing.T) { @@ -301,11 +306,11 @@ func TestESIndexAndSearch(t *testing.T) { } indexer := elasticsearch.NewIndexer(u, "gitea_codes") - if _, err := indexer.Init(context.Background()); err != nil { + if _, err := indexer.Init(t.Context()); err != nil { if indexer != nil { indexer.Close() } - assert.FailNow(t, "Unable to init ES indexer Error: %v", err) + require.NoError(t, err, "Unable to init ES indexer") } defer indexer.Close() @@ -322,9 +327,9 @@ func setupRepositoryIndexes(ctx context.Context, indexer internal.Indexer) error return nil } -func tearDownRepositoryIndexes(indexer internal.Indexer) error { +func tearDownRepositoryIndexes(ctx context.Context, indexer internal.Indexer) error { for _, repoID := range repositoriesToSearch() { - if err := indexer.Delete(context.Background(), repoID); err != nil { + if err := indexer.Delete(ctx, repoID); err != nil { return err } } diff --git a/modules/indexer/code/internal/indexer.go b/modules/indexer/code/internal/indexer.go index c259fcd26e..d58b028124 100644 --- a/modules/indexer/code/internal/indexer.go +++ b/modules/indexer/code/internal/indexer.go @@ -5,10 +5,11 @@ package internal import ( "context" - "fmt" + "errors" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/internal" ) @@ -18,6 +19,7 @@ type Indexer interface { Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error Delete(ctx context.Context, repoID int64) error Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) + SupportedSearchModes() []indexer.SearchMode } type SearchOptions struct { @@ -25,7 +27,7 @@ type SearchOptions struct { Keyword string Language string - IsKeywordFuzzy bool + SearchMode indexer.SearchModeType db.Paginator } @@ -41,14 +43,18 @@ type dummyIndexer struct { internal.Indexer } +func (d *dummyIndexer) SupportedSearchModes() []indexer.SearchMode { + return nil +} + func (d *dummyIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Delete(ctx context.Context, repoID int64) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) { - return 0, nil, nil, fmt.Errorf("indexer is not ready") + return 0, nil, nil, errors.New("indexer is not ready") } diff --git a/modules/indexer/code/internal/util.go b/modules/indexer/code/internal/util.go index 5b95783d9f..fa958be473 100644 --- a/modules/indexer/code/internal/util.go +++ b/modules/indexer/code/internal/util.go @@ -10,9 +10,7 @@ import ( "code.gitea.io/gitea/modules/log" ) -const ( - filenameMatchNumberOfLines = 7 // Copied from github search -) +const filenameMatchNumberOfLines = 7 // Copied from GitHub search func FilenameIndexerID(repoID int64, filename string) string { return internal.Base36(repoID) + "_" + filename @@ -35,7 +33,7 @@ func FilenameOfIndexerID(indexerID string) string { return indexerID[index+1:] } -// Given the contents of file, returns the boundaries of its first seven lines. +// FilenameMatchIndexPos returns the boundaries of its first seven lines. func FilenameMatchIndexPos(content string) (int, int) { count := 1 for i, c := range content { diff --git a/modules/indexer/code/search.go b/modules/indexer/code/search.go index 74c957dde6..a7a5d7d2e3 100644 --- a/modules/indexer/code/search.go +++ b/modules/indexer/code/search.go @@ -77,7 +77,7 @@ func HighlightSearchResultCode(filename, language string, lineNums []int, code s // The lineNums outputted by highlight.Code might not match the original lineNums, because "highlight" removes the last `\n` lines := make([]*ResultLine, min(len(highlightedLines), len(lineNums))) - for i := 0; i < len(lines); i++ { + for i := range lines { lines[i] = &ResultLine{ Num: lineNums[i], FormattedContent: template.HTML(highlightedLines[i]), @@ -129,7 +129,6 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res } // PerformSearch perform a search on a repository -// if isFuzzy is true set the Damerau-Levenshtein distance from 0 to 2 func PerformSearch(ctx context.Context, opts *SearchOptions) (int, []*Result, []*SearchResultLanguages, error) { if opts == nil || len(opts.Keyword) == 0 { return 0, nil, nil, nil diff --git a/modules/indexer/indexer.go b/modules/indexer/indexer.go new file mode 100644 index 0000000000..1e0f81de89 --- /dev/null +++ b/modules/indexer/indexer.go @@ -0,0 +1,54 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package indexer + +type SearchModeType string + +const ( + SearchModeExact SearchModeType = "exact" + SearchModeWords SearchModeType = "words" + SearchModeFuzzy SearchModeType = "fuzzy" + SearchModeRegexp SearchModeType = "regexp" +) + +type SearchMode struct { + ModeValue SearchModeType + TooltipTrKey string + TitleTrKey string +} + +func SearchModesExactWords() []SearchMode { + return []SearchMode{ + { + ModeValue: SearchModeExact, + TooltipTrKey: "search.exact_tooltip", + TitleTrKey: "search.exact", + }, + { + ModeValue: SearchModeWords, + TooltipTrKey: "search.words_tooltip", + TitleTrKey: "search.words", + }, + } +} + +func SearchModesExactWordsFuzzy() []SearchMode { + return append(SearchModesExactWords(), []SearchMode{ + { + ModeValue: SearchModeFuzzy, + TooltipTrKey: "search.fuzzy_tooltip", + TitleTrKey: "search.fuzzy", + }, + }...) +} + +func GitGrepSupportedSearchModes() []SearchMode { + return append(SearchModesExactWords(), []SearchMode{ + { + ModeValue: SearchModeRegexp, + TooltipTrKey: "search.regexp_tooltip", + TitleTrKey: "search.regexp", + }, + }...) +} diff --git a/modules/indexer/internal/bleve/indexer.go b/modules/indexer/internal/bleve/indexer.go index 01e53ca636..9d1e24a874 100644 --- a/modules/indexer/internal/bleve/indexer.go +++ b/modules/indexer/internal/bleve/indexer.go @@ -5,7 +5,7 @@ package bleve import ( "context" - "fmt" + "errors" "code.gitea.io/gitea/modules/indexer/internal" "code.gitea.io/gitea/modules/log" @@ -39,11 +39,11 @@ func NewIndexer(indexDir string, version int, mappingGetter func() (mapping.Inde // Init initializes the indexer func (i *Indexer) Init(_ context.Context) (bool, error) { if i == nil { - return false, fmt.Errorf("cannot init nil indexer") + return false, errors.New("cannot init nil indexer") } if i.Indexer != nil { - return false, fmt.Errorf("indexer is already initialized") + return false, errors.New("indexer is already initialized") } indexer, version, err := openIndexer(i.indexDir, i.version) @@ -83,10 +83,10 @@ func (i *Indexer) Init(_ context.Context) (bool, error) { // Ping checks if the indexer is available func (i *Indexer) Ping(_ context.Context) error { if i == nil { - return fmt.Errorf("cannot ping nil indexer") + return errors.New("cannot ping nil indexer") } if i.Indexer == nil { - return fmt.Errorf("indexer is not initialized") + return errors.New("indexer is not initialized") } return nil } diff --git a/modules/indexer/internal/bleve/query.go b/modules/indexer/internal/bleve/query.go index 1b18ca1a77..8895ae2c64 100644 --- a/modules/indexer/internal/bleve/query.go +++ b/modules/indexer/internal/bleve/query.go @@ -28,6 +28,16 @@ func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query return q } +// MatchAndQuery generates a match query for the given phrase, field and analyzer +func MatchAndQuery(matchPhrase, field, analyzer string, fuzziness int) *query.MatchQuery { + q := bleve.NewMatchQuery(matchPhrase) + q.FieldVal = field + q.Analyzer = analyzer + q.Fuzziness = fuzziness + q.Operator = query.MatchQueryOperatorAnd + return q +} + // BoolFieldQuery generates a bool field query for the given value and field func BoolFieldQuery(value bool, field string) *query.BoolFieldQuery { q := bleve.NewBoolFieldQuery(value) diff --git a/modules/indexer/internal/bleve/util.go b/modules/indexer/internal/bleve/util.go index a0c3dc4ad4..b6daa9e14b 100644 --- a/modules/indexer/internal/bleve/util.go +++ b/modules/indexer/internal/bleve/util.go @@ -9,6 +9,7 @@ import ( "unicode" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "github.com/blevesearch/bleve/v2" @@ -54,9 +55,9 @@ func openIndexer(path string, latestVersion int) (bleve.Index, int, error) { return index, 0, nil } -// This method test the GuessFuzzinessByKeyword method. The fuzziness is based on the levenshtein distance and determines how many chars -// may be different on two string and they still be considered equivalent. -// Given a phrasse, its shortest word determines its fuzziness. If a phrase uses CJK (eg: `갃갃갃` `å•Šå•Šå•Š`), the fuzziness is zero. +// GuessFuzzinessByKeyword guesses fuzziness based on the levenshtein distance and determines how many chars +// may be different on two string, and they still be considered equivalent. +// Given a phrase, its shortest word determines its fuzziness. If a phrase uses CJK (eg: `갃갃갃` `å•Šå•Šå•Š`), the fuzziness is zero. func GuessFuzzinessByKeyword(s string) int { tokenizer := unicode_tokenizer.NewUnicodeTokenizer() tokens := tokenizer.Tokenize([]byte(s)) @@ -85,5 +86,5 @@ func guessFuzzinessByKeyword(s string) int { return 0 } } - return min(maxFuzziness, len(s)/4) + return min(min(setting.Indexer.TypeBleveMaxFuzzniess, maxFuzziness), len(s)/4) } diff --git a/modules/indexer/internal/bleve/util_test.go b/modules/indexer/internal/bleve/util_test.go index 8f7844464e..1a7e4db0f4 100644 --- a/modules/indexer/internal/bleve/util_test.go +++ b/modules/indexer/internal/bleve/util_test.go @@ -7,10 +7,15 @@ import ( "fmt" "testing" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" ) func TestBleveGuessFuzzinessByKeyword(t *testing.T) { + defer test.MockVariableValue(&setting.Indexer.TypeBleveMaxFuzzniess, 2)() + scenarios := []struct { Input string Fuzziness int // See util.go for the definition of fuzziness in this particular context @@ -46,7 +51,7 @@ func TestBleveGuessFuzzinessByKeyword(t *testing.T) { } for _, scenario := range scenarios { - t.Run(fmt.Sprintf("ensure fuzziness of '%s' is '%d'", scenario.Input, scenario.Fuzziness), func(t *testing.T) { + t.Run(fmt.Sprintf("Fuziniess:%s=%d", scenario.Input, scenario.Fuzziness), func(t *testing.T) { assert.Equal(t, scenario.Fuzziness, GuessFuzzinessByKeyword(scenario.Input)) }) } diff --git a/modules/indexer/internal/elasticsearch/indexer.go b/modules/indexer/internal/elasticsearch/indexer.go index 395eea3bce..265ce26585 100644 --- a/modules/indexer/internal/elasticsearch/indexer.go +++ b/modules/indexer/internal/elasticsearch/indexer.go @@ -5,6 +5,7 @@ package elasticsearch import ( "context" + "errors" "fmt" "code.gitea.io/gitea/modules/indexer/internal" @@ -36,10 +37,10 @@ func NewIndexer(url, indexName string, version int, mapping string) *Indexer { // Init initializes the indexer func (i *Indexer) Init(ctx context.Context) (bool, error) { if i == nil { - return false, fmt.Errorf("cannot init nil indexer") + return false, errors.New("cannot init nil indexer") } if i.Client != nil { - return false, fmt.Errorf("indexer is already initialized") + return false, errors.New("indexer is already initialized") } client, err := i.initClient() @@ -66,10 +67,10 @@ func (i *Indexer) Init(ctx context.Context) (bool, error) { // Ping checks if the indexer is available func (i *Indexer) Ping(ctx context.Context) error { if i == nil { - return fmt.Errorf("cannot ping nil indexer") + return errors.New("cannot ping nil indexer") } if i.Client == nil { - return fmt.Errorf("indexer is not initialized") + return errors.New("indexer is not initialized") } resp, err := i.Client.ClusterHealth().Do(ctx) diff --git a/modules/indexer/internal/indexer.go b/modules/indexer/internal/indexer.go index c7f356da1e..3442bbaff2 100644 --- a/modules/indexer/internal/indexer.go +++ b/modules/indexer/internal/indexer.go @@ -5,7 +5,7 @@ package internal import ( "context" - "fmt" + "errors" ) // Indexer defines an basic indexer interface @@ -27,11 +27,11 @@ func NewDummyIndexer() Indexer { type dummyIndexer struct{} func (d *dummyIndexer) Init(ctx context.Context) (bool, error) { - return false, fmt.Errorf("indexer is not ready") + return false, errors.New("indexer is not ready") } func (d *dummyIndexer) Ping(ctx context.Context) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Close() {} diff --git a/modules/indexer/internal/meilisearch/indexer.go b/modules/indexer/internal/meilisearch/indexer.go index 01bb49bbfc..65db75bb55 100644 --- a/modules/indexer/internal/meilisearch/indexer.go +++ b/modules/indexer/internal/meilisearch/indexer.go @@ -5,6 +5,7 @@ package meilisearch import ( "context" + "errors" "fmt" "github.com/meilisearch/meilisearch-go" @@ -33,11 +34,11 @@ func NewIndexer(url, apiKey, indexName string, version int, settings *meilisearc // Init initializes the indexer func (i *Indexer) Init(_ context.Context) (bool, error) { if i == nil { - return false, fmt.Errorf("cannot init nil indexer") + return false, errors.New("cannot init nil indexer") } if i.Client != nil { - return false, fmt.Errorf("indexer is already initialized") + return false, errors.New("indexer is already initialized") } i.Client = meilisearch.New(i.url, meilisearch.WithAPIKey(i.apiKey)) @@ -62,10 +63,10 @@ func (i *Indexer) Init(_ context.Context) (bool, error) { // Ping checks if the indexer is available func (i *Indexer) Ping(ctx context.Context) error { if i == nil { - return fmt.Errorf("cannot ping nil indexer") + return errors.New("cannot ping nil indexer") } if i.Client == nil { - return fmt.Errorf("indexer is not initialized") + return errors.New("indexer is not initialized") } resp, err := i.Client.Health() if err != nil { diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index bf51bd6c14..39d96cab98 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -5,10 +5,14 @@ package bleve import ( "context" + "strconv" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/util" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" @@ -120,6 +124,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWordsFuzzy() +} + // NewIndexer creates a new bleve local indexer func NewIndexer(indexDir string) *Indexer { inner := inner_bleve.NewIndexer(indexDir, issueIndexerLatestVersion, generateIssueIndexMapping) @@ -157,16 +165,24 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( var queries []query.Query if options.Keyword != "" { - fuzziness := 0 - if options.IsFuzzyKeyword { - fuzziness = inner_bleve.GuessFuzzinessByKeyword(options.Keyword) + searchMode := util.IfZero(options.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeWords || searchMode == indexer.SearchModeFuzzy { + fuzziness := 0 + if searchMode == indexer.SearchModeFuzzy { + fuzziness = inner_bleve.GuessFuzzinessByKeyword(options.Keyword) + } + queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ + inner_bleve.MatchAndQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness), + inner_bleve.MatchAndQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness), + inner_bleve.MatchAndQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness), + }...)) + } else /* exact */ { + queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ + inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, 0), + inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, 0), + inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, 0), + }...)) } - - queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ - inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness), - inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness), - inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness), - }...)) } if len(options.RepoIDs) > 0 || options.AllPublic { @@ -232,12 +248,20 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectColumnID.Value(), "project_board_id")) } - if options.PosterID.Has() { - queries = append(queries, inner_bleve.NumericEqualityQuery(options.PosterID.Value(), "poster_id")) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + queries = append(queries, inner_bleve.NumericEqualityQuery(posterIDInt64, "poster_id")) } - if options.AssigneeID.Has() { - queries = append(queries, inner_bleve.NumericEqualityQuery(options.AssigneeID.Value(), "assignee_id")) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + queries = append(queries, inner_bleve.NumericRangeInclusiveQuery(optional.Some[int64](1), optional.None[int64](), "assignee_id")) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + queries = append(queries, inner_bleve.NumericEqualityQuery(assigneeIDInt64, "assignee_id")) + } } if options.MentionID.Has() { diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 6c9cfcf670..50951f9c88 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -5,29 +5,35 @@ package db import ( "context" + "strings" + "sync" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_db "code.gitea.io/gitea/modules/indexer/internal/db" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/util" "xorm.io/builder" ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = (*Indexer)(nil) // Indexer implements Indexer interface to use database's like search type Indexer struct { indexer_internal.Indexer } -func NewIndexer() *Indexer { - return &Indexer{ - Indexer: &inner_db.Indexer{}, - } +func (i *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() } +var GetIndexer = sync.OnceValue(func() *Indexer { + return &Indexer{Indexer: &inner_db.Indexer{}} +}) + // Index dummy function func (i *Indexer) Index(_ context.Context, _ ...*internal.IndexerData) error { return nil @@ -38,6 +44,26 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error { return nil } +func buildMatchQuery(mode indexer.SearchModeType, colName, keyword string) builder.Cond { + if mode == indexer.SearchModeExact { + return db.BuildCaseInsensitiveLike(colName, keyword) + } + + // match words + cond := builder.NewCond() + fields := strings.Fields(keyword) + if len(fields) == 0 { + return builder.Expr("1=1") + } + for _, field := range fields { + if field == "" { + continue + } + cond = cond.And(db.BuildCaseInsensitiveLike(colName, field)) + } + return cond +} + // Search searches for issues func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { // FIXME: I tried to avoid importing models here, but it seems to be impossible. @@ -58,16 +84,16 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( repoCond = builder.Eq{"repo_id": options.RepoIDs[0]} } subQuery := builder.Select("id").From("issue").Where(repoCond) - + searchMode := util.IfZero(options.SearchMode, i.SupportedSearchModes()[0].ModeValue) cond = builder.Or( - db.BuildCaseInsensitiveLike("issue.name", options.Keyword), - db.BuildCaseInsensitiveLike("issue.content", options.Keyword), + buildMatchQuery(searchMode, "issue.name", options.Keyword), + buildMatchQuery(searchMode, "issue.content", options.Keyword), builder.In("issue.id", builder.Select("issue_id"). From("comment"). Where(builder.And( builder.Eq{"type": issue_model.CommentTypeComment}, builder.In("issue_id", subQuery), - db.BuildCaseInsensitiveLike("content", options.Keyword), + buildMatchQuery(searchMode, "content", options.Keyword), )), ), ) @@ -95,7 +121,11 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( }, nil } - ids, total, err := issue_model.IssueIDs(ctx, opt, cond) + return i.FindWithIssueOptions(ctx, opt, cond) +} + +func (i *Indexer) FindWithIssueOptions(ctx context.Context, opt *issue_model.IssuesOptions, otherConds ...builder.Cond) (*internal.SearchResult, error) { + ids, total, err := issue_model.IssueIDs(ctx, opt, otherConds...) if err != nil { return nil, err } diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index 42834f6e88..380a25dc23 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -6,6 +6,7 @@ package db import ( "context" "fmt" + "strings" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" @@ -34,7 +35,11 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m case internal.SortByDeadlineAsc: sortType = "nearduedate" default: - sortType = "newest" + if strings.HasPrefix(string(options.SortBy), issue_model.ScopeSortPrefix) { + sortType = string(options.SortBy) + } else { + sortType = "newest" + } } // See the comment of issues_model.SearchOptions for the reason why we need to convert @@ -54,7 +59,7 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m RepoIDs: options.RepoIDs, AllPublic: options.AllPublic, RepoCond: nil, - AssigneeID: optional.Some(convertID(options.AssigneeID)), + AssigneeID: options.AssigneeID, PosterID: options.PosterID, MentionedID: convertID(options.MentionID), ReviewRequestedID: convertID(options.ReviewRequestedID), @@ -68,14 +73,13 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m ExcludedLabelNames: nil, IncludeMilestones: nil, SortType: sortType, - IssueIDs: nil, UpdatedAfterUnix: options.UpdatedAfterUnix.Value(), UpdatedBeforeUnix: options.UpdatedBeforeUnix.Value(), PriorityRepoID: 0, IsArchived: options.IsArchived, - Org: nil, + Owner: nil, Team: nil, - User: nil, + Doer: nil, } if len(options.MilestoneIDs) == 1 && options.MilestoneIDs[0] == 0 { diff --git a/modules/indexer/issues/dboptions.go b/modules/indexer/issues/dboptions.go index 4f6ad96d22..f17724664d 100644 --- a/modules/indexer/issues/dboptions.go +++ b/modules/indexer/issues/dboptions.go @@ -4,12 +4,19 @@ package issues import ( + "strings" + "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/indexer/issues/internal" "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/setting" ) func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOptions { + if opts.IssueIDs != nil { + setting.PanicInDevOrTesting("Indexer SearchOptions doesn't support IssueIDs") + } searchOpt := &SearchOptions{ Keyword: keyword, RepoIDs: opts.RepoIDs, @@ -45,11 +52,7 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp searchOpt.ProjectID = optional.Some[int64](0) // Those issues with no project(projectid==0) } - if opts.AssigneeID.Value() == db.NoConditionID { - searchOpt.AssigneeID = optional.Some[int64](0) // FIXME: this is inconsistent from other places, 0 means "no assignee" - } else if opts.AssigneeID.Value() != 0 { - searchOpt.AssigneeID = opts.AssigneeID - } + searchOpt.AssigneeID = opts.AssigneeID // See the comment of issues_model.SearchOptions for the reason why we need to convert convertID := func(id int64) optional.Option[int64] { @@ -99,7 +102,11 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp // Unsupported sort type for search fallthrough default: - searchOpt.SortBy = SortByUpdatedDesc + if strings.HasPrefix(opts.SortType, issues_model.ScopeSortPrefix) { + searchOpt.SortBy = internal.SortBy(opts.SortType) + } else { + searchOpt.SortBy = SortByUpdatedDesc + } } return searchOpt diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 4c293f3f2a..9d627466ef 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -5,14 +5,15 @@ package elasticsearch import ( "context" - "fmt" "strconv" "strings" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/util" "github.com/olivere/elastic/v7" ) @@ -33,6 +34,11 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + // TODO: es supports fuzzy search, but our code doesn't at the moment, and actually the default fuzziness is already "AUTO" + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new elasticsearch indexer func NewIndexer(url, indexerName string) *Indexer { inner := inner_elasticsearch.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping) @@ -89,7 +95,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er issue := issues[0] _, err := b.inner.Client.Index(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", issue.ID)). + Id(strconv.FormatInt(issue.ID, 10)). BodyJson(issue). Do(ctx) return err @@ -100,7 +106,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er reqs = append(reqs, elastic.NewBulkIndexRequest(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", issue.ID)). + Id(strconv.FormatInt(issue.ID, 10)). Doc(issue), ) } @@ -119,7 +125,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { } else if len(ids) == 1 { _, err := b.inner.Client.Delete(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", ids[0])). + Id(strconv.FormatInt(ids[0], 10)). Do(ctx) return err } @@ -129,7 +135,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { reqs = append(reqs, elastic.NewBulkDeleteRequest(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", id)), + Id(strconv.FormatInt(id, 10)), ) } @@ -146,12 +152,12 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query := elastic.NewBoolQuery() if options.Keyword != "" { - searchType := esMultiMatchTypePhrasePrefix - if options.IsFuzzyKeyword { - searchType = esMultiMatchTypeBestFields + searchMode := util.IfZero(options.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeExact { + query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypePhrasePrefix)) + } else /* words */ { + query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypeBestFields).Operator("and")) } - - query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(searchType)) } if len(options.RepoIDs) > 0 { @@ -205,12 +211,22 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query.Must(elastic.NewTermQuery("project_board_id", options.ProjectColumnID.Value())) } - if options.PosterID.Has() { - query.Must(elastic.NewTermQuery("poster_id", options.PosterID.Value())) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + query.Must(elastic.NewTermQuery("poster_id", posterIDInt64)) } - if options.AssigneeID.Has() { - query.Must(elastic.NewTermQuery("assignee_id", options.AssigneeID.Value())) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + q := elastic.NewRangeQuery("assignee_id") + q.Gte(1) + query.Must(q) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + query.Must(elastic.NewTermQuery("assignee_id", assigneeIDInt64)) + } } if options.MentionID.Has() { diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_test.go b/modules/indexer/issues/elasticsearch/elasticsearch_test.go index ffd85b1aa1..dc329c07dd 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_test.go @@ -11,6 +11,8 @@ import ( "time" "code.gitea.io/gitea/modules/indexer/issues/internal/tests" + + "github.com/stretchr/testify/require" ) func TestElasticsearchIndexer(t *testing.T) { @@ -26,20 +28,10 @@ func TestElasticsearchIndexer(t *testing.T) { } } - ok := false - for i := 0; i < 60; i++ { + require.Eventually(t, func() bool { resp, err := http.Get(url) - if err == nil && resp.StatusCode == http.StatusOK { - ok = true - break - } - t.Logf("Waiting for elasticsearch to be up: %v", err) - time.Sleep(time.Second) - } - if !ok { - t.Fatalf("Failed to wait for elasticsearch to be up") - return - } + return err == nil && resp.StatusCode == http.StatusOK + }, time.Minute, time.Second, "Expected elasticsearch to be up") indexer := NewIndexer(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) defer indexer.Close() diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index c82dc0867e..8f25c84b76 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -14,6 +14,7 @@ import ( db_model "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/issues/bleve" "code.gitea.io/gitea/modules/indexer/issues/db" "code.gitea.io/gitea/modules/indexer/issues/elasticsearch" @@ -102,7 +103,7 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err) } case "db": - issueIndexer = db.NewIndexer() + issueIndexer = db.GetIndexer() case "meilisearch": issueIndexer = meilisearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName) existed, err = issueIndexer.Init(ctx) @@ -216,7 +217,7 @@ func PopulateIssueIndexer(ctx context.Context) error { return fmt.Errorf("shutdown before completion: %w", ctx.Err()) default: } - repos, _, err := repo_model.SearchRepositoryByName(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{ ListOptions: db_model.ListOptions{Page: page, PageSize: repo_model.RepositoryListDefaultPageSize}, OrderBy: db_model.SearchOrderByID, Private: true, @@ -281,7 +282,7 @@ const ( // SearchIssues search issues by options. func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, error) { - indexer := *globalIndexer.Load() + ix := *globalIndexer.Load() if opts.Keyword == "" || opts.IsKeywordNumeric() { // This is a conservative shortcut. @@ -290,20 +291,22 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, err // So if the user creates an issue and list issues immediately, the issue may not be listed because the indexer needs time to index the issue. // Even worse, the external indexer like elastic search may not be available for a while, // and the user may not be able to list issues completely until it is available again. - indexer = db.NewIndexer() + ix = db.GetIndexer() } - result, err := indexer.Search(ctx, opts) + result, err := ix.Search(ctx, opts) if err != nil { return nil, 0, err } + return SearchResultToIDSlice(result), result.Total, nil +} +func SearchResultToIDSlice(result *internal.SearchResult) []int64 { ret := make([]int64, 0, len(result.Hits)) for _, hit := range result.Hits { ret = append(ret, hit.ID) } - - return ret, result.Total, nil + return ret } // CountIssues counts issues by options. It is a shortcut of SearchIssues(ctx, opts) but only returns the total count. @@ -313,3 +316,11 @@ func CountIssues(ctx context.Context, opts *SearchOptions) (int64, error) { _, total, err := SearchIssues(ctx, opts) return total, err } + +func SupportedSearchModes() []indexer.SearchMode { + gi := globalIndexer.Load() + if gi == nil { + return nil + } + return (*gi).SupportedSearchModes() +} diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index 06a6a46c23..3e38ac49b7 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -4,7 +4,6 @@ package issues import ( - "context" "testing" "code.gitea.io/gitea/models/db" @@ -19,6 +18,7 @@ import ( _ "code.gitea.io/gitea/models/activities" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMain(m *testing.M) { @@ -26,7 +26,7 @@ func TestMain(m *testing.M) { } func TestDBSearchIssues(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) + require.NoError(t, unittest.PrepareTestDatabase()) setting.Indexer.IssueType = "db" InitIssueIndexer(true) @@ -44,6 +44,7 @@ func TestDBSearchIssues(t *testing.T) { t.Run("search issues with order", searchIssueWithOrder) t.Run("search issues in project", searchIssueInProject) t.Run("search issues with paginator", searchIssueWithPaginator) + t.Run("search issues with any assignee", searchIssueWithAnyAssignee) } func searchIssueWithKeyword(t *testing.T) { @@ -82,11 +83,11 @@ func searchIssueWithKeyword(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } - assert.Equal(t, test.expectedIDs, issueIDs) + t.Run(test.opts.Keyword, func(t *testing.T) { + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) + assert.Equal(t, test.expectedIDs, issueIDs) + }) } } @@ -119,10 +120,8 @@ func searchIssueByIndex(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -165,10 +164,8 @@ func searchIssueInRepo(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -180,19 +177,19 @@ func searchIssueByID(t *testing.T) { }{ { opts: SearchOptions{ - PosterID: optional.Some(int64(1)), + PosterID: "1", }, expectedIDs: []int64{11, 6, 3, 2, 1}, }, { opts: SearchOptions{ - AssigneeID: optional.Some(int64(1)), + AssigneeID: "1", }, expectedIDs: []int64{6, 1}, }, { - // NOTE: This tests no assignees filtering and also ToSearchOptions() to ensure it will set AssigneeID to 0 when it is passed as -1. - opts: *ToSearchOptions("", &issues.IssuesOptions{AssigneeID: optional.Some(db.NoConditionID)}), + // NOTE: This tests no assignees filtering and also ToSearchOptions() to ensure it handles the filter correctly + opts: *ToSearchOptions("", &issues.IssuesOptions{AssigneeID: "(none)"}), expectedIDs: []int64{22, 21, 16, 15, 14, 13, 12, 11, 20, 5, 19, 18, 10, 7, 4, 9, 8, 3, 2}, }, { @@ -237,10 +234,8 @@ func searchIssueByID(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -264,10 +259,8 @@ func searchIssueIsPull(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -291,10 +284,8 @@ func searchIssueIsClosed(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -318,10 +309,8 @@ func searchIssueIsArchived(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -345,10 +334,8 @@ func searchIssueByMilestoneID(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -378,10 +365,8 @@ func searchIssueByLabelID(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -399,10 +384,8 @@ func searchIssueByTime(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -420,10 +403,8 @@ func searchIssueWithOrder(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -453,10 +434,8 @@ func searchIssueInProject(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } } @@ -478,10 +457,30 @@ func searchIssueWithPaginator(t *testing.T) { }, } for _, test := range tests { - issueIDs, total, err := SearchIssues(context.TODO(), &test.opts) - if !assert.NoError(t, err) { - return - } + issueIDs, total, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) + assert.Equal(t, test.expectedIDs, issueIDs) + assert.Equal(t, test.expectedTotal, total) + } +} + +func searchIssueWithAnyAssignee(t *testing.T) { + tests := []struct { + opts SearchOptions + expectedIDs []int64 + expectedTotal int64 + }{ + { + SearchOptions{ + AssigneeID: "(any)", + }, + []int64{17, 6, 1}, + 3, + }, + } + for _, test := range tests { + issueIDs, total, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) assert.Equal(t, test.expectedTotal, total) } diff --git a/modules/indexer/issues/internal/indexer.go b/modules/indexer/issues/internal/indexer.go index 95740bc598..59c6f48485 100644 --- a/modules/indexer/issues/internal/indexer.go +++ b/modules/indexer/issues/internal/indexer.go @@ -5,8 +5,9 @@ package internal import ( "context" - "fmt" + "errors" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/internal" ) @@ -16,6 +17,7 @@ type Indexer interface { Index(ctx context.Context, issue ...*IndexerData) error Delete(ctx context.Context, ids ...int64) error Search(ctx context.Context, options *SearchOptions) (*SearchResult, error) + SupportedSearchModes() []indexer.SearchMode } // NewDummyIndexer returns a dummy indexer @@ -29,14 +31,18 @@ type dummyIndexer struct { internal.Indexer } +func (d *dummyIndexer) SupportedSearchModes() []indexer.SearchMode { + return nil +} + func (d *dummyIndexer) Index(_ context.Context, _ ...*IndexerData) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Delete(_ context.Context, _ ...int64) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Search(_ context.Context, _ *SearchOptions) (*SearchResult, error) { - return nil, fmt.Errorf("indexer is not ready") + return nil, errors.New("indexer is not ready") } diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 09dcbf4804..0d4f0f727d 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -7,6 +7,7 @@ import ( "strconv" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/timeutil" ) @@ -77,7 +78,7 @@ type SearchResult struct { type SearchOptions struct { Keyword string // keyword to search - IsFuzzyKeyword bool // if false the levenshtein distance is 0 + SearchMode indexer.SearchModeType RepoIDs []int64 // repository IDs which the issues belong to AllPublic bool // if include all public repositories @@ -96,9 +97,8 @@ type SearchOptions struct { ProjectID optional.Option[int64] // project the issues belong to ProjectColumnID optional.Option[int64] // project column the issues belong to - PosterID optional.Option[int64] // poster of the issues - - AssigneeID optional.Option[int64] // assignee of the issues, zero means no assignee + PosterID string // poster of the issues, "(none)" or "(any)" or a user ID + AssigneeID string // assignee of the issues, "(none)" or "(any)" or a user ID MentionID optional.Option[int64] // mentioned user of the issues diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 94ce8520bf..7aebbbcd58 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -8,7 +8,6 @@ package tests import ( - "context" "fmt" "slices" "testing" @@ -24,10 +23,10 @@ import ( ) func TestIndexer(t *testing.T, indexer internal.Indexer) { - _, err := indexer.Init(context.Background()) + _, err := indexer.Init(t.Context()) require.NoError(t, err) - require.NoError(t, indexer.Ping(context.Background())) + require.NoError(t, indexer.Ping(t.Context())) var ( ids []int64 @@ -39,32 +38,32 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { ids = append(ids, v.ID) data[v.ID] = v } - require.NoError(t, indexer.Index(context.Background(), d...)) - require.NoError(t, waitData(indexer, int64(len(data)))) + require.NoError(t, indexer.Index(t.Context(), d...)) + waitData(t, indexer, int64(len(data))) } defer func() { - require.NoError(t, indexer.Delete(context.Background(), ids...)) + require.NoError(t, indexer.Delete(t.Context(), ids...)) }() for _, c := range cases { t.Run(c.Name, func(t *testing.T) { if len(c.ExtraData) > 0 { - require.NoError(t, indexer.Index(context.Background(), c.ExtraData...)) + require.NoError(t, indexer.Index(t.Context(), c.ExtraData...)) for _, v := range c.ExtraData { data[v.ID] = v } - require.NoError(t, waitData(indexer, int64(len(data)))) + waitData(t, indexer, int64(len(data))) defer func() { for _, v := range c.ExtraData { - require.NoError(t, indexer.Delete(context.Background(), v.ID)) + require.NoError(t, indexer.Delete(t.Context(), v.ID)) delete(data, v.ID) } - require.NoError(t, waitData(indexer, int64(len(data)))) + waitData(t, indexer, int64(len(data))) }() } - result, err := indexer.Search(context.Background(), c.SearchOptions) + result, err := indexer.Search(t.Context(), c.SearchOptions) require.NoError(t, err) if c.Expected != nil { @@ -80,7 +79,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { // test counting c.SearchOptions.Paginator = &db.ListOptions{PageSize: 0} - countResult, err := indexer.Search(context.Background(), c.SearchOptions) + countResult, err := indexer.Search(t.Context(), c.SearchOptions) require.NoError(t, err) assert.Empty(t, countResult.Hits) assert.Equal(t, result.Total, countResult.Total) @@ -93,7 +92,7 @@ var cases = []*testIndexerCase{ Name: "default", SearchOptions: &internal.SearchOptions{}, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) }, }, @@ -379,7 +378,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - PosterID: optional.Some(int64(1)), + PosterID: "1", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -397,7 +396,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - AssigneeID: optional.Some(int64(1)), + AssigneeID: "1", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -415,7 +414,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - AssigneeID: optional.Some(int64(0)), + AssigneeID: "(none)", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -526,7 +525,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCreatedDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -542,7 +541,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByUpdatedDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -558,7 +557,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCommentsDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -574,7 +573,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByDeadlineDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -590,7 +589,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCreatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -606,7 +605,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByUpdatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -622,7 +621,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCommentsAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -638,7 +637,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByDeadlineAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -647,6 +646,21 @@ var cases = []*testIndexerCase{ } }, }, + { + Name: "SearchAnyAssignee", + SearchOptions: &internal.SearchOptions{ + AssigneeID: "(any)", + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Len(t, result.Hits, 180) + for _, v := range result.Hits { + assert.GreaterOrEqual(t, data[v.ID].AssigneeID, int64(1)) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.AssigneeID >= 1 + }), result.Total) + }, + }, } type testIndexerCase struct { @@ -736,22 +750,10 @@ func countIndexerData(data map[int64]*internal.IndexerData, f func(v *internal.I // waitData waits for the indexer to index all data. // Some engines like Elasticsearch index data asynchronously, so we need to wait for a while. -func waitData(indexer internal.Indexer, total int64) error { - var actual int64 - for i := 0; i < 100; i++ { - result, err := indexer.Search(context.Background(), &internal.SearchOptions{ - Paginator: &db.ListOptions{ - PageSize: 0, - }, - }) - if err != nil { - return err - } - actual = result.Total - if actual == total { - return nil - } - time.Sleep(100 * time.Millisecond) - } - return fmt.Errorf("waitData: expected %d, actual %d", total, actual) +func waitData(t *testing.T, indexer internal.Indexer, total int64) { + assert.Eventually(t, func() bool { + result, err := indexer.Search(t.Context(), &internal.SearchOptions{Paginator: &db.ListOptions{}}) + require.NoError(t, err) + return result.Total == total + }, 10*time.Second, 100*time.Millisecond, "expected total=%d", total) } diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 1066e96272..759a98473f 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_meilisearch "code.gitea.io/gitea/modules/indexer/internal/meilisearch" "code.gitea.io/gitea/modules/indexer/issues/internal" @@ -35,6 +36,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_meilisearch.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new meilisearch indexer func NewIndexer(url, apiKey, indexerName string) *Indexer { settings := &meilisearch.Settings{ @@ -182,12 +187,20 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query.And(inner_meilisearch.NewFilterEq("project_board_id", options.ProjectColumnID.Value())) } - if options.PosterID.Has() { - query.And(inner_meilisearch.NewFilterEq("poster_id", options.PosterID.Value())) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + query.And(inner_meilisearch.NewFilterEq("poster_id", posterIDInt64)) } - if options.AssigneeID.Has() { - query.And(inner_meilisearch.NewFilterEq("assignee_id", options.AssigneeID.Value())) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + query.And(inner_meilisearch.NewFilterGte("assignee_id", 1)) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + query.And(inner_meilisearch.NewFilterEq("assignee_id", assigneeIDInt64)) + } } if options.MentionID.Has() { @@ -230,9 +243,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( limit = 1 } - keyword := options.Keyword - if !options.IsFuzzyKeyword { - // to make it non fuzzy ("typo tolerance" in meilisearch terms), we have to quote the keyword(s) + keyword := options.Keyword // default to match "words" + if options.SearchMode == indexer.SearchModeExact { // https://www.meilisearch.com/docs/reference/api/search#phrase-search keyword = doubleQuoteKeyword(keyword) } diff --git a/modules/indexer/issues/meilisearch/meilisearch_test.go b/modules/indexer/issues/meilisearch/meilisearch_test.go index 4666df136a..2fea4004cb 100644 --- a/modules/indexer/issues/meilisearch/meilisearch_test.go +++ b/modules/indexer/issues/meilisearch/meilisearch_test.go @@ -15,6 +15,7 @@ import ( "github.com/meilisearch/meilisearch-go" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMeilisearchIndexer(t *testing.T) { @@ -32,20 +33,10 @@ func TestMeilisearchIndexer(t *testing.T) { key = os.Getenv("TEST_MEILISEARCH_KEY") } - ok := false - for i := 0; i < 60; i++ { + require.Eventually(t, func() bool { resp, err := http.Get(url) - if err == nil && resp.StatusCode == http.StatusOK { - ok = true - break - } - t.Logf("Waiting for meilisearch to be up: %v", err) - time.Sleep(time.Second) - } - if !ok { - t.Fatalf("Failed to wait for meilisearch to be up") - return - } + return err == nil && resp.StatusCode == http.StatusOK + }, time.Minute, time.Second, "Expected meilisearch to be up") indexer := NewIndexer(url, key, fmt.Sprintf("test_meilisearch_indexer_%d", time.Now().Unix())) defer indexer.Close() @@ -83,13 +74,13 @@ func TestConvertHits(t *testing.T) { } hits, err := convertHits(validResponse) assert.NoError(t, err) - assert.EqualValues(t, []internal.Match{{ID: 11}, {ID: 22}, {ID: 33}}, hits) + assert.Equal(t, []internal.Match{{ID: 11}, {ID: 22}, {ID: 33}}, hits) } func TestDoubleQuoteKeyword(t *testing.T) { - assert.EqualValues(t, "", doubleQuoteKeyword("")) - assert.EqualValues(t, `"a" "b" "c"`, doubleQuoteKeyword("a b c")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword(`a "" "d" """g`)) + assert.Empty(t, doubleQuoteKeyword("")) + assert.Equal(t, `"a" "b" "c"`, doubleQuoteKeyword("a b c")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword(`a "" "d" """g`)) } diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index deb19adc49..19d835a1d8 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -92,6 +92,11 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD projectID = issue.Project.ID } + projectColumnID, err := issue.ProjectColumnID(ctx) + if err != nil { + return nil, false, err + } + return &internal.IndexerData{ ID: issue.ID, RepoID: issue.RepoID, @@ -106,7 +111,7 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD NoLabel: len(labels) == 0, MilestoneID: issue.MilestoneID, ProjectID: projectID, - ProjectColumnID: issue.ProjectColumnID(ctx), + ProjectColumnID: projectColumnID, PosterID: issue.PosterID, AssigneeID: issue.AssigneeID, MentionIDs: mentionIDs, diff --git a/modules/indexer/stats/db.go b/modules/indexer/stats/db.go index 98a977c700..199d493e97 100644 --- a/modules/indexer/stats/db.go +++ b/modules/indexer/stats/db.go @@ -8,6 +8,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/languagestats" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" @@ -49,10 +50,10 @@ func (db *DBIndexer) Index(id int64) error { commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch) if err != nil { if git.IsErrBranchNotExist(err) || git.IsErrNotExist(err) || setting.IsInTesting { - log.Debug("Unable to get commit ID for default branch %s in %s ... skipping this repository", repo.DefaultBranch, repo.RepoPath()) + log.Debug("Unable to get commit ID for default branch %s in %s ... skipping this repository", repo.DefaultBranch, repo.FullName()) return nil } - log.Error("Unable to get commit ID for default branch %s in %s. Error: %v", repo.DefaultBranch, repo.RepoPath(), err) + log.Error("Unable to get commit ID for default branch %s in %s. Error: %v", repo.DefaultBranch, repo.FullName(), err) return err } @@ -62,20 +63,20 @@ func (db *DBIndexer) Index(id int64) error { } // Calculate and save language statistics to database - stats, err := gitRepo.GetLanguageStats(commitID) + stats, err := languagestats.GetLanguageStats(gitRepo, commitID) if err != nil { if !setting.IsInTesting { - log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err) + log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err) } return err } err = repo_model.UpdateLanguageStats(ctx, repo, commitID, stats) if err != nil { - log.Error("Unable to update language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err) + log.Error("Unable to update language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err) return err } - log.Debug("DBIndexer completed language stats for ID %s for default branch %s in %s. stats count: %d", commitID, repo.DefaultBranch, repo.RepoPath(), len(stats)) + log.Debug("DBIndexer completed language stats for ID %s for default branch %s in %s. stats count: %d", commitID, repo.DefaultBranch, repo.FullName(), len(stats)) return nil } diff --git a/modules/indexer/stats/indexer_test.go b/modules/indexer/stats/indexer_test.go index 5be45d7a3b..d32a8bf151 100644 --- a/modules/indexer/stats/indexer_test.go +++ b/modules/indexer/stats/indexer_test.go @@ -4,7 +4,6 @@ package stats import ( - "context" "testing" "time" @@ -40,7 +39,7 @@ func TestRepoStatsIndex(t *testing.T) { err = UpdateRepoIndexer(repo) assert.NoError(t, err) - assert.NoError(t, queue.GetManager().FlushAll(context.Background(), 5*time.Second)) + assert.NoError(t, queue.GetManager().FlushAll(t.Context(), 5*time.Second)) status, err := repo_model.GetIndexerStatus(db.DefaultContext, repo, repo_model.RepoIndexerTypeStats) assert.NoError(t, err) diff --git a/modules/indexer/stats/queue.go b/modules/indexer/stats/queue.go index d002bd57cf..69cde321d8 100644 --- a/modules/indexer/stats/queue.go +++ b/modules/indexer/stats/queue.go @@ -4,7 +4,7 @@ package stats import ( - "fmt" + "errors" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" @@ -31,7 +31,7 @@ func handler(items ...int64) []int64 { func initStatsQueue() error { statsQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_stats_update", handler) if statsQueue == nil { - return fmt.Errorf("unable to create repo_stats_update queue") + return errors.New("unable to create repo_stats_update queue") } go graceful.GetManager().RunWithCancel(statsQueue) return nil diff --git a/modules/issue/template/template.go b/modules/issue/template/template.go index a2b9d6b33e..192aaf8e01 100644 --- a/modules/issue/template/template.go +++ b/modules/issue/template/template.go @@ -4,9 +4,11 @@ package template import ( + "errors" "fmt" "net/url" "regexp" + "slices" "strconv" "strings" @@ -31,17 +33,17 @@ func Validate(template *api.IssueTemplate) error { func validateMetadata(template *api.IssueTemplate) error { if strings.TrimSpace(template.Name) == "" { - return fmt.Errorf("'name' is required") + return errors.New("'name' is required") } if strings.TrimSpace(template.About) == "" { - return fmt.Errorf("'about' is required") + return errors.New("'about' is required") } return nil } func validateYaml(template *api.IssueTemplate) error { if len(template.Fields) == 0 { - return fmt.Errorf("'body' is required") + return errors.New("'body' is required") } ids := make(container.Set[string]) for idx, field := range template.Fields { @@ -401,7 +403,7 @@ func (f *valuedField) Render() string { } func (f *valuedField) Value() string { - return strings.TrimSpace(f.Get(fmt.Sprintf("form-field-%s", f.ID))) + return strings.TrimSpace(f.Get("form-field-" + f.ID)) } func (f *valuedField) Options() []*valuedOption { @@ -444,14 +446,9 @@ func (o *valuedOption) Label() string { func (o *valuedOption) IsChecked() bool { switch o.field.Type { case api.IssueFormFieldTypeDropdown: - checks := strings.Split(o.field.Get(fmt.Sprintf("form-field-%s", o.field.ID)), ",") + checks := strings.Split(o.field.Get("form-field-"+o.field.ID), ",") idx := strconv.Itoa(o.index) - for _, v := range checks { - if v == idx { - return true - } - } - return false + return slices.Contains(checks, idx) case api.IssueFormFieldTypeCheckboxes: return o.field.Get(fmt.Sprintf("form-field-%s-%d", o.field.ID, o.index)) == "on" } diff --git a/modules/issue/template/template_test.go b/modules/issue/template/template_test.go index 689a285b47..7fec9431b6 100644 --- a/modules/issue/template/template_test.go +++ b/modules/issue/template/template_test.go @@ -904,7 +904,7 @@ Option 1 of dropdown, Option 2 of dropdown t.Fatal(err) } if got := RenderToMarkdown(template, tt.args.values); got != tt.want { - assert.EqualValues(t, tt.want, got) + assert.Equal(t, tt.want, got) } }) } @@ -957,9 +957,8 @@ func Test_minQuotes(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := minQuotes(tt.args.value); got != tt.want { - t.Errorf("minQuotes() = %v, want %v", got, tt.want) - } + got := minQuotes(tt.args.value) + assert.Equal(t, tt.want, got) }) } } diff --git a/modules/issue/template/unmarshal.go b/modules/issue/template/unmarshal.go index 0fc13d7ddf..1d8e9dd02d 100644 --- a/modules/issue/template/unmarshal.go +++ b/modules/issue/template/unmarshal.go @@ -109,7 +109,7 @@ func unmarshal(filename string, content []byte) (*api.IssueTemplate, error) { it.Content = string(content) it.Name = path.Base(it.FileName) // paths in Git are always '/' separated - do not use filepath! - it.About, _ = util.SplitStringAtByteN(it.Content, 80) + it.About = util.EllipsisDisplayString(it.Content, 80) } else { it.Content = templateBody if it.About == "" { diff --git a/modules/json/json.go b/modules/json/json.go index 34568c75c6..444dc8526a 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -3,11 +3,10 @@ package json -// Allow "encoding/json" import. import ( "bytes" "encoding/binary" - "encoding/json" //nolint:depguard + "encoding/json" //nolint:depguard // this package wraps it "io" jsoniter "github.com/json-iterator/go" @@ -145,6 +144,12 @@ func Valid(data []byte) bool { // UnmarshalHandleDoubleEncode - due to a bug in xorm (see https://gitea.com/xorm/xorm/pulls/1957) - it's // possible that a Blob may be double encoded or gain an unwanted prefix of 0xff 0xfe. func UnmarshalHandleDoubleEncode(bs []byte, v any) error { + if len(bs) == 0 { + // json.Unmarshal will report errors if input is empty (nil or zero-length) + // It seems that XORM ignores the nil but still passes zero-length string into this function + // To be consistent, we should treat all empty inputs as success + return nil + } err := json.Unmarshal(bs, v) if err != nil { ok := true diff --git a/modules/json/json_test.go b/modules/json/json_test.go new file mode 100644 index 0000000000..ace7167913 --- /dev/null +++ b/modules/json/json_test.go @@ -0,0 +1,18 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package json + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGiteaDBJSONUnmarshal(t *testing.T) { + var m map[any]any + err := UnmarshalHandleDoubleEncode(nil, &m) + assert.NoError(t, err) + err = UnmarshalHandleDoubleEncode([]byte(""), &m) + assert.NoError(t, err) +} diff --git a/modules/label/label.go b/modules/label/label.go index d3ef0e1dc9..3e68c4d26e 100644 --- a/modules/label/label.go +++ b/modules/label/label.go @@ -7,19 +7,24 @@ import ( "fmt" "regexp" "strings" -) + "sync" -// colorPattern is a regexp which can validate label color -var colorPattern = regexp.MustCompile("^#?(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{3})$") + "code.gitea.io/gitea/modules/util" +) // Label represents label information loaded from template type Label struct { - Name string `yaml:"name"` - Color string `yaml:"color"` - Description string `yaml:"description,omitempty"` - Exclusive bool `yaml:"exclusive,omitempty"` + Name string `yaml:"name"` + Color string `yaml:"color"` + Description string `yaml:"description,omitempty"` + Exclusive bool `yaml:"exclusive,omitempty"` + ExclusiveOrder int `yaml:"exclusive_order,omitempty"` } +var colorPattern = sync.OnceValue(func() *regexp.Regexp { + return regexp.MustCompile(`^#([\da-fA-F]{3}|[\da-fA-F]{6})$`) +}) + // NormalizeColor normalizes a color string to a 6-character hex code func NormalizeColor(color string) (string, error) { // normalize case @@ -30,8 +35,8 @@ func NormalizeColor(color string) (string, error) { color = "#" + color } - if !colorPattern.MatchString(color) { - return "", fmt.Errorf("bad color code: %s", color) + if !colorPattern().MatchString(color) { + return "", util.NewInvalidArgumentErrorf("invalid color: %s", color) } // convert 3-character shorthand into 6-character version diff --git a/modules/label/parser.go b/modules/label/parser.go index 511bac823f..2a10152062 100644 --- a/modules/label/parser.go +++ b/modules/label/parser.go @@ -72,7 +72,7 @@ func parseYamlFormat(fileName string, data []byte) ([]*Label, error) { func parseLegacyFormat(fileName string, data []byte) ([]*Label, error) { lines := strings.Split(string(data), "\n") list := make([]*Label, 0, len(lines)) - for i := 0; i < len(lines); i++ { + for i := range lines { line := strings.TrimSpace(lines[i]) if len(line) == 0 { continue @@ -108,7 +108,7 @@ func LoadTemplateDescription(fileName string) (string, error) { return "", err } - for i := 0; i < len(list); i++ { + for i := range list { if i > 0 { buf.WriteString(", ") } diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 3acd23b8f7..4b51193846 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -70,12 +70,16 @@ func (c *HTTPClient) transferNames() []string { func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { log.Trace("BATCH operation with objects: %v", objects) - url := fmt.Sprintf("%s/objects/batch", c.endpoint) + url := c.endpoint + "/objects/batch" + // Original: In some lfs server implementations, they require the ref attribute. #32838 // `ref` is an "optional object describing the server ref that the objects belong to" - // but some (incorrect) lfs servers require it, so maybe adding an empty ref here doesn't break the correct ones. + // but some (incorrect) lfs servers like aliyun require it, so maybe adding an empty ref here doesn't break the correct ones. // https://github.com/git-lfs/git-lfs/blob/a32a02b44bf8a511aa14f047627c49e1a7fd5021/docs/api/batch.md?plain=1#L37 - request := &BatchRequest{operation, c.transferNames(), &Reference{}, objects} + // + // UPDATE: it can't use "empty ref" here because it breaks others like https://github.com/go-gitea/gitea/issues/33453 + request := &BatchRequest{operation, c.transferNames(), nil, objects} + payload := new(bytes.Buffer) err := json.NewEncoder(payload).Encode(request) if err != nil { diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index aa7e3c45c4..179bcdb29a 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -31,7 +31,7 @@ func (a *DummyTransferAdapter) Name() string { } func (a *DummyTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) { - return io.NopCloser(bytes.NewBufferString("dummy")), nil + return io.NopCloser(strings.NewReader("dummy")), nil } func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error { @@ -49,7 +49,7 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { if strings.Contains(url, "status-not-ok") { return &http.Response{StatusCode: http.StatusBadRequest} } else if strings.Contains(url, "invalid-json-response") { - return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBufferString("invalid json"))} + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("invalid json"))} } else if strings.Contains(url, "valid-batch-request-download") { batchResponse = &BatchResponse{ Transfer: "dummy", @@ -248,7 +248,7 @@ func TestHTTPClientDownload(t *testing.T) { }, } - err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { + err := client.Download(t.Context(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { if objectError != nil { return objectError } @@ -348,7 +348,7 @@ func TestHTTPClientUpload(t *testing.T) { }, } - err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) { + err := client.Upload(t.Context(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) { return io.NopCloser(new(bytes.Buffer)), objectError }) if c.expectedError != "" { diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index ebde20f826..9c95613057 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -15,15 +15,13 @@ import ( "strings" ) +// spec: https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md const ( - blobSizeCutoff = 1024 + MetaFileMaxSize = 1024 // spec says the maximum size of a pointer file must be smaller than 1024 - // MetaFileIdentifier is the string appearing at the first line of LFS pointer files. - // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" + MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" // the first line of a pointer file - // MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. - MetaFileOidPrefix = "oid sha256:" + MetaFileOidPrefix = "oid sha256:" // spec says the only supported hash is sha256 at the moment ) var ( @@ -39,7 +37,7 @@ var ( // ReadPointer tries to read LFS pointer data from the reader func ReadPointer(reader io.Reader) (Pointer, error) { - buf := make([]byte, blobSizeCutoff) + buf := make([]byte, MetaFileMaxSize) n, err := io.ReadFull(reader, buf) if err != nil && err != io.ErrUnexpectedEOF { return Pointer{}, err @@ -65,6 +63,7 @@ func ReadPointerFromBuffer(buf []byte) (Pointer, error) { return p, ErrInvalidStructure } + // spec says "key/value pairs MUST be sorted alphabetically in ascending order (version is exception and must be the first)" oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix) if len(oid) != 64 || !oidPattern.MatchString(oid) { return p, ErrInvalidOIDFormat diff --git a/modules/lfs/pointer_scanner_gogit.go b/modules/lfs/pointer_scanner_gogit.go index f4302c23bc..e153b8e24e 100644 --- a/modules/lfs/pointer_scanner_gogit.go +++ b/modules/lfs/pointer_scanner_gogit.go @@ -31,7 +31,7 @@ func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan c default: } - if blob.Size > blobSizeCutoff { + if blob.Size > MetaFileMaxSize { return nil } diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index a430b71a5f..ef72d76db4 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -5,7 +5,6 @@ package lfs import ( "bytes" - "context" "io" "net/http" "strings" @@ -33,7 +32,7 @@ func TestBasicTransferAdapter(t *testing.T) { if strings.Contains(url, "download-request") { assert.Equal(t, "GET", req.Method) - return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBufferString("dummy"))} + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("dummy"))} } else if strings.Contains(url, "upload-request") { assert.Equal(t, "PUT", req.Method) assert.Equal(t, "application/octet-stream", req.Header.Get("Content-Type")) @@ -94,7 +93,7 @@ func TestBasicTransferAdapter(t *testing.T) { } for n, c := range cases { - _, err := a.Download(context.Background(), c.link) + _, err := a.Download(t.Context(), c.link) if len(c.expectederror) > 0 { assert.Contains(t, err.Error(), c.expectederror, "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { @@ -127,7 +126,7 @@ func TestBasicTransferAdapter(t *testing.T) { } for n, c := range cases { - err := a.Upload(context.Background(), c.link, p, bytes.NewBufferString("dummy")) + err := a.Upload(t.Context(), c.link, p, strings.NewReader("dummy")) if len(c.expectederror) > 0 { assert.Contains(t, err.Error(), c.expectederror, "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { @@ -160,7 +159,7 @@ func TestBasicTransferAdapter(t *testing.T) { } for n, c := range cases { - err := a.Verify(context.Background(), c.link, p) + err := a.Verify(t.Context(), c.link, p) if len(c.expectederror) > 0 { assert.Contains(t, err.Error(), c.expectederror, "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { diff --git a/modules/lfstransfer/backend/backend.go b/modules/lfstransfer/backend/backend.go index 2b1fe49fda..dd4108ea56 100644 --- a/modules/lfstransfer/backend/backend.go +++ b/modules/lfstransfer/backend/backend.go @@ -4,7 +4,6 @@ package backend import ( - "bytes" "context" "encoding/base64" "fmt" @@ -29,7 +28,7 @@ var Capabilities = []string{ "locking", } -var _ transfer.Backend = &GiteaBackend{} +var _ transfer.Backend = (*GiteaBackend)(nil) // GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API type GiteaBackend struct { @@ -48,7 +47,7 @@ func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (t return nil, err } server = server.JoinPath("api/internal/repo", repo, "info/lfs") - return &GiteaBackend{ctx: ctx, server: server, op: op, authToken: token, internalAuth: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil + return &GiteaBackend{ctx: ctx, server: server, op: op, authToken: token, internalAuth: "Bearer " + setting.InternalToken, logger: logger}, nil } // Batch implements transfer.Backend @@ -71,24 +70,23 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans g.logger.Log("json marshal error", err) return nil, err } - url := g.server.JoinPath("objects/batch").String() headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + req := newInternalRequestLFS(g.ctx, g.server.JoinPath("objects/batch").String(), http.MethodPost, headers, bodyBytes) resp, err := req.Response() if err != nil { g.logger.Log("http request error", err) return nil, err } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode)) return nil, statusCodeToErr(resp.StatusCode) } - defer resp.Body.Close() respBytes, err := io.ReadAll(resp.Body) if err != nil { g.logger.Log("http read error", err) @@ -158,8 +156,7 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans return pointers, nil } -// Download implements transfer.Backend. The returned reader must be closed by the -// caller. +// Download implements transfer.Backend. The returned reader must be closed by the caller. func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, int64, error) { idMapStr, exists := args[argID] if !exists { @@ -181,31 +178,30 @@ func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, g.logger.Log("argument id incorrect") return nil, 0, transfer.ErrCorruptData } - url := action.Href headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeOctetStream, } - req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil) + req := newInternalRequestLFS(g.ctx, toInternalLFSURL(action.Href), http.MethodGet, headers, nil) resp, err := req.Response() if err != nil { - return nil, 0, err + return nil, 0, fmt.Errorf("failed to get response: %w", err) } + // no need to close the body here by "defer resp.Body.Close()", see below if resp.StatusCode != http.StatusOK { return nil, 0, statusCodeToErr(resp.StatusCode) } - defer resp.Body.Close() - respBytes, err := io.ReadAll(resp.Body) + + respSize, err := strconv.ParseInt(resp.Header.Get("X-Gitea-LFS-Content-Length"), 10, 64) if err != nil { - return nil, 0, err + return nil, 0, fmt.Errorf("failed to parse content length: %w", err) } - respSize := int64(len(respBytes)) - respBuf := io.NopCloser(bytes.NewBuffer(respBytes)) - return respBuf, respSize, nil + // transfer.Backend will check io.Closer interface and close this Body reader + return resp.Body, respSize, nil } -// StartUpload implements transfer.Backend. +// Upload implements transfer.Backend. func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer.Args) error { idMapStr, exists := args[argID] if !exists { @@ -227,22 +223,20 @@ func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer g.logger.Log("argument id incorrect") return transfer.ErrCorruptData } - url := action.Href headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerContentType: mimeOctetStream, headerContentLength: strconv.FormatInt(size, 10), } - reqBytes, err := io.ReadAll(r) - if err != nil { - return err - } - req := newInternalRequest(g.ctx, url, http.MethodPut, headers, reqBytes) + + req := newInternalRequestLFS(g.ctx, toInternalLFSURL(action.Href), http.MethodPut, headers, nil) + req.Body(r) resp, err := req.Response() if err != nil { return err } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return statusCodeToErr(resp.StatusCode) } @@ -277,18 +271,18 @@ func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (trans // the server sent no verify action return transfer.SuccessStatus(), nil } - url := action.Href headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + req := newInternalRequestLFS(g.ctx, toInternalLFSURL(action.Href), http.MethodPost, headers, bodyBytes) resp, err := req.Response() if err != nil { return transfer.NewStatus(transfer.StatusInternalServerError), err } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return transfer.NewStatus(uint32(resp.StatusCode), http.StatusText(resp.StatusCode)), statusCodeToErr(resp.StatusCode) } diff --git a/modules/lfstransfer/backend/lock.go b/modules/lfstransfer/backend/lock.go index f094cce1db..2c3c16a9bb 100644 --- a/modules/lfstransfer/backend/lock.go +++ b/modules/lfstransfer/backend/lock.go @@ -5,6 +5,7 @@ package backend import ( "context" + "errors" "fmt" "io" "net/http" @@ -43,14 +44,13 @@ func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) { g.logger.Log("json marshal error", err) return nil, err } - url := g.server.String() headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + req := newInternalRequestLFS(g.ctx, g.server.String(), http.MethodPost, headers, bodyBytes) resp, err := req.Response() if err != nil { g.logger.Log("http request error", err) @@ -75,7 +75,7 @@ func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) { if respBody.Lock == nil { g.logger.Log("api returned nil lock") - return nil, fmt.Errorf("api returned nil lock") + return nil, errors.New("api returned nil lock") } respLock := respBody.Lock owner := userUnknown @@ -95,14 +95,13 @@ func (g *giteaLockBackend) Unlock(lock transfer.Lock) error { g.logger.Log("json marshal error", err) return err } - url := g.server.JoinPath(lock.ID(), "unlock").String() headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) + req := newInternalRequestLFS(g.ctx, g.server.JoinPath(lock.ID(), "unlock").String(), http.MethodPost, headers, bodyBytes) resp, err := req.Response() if err != nil { g.logger.Log("http request error", err) @@ -176,16 +175,15 @@ func (g *giteaLockBackend) Range(cursor string, limit int, iter func(transfer.Lo } func (g *giteaLockBackend) queryLocks(v url.Values) ([]transfer.Lock, string, error) { - urlq := g.server.JoinPath() // get a copy - urlq.RawQuery = v.Encode() - url := urlq.String() + serverURLWithQuery := g.server.JoinPath() // get a copy + serverURLWithQuery.RawQuery = v.Encode() headers := map[string]string{ headerAuthorization: g.authToken, headerGiteaInternalAuth: g.internalAuth, headerAccept: mimeGitLFS, headerContentType: mimeGitLFS, } - req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil) + req := newInternalRequestLFS(g.ctx, serverURLWithQuery.String(), http.MethodGet, headers, nil) resp, err := req.Response() if err != nil { g.logger.Log("http request error", err) @@ -266,7 +264,7 @@ func (g *giteaLock) CurrentUser() (string, error) { // AsLockSpec implements transfer.Lock func (g *giteaLock) AsLockSpec(ownerID bool) ([]string, error) { msgs := []string{ - fmt.Sprintf("lock %s", g.ID()), + "lock " + g.ID(), fmt.Sprintf("path %s %s", g.ID(), g.Path()), fmt.Sprintf("locked-at %s %s", g.ID(), g.FormattedTimestamp()), fmt.Sprintf("ownername %s %s", g.ID(), g.OwnerName()), @@ -288,9 +286,9 @@ func (g *giteaLock) AsLockSpec(ownerID bool) ([]string, error) { // AsArguments implements transfer.Lock func (g *giteaLock) AsArguments() []string { return []string{ - fmt.Sprintf("id=%s", g.ID()), - fmt.Sprintf("path=%s", g.Path()), - fmt.Sprintf("locked-at=%s", g.FormattedTimestamp()), - fmt.Sprintf("ownername=%s", g.OwnerName()), + "id=" + g.ID(), + "path=" + g.Path(), + "locked-at=" + g.FormattedTimestamp(), + "ownername=" + g.OwnerName(), } } diff --git a/modules/lfstransfer/backend/util.go b/modules/lfstransfer/backend/util.go index cffefef375..afe02f799c 100644 --- a/modules/lfstransfer/backend/util.go +++ b/modules/lfstransfer/backend/util.go @@ -5,15 +5,16 @@ package backend import ( "context" - "crypto/tls" "fmt" - "net" + "io" "net/http" - "time" + "net/url" + "strings" "code.gitea.io/gitea/modules/httplib" - "code.gitea.io/gitea/modules/proxyprotocol" + "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "github.com/charmbracelet/git-lfs-transfer/transfer" ) @@ -60,8 +61,7 @@ const ( // Operations enum const ( - opNone = iota - opDownload + opDownload = iota + 1 opUpload ) @@ -89,53 +89,61 @@ func statusCodeToErr(code int) error { } } -func newInternalRequest(ctx context.Context, url, method string, headers map[string]string, body []byte) *httplib.Request { - req := httplib.NewRequest(url, method). - SetContext(ctx). - SetTimeout(10*time.Second, 60*time.Second). - SetTLSClientConfig(&tls.Config{ - InsecureSkipVerify: true, - }) - - if setting.Protocol == setting.HTTPUnix { - req.SetTransport(&http.Transport{ - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - var d net.Dialer - conn, err := d.DialContext(ctx, "unix", setting.HTTPAddr) - if err != nil { - return conn, err - } - if setting.LocalUseProxyProtocol { - if err = proxyprotocol.WriteLocalHeader(conn); err != nil { - _ = conn.Close() - return nil, err - } - } - return conn, err - }, - }) - } else if setting.LocalUseProxyProtocol { - req.SetTransport(&http.Transport{ - DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - var d net.Dialer - conn, err := d.DialContext(ctx, network, address) - if err != nil { - return conn, err - } - if err = proxyprotocol.WriteLocalHeader(conn); err != nil { - _ = conn.Close() - return nil, err - } - return conn, err - }, - }) +func toInternalLFSURL(s string) string { + pos1 := strings.Index(s, "://") + if pos1 == -1 { + return "" } + appSubURLWithSlash := setting.AppSubURL + "/" + pos2 := strings.Index(s[pos1+3:], appSubURLWithSlash) + if pos2 == -1 { + return "" + } + routePath := s[pos1+3+pos2+len(appSubURLWithSlash):] + fields := strings.SplitN(routePath, "/", 3) + if len(fields) < 3 || !strings.HasPrefix(fields[2], "info/lfs") { + return "" + } + return setting.LocalURL + "api/internal/repo/" + routePath +} + +func isInternalLFSURL(s string) bool { + if !strings.HasPrefix(s, setting.LocalURL) { + return false + } + u, err := url.Parse(s) + if err != nil { + return false + } + routePath := util.PathJoinRelX(u.Path) + subRoutePath, cut := strings.CutPrefix(routePath, "api/internal/repo/") + if !cut { + return false + } + fields := strings.SplitN(subRoutePath, "/", 3) + if len(fields) < 3 || !strings.HasPrefix(fields[2], "info/lfs") { + return false + } + return true +} +func newInternalRequestLFS(ctx context.Context, internalURL, method string, headers map[string]string, body any) *httplib.Request { + if !isInternalLFSURL(internalURL) { + return nil + } + req := private.NewInternalRequest(ctx, internalURL, method) + req.SetReadWriteTimeout(0) for k, v := range headers { req.Header(k, v) } - - req.Body(body) - + switch body := body.(type) { + case nil: // do nothing + case []byte: + req.Body(body) // []byte + case io.Reader: + req.Body(body) // io.Reader or io.ReadCloser + default: + panic(fmt.Sprintf("unsupported request body type %T", body)) + } return req } diff --git a/modules/lfstransfer/backend/util_test.go b/modules/lfstransfer/backend/util_test.go new file mode 100644 index 0000000000..408b53c369 --- /dev/null +++ b/modules/lfstransfer/backend/util_test.go @@ -0,0 +1,53 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package backend + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" +) + +func TestToInternalLFSURL(t *testing.T) { + defer test.MockVariableValue(&setting.LocalURL, "http://localurl/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + cases := []struct { + url string + expected string + }{ + {"http://appurl/any", ""}, + {"http://appurl/sub/any", ""}, + {"http://appurl/sub/owner/repo/any", ""}, + {"http://appurl/sub/owner/repo/info/any", ""}, + {"http://appurl/sub/owner/repo/info/lfs/any", "http://localurl/api/internal/repo/owner/repo/info/lfs/any"}, + } + for _, c := range cases { + assert.Equal(t, c.expected, toInternalLFSURL(c.url), c.url) + } +} + +func TestIsInternalLFSURL(t *testing.T) { + defer test.MockVariableValue(&setting.LocalURL, "http://localurl/")() + defer test.MockVariableValue(&setting.InternalToken, "mock-token")() + cases := []struct { + url string + expected bool + }{ + {"", false}, + {"http://otherurl/api/internal/repo/owner/repo/info/lfs/any", false}, + {"http://localurl/api/internal/repo/owner/repo/info/lfs/any", true}, + {"http://localurl/api/internal/repo/owner/repo/info", false}, + {"http://localurl/api/internal/misc/owner/repo/info/lfs/any", false}, + {"http://localurl/api/internal/owner/repo/info/lfs/any", false}, + {"http://localurl/api/internal/foo/bar", false}, + } + for _, c := range cases { + req := newInternalRequestLFS(t.Context(), c.url, "GET", nil, nil) + assert.Equal(t, c.expected, req != nil, c.url) + assert.Equal(t, c.expected, isInternalLFSURL(c.url), c.url) + } +} diff --git a/modules/log/event_format.go b/modules/log/event_format.go index 8fda0a4980..4cf471d223 100644 --- a/modules/log/event_format.go +++ b/modules/log/event_format.go @@ -125,15 +125,19 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms buf = append(buf, resetBytes...) } } - if flags&(Lshortfile|Llongfile) != 0 { + if flags&(Lshortfile|Llongfile) != 0 && event.Filename != "" { if mode.Colorize { buf = append(buf, fgGreenBytes...) } file := event.Filename if flags&Lmedfile == Lmedfile { - startIndex := len(file) - 20 - if startIndex > 0 { - file = "..." + file[startIndex:] + fileLen := len(file) + const softLimit = 20 + if fileLen > softLimit { + slashIndex := strings.LastIndexByte(file[:fileLen-softLimit], '/') + if slashIndex != -1 { + file = ".../" + file[slashIndex+1:] + } } } else if flags&Lshortfile != 0 { startIndex := strings.LastIndexByte(file, '/') @@ -157,14 +161,22 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms if mode.Colorize { buf = append(buf, fgGreenBytes...) } - funcname := event.Caller + funcName := event.Caller + shortFuncName := funcName if flags&Lshortfuncname != 0 { - lastIndex := strings.LastIndexByte(funcname, '.') - if lastIndex > 0 && len(funcname) > lastIndex+1 { - funcname = funcname[lastIndex+1:] + // funcName = "code.gitea.io/gitea/modules/foo/bar.MyFunc.func1.2()" + slashPos := strings.LastIndexByte(funcName, '/') + dotPos := strings.IndexByte(funcName[slashPos+1:], '.') + if dotPos > 0 { + // shortFuncName = "MyFunc.func1.2()" + shortFuncName = funcName[slashPos+1+dotPos+1:] + if strings.Contains(shortFuncName, ".") { + shortFuncName = strings.ReplaceAll(shortFuncName, ".func", ".") + } } + funcName = shortFuncName } - buf = append(buf, funcname...) + buf = append(buf, funcName...) if mode.Colorize { buf = append(buf, resetBytes...) } @@ -200,7 +212,7 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms } } if hasColorValue { - msg = []byte(fmt.Sprintf(msgFormat, msgArgs...)) + msg = fmt.Appendf(nil, msgFormat, msgArgs...) } } // try to re-use the pre-formatted simple text message @@ -231,8 +243,8 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms buf = append(buf, msg...) if event.Stacktrace != "" && mode.StacktraceLevel <= event.Level { - lines := bytes.Split([]byte(event.Stacktrace), []byte("\n")) - for _, line := range lines { + lines := bytes.SplitSeq([]byte(event.Stacktrace), []byte("\n")) + for line := range lines { buf = append(buf, "\n\t"...) buf = append(buf, line...) } diff --git a/modules/log/event_writer_base.go b/modules/log/event_writer_base.go index c327c48ca2..9189ca4e90 100644 --- a/modules/log/event_writer_base.go +++ b/modules/log/event_writer_base.go @@ -105,7 +105,7 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) { case io.WriterTo: _, err = msg.WriteTo(b.OutputWriteCloser) default: - _, err = b.OutputWriteCloser.Write([]byte(fmt.Sprint(msg))) + _, err = fmt.Fprint(b.OutputWriteCloser, msg) } if err != nil { FallbackErrorf("unable to write log message of %q (%v): %v", b.Name, err, event.Msg) diff --git a/modules/log/event_writer_conn_test.go b/modules/log/event_writer_conn_test.go index 69e87aa8c4..2aff37812d 100644 --- a/modules/log/event_writer_conn_test.go +++ b/modules/log/event_writer_conn_test.go @@ -4,7 +4,6 @@ package log import ( - "context" "fmt" "io" "net" @@ -40,7 +39,7 @@ func TestConnLogger(t *testing.T) { level := INFO flags := LstdFlags | LUTC | Lfuncname - logger := NewLoggerWithWriters(context.Background(), "test", NewEventWriterConn("test-conn", WriterMode{ + logger := NewLoggerWithWriters(t.Context(), "test", NewEventWriterConn("test-conn", WriterMode{ Level: level, Prefix: prefix, Flags: FlagsFromBits(flags), diff --git a/modules/log/flags.go b/modules/log/flags.go index 8064c91745..f409261150 100644 --- a/modules/log/flags.go +++ b/modules/log/flags.go @@ -123,7 +123,7 @@ func FlagsFromString(from string, def ...uint32) Flags { return Flags{defined: true, flags: def[0]} } flags := uint32(0) - for _, flag := range strings.Split(strings.ToLower(from), ",") { + for flag := range strings.SplitSeq(strings.ToLower(from), ",") { flags |= flagFromString[strings.TrimSpace(flag)] } return Flags{defined: true, flags: flags} diff --git a/modules/log/flags_test.go b/modules/log/flags_test.go index 03972a9fb0..6eb65d8114 100644 --- a/modules/log/flags_test.go +++ b/modules/log/flags_test.go @@ -12,19 +12,19 @@ import ( ) func TestFlags(t *testing.T) { - assert.EqualValues(t, Ldefault, Flags{}.Bits()) + assert.Equal(t, Ldefault, Flags{}.Bits()) assert.EqualValues(t, 0, FlagsFromString("").Bits()) - assert.EqualValues(t, Lgopid, FlagsFromString("", Lgopid).Bits()) + assert.Equal(t, Lgopid, FlagsFromString("", Lgopid).Bits()) assert.EqualValues(t, 0, FlagsFromString("none", Lgopid).Bits()) - assert.EqualValues(t, Ldate|Ltime, FlagsFromString("date,time", Lgopid).Bits()) + assert.Equal(t, Ldate|Ltime, FlagsFromString("date,time", Lgopid).Bits()) - assert.EqualValues(t, "stdflags", FlagsFromString("stdflags").String()) - assert.EqualValues(t, "medfile", FlagsFromString("medfile").String()) + assert.Equal(t, "stdflags", FlagsFromString("stdflags").String()) + assert.Equal(t, "medfile", FlagsFromString("medfile").String()) bs, err := json.Marshal(FlagsFromString("utc,level")) assert.NoError(t, err) - assert.EqualValues(t, `"level,utc"`, string(bs)) + assert.Equal(t, `"level,utc"`, string(bs)) var flags Flags assert.NoError(t, json.Unmarshal(bs, &flags)) - assert.EqualValues(t, LUTC|Llevel, flags.Bits()) + assert.Equal(t, LUTC|Llevel, flags.Bits()) } diff --git a/modules/log/level_test.go b/modules/log/level_test.go index cd18a807d8..0e59af6cb7 100644 --- a/modules/log/level_test.go +++ b/modules/log/level_test.go @@ -32,11 +32,11 @@ func TestLevelMarshalUnmarshalJSON(t *testing.T) { assert.NoError(t, err) assert.Equal(t, INFO, testLevel.Level) - err = json.Unmarshal([]byte(fmt.Sprintf(`{"level":%d}`, 2)), &testLevel) + err = json.Unmarshal(fmt.Appendf(nil, `{"level":%d}`, 2), &testLevel) assert.NoError(t, err) assert.Equal(t, INFO, testLevel.Level) - err = json.Unmarshal([]byte(fmt.Sprintf(`{"level":%d}`, 10012)), &testLevel) + err = json.Unmarshal(fmt.Appendf(nil, `{"level":%d}`, 10012), &testLevel) assert.NoError(t, err) assert.Equal(t, INFO, testLevel.Level) @@ -51,5 +51,5 @@ func TestLevelMarshalUnmarshalJSON(t *testing.T) { } func makeTestLevelBytes(level string) []byte { - return []byte(fmt.Sprintf(`{"level":"%s"}`, level)) + return fmt.Appendf(nil, `{"level":"%s"}`, level) } diff --git a/modules/log/logger.go b/modules/log/logger.go index a833b6ef0f..8b89e0eb5a 100644 --- a/modules/log/logger.go +++ b/modules/log/logger.go @@ -24,7 +24,7 @@ package log // BaseLogger provides the basic logging functions type BaseLogger interface { - Log(skip int, level Level, format string, v ...any) + Log(skip int, event *Event, format string, v ...any) GetLevel() Level } @@ -45,6 +45,6 @@ type Logger interface { LevelLogger } -type LogStringer interface { //nolint:revive +type LogStringer interface { //nolint:revive // export stutter LogString() string } diff --git a/modules/log/logger_global.go b/modules/log/logger_global.go index 6ce8b70fed..07c25cd62f 100644 --- a/modules/log/logger_global.go +++ b/modules/log/logger_global.go @@ -18,7 +18,7 @@ func GetLevel() Level { } func Log(skip int, level Level, format string, v ...any) { - GetLogger(DEFAULT).Log(skip+1, level, format, v...) + GetLogger(DEFAULT).Log(skip+1, &Event{Level: level}, format, v...) } func Trace(format string, v ...any) { diff --git a/modules/log/logger_impl.go b/modules/log/logger_impl.go index 76dd5f43fb..551c1454aa 100644 --- a/modules/log/logger_impl.go +++ b/modules/log/logger_impl.go @@ -5,6 +5,7 @@ package log import ( "context" + "reflect" "runtime" "strings" "sync" @@ -175,29 +176,42 @@ func (l *LoggerImpl) IsEnabled() bool { return l.level.Load() < int32(FATAL) && len(l.eventWriters) > 0 } +func asLogStringer(v any) LogStringer { + if s, ok := v.(LogStringer); ok { + return s + } else if a := reflect.ValueOf(v); a.Kind() == reflect.Struct { + // in case the receiver is a pointer, but the value is a struct + vp := reflect.New(a.Type()) + vp.Elem().Set(a) + if s, ok := vp.Interface().(LogStringer); ok { + return s + } + } + return nil +} + // Log prepares the log event, if the level matches, the event will be sent to the writers -func (l *LoggerImpl) Log(skip int, level Level, format string, logArgs ...any) { - if Level(l.level.Load()) > level { +func (l *LoggerImpl) Log(skip int, event *Event, format string, logArgs ...any) { + if Level(l.level.Load()) > event.Level { return } - event := &Event{ - Time: time.Now(), - Level: level, - Caller: "?()", + if event.Time.IsZero() { + event.Time = time.Now() } - - pc, filename, line, ok := runtime.Caller(skip + 1) - if ok { - fn := runtime.FuncForPC(pc) - if fn != nil { - event.Caller = fn.Name() + "()" + if event.Caller == "" { + pc, filename, line, ok := runtime.Caller(skip + 1) + if ok { + fn := runtime.FuncForPC(pc) + if fn != nil { + fnName := fn.Name() + event.Caller = strings.ReplaceAll(fnName, "[...]", "") + "()" // generic function names are "foo[...]" + } + } + event.Filename, event.Line = strings.TrimPrefix(filename, projectPackagePrefix), line + if l.stacktraceLevel.Load() <= int32(event.Level) { + event.Stacktrace = Stack(skip + 1) } - } - event.Filename, event.Line = strings.TrimPrefix(filename, projectPackagePrefix), line - - if l.stacktraceLevel.Load() <= int32(level) { - event.Stacktrace = Stack(skip + 1) } // get a simple text message without color @@ -207,11 +221,11 @@ func (l *LoggerImpl) Log(skip int, level Level, format string, logArgs ...any) { // handle LogStringer values for i, v := range msgArgs { if cv, ok := v.(*ColoredValue); ok { - if s, ok := cv.v.(LogStringer); ok { - cv.v = logStringFormatter{v: s} + if ls := asLogStringer(cv.v); ls != nil { + cv.v = logStringFormatter{v: ls} } - } else if s, ok := v.(LogStringer); ok { - msgArgs[i] = logStringFormatter{v: s} + } else if ls := asLogStringer(v); ls != nil { + msgArgs[i] = logStringFormatter{v: ls} } } diff --git a/modules/log/logger_test.go b/modules/log/logger_test.go index 0de14eb411..a74139dc51 100644 --- a/modules/log/logger_test.go +++ b/modules/log/logger_test.go @@ -4,7 +4,7 @@ package log import ( - "context" + "regexp" "sync" "testing" "time" @@ -35,11 +35,11 @@ func (d *dummyWriter) Close() error { return nil } -func (d *dummyWriter) GetLogs() []string { +func (d *dummyWriter) FetchLogs() []string { d.mu.Lock() defer d.mu.Unlock() - logs := make([]string, len(d.logs)) - copy(logs, d.logs) + logs := d.logs + d.logs = nil return logs } @@ -53,20 +53,20 @@ func newDummyWriter(name string, level Level, delay time.Duration) *dummyWriter } func TestLogger(t *testing.T) { - logger := NewLoggerWithWriters(context.Background(), "test") + logger := NewLoggerWithWriters(t.Context(), "test") dump := logger.DumpWriters() assert.Empty(t, dump) - assert.EqualValues(t, NONE, logger.GetLevel()) + assert.Equal(t, NONE, logger.GetLevel()) assert.False(t, logger.IsEnabled()) w1 := newDummyWriter("dummy-1", DEBUG, 0) logger.AddWriters(w1) - assert.EqualValues(t, DEBUG, logger.GetLevel()) + assert.Equal(t, DEBUG, logger.GetLevel()) w2 := newDummyWriter("dummy-2", WARN, 200*time.Millisecond) logger.AddWriters(w2) - assert.EqualValues(t, DEBUG, logger.GetLevel()) + assert.Equal(t, DEBUG, logger.GetLevel()) dump = logger.DumpWriters() assert.Len(t, dump, 2) @@ -77,18 +77,18 @@ func TestLogger(t *testing.T) { // w2 is slow, so only w1 has logs time.Sleep(100 * time.Millisecond) - assert.Equal(t, []string{"debug-level\n", "error-level\n"}, w1.GetLogs()) - assert.Equal(t, []string{}, w2.GetLogs()) + assert.Equal(t, []string{"debug-level\n", "error-level\n"}, w1.FetchLogs()) + assert.Empty(t, w2.FetchLogs()) logger.Close() // after Close, all logs are flushed - assert.Equal(t, []string{"debug-level\n", "error-level\n"}, w1.GetLogs()) - assert.Equal(t, []string{"error-level\n"}, w2.GetLogs()) + assert.Empty(t, w1.FetchLogs()) + assert.Equal(t, []string{"error-level\n"}, w2.FetchLogs()) } func TestLoggerPause(t *testing.T) { - logger := NewLoggerWithWriters(context.Background(), "test") + logger := NewLoggerWithWriters(t.Context(), "test") w1 := newDummyWriter("dummy-1", DEBUG, 0) logger.AddWriters(w1) @@ -98,12 +98,12 @@ func TestLoggerPause(t *testing.T) { logger.Info("info-level") time.Sleep(100 * time.Millisecond) - assert.Equal(t, []string{}, w1.GetLogs()) + assert.Empty(t, w1.FetchLogs()) GetManager().ResumeAll() time.Sleep(100 * time.Millisecond) - assert.Equal(t, []string{"info-level\n"}, w1.GetLogs()) + assert.Equal(t, []string{"info-level\n"}, w1.FetchLogs()) logger.Close() } @@ -116,21 +116,54 @@ func (t testLogString) LogString() string { return "log-string" } -func TestLoggerLogString(t *testing.T) { - logger := NewLoggerWithWriters(context.Background(), "test") +type testLogStringPtrReceiver struct { + Field string +} - w1 := newDummyWriter("dummy-1", DEBUG, 0) - w1.Mode.Colorize = true - logger.AddWriters(w1) +func (t *testLogStringPtrReceiver) LogString() string { + return "log-string-ptr-receiver" +} - logger.Info("%s %s %#v %v", testLogString{}, &testLogString{}, testLogString{Field: "detail"}, NewColoredValue(testLogString{}, FgRed)) - logger.Close() +func genericFunc[T any](logger Logger, v T) { + logger.Info("from genericFunc: %v", v) +} - assert.Equal(t, []string{"log-string log-string log.testLogString{Field:\"detail\"} \x1b[31mlog-string\x1b[0m\n"}, w1.GetLogs()) +func TestLoggerOutput(t *testing.T) { + t.Run("LogString", func(t *testing.T) { + logger := NewLoggerWithWriters(t.Context(), "test") + w1 := newDummyWriter("dummy-1", DEBUG, 0) + w1.Mode.Colorize = true + logger.AddWriters(w1) + logger.Info("%s %s %#v %v", testLogString{}, &testLogString{}, testLogString{Field: "detail"}, NewColoredValue(testLogString{}, FgRed)) + logger.Info("%s %s %#v %v", testLogStringPtrReceiver{}, &testLogStringPtrReceiver{}, testLogStringPtrReceiver{Field: "detail"}, NewColoredValue(testLogStringPtrReceiver{}, FgRed)) + logger.Close() + + assert.Equal(t, []string{ + "log-string log-string log.testLogString{Field:\"detail\"} \x1b[31mlog-string\x1b[0m\n", + "log-string-ptr-receiver log-string-ptr-receiver &log.testLogStringPtrReceiver{Field:\"detail\"} \x1b[31mlog-string-ptr-receiver\x1b[0m\n", + }, w1.FetchLogs()) + }) + + t.Run("Caller", func(t *testing.T) { + logger := NewLoggerWithWriters(t.Context(), "test") + w1 := newDummyWriter("dummy-1", DEBUG, 0) + w1.EventWriterBaseImpl.Mode.Flags.flags = Lmedfile | Lshortfuncname + logger.AddWriters(w1) + anonymousFunc := func(logger Logger) { + logger.Info("from anonymousFunc") + } + genericFunc(logger, "123") + anonymousFunc(logger) + logger.Close() + logs := w1.FetchLogs() + assert.Len(t, logs, 2) + assert.Regexp(t, `modules/log/logger_test.go:\w+:`+regexp.QuoteMeta(`genericFunc() from genericFunc: 123`), logs[0]) + assert.Regexp(t, `modules/log/logger_test.go:\w+:`+regexp.QuoteMeta(`TestLoggerOutput.2.1() from anonymousFunc`), logs[1]) + }) } func TestLoggerExpressionFilter(t *testing.T) { - logger := NewLoggerWithWriters(context.Background(), "test") + logger := NewLoggerWithWriters(t.Context(), "test") w1 := newDummyWriter("dummy-1", DEBUG, 0) w1.Mode.Expression = "foo.*" @@ -142,5 +175,5 @@ func TestLoggerExpressionFilter(t *testing.T) { logger.SendLogEvent(&Event{Level: INFO, Filename: "foo.go", MsgSimpleText: "by filename"}) logger.Close() - assert.Equal(t, []string{"foo\n", "foo bar\n", "by filename\n"}, w1.GetLogs()) + assert.Equal(t, []string{"foo\n", "foo bar\n", "by filename\n"}, w1.FetchLogs()) } diff --git a/modules/log/manager_test.go b/modules/log/manager_test.go index b8fbf84613..beddbccb73 100644 --- a/modules/log/manager_test.go +++ b/modules/log/manager_test.go @@ -37,6 +37,6 @@ func TestSharedWorker(t *testing.T) { m.Close() - logs := w.(*dummyWriter).GetLogs() + logs := w.(*dummyWriter).FetchLogs() assert.Equal(t, []string{"msg-1\n", "msg-2\n", "msg-3\n"}, logs) } diff --git a/modules/log/misc.go b/modules/log/misc.go index ae4ce04cf3..c9d230e4ac 100644 --- a/modules/log/misc.go +++ b/modules/log/misc.go @@ -19,8 +19,8 @@ func BaseLoggerToGeneralLogger(b BaseLogger) Logger { var _ Logger = (*baseToLogger)(nil) -func (s *baseToLogger) Log(skip int, level Level, format string, v ...any) { - s.base.Log(skip+1, level, format, v...) +func (s *baseToLogger) Log(skip int, event *Event, format string, v ...any) { + s.base.Log(skip+1, event, format, v...) } func (s *baseToLogger) GetLevel() Level { @@ -32,27 +32,27 @@ func (s *baseToLogger) LevelEnabled(level Level) bool { } func (s *baseToLogger) Trace(format string, v ...any) { - s.base.Log(1, TRACE, format, v...) + s.base.Log(1, &Event{Level: TRACE}, format, v...) } func (s *baseToLogger) Debug(format string, v ...any) { - s.base.Log(1, DEBUG, format, v...) + s.base.Log(1, &Event{Level: DEBUG}, format, v...) } func (s *baseToLogger) Info(format string, v ...any) { - s.base.Log(1, INFO, format, v...) + s.base.Log(1, &Event{Level: INFO}, format, v...) } func (s *baseToLogger) Warn(format string, v ...any) { - s.base.Log(1, WARN, format, v...) + s.base.Log(1, &Event{Level: WARN}, format, v...) } func (s *baseToLogger) Error(format string, v ...any) { - s.base.Log(1, ERROR, format, v...) + s.base.Log(1, &Event{Level: ERROR}, format, v...) } func (s *baseToLogger) Critical(format string, v ...any) { - s.base.Log(1, CRITICAL, format, v...) + s.base.Log(1, &Event{Level: CRITICAL}, format, v...) } type PrintfLogger struct { diff --git a/modules/markup/asciicast/asciicast.go b/modules/markup/asciicast/asciicast.go index 1d0d631650..d86d61d7c4 100644 --- a/modules/markup/asciicast/asciicast.go +++ b/modules/markup/asciicast/asciicast.go @@ -46,7 +46,7 @@ func (Renderer) Render(ctx *markup.RenderContext, _ io.Reader, output io.Writer) setting.AppSubURL, url.PathEscape(ctx.RenderOptions.Metas["user"]), url.PathEscape(ctx.RenderOptions.Metas["repo"]), - ctx.RenderOptions.Metas["BranchNameSubURL"], + ctx.RenderOptions.Metas["RefTypeNameSubURL"], url.PathEscape(ctx.RenderOptions.RelativePath), ) return ctx.RenderInternal.FormatWithSafeAttrs(output, `<div class="%s" %s="%s"></div>`, playerClassName, playerSrcAttr, rawURL) diff --git a/modules/markup/common/footnote.go b/modules/markup/common/footnote.go index 4406803694..1ece436c66 100644 --- a/modules/markup/common/footnote.go +++ b/modules/markup/common/footnote.go @@ -53,7 +53,7 @@ type FootnoteLink struct { // Dump implements Node.Dump. func (n *FootnoteLink) Dump(source []byte, level int) { m := map[string]string{} - m["Index"] = fmt.Sprintf("%v", n.Index) + m["Index"] = strconv.Itoa(n.Index) m["Name"] = fmt.Sprintf("%v", n.Name) ast.DumpHelper(n, source, level, m, nil) } @@ -85,7 +85,7 @@ type FootnoteBackLink struct { // Dump implements Node.Dump. func (n *FootnoteBackLink) Dump(source []byte, level int) { m := map[string]string{} - m["Index"] = fmt.Sprintf("%v", n.Index) + m["Index"] = strconv.Itoa(n.Index) m["Name"] = fmt.Sprintf("%v", n.Name) ast.DumpHelper(n, source, level, m, nil) } @@ -151,7 +151,7 @@ type FootnoteList struct { // Dump implements Node.Dump. func (n *FootnoteList) Dump(source []byte, level int) { m := map[string]string{} - m["Count"] = fmt.Sprintf("%v", n.Count) + m["Count"] = strconv.Itoa(n.Count) ast.DumpHelper(n, source, level, m, nil) } @@ -197,7 +197,7 @@ func (b *footnoteBlockParser) Open(parent ast.Node, reader text.Reader, pc parse return nil, parser.NoChildren } open := pos + 1 - closure := util.FindClosure(line[pos+1:], '[', ']', false, false) //nolint + closure := util.FindClosure(line[pos+1:], '[', ']', false, false) //nolint:staticcheck // deprecated function closes := pos + 1 + closure next := closes + 1 if closure > -1 { @@ -287,7 +287,7 @@ func (s *footnoteParser) Parse(parent ast.Node, block text.Reader, pc parser.Con return nil } open := pos - closure := util.FindClosure(line[pos:], '[', ']', false, false) //nolint + closure := util.FindClosure(line[pos:], '[', ']', false, false) //nolint:staticcheck // deprecated function if closure < 0 { return nil } @@ -409,9 +409,9 @@ func (r *FootnoteHTMLRenderer) renderFootnoteLink(w util.BufWriter, source []byt _, _ = w.Write(n.Name) _, _ = w.WriteString(`"><a href="#fn:`) _, _ = w.Write(n.Name) - _, _ = w.WriteString(`" class="footnote-ref" role="doc-noteref">`) + _, _ = w.WriteString(`" class="footnote-ref" role="doc-noteref">`) // FIXME: here and below, need to keep the classes _, _ = w.WriteString(is) - _, _ = w.WriteString(`</a></sup>`) + _, _ = w.WriteString(` </a></sup>`) // the style doesn't work at the moment, so add a space to separate the names } return ast.WalkContinue, nil } diff --git a/modules/markup/common/linkify.go b/modules/markup/common/linkify.go index 52888958fa..3eecb97eac 100644 --- a/modules/markup/common/linkify.go +++ b/modules/markup/common/linkify.go @@ -85,9 +85,10 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont } else if lastChar == ')' { closing := 0 for i := m[1] - 1; i >= m[0]; i-- { - if line[i] == ')' { + switch line[i] { + case ')': closing++ - } else if line[i] == '(' { + case '(': closing-- } } diff --git a/modules/markup/console/console.go b/modules/markup/console/console.go index 06f3acfa68..492579b0a5 100644 --- a/modules/markup/console/console.go +++ b/modules/markup/console/console.go @@ -6,13 +6,14 @@ package console import ( "bytes" "io" - "path" + "unicode/utf8" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" trend "github.com/buildkite/terminal-to-html/v3" - "github.com/go-enry/go-enry/v2" ) func init() { @@ -22,6 +23,8 @@ func init() { // Renderer implements markup.Renderer type Renderer struct{} +var _ markup.RendererContentDetector = (*Renderer)(nil) + // Name implements markup.Renderer func (Renderer) Name() string { return "console" @@ -40,15 +43,36 @@ func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { } // CanRender implements markup.RendererContentDetector -func (Renderer) CanRender(filename string, input io.Reader) bool { - buf, err := io.ReadAll(input) - if err != nil { +func (Renderer) CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool { + if !sniffedType.IsTextPlain() { return false } - if enry.GetLanguage(path.Base(filename), buf) != enry.OtherLanguage { + + s := util.UnsafeBytesToString(prefetchBuf) + rs := []rune(s) + cnt := 0 + firstErrPos := -1 + isCtrlSep := func(p int) bool { + return p < len(rs) && (rs[p] == ';' || rs[p] == 'm') + } + for i, c := range rs { + if c == 0 { + return false + } + if c == '\x1b' { + match := i+1 < len(rs) && rs[i+1] == '[' + if match && (isCtrlSep(i+2) || isCtrlSep(i+3) || isCtrlSep(i+4) || isCtrlSep(i+5)) { + cnt++ + } + } + if c == utf8.RuneError && firstErrPos == -1 { + firstErrPos = i + } + } + if firstErrPos != -1 && firstErrPos != len(rs)-1 { return false } - return bytes.ContainsRune(buf, '\x1b') + return cnt >= 2 // only render it as console output if there are at least two escape sequences } // Render renders terminal colors to HTML with all specific handling stuff. diff --git a/modules/markup/console/console_test.go b/modules/markup/console/console_test.go index e1f0da1f01..d1192bebc2 100644 --- a/modules/markup/console/console_test.go +++ b/modules/markup/console/console_test.go @@ -4,28 +4,43 @@ package console import ( - "context" "strings" "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/typesniffer" "github.com/stretchr/testify/assert" ) func TestRenderConsole(t *testing.T) { - var render Renderer - kases := map[string]string{ - "\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok": "<span class=\"term-fg37 term-bg40\">npm</span> <span class=\"term-fg32\">info</span> <span class=\"term-fg35\">it worked if it ends with</span> ok", + cases := []struct { + input string + expected string + }{ + {"\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok", `<span class="term-fg37 term-bg40">npm</span> <span class="term-fg32">info</span> <span class="term-fg35">it worked if it ends with</span> ok`}, + {"\x1b[1;2m \x1b[123m å•Š", `<span class="term-fg2"> å•Š</span>`}, + {"\x1b[1;2m \x1b[123m \xef", `<span class="term-fg2"> �</span>`}, + {"\x1b[1;2m \x1b[123m \xef \xef", ``}, + {"\x1b[12", ``}, + {"\x1b[1", ``}, + {"\x1b[FOO\x1b[", ``}, + {"\x1b[mFOO\x1b[m", `FOO`}, } - for k, v := range kases { + var render Renderer + for i, c := range cases { var buf strings.Builder - canRender := render.CanRender("test", strings.NewReader(k)) - assert.True(t, canRender) + st := typesniffer.DetectContentType([]byte(c.input)) + canRender := render.CanRender("test", st, []byte(c.input)) + if c.expected == "" { + assert.False(t, canRender, "case %d: expected not to render", i) + continue + } - err := render.Render(markup.NewRenderContext(context.Background()), strings.NewReader(k), &buf) + assert.True(t, canRender) + err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(c.input), &buf) assert.NoError(t, err) - assert.EqualValues(t, v, buf.String()) + assert.Equal(t, c.expected, buf.String()) } } diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index 4c47170c30..fff7f0baca 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -4,7 +4,6 @@ package markup import ( - "context" "strings" "testing" @@ -24,8 +23,8 @@ func TestRenderCSV(t *testing.T) { for k, v := range kases { var buf strings.Builder - err := render.Render(markup.NewRenderContext(context.Background()), strings.NewReader(k), &buf) + err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(k), &buf) assert.NoError(t, err) - assert.EqualValues(t, v, buf.String()) + assert.Equal(t, v, buf.String()) } } diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 03242e569e..39861ade12 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -12,11 +12,9 @@ import ( "runtime" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) // RegisterRenderers registers all supported third part renderers according settings @@ -77,27 +75,22 @@ func envMark(envName string) string { // Render renders the data of the document to HTML via the external tool. func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { - var ( - command = strings.NewReplacer( - envMark("GITEA_PREFIX_SRC"), ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault), - envMark("GITEA_PREFIX_RAW"), ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw), - ).Replace(p.Command) - commands = strings.Fields(command) - args = commands[1:] - ) + baseLinkSrc := ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault) + baseLinkRaw := ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw) + command := strings.NewReplacer( + envMark("GITEA_PREFIX_SRC"), baseLinkSrc, + envMark("GITEA_PREFIX_RAW"), baseLinkRaw, + ).Replace(p.Command) + commands := strings.Fields(command) + args := commands[1:] if p.IsInputFile { // write to temp file - f, err := os.CreateTemp("", "gitea_input") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("gitea_input") if err != nil { return fmt.Errorf("%s create temp file when rendering %s failed: %w", p.Name(), p.Command, err) } - tmpPath := f.Name() - defer func() { - if err := util.Remove(tmpPath); err != nil { - log.Warn("Unable to remove temporary file: %s: Error: %v", tmpPath, err) - } - }() + defer cleanup() _, err = io.Copy(f, input) if err != nil { @@ -112,14 +105,14 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. args = append(args, f.Name()) } - processCtx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Render [%s] for %s", commands[0], ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault))) + processCtx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Render [%s] for %s", commands[0], baseLinkSrc)) defer finished() cmd := exec.CommandContext(processCtx, commands[0], args...) cmd.Env = append( os.Environ(), - "GITEA_PREFIX_SRC="+ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault), - "GITEA_PREFIX_RAW="+ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw), + "GITEA_PREFIX_SRC="+baseLinkSrc, + "GITEA_PREFIX_RAW="+baseLinkRaw, ) if !p.IsInputFile { cmd.Stdin = input diff --git a/modules/markup/html.go b/modules/markup/html.go index bb12febf27..51afd4be00 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "regexp" + "slices" "strings" "sync" @@ -32,7 +33,6 @@ type globalVarsType struct { comparePattern *regexp.Regexp fullURLPattern *regexp.Regexp emailRegex *regexp.Regexp - blackfridayExtRegex *regexp.Regexp emojiShortCodeRegex *regexp.Regexp issueFullPattern *regexp.Regexp filesChangedFullPattern *regexp.Regexp @@ -47,7 +47,7 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // NOTE: All below regex matching do not perform any extra validation. // Thus a link is produced even if the linked entity does not exist. // While fast, this is also incorrect and lead to false positives. - // TODO: fix invalid linking issue + // TODO: fix invalid linking issue (update: stale TODO, what issues? maybe no TODO anymore) // valid chars in encoded path and parameter: [-+~_%.a-zA-Z0-9/] @@ -72,10 +72,8 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // it is still accepted by the CommonMark specification, as well as the HTML5 spec: // http://spec.commonmark.org/0.28/#email-address // https://html.spec.whatwg.org/multipage/input.html#e-mail-state-(type%3Demail) - v.emailRegex = regexp.MustCompile("(?:\\s|^|\\(|\\[)([a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)(?:\\s|$|\\)|\\]|;|,|\\?|!|\\.(\\s|$))") - - // blackfridayExtRegex is for blackfriday extensions create IDs like fn:user-content-footnote - v.blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`) + // At the moment, we use stricter rule for rendering purpose: only allow the "name" part starting after the word boundary + v.emailRegex = regexp.MustCompile(`\b([-\w.!#$%&'*+/=?^{|}~]*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)\b`) // emojiShortCodeRegex find emoji by alias like :smile: v.emojiShortCodeRegex = regexp.MustCompile(`:[-+\w]+:`) @@ -89,22 +87,18 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // codePreviewPattern matches "http://domain/.../{owner}/{repo}/src/commit/{commit}/{filepath}#L10-L20" v.codePreviewPattern = regexp.MustCompile(`https?://\S+/([^\s/]+)/([^\s/]+)/src/commit/([0-9a-f]{7,64})(/\S+)#(L\d+(-L\d+)?)`) - v.tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`) + // cleans: "<foo/bar", "<any words/", ("<html", "<head", "<script", "<style", "<?", "<%") + v.tagCleaner = regexp.MustCompile(`(?i)<(/?\w+/\w+|/[\w ]+/|/?(html|head|script|style|%|\?)\b)`) v.nulCleaner = strings.NewReplacer("\000", "") return v }) -// IsFullURLBytes reports whether link fits valid format. -func IsFullURLBytes(link []byte) bool { - return globalVars().fullURLPattern.Match(link) -} - func IsFullURLString(link string) bool { return globalVars().fullURLPattern.MatchString(link) } func IsNonEmptyRelativePath(link string) bool { - return link != "" && !IsFullURLString(link) && link[0] != '/' && link[0] != '?' && link[0] != '#' + return link != "" && !IsFullURLString(link) && link[0] != '?' && link[0] != '#' } // CustomLinkURLSchemes allows for additional schemes to be detected when parsing links within text @@ -116,13 +110,7 @@ func CustomLinkURLSchemes(schemes []string) { if !validScheme.MatchString(s) { continue } - without := false - for _, sna := range xurls.SchemesNoAuthority { - if s == sna { - without = true - break - } - } + without := slices.Contains(xurls.SchemesNoAuthority, s) if without { s += ":" } else { @@ -260,7 +248,7 @@ func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output node, err := html.Parse(io.MultiReader( // prepend "<html><body>" strings.NewReader("<html><body>"), - // Strip out nuls - they're always invalid + // strip out NULLs (they're always invalid), and escape known tags bytes.NewReader(globalVars().tagCleaner.ReplaceAll([]byte(globalVars().nulCleaner.Replace(string(rawHTML))), []byte("<$1"))), // close the tags strings.NewReader("</body></html>"), @@ -316,44 +304,39 @@ func isEmojiNode(node *html.Node) bool { } func visitNode(ctx *RenderContext, procs []processor, node *html.Node) *html.Node { - // Add user-content- to IDs and "#" links if they don't already have them - for idx, attr := range node.Attr { - val := strings.TrimPrefix(attr.Val, "#") - notHasPrefix := !(strings.HasPrefix(val, "user-content-") || globalVars().blackfridayExtRegex.MatchString(val)) - - if attr.Key == "id" && notHasPrefix { - node.Attr[idx].Val = "user-content-" + attr.Val - } - - if attr.Key == "href" && strings.HasPrefix(attr.Val, "#") && notHasPrefix { - node.Attr[idx].Val = "#user-content-" + val - } - } - - switch node.Type { - case html.TextNode: + if node.Type == html.TextNode { for _, proc := range procs { proc(ctx, node) // it might add siblings } + return node.NextSibling + } + if node.Type != html.ElementNode { + return node.NextSibling + } - case html.ElementNode: - if isEmojiNode(node) { - // TextNode emoji will be converted to `<span class="emoji">`, then the next iteration will visit the "span" - // if we don't stop it, it will go into the TextNode again and create an infinite recursion - return node.NextSibling - } else if node.Data == "code" || node.Data == "pre" { - return node.NextSibling // ignore code and pre nodes - } else if node.Data == "img" { - return visitNodeImg(ctx, node) - } else if node.Data == "video" { - return visitNodeVideo(ctx, node) - } else if node.Data == "a" { - procs = emojiProcessors // Restrict text in links to emojis - } - for n := node.FirstChild; n != nil; { - n = visitNode(ctx, procs, n) - } - default: + processNodeAttrID(node) + processFootnoteNode(ctx, node) // FIXME: the footnote processing should be done in the "footnote.go" renderer directly + + if isEmojiNode(node) { + // TextNode emoji will be converted to `<span class="emoji">`, then the next iteration will visit the "span" + // if we don't stop it, it will go into the TextNode again and create an infinite recursion + return node.NextSibling + } else if node.Data == "code" || node.Data == "pre" { + return node.NextSibling // ignore code and pre nodes + } else if node.Data == "img" { + return visitNodeImg(ctx, node) + } else if node.Data == "video" { + return visitNodeVideo(ctx, node) + } + + if node.Data == "a" { + processNodeA(ctx, node) + // only use emoji processors for the content in the "A" tag, + // because the content there is not processable, for example: the content is a commit id or a full URL. + procs = emojiProcessors + } + for n := node.FirstChild; n != nil; { + n = visitNode(ctx, procs, n) } return node.NextSibling } diff --git a/modules/markup/html_commit.go b/modules/markup/html_commit.go index 358e7b06ba..fe7a034967 100644 --- a/modules/markup/html_commit.go +++ b/modules/markup/html_commit.go @@ -8,6 +8,7 @@ import ( "strings" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/references" "code.gitea.io/gitea/modules/util" "golang.org/x/net/html" @@ -42,7 +43,6 @@ func createCodeLink(href, content, class string) *html.Node { code := &html.Node{ Type: html.ElementNode, Data: atom.Code.String(), - Attr: []html.Attribute{{Key: "class", Val: "nohighlight"}}, } code.AppendChild(text) @@ -62,7 +62,7 @@ func anyHashPatternExtract(s string) (ret anyHashPatternResult, ok bool) { // if url ends in '.', it's very likely that it is not part of the actual url but used to finish a sentence. ret.PosEnd-- ret.FullURL = ret.FullURL[:len(ret.FullURL)-1] - for i := 0; i < len(m); i++ { + for i := range m { m[i] = min(m[i], ret.PosEnd) } } @@ -188,9 +188,27 @@ func hashCurrentPatternProcessor(ctx *RenderContext, node *html.Node) { continue } - link := ctx.RenderHelper.ResolveLink(util.URLJoin(ctx.RenderOptions.Metas["user"], ctx.RenderOptions.Metas["repo"], "commit", hash), LinkTypeApp) + link := "/:root/" + util.URLJoin(ctx.RenderOptions.Metas["user"], ctx.RenderOptions.Metas["repo"], "commit", hash) replaceContent(node, m[2], m[3], createCodeLink(link, base.ShortSha(hash), "commit")) start = 0 node = node.NextSibling.NextSibling } } + +func commitCrossReferencePatternProcessor(ctx *RenderContext, node *html.Node) { + next := node.NextSibling + + for node != nil && node != next { + found, ref := references.FindRenderizableCommitCrossReference(node.Data) + if !found { + return + } + + refText := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) + linkHref := "/:root/" + util.URLJoin(ref.Owner, ref.Name, "commit", ref.CommitSha) + link := createLink(ctx, linkHref, refText, "commit") + + replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link) + node = node.NextSibling.NextSibling + } +} diff --git a/modules/markup/html_email.go b/modules/markup/html_email.go index cbfae8b829..cf18e99d98 100644 --- a/modules/markup/html_email.go +++ b/modules/markup/html_email.go @@ -3,7 +3,11 @@ package markup -import "golang.org/x/net/html" +import ( + "strings" + + "golang.org/x/net/html" +) // emailAddressProcessor replaces raw email addresses with a mailto: link. func emailAddressProcessor(ctx *RenderContext, node *html.Node) { @@ -14,6 +18,14 @@ func emailAddressProcessor(ctx *RenderContext, node *html.Node) { return } + var nextByte byte + if len(node.Data) > m[3] { + nextByte = node.Data[m[3]] + } + if strings.IndexByte(":/", nextByte) != -1 { + // for cases: "git@gitea.com:owner/repo.git", "https://git@gitea.com/owner/repo.git" + return + } mail := node.Data[m[2]:m[3]] replaceContent(node, m[2], m[3], createLink(ctx, "mailto:"+mail, mail, "" /*mailto*/)) node = node.NextSibling.NextSibling diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index 159d712955..467cc509d0 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -107,7 +107,7 @@ func TestRender_IssueIndexPattern2(t *testing.T) { isExternal := false if marker == "!" { path = "pulls" - prefix = "http://localhost:3000/someUser/someRepo/pulls/" + prefix = "/someUser/someRepo/pulls/" } else { path = "issues" prefix = "https://someurl.com/someUser/someRepo/" @@ -116,7 +116,7 @@ func TestRender_IssueIndexPattern2(t *testing.T) { links := make([]any, len(indices)) for i, index := range indices { - links[i] = numericIssueLink(util.URLJoin(TestRepoURL, path), "ref-issue", index, marker) + links[i] = numericIssueLink(util.URLJoin("/test-owner/test-repo", path), "ref-issue", index, marker) } expectedNil := fmt.Sprintf(expectedFmt, links...) testRenderIssueIndexPattern(t, s, expectedNil, NewTestRenderContext(TestAppURL, localMetas)) @@ -293,13 +293,13 @@ func TestRender_AutoLink(t *testing.T) { // render valid commit URLs tmp := util.URLJoin(TestRepoURL, "commit", "d8a994ef243349f321568f9e36d5c3f444b99cae") - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24</code></a>") tmp += "#diff-2" - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24 (diff-2)</code></a>") // render other commit URLs tmp = "https://external-link.gitea.io/go-gitea/gitea/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2" - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24 (diff-2)</code></a>") } func TestRender_FullIssueURLs(t *testing.T) { @@ -405,10 +405,10 @@ func TestRegExp_anySHA1Pattern(t *testing.T) { if v.CommitID == "" { assert.False(t, ok) } else { - assert.EqualValues(t, strings.TrimSuffix(k, "."), ret.FullURL) - assert.EqualValues(t, v.CommitID, ret.CommitID) - assert.EqualValues(t, v.SubPath, ret.SubPath) - assert.EqualValues(t, v.QueryHash, ret.QueryHash) + assert.Equal(t, strings.TrimSuffix(k, "."), ret.FullURL) + assert.Equal(t, v.CommitID, ret.CommitID) + assert.Equal(t, v.SubPath, ret.SubPath) + assert.Equal(t, v.QueryHash, ret.QueryHash) } } } diff --git a/modules/markup/html_issue.go b/modules/markup/html_issue.go index e64ec76c3d..85bec5db20 100644 --- a/modules/markup/html_issue.go +++ b/modules/markup/html_issue.go @@ -4,9 +4,9 @@ package markup import ( + "strconv" "strings" - "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/references" @@ -16,8 +16,16 @@ import ( "code.gitea.io/gitea/modules/util" "golang.org/x/net/html" + "golang.org/x/net/html/atom" ) +type RenderIssueIconTitleOptions struct { + OwnerName string + RepoName string + LinkHref string + IssueIndex int64 +} + func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) { if ctx.RenderOptions.Metas == nil { return @@ -66,6 +74,27 @@ func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) { } } +func createIssueLinkContentWithSummary(ctx *RenderContext, linkHref string, ref *references.RenderizableReference) *html.Node { + if DefaultRenderHelperFuncs.RenderRepoIssueIconTitle == nil { + return nil + } + issueIndex, _ := strconv.ParseInt(ref.Issue, 10, 64) + h, err := DefaultRenderHelperFuncs.RenderRepoIssueIconTitle(ctx, RenderIssueIconTitleOptions{ + OwnerName: ref.Owner, + RepoName: ref.Name, + LinkHref: ctx.RenderHelper.ResolveLink(linkHref, LinkTypeDefault), + IssueIndex: issueIndex, + }) + if err != nil { + log.Error("RenderRepoIssueIconTitle failed: %v", err) + return nil + } + if h == "" { + return nil + } + return &html.Node{Type: html.RawNode, Data: string(ctx.RenderInternal.ProtectSafeAttrs(h))} +} + func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { if ctx.RenderOptions.Metas == nil { return @@ -76,32 +105,28 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { // old logic: crossLinkOnly := ctx.RenderOptions.Metas["mode"] == "document" && !ctx.IsWiki crossLinkOnly := ctx.RenderOptions.Metas["markupAllowShortIssuePattern"] != "true" - var ( - found bool - ref *references.RenderizableReference - ) + var ref *references.RenderizableReference next := node.NextSibling - for node != nil && node != next { _, hasExtTrackFormat := ctx.RenderOptions.Metas["format"] // Repos with external issue trackers might still need to reference local PRs // We need to concern with the first one that shows up in the text, whichever it is isNumericStyle := ctx.RenderOptions.Metas["style"] == "" || ctx.RenderOptions.Metas["style"] == IssueNameStyleNumeric - foundNumeric, refNumeric := references.FindRenderizableReferenceNumeric(node.Data, hasExtTrackFormat && !isNumericStyle, crossLinkOnly) + refNumeric := references.FindRenderizableReferenceNumeric(node.Data, hasExtTrackFormat && !isNumericStyle, crossLinkOnly) switch ctx.RenderOptions.Metas["style"] { case "", IssueNameStyleNumeric: - found, ref = foundNumeric, refNumeric + ref = refNumeric case IssueNameStyleAlphanumeric: - found, ref = references.FindRenderizableReferenceAlphanumeric(node.Data) + ref = references.FindRenderizableReferenceAlphanumeric(node.Data) case IssueNameStyleRegexp: pattern, err := regexplru.GetCompiled(ctx.RenderOptions.Metas["regexp"]) if err != nil { return } - found, ref = references.FindRenderizableReferenceRegexp(node.Data, pattern) + ref = references.FindRenderizableReferenceRegexp(node.Data, pattern) } // Repos with external issue trackers might still need to reference local PRs @@ -109,17 +134,17 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { if hasExtTrackFormat && !isNumericStyle && refNumeric != nil { // If numeric (PR) was found, and it was BEFORE the non-numeric pattern, use that // Allow a free-pass when non-numeric pattern wasn't found. - if found && (ref == nil || refNumeric.RefLocation.Start < ref.RefLocation.Start) { - found = foundNumeric + if ref == nil || refNumeric.RefLocation.Start < ref.RefLocation.Start { ref = refNumeric } } - if !found { + + if ref == nil { return } var link *html.Node - reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End] + refText := node.Data[ref.RefLocation.Start:ref.RefLocation.End] if hasExtTrackFormat && !ref.IsPull { ctx.RenderOptions.Metas["index"] = ref.Issue @@ -129,18 +154,23 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { log.Error("unable to expand template vars for ref %s, err: %v", ref.Issue, err) } - link = createLink(ctx, res, reftext, "ref-issue ref-external-issue") + link = createLink(ctx, res, refText, "ref-issue ref-external-issue") } else { // Path determines the type of link that will be rendered. It's unknown at this point whether // the linked item is actually a PR or an issue. Luckily it's of no real consequence because // Gitea will redirect on click as appropriate. + issueOwner := util.Iif(ref.Owner == "", ctx.RenderOptions.Metas["user"], ref.Owner) + issueRepo := util.Iif(ref.Owner == "", ctx.RenderOptions.Metas["repo"], ref.Name) issuePath := util.Iif(ref.IsPull, "pulls", "issues") - if ref.Owner == "" { - linkHref := ctx.RenderHelper.ResolveLink(util.URLJoin(ctx.RenderOptions.Metas["user"], ctx.RenderOptions.Metas["repo"], issuePath, ref.Issue), LinkTypeApp) - link = createLink(ctx, linkHref, reftext, "ref-issue") - } else { - linkHref := ctx.RenderHelper.ResolveLink(util.URLJoin(ref.Owner, ref.Name, issuePath, ref.Issue), LinkTypeApp) - link = createLink(ctx, linkHref, reftext, "ref-issue") + linkHref := "/:root/" + util.URLJoin(issueOwner, issueRepo, issuePath, ref.Issue) + + // at the moment, only render the issue index in a full line (or simple line) as icon+title + // otherwise it would be too noisy for "take #1 as an example" in a sentence + if node.Parent.DataAtom == atom.Li && ref.RefLocation.Start < 20 && ref.RefLocation.End == len(node.Data) { + link = createIssueLinkContentWithSummary(ctx, linkHref, ref) + } + if link == nil { + link = createLink(ctx, linkHref, refText, "ref-issue") } } @@ -168,21 +198,3 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { node = node.NextSibling.NextSibling.NextSibling.NextSibling } } - -func commitCrossReferencePatternProcessor(ctx *RenderContext, node *html.Node) { - next := node.NextSibling - - for node != nil && node != next { - found, ref := references.FindRenderizableCommitCrossReference(node.Data) - if !found { - return - } - - reftext := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) - linkHref := ctx.RenderHelper.ResolveLink(util.URLJoin(ref.Owner, ref.Name, "commit", ref.CommitSha), LinkTypeApp) - link := createLink(ctx, linkHref, reftext, "commit") - - replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link) - node = node.NextSibling.NextSibling - } -} diff --git a/modules/markup/html_issue_test.go b/modules/markup/html_issue_test.go new file mode 100644 index 0000000000..39cd9dcf6a --- /dev/null +++ b/modules/markup/html_issue_test.go @@ -0,0 +1,91 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup_test + +import ( + "context" + "html/template" + "strings" + "testing" + + "code.gitea.io/gitea/modules/htmlutil" + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/markdown" + testModule "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRender_IssueList(t *testing.T) { + defer testModule.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + markup.Init(&markup.RenderHelperFuncs{ + RenderRepoIssueIconTitle: func(ctx context.Context, opts markup.RenderIssueIconTitleOptions) (template.HTML, error) { + return htmlutil.HTMLFormat("<div>issue #%d</div>", opts.IssueIndex), nil + }, + }) + + test := func(input, expected string) { + rctx := markup.NewTestRenderContext(markup.TestAppURL, map[string]string{ + "user": "test-user", "repo": "test-repo", + "markupAllowShortIssuePattern": "true", + "footnoteContextId": "12345", + }) + out, err := markdown.RenderString(rctx, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(out))) + } + + t.Run("NormalIssueRef", func(t *testing.T) { + test( + "#12345", + `<p><a href="/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a></p>`, + ) + }) + + t.Run("ListIssueRef", func(t *testing.T) { + test( + "* #12345", + `<ul> +<li><div>issue #12345</div></li> +</ul>`, + ) + }) + + t.Run("ListIssueRefNormal", func(t *testing.T) { + test( + "* foo #12345 bar", + `<ul> +<li>foo <a href="/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a> bar</li> +</ul>`, + ) + }) + + t.Run("ListTodoIssueRef", func(t *testing.T) { + test( + "* [ ] #12345", + `<ul> +<li class="task-list-item"><input type="checkbox" disabled="" data-source-position="2"/><div>issue #12345</div></li> +</ul>`, + ) + }) + + t.Run("IssueFootnote", func(t *testing.T) { + test( + "foo[^1][^2]\n\n[^1]: bar\n[^2]: baz", + `<p>foo<sup id="fnref:user-content-1-12345"><a href="#fn:user-content-1-12345" rel="nofollow">1 </a></sup><sup id="fnref:user-content-2-12345"><a href="#fn:user-content-2-12345" rel="nofollow">2 </a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-1-12345"> +<p>bar <a href="#fnref:user-content-1-12345" rel="nofollow">↩︎</a></p> +</li> +<li id="fn:user-content-2-12345"> +<p>baz <a href="#fnref:user-content-2-12345" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div>`, + ) + }) +} diff --git a/modules/markup/html_link.go b/modules/markup/html_link.go index 5fd38b63cd..43faef1681 100644 --- a/modules/markup/html_link.go +++ b/modules/markup/html_link.go @@ -9,6 +9,7 @@ import ( "strings" "code.gitea.io/gitea/modules/markup/common" + "code.gitea.io/gitea/modules/util" "golang.org/x/net/html" "golang.org/x/net/html/atom" @@ -30,8 +31,8 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { // It makes page handling terrible, but we prefer GitHub syntax // And fall back to MediaWiki only when it is obvious from the look // Of text and link contents - sl := strings.Split(content, "|") - for _, v := range sl { + sl := strings.SplitSeq(content, "|") + for v := range sl { if equalPos := strings.IndexByte(v, '='); equalPos == -1 { // There is no equal in this argument; this is a mandatory arg if props["name"] == "" { @@ -124,7 +125,6 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { } } if image { - link = ctx.RenderHelper.ResolveLink(link, LinkTypeMedia) title := props["title"] if title == "" { title = props["alt"] @@ -150,7 +150,6 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { childNode.Attr = childNode.Attr[:2] } } else { - link = ctx.RenderHelper.ResolveLink(link, LinkTypeDefault) childNode.Type = html.TextNode childNode.Data = name } @@ -171,6 +170,10 @@ func linkProcessor(ctx *RenderContext, node *html.Node) { } uri := node.Data[m[0]:m[1]] + remaining := node.Data[m[1]:] + if util.IsLikelyEllipsisLeftPart(remaining) { + return + } replaceContent(node, m[0], m[1], createLink(ctx, uri, uri, "" /*link*/)) node = node.NextSibling.NextSibling } diff --git a/modules/markup/html_mention.go b/modules/markup/html_mention.go index fffa12e7b7..f97c034cf3 100644 --- a/modules/markup/html_mention.go +++ b/modules/markup/html_mention.go @@ -33,7 +33,7 @@ func mentionProcessor(ctx *RenderContext, node *html.Node) { if ok && strings.Contains(mention, "/") { mentionOrgAndTeam := strings.Split(mention, "/") if mentionOrgAndTeam[0][1:] == ctx.RenderOptions.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") { - link := ctx.RenderHelper.ResolveLink(util.URLJoin("org", ctx.RenderOptions.Metas["org"], "teams", mentionOrgAndTeam[1]), LinkTypeApp) + link := "/:root/" + util.URLJoin("org", ctx.RenderOptions.Metas["org"], "teams", mentionOrgAndTeam[1]) replaceContent(node, loc.Start, loc.End, createLink(ctx, link, mention, "" /*mention*/)) node = node.NextSibling.NextSibling start = 0 @@ -45,7 +45,7 @@ func mentionProcessor(ctx *RenderContext, node *html.Node) { mentionedUsername := mention[1:] if DefaultRenderHelperFuncs != nil && DefaultRenderHelperFuncs.IsUsernameMentionable(ctx, mentionedUsername) { - link := ctx.RenderHelper.ResolveLink(mentionedUsername, LinkTypeApp) + link := "/:root/" + mentionedUsername replaceContent(node, loc.Start, loc.End, createLink(ctx, link, mention, "" /*mention*/)) node = node.NextSibling.NextSibling start = 0 diff --git a/modules/markup/html_node.go b/modules/markup/html_node.go index 6e8ca67900..4eb78fdd2b 100644 --- a/modules/markup/html_node.go +++ b/modules/markup/html_node.go @@ -4,42 +4,105 @@ package markup import ( + "strings" + "golang.org/x/net/html" ) +func isAnchorIDUserContent(s string) bool { + // blackfridayExtRegex is for blackfriday extensions create IDs like fn:user-content-footnote + // old logic: blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`) + return strings.HasPrefix(s, "user-content-") || strings.Contains(s, ":user-content-") +} + +func isAnchorIDFootnote(s string) bool { + return strings.HasPrefix(s, "fnref:user-content-") || strings.HasPrefix(s, "fn:user-content-") +} + +func isAnchorHrefFootnote(s string) bool { + return strings.HasPrefix(s, "#fnref:user-content-") || strings.HasPrefix(s, "#fn:user-content-") +} + +func processNodeAttrID(node *html.Node) { + // Add user-content- to IDs and "#" links if they don't already have them, + // and convert the link href to a relative link to the host root + for idx, attr := range node.Attr { + if attr.Key == "id" { + if !isAnchorIDUserContent(attr.Val) { + node.Attr[idx].Val = "user-content-" + attr.Val + } + } + } +} + +func processFootnoteNode(ctx *RenderContext, node *html.Node) { + for idx, attr := range node.Attr { + if (attr.Key == "id" && isAnchorIDFootnote(attr.Val)) || + (attr.Key == "href" && isAnchorHrefFootnote(attr.Val)) { + if footnoteContextID := ctx.RenderOptions.Metas["footnoteContextId"]; footnoteContextID != "" { + node.Attr[idx].Val = attr.Val + "-" + footnoteContextID + } + continue + } + } +} + +func processNodeA(ctx *RenderContext, node *html.Node) { + for idx, attr := range node.Attr { + if attr.Key == "href" { + if anchorID, ok := strings.CutPrefix(attr.Val, "#"); ok { + if !isAnchorIDUserContent(attr.Val) { + node.Attr[idx].Val = "#user-content-" + anchorID + } + } else { + node.Attr[idx].Val = ctx.RenderHelper.ResolveLink(attr.Val, LinkTypeDefault) + } + } + } +} + func visitNodeImg(ctx *RenderContext, img *html.Node) (next *html.Node) { next = img.NextSibling - for i, attr := range img.Attr { - if attr.Key != "src" { + attrSrc, hasLazy := "", false + for i, imgAttr := range img.Attr { + hasLazy = hasLazy || imgAttr.Key == "loading" && imgAttr.Val == "lazy" + if imgAttr.Key != "src" { + attrSrc = imgAttr.Val continue } - if IsNonEmptyRelativePath(attr.Val) { - attr.Val = ctx.RenderHelper.ResolveLink(attr.Val, LinkTypeMedia) + imgSrcOrigin := imgAttr.Val + isLinkable := imgSrcOrigin != "" && !strings.HasPrefix(imgSrcOrigin, "data:") - // By default, the "<img>" tag should also be clickable, - // because frontend use `<img>` to paste the re-scaled image into the markdown, - // so it must match the default markdown image behavior. - hasParentAnchor := false - for p := img.Parent; p != nil; p = p.Parent { - if hasParentAnchor = p.Type == html.ElementNode && p.Data == "a"; hasParentAnchor { - break - } - } - if !hasParentAnchor { - imgA := &html.Node{Type: html.ElementNode, Data: "a", Attr: []html.Attribute{ - {Key: "href", Val: attr.Val}, - {Key: "target", Val: "_blank"}, - }} - parent := img.Parent - imgNext := img.NextSibling - parent.RemoveChild(img) - parent.InsertBefore(imgA, imgNext) - imgA.AppendChild(img) + // By default, the "<img>" tag should also be clickable, + // because frontend uses `<img>` to paste the re-scaled image into the Markdown, + // so it must match the default Markdown image behavior. + cnt := 0 + for p := img.Parent; isLinkable && p != nil && cnt < 2; p = p.Parent { + if hasParentAnchor := p.Type == html.ElementNode && p.Data == "a"; hasParentAnchor { + isLinkable = false + break } + cnt++ } - attr.Val = camoHandleLink(attr.Val) - img.Attr[i] = attr + if isLinkable { + wrapper := &html.Node{Type: html.ElementNode, Data: "a", Attr: []html.Attribute{ + {Key: "href", Val: ctx.RenderHelper.ResolveLink(imgSrcOrigin, LinkTypeDefault)}, + {Key: "target", Val: "_blank"}, + }} + parent := img.Parent + imgNext := img.NextSibling + parent.RemoveChild(img) + parent.InsertBefore(wrapper, imgNext) + wrapper.AppendChild(img) + } + + imgAttr.Val = ctx.RenderHelper.ResolveLink(imgSrcOrigin, LinkTypeMedia) + imgAttr.Val = camoHandleLink(imgAttr.Val) + img.Attr[i] = imgAttr + } + if !RenderBehaviorForTesting.DisableAdditionalAttributes && !hasLazy && !strings.HasPrefix(attrSrc, "data:") { + img.Attr = append(img.Attr, html.Attribute{Key: "loading", Val: "lazy"}) } return next } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 54bd91f3b3..5fdbf43f7c 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -35,6 +35,7 @@ func TestRender_Commits(t *testing.T) { sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" repo := markup.TestAppURL + testRepoOwnerName + "/" + testRepoName + "/" commit := util.URLJoin(repo, "commit", sha) + commitPath := "/user13/repo11/commit/" + sha tree := util.URLJoin(repo, "tree", sha, "src") file := util.URLJoin(repo, "commit", sha, "example.txt") @@ -44,9 +45,9 @@ func TestRender_Commits(t *testing.T) { commitCompare := util.URLJoin(repo, "compare", sha+"..."+sha) commitCompareWithHash := commitCompare + "#L2" - test(sha, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) - test(sha[:7], `<p><a href="`+commit[:len(commit)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`) - test(sha[:39], `<p><a href="`+commit[:len(commit)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(sha, `<p><a href="`+commitPath+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(sha[:7], `<p><a href="`+commitPath[:len(commitPath)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`) + test(sha[:39], `<p><a href="`+commitPath[:len(commitPath)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test(commit, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test(tree, `<p><a href="`+tree+`" rel="nofollow"><code>65f1bf27bc/src</code></a></p>`) @@ -57,13 +58,13 @@ func TestRender_Commits(t *testing.T) { test(commitCompare, `<p><a href="`+commitCompare+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc</code></a></p>`) test(commitCompareWithHash, `<p><a href="`+commitCompareWithHash+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc (L2)</code></a></p>`) - test("commit "+sha, `<p>commit <a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test("commit "+sha, `<p>commit <a href="`+commitPath+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test("/home/gitea/"+sha, "<p>/home/gitea/"+sha+"</p>") test("deadbeef", `<p>deadbeef</p>`) test("d27ace93", `<p>d27ace93</p>`) test(sha[:14]+".x", `<p>`+sha[:14]+`.x</p>`) - expected14 := `<a href="` + commit[:len(commit)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>` + expected14 := `<a href="` + commitPath[:len(commitPath)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>` test(sha[:14]+".", `<p>`+expected14+`.</p>`) test(sha[:14]+",", `<p>`+expected14+`,</p>`) test("["+sha[:14]+"]", `<p>[`+expected14+`]</p>`) @@ -80,10 +81,10 @@ func TestRender_CrossReferences(t *testing.T) { test( "test-owner/test-repo#12345", - `<p><a href="`+util.URLJoin(markup.TestAppURL, "test-owner", "test-repo", "issues", "12345")+`" class="ref-issue" rel="nofollow">test-owner/test-repo#12345</a></p>`) + `<p><a href="/test-owner/test-repo/issues/12345" class="ref-issue" rel="nofollow">test-owner/test-repo#12345</a></p>`) test( "go-gitea/gitea#12345", - `<p><a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) + `<p><a href="/go-gitea/gitea/issues/12345" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) test( "/home/gitea/go-gitea/gitea#12345", `<p>/home/gitea/go-gitea/gitea#12345</p>`) @@ -206,6 +207,16 @@ func TestRender_links(t *testing.T) { test( "ftps://gitea.com", `<p>ftps://gitea.com</p>`) + + t.Run("LinkEllipsis", func(t *testing.T) { + input := util.EllipsisDisplayString("http://10.1.2.3", 12) + assert.Equal(t, "http://10…", input) + test(input, "<p>http://10…</p>") + + input = util.EllipsisDisplayString("http://10.1.2.3", 13) + assert.Equal(t, "http://10.…", input) + test(input, "<p>http://10.…</p>") + }) } func TestRender_email(t *testing.T) { @@ -214,10 +225,10 @@ func TestRender_email(t *testing.T) { test := func(input, expected string) { res, err := markup.RenderString(markup.NewTestRenderContext().WithRelativePath("a.md"), input) assert.NoError(t, err) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res)) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res), "input: %s", input) } - // Text that should be turned into email link + // Text that should be turned into email link test( "info@gitea.com", `<p><a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a></p>`) @@ -249,28 +260,48 @@ func TestRender_email(t *testing.T) { <a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>? <a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>!</p>`) + // match GitHub behavior + test("email@domain@domain.com", `<p>email@<a href="mailto:domain@domain.com" rel="nofollow">domain@domain.com</a></p>`) + + // match GitHub behavior + test(`"info@gitea.com"`, `<p>"<a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a>"</p>`) + // Test that should *not* be turned into email links test( - "\"info@gitea.com\"", - `<p>"info@gitea.com"</p>`) - test( "/home/gitea/mailstore/info@gitea/com", `<p>/home/gitea/mailstore/info@gitea/com</p>`) test( "git@try.gitea.io:go-gitea/gitea.git", `<p>git@try.gitea.io:go-gitea/gitea.git</p>`) test( + "https://foo:bar@gitea.io", + `<p><a href="https://foo:bar@gitea.io" rel="nofollow">https://foo:bar@gitea.io</a></p>`) + test( "gitea@3", `<p>gitea@3</p>`) test( "gitea@gmail.c", `<p>gitea@gmail.c</p>`) test( - "email@domain@domain.com", - `<p>email@domain@domain.com</p>`) - test( "email@domain..com", `<p>email@domain..com</p>`) + + cases := []struct { + input, expected string + }{ + // match GitHub behavior + {"?a@d.zz", `<p>?<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + {"*a@d.zz", `<p>*<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + {"~a@d.zz", `<p>~<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + + // the following cases don't match GitHub behavior, but they are valid email addresses ... + // maybe we should reduce the candidate characters for the "name" part in the future + {"a*a@d.zz", `<p><a href="mailto:a*a@d.zz" rel="nofollow">a*a@d.zz</a></p>`}, + {"a~a@d.zz", `<p><a href="mailto:a~a@d.zz" rel="nofollow">a~a@d.zz</a></p>`}, + } + for _, c := range cases { + test(c.input, c.expected) + } } func TestRender_emoji(t *testing.T) { @@ -458,7 +489,7 @@ func Test_ParseClusterFuzz(t *testing.T) { assert.NotContains(t, res.String(), "<html") } -func TestPostProcess_RenderDocument(t *testing.T) { +func TestPostProcess(t *testing.T) { setting.StaticURLPrefix = markup.TestAppURL // can't run standalone defer testModule.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() @@ -469,7 +500,7 @@ func TestPostProcess_RenderDocument(t *testing.T) { assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res.String())) } - // Issue index shouldn't be post processing in a document. + // Issue index shouldn't be post-processing in a document. test( "#1", "#1") @@ -477,9 +508,9 @@ func TestPostProcess_RenderDocument(t *testing.T) { // But cross-referenced issue index should work. test( "go-gitea/gitea#12345", - `<a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue">go-gitea/gitea#12345</a>`) + `<a href="/go-gitea/gitea/issues/12345" class="ref-issue">go-gitea/gitea#12345</a>`) - // Test that other post processing still works. + // Test that other post-processing still works. test( ":gitea:", `<span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span>`) @@ -488,6 +519,16 @@ func TestPostProcess_RenderDocument(t *testing.T) { `Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle`) test("http://localhost:3000/person/repo/issues/4#issuecomment-1234", `<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234" class="ref-issue">person/repo#4 (comment)</a>`) + + // special tags, GitHub's behavior, and for unclosed tags, output as text content as much as possible + test("<script>a", `<script>a`) + test("<script>a</script>", `<script>a</script>`) + test("<STYLE>a", `<STYLE>a`) + test("<style>a</STYLE>", `<style>a</STYLE>`) + + // other special tags, our special behavior + test("<?php\nfoo", "<?php\nfoo") + test("<%asp\nfoo", "<%asp\nfoo") } func TestIssue16020(t *testing.T) { @@ -512,7 +553,7 @@ func BenchmarkEmojiPostprocess(b *testing.B) { data += data } b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { var res strings.Builder err := markup.PostProcessDefault(markup.NewTestRenderContext(localMetas), strings.NewReader(data), &res) assert.NoError(b, err) @@ -533,7 +574,7 @@ func TestIssue18471(t *testing.T) { err := markup.PostProcessDefault(markup.NewTestRenderContext(localMetas), strings.NewReader(data), &res) assert.NoError(t, err) - assert.Equal(t, `<a href="http://domain/org/repo/compare/783b039...da951ce" class="compare"><code class="nohighlight">783b039...da951ce</code></a>`, res.String()) + assert.Equal(t, `<a href="http://domain/org/repo/compare/783b039...da951ce" class="compare"><code>783b039...da951ce</code></a>`, res.String()) } func TestIsFullURL(t *testing.T) { diff --git a/modules/markup/internal/internal_test.go b/modules/markup/internal/internal_test.go index 98ff3bc079..590bcbb67f 100644 --- a/modules/markup/internal/internal_test.go +++ b/modules/markup/internal/internal_test.go @@ -35,7 +35,7 @@ func TestRenderInternal(t *testing.T) { assert.EqualValues(t, c.protected, protected) _, _ = io.WriteString(in, string(protected)) _ = in.Close() - assert.EqualValues(t, c.recovered, out.String()) + assert.Equal(t, c.recovered, out.String()) } var r1, r2 RenderInternal @@ -44,11 +44,11 @@ func TestRenderInternal(t *testing.T) { _ = r1.init("sec", nil) protected = r1.ProtectSafeAttrs(`<div class="test"></div>`) assert.EqualValues(t, `<div data-attr-class="sec:test"></div>`, protected) - assert.EqualValues(t, "data-attr-class", r1.SafeAttr("class")) - assert.EqualValues(t, "sec:val", r1.SafeValue("val")) + assert.Equal(t, "data-attr-class", r1.SafeAttr("class")) + assert.Equal(t, "sec:val", r1.SafeValue("val")) recovered, ok := r1.RecoverProtectedValue("sec:val") assert.True(t, ok) - assert.EqualValues(t, "val", recovered) + assert.Equal(t, "val", recovered) recovered, ok = r1.RecoverProtectedValue("other:val") assert.False(t, ok) assert.Empty(t, recovered) @@ -57,5 +57,5 @@ func TestRenderInternal(t *testing.T) { in2 := r2.init("sec-other", out2) _, _ = io.WriteString(in2, string(protected)) _ = in2.Close() - assert.EqualValues(t, `<div data-attr-class="sec:test"></div>`, out2.String(), "different secureID should not recover the value") + assert.Equal(t, `<div data-attr-class="sec:test"></div>`, out2.String(), "different secureID should not recover the value") } diff --git a/modules/markup/internal/renderinternal.go b/modules/markup/internal/renderinternal.go index 4d58f160a9..7a3e37b120 100644 --- a/modules/markup/internal/renderinternal.go +++ b/modules/markup/internal/renderinternal.go @@ -76,7 +76,7 @@ func (r *RenderInternal) ProtectSafeAttrs(content template.HTML) template.HTML { return template.HTML(reAttrClass().ReplaceAllString(string(content), `$1 data-attr-class="`+r.secureIDPrefix+`$2"$3`)) } -func (r *RenderInternal) FormatWithSafeAttrs(w io.Writer, fmt string, a ...any) error { +func (r *RenderInternal) FormatWithSafeAttrs(w io.Writer, fmt template.HTML, a ...any) error { _, err := w.Write([]byte(r.ProtectSafeAttrs(htmlutil.HTMLFormat(fmt, a...)))) return err } diff --git a/modules/markup/markdown/ast.go b/modules/markup/markdown/ast.go index ca165b1ba0..f29f883734 100644 --- a/modules/markup/markdown/ast.go +++ b/modules/markup/markdown/ast.go @@ -4,6 +4,7 @@ package markdown import ( + "html/template" "strconv" "github.com/yuin/goldmark/ast" @@ -29,9 +30,7 @@ func (n *Details) Kind() ast.NodeKind { // NewDetails returns a new Paragraph node. func NewDetails() *Details { - return &Details{ - BaseBlock: ast.BaseBlock{}, - } + return &Details{} } // Summary is a block that contains the summary of details block @@ -54,9 +53,7 @@ func (n *Summary) Kind() ast.NodeKind { // NewSummary returns a new Summary node. func NewSummary() *Summary { - return &Summary{ - BaseBlock: ast.BaseBlock{}, - } + return &Summary{} } // TaskCheckBoxListItem is a block that represents a list item of a markdown block with a checkbox @@ -95,29 +92,6 @@ type Icon struct { Name []byte } -// Dump implements Node.Dump . -func (n *Icon) Dump(source []byte, level int) { - m := map[string]string{} - m["Name"] = string(n.Name) - ast.DumpHelper(n, source, level, m, nil) -} - -// KindIcon is the NodeKind for Icon -var KindIcon = ast.NewNodeKind("Icon") - -// Kind implements Node.Kind. -func (n *Icon) Kind() ast.NodeKind { - return KindIcon -} - -// NewIcon returns a new Paragraph node. -func NewIcon(name string) *Icon { - return &Icon{ - BaseInline: ast.BaseInline{}, - Name: []byte(name), - } -} - // ColorPreview is an inline for a color preview type ColorPreview struct { ast.BaseInline @@ -175,3 +149,24 @@ func NewAttention(attentionType string) *Attention { AttentionType: attentionType, } } + +var KindRawHTML = ast.NewNodeKind("RawHTML") + +type RawHTML struct { + ast.BaseBlock + rawHTML template.HTML +} + +func (n *RawHTML) Dump(source []byte, level int) { + m := map[string]string{} + m["RawHTML"] = string(n.rawHTML) + ast.DumpHelper(n, source, level, m, nil) +} + +func (n *RawHTML) Kind() ast.NodeKind { + return KindRawHTML +} + +func NewRawHTML(rawHTML template.HTML) *RawHTML { + return &RawHTML{rawHTML: rawHTML} +} diff --git a/modules/markup/markdown/convertyaml.go b/modules/markup/markdown/convertyaml.go index 1675b68be2..04664a9c1d 100644 --- a/modules/markup/markdown/convertyaml.go +++ b/modules/markup/markdown/convertyaml.go @@ -4,23 +4,22 @@ package markdown import ( + "strings" + + "code.gitea.io/gitea/modules/htmlutil" + "code.gitea.io/gitea/modules/svg" + "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" "gopkg.in/yaml.v3" ) func nodeToTable(meta *yaml.Node) ast.Node { - for { - if meta == nil { - return nil - } - switch meta.Kind { - case yaml.DocumentNode: - meta = meta.Content[0] - continue - default: - } - break + for meta != nil && meta.Kind == yaml.DocumentNode { + meta = meta.Content[0] + } + if meta == nil { + return nil } switch meta.Kind { case yaml.MappingNode: @@ -72,12 +71,28 @@ func sequenceNodeToTable(meta *yaml.Node) ast.Node { return table } -func nodeToDetails(meta *yaml.Node, icon string) ast.Node { +func nodeToDetails(g *ASTTransformer, meta *yaml.Node) ast.Node { + for meta != nil && meta.Kind == yaml.DocumentNode { + meta = meta.Content[0] + } + if meta == nil { + return nil + } + if meta.Kind != yaml.MappingNode { + return nil + } + var keys []string + for i := 0; i < len(meta.Content); i += 2 { + if meta.Content[i].Kind == yaml.ScalarNode { + keys = append(keys, meta.Content[i].Value) + } + } details := NewDetails() + details.SetAttributeString(g.renderInternal.SafeAttr("class"), g.renderInternal.SafeValue("frontmatter-content")) summary := NewSummary() - summary.AppendChild(summary, NewIcon(icon)) + summaryInnerHTML := htmlutil.HTMLFormat("%s %s", svg.RenderHTML("octicon-table", 12), strings.Join(keys, ", ")) + summary.AppendChild(summary, NewRawHTML(summaryInnerHTML)) details.AppendChild(details, summary) details.AppendChild(details, nodeToTable(meta)) - return details } diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 69c2a96ff1..b28fa9824e 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -5,14 +5,10 @@ package markdown import ( "fmt" - "regexp" - "strings" - "sync" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/internal" - "code.gitea.io/gitea/modules/setting" "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" @@ -51,7 +47,7 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa tocList := make([]Header, 0, 20) if rc.yamlNode != nil { - metaNode := rc.toMetaNode() + metaNode := rc.toMetaNode(g) if metaNode != nil { node.InsertBefore(node, firstChild, metaNode) } @@ -68,23 +64,12 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa g.transformHeading(ctx, v, reader, &tocList) case *ast.Paragraph: g.applyElementDir(v) - case *ast.Image: - g.transformImage(ctx, v) - case *ast.Link: - g.transformLink(ctx, v) case *ast.List: g.transformList(ctx, v, rc) case *ast.Text: if v.SoftLineBreak() && !v.HardLineBreak() { - // TODO: this was a quite unclear part, old code: `if metas["mode"] != "document" { use comment link break setting }` - // many places render non-comment contents with no mode=document, then these contents also use comment's hard line break setting - // especially in many tests. - markdownLineBreakStyle := ctx.RenderOptions.Metas["markdownLineBreakStyle"] - if markdownLineBreakStyle == "comment" { - v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInComments) - } else if markdownLineBreakStyle == "document" { - v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInDocuments) - } + newLineHardBreak := ctx.RenderOptions.Metas["markdownNewLineHardBreak"] == "true" + v.SetHardLineBreak(newLineHardBreak) } case *ast.CodeSpan: g.transformCodeSpan(ctx, v, reader) @@ -111,11 +96,6 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa } } -// it is copied from old code, which is quite doubtful whether it is correct -var reValidIconName = sync.OnceValue(func() *regexp.Regexp { - return regexp.MustCompile(`^[-\w]+$`) // old: regexp.MustCompile("^[a-z ]+$") -}) - // NewHTMLRenderer creates a HTMLRenderer to render in the gitea form. func NewHTMLRenderer(renderInternal *internal.RenderInternal, opts ...html.Option) renderer.NodeRenderer { r := &HTMLRenderer{ @@ -140,11 +120,11 @@ func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { reg.Register(ast.KindDocument, r.renderDocument) reg.Register(KindDetails, r.renderDetails) reg.Register(KindSummary, r.renderSummary) - reg.Register(KindIcon, r.renderIcon) reg.Register(ast.KindCodeSpan, r.renderCodeSpan) reg.Register(KindAttention, r.renderAttention) reg.Register(KindTaskCheckBoxListItem, r.renderTaskCheckBoxListItem) reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox) + reg.Register(KindRawHTML, r.renderRawHTML) } func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { @@ -155,7 +135,7 @@ func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast. if entering { _, err = w.WriteString("<div") if err == nil { - _, err = w.WriteString(fmt.Sprintf(` lang=%q`, val)) + _, err = fmt.Fprintf(w, ` lang=%q`, val) } if err == nil { _, err = w.WriteRune('>') @@ -206,30 +186,14 @@ func (r *HTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.N return ast.WalkContinue, nil } -func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *HTMLRenderer) renderRawHTML(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } - - n := node.(*Icon) - - name := strings.TrimSpace(strings.ToLower(string(n.Name))) - - if len(name) == 0 { - // skip this - return ast.WalkContinue, nil - } - - if !reValidIconName().MatchString(name) { - // skip this - return ast.WalkContinue, nil - } - - // FIXME: the "icon xxx" is from Fomantic UI, it's really questionable whether it still works correctly - err := r.renderInternal.FormatWithSafeAttrs(w, `<i class="icon %s"></i>`, name) + n := node.(*RawHTML) + _, err := w.WriteString(string(r.renderInternal.ProtectSafeAttrs(n.rawHTML))) if err != nil { return ast.WalkStop, err } - return ast.WalkContinue, nil } diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index b5fffccdb9..3b788432ba 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -5,7 +5,7 @@ package markdown import ( - "fmt" + "errors" "html/template" "io" "strings" @@ -48,7 +48,7 @@ func (l *limitWriter) Write(data []byte) (int, error) { if err != nil { return n, err } - return n, fmt.Errorf("rendered content too large - truncating render") + return n, errors.New("rendered content too large - truncating render") } n, err := l.w.Write(data) l.sum += int64(n) @@ -86,20 +86,15 @@ func (r *GlodmarkRender) highlightingRenderer(w util.BufWriter, c highlighting.C preClasses += " is-loading" } - err := r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<pre class="%s">`, preClasses) - if err != nil { - return - } - // include language-x class as part of commonmark spec, "chroma" class is used to highlight the code // the "display" class is used by "js/markup/math.ts" to render the code element as a block // the "math.ts" strictly depends on the structure: <pre class="code-block is-loading"><code class="language-math display">...</code></pre> - err = r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<code class="chroma language-%s display">`, languageStr) + err := r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<div class="code-block-container code-overflow-scroll"><pre class="%s"><code class="chroma language-%s display">`, preClasses, languageStr) if err != nil { return } } else { - _, err := w.WriteString("</code></pre>") + _, err := w.WriteString("</code></pre></div>") if err != nil { return } @@ -126,11 +121,11 @@ func SpecializedMarkdown(ctx *markup.RenderContext) *GlodmarkRender { highlighting.WithWrapperRenderer(r.highlightingRenderer), ), math.NewExtension(&ctx.RenderInternal, math.Options{ - Enabled: setting.Markdown.EnableMath, - ParseDollarInline: true, - ParseDollarBlock: true, - ParseSquareBlock: true, // TODO: this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping, it should be deprecated in the future (by some config options) - // ParseBracketInline: true, // TODO: this is also a bad syntax "\( ... \)", it also conflicts, it should be deprecated in the future + Enabled: setting.Markdown.EnableMath, + ParseInlineDollar: setting.Markdown.MathCodeBlockOptions.ParseInlineDollar, + ParseInlineParentheses: setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses, // this is a bad syntax "\( ... \)", it conflicts with normal markdown escaping + ParseBlockDollar: setting.Markdown.MathCodeBlockOptions.ParseBlockDollar, + ParseBlockSquareBrackets: setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets, // this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping }), meta.Meta, ), @@ -159,6 +154,14 @@ func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error limit: setting.UI.MaxDisplayFileSize * 3, } + // FIXME: Don't read all to memory, but goldmark doesn't support + buf, err := io.ReadAll(input) + if err != nil { + log.Error("Unable to ReadAll: %v", err) + return err + } + buf = giteautil.NormalizeEOL(buf) + // FIXME: should we include a timeout to abort the renderer if it takes too long? defer func() { err := recover() @@ -166,35 +169,20 @@ func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error return } - log.Warn("Unable to render markdown due to panic in goldmark: %v", err) - if (!setting.IsProd && !setting.IsInTesting) || log.IsDebug() { - log.Error("Panic in markdown: %v\n%s", err, log.Stack(2)) - } + log.Error("Panic in markdown: %v\n%s", err, log.Stack(2)) + escapedHTML := template.HTMLEscapeString(giteautil.UnsafeBytesToString(buf)) + _, _ = output.Write(giteautil.UnsafeStringToBytes(escapedHTML)) }() - // FIXME: Don't read all to memory, but goldmark doesn't support pc := newParserContext(ctx) - buf, err := io.ReadAll(input) - if err != nil { - log.Error("Unable to ReadAll: %v", err) - return err - } - buf = giteautil.NormalizeEOL(buf) // Preserve original length. bufWithMetadataLength := len(buf) - rc := &RenderConfig{ - Meta: markup.RenderMetaAsDetails, - Icon: "table", - Lang: "", - } + rc := &RenderConfig{Meta: markup.RenderMetaAsDetails} buf, _ = ExtractMetadataBytes(buf, rc) - metaLength := bufWithMetadataLength - len(buf) - if metaLength < 0 { - metaLength = 0 - } + metaLength := max(bufWithMetadataLength-len(buf), 0) rc.metaLength = metaLength pc.Set(renderConfigKey, rc) diff --git a/modules/markup/markdown/markdown_attention_test.go b/modules/markup/markdown/markdown_attention_test.go index f6ec775b2c..7b54653ec0 100644 --- a/modules/markup/markdown/markdown_attention_test.go +++ b/modules/markup/markdown/markdown_attention_test.go @@ -23,6 +23,11 @@ func TestAttention(t *testing.T) { defer svg.MockIcon("octicon-alert")() defer svg.MockIcon("octicon-stop")() + test := func(input, expected string) { + result, err := markdown.RenderString(markup.NewTestRenderContext(), input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(result))) + } renderAttention := func(attention, icon string) string { tmpl := `<blockquote class="attention-header attention-{attention}"><p><svg class="attention-icon attention-{attention} svg {icon}" width="16" height="16"></svg><strong class="attention-{attention}">{Attention}</strong></p>` tmpl = strings.ReplaceAll(tmpl, "{attention}", attention) @@ -31,12 +36,6 @@ func TestAttention(t *testing.T) { return tmpl } - test := func(input, expected string) { - result, err := markdown.RenderString(markup.NewTestRenderContext(), input) - assert.NoError(t, err) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(result))) - } - test(` > [!NOTE] > text @@ -53,4 +52,7 @@ func TestAttention(t *testing.T) { // legacy GitHub style test(`> **warning**`, renderAttention("warning", "octicon-alert")+"\n</blockquote>") + + // edge case (it used to cause panic) + test(">\ntext", "<blockquote>\n</blockquote>\n<p>text</p>") } diff --git a/modules/markup/markdown/markdown_benchmark_test.go b/modules/markup/markdown/markdown_benchmark_test.go index 0f7e3eea6f..e08612f064 100644 --- a/modules/markup/markdown/markdown_benchmark_test.go +++ b/modules/markup/markdown/markdown_benchmark_test.go @@ -12,14 +12,14 @@ import ( func BenchmarkSpecializedMarkdown(b *testing.B) { // 240856 4719 ns/op - for i := 0; i < b.N; i++ { + for b.Loop() { markdown.SpecializedMarkdown(&markup.RenderContext{}) } } func BenchmarkMarkdownRender(b *testing.B) { // 23202 50840 ns/op - for i := 0; i < b.N; i++ { + for b.Loop() { _, _ = markdown.RenderString(markup.NewTestRenderContext(), "https://example.com\n- a\n- b\n") } } diff --git a/modules/markup/markdown/markdown_math_test.go b/modules/markup/markdown/markdown_math_test.go index 813f050965..a75f18d36a 100644 --- a/modules/markup/markdown/markdown_math_test.go +++ b/modules/markup/markdown/markdown_math_test.go @@ -8,6 +8,8 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" ) @@ -15,6 +17,7 @@ import ( const nl = "\n" func TestMathRender(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseInlineParentheses: true} testcases := []struct { testcase string expected string @@ -69,7 +72,7 @@ func TestMathRender(t *testing.T) { }, { "$$a$$", - `<code class="language-math display">a</code>` + nl, + `<p><code class="language-math">a</code></p>` + nl, }, { "$$a$$ test", @@ -111,6 +114,7 @@ func TestMathRender(t *testing.T) { } func TestMathRenderBlockIndent(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseBlockDollar: true, ParseBlockSquareBrackets: true} testcases := []struct { name string testcase string @@ -243,3 +247,64 @@ x }) } } + +func TestMathRenderOptions(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{} + defer test.MockVariableValue(&setting.Markdown.MathCodeBlockOptions) + test := func(t *testing.T, expected, input string) { + res, err := RenderString(markup.NewTestRenderContext(), input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(res)), "input: %s", input) + } + + // default (non-conflict) inline syntax + test(t, `<p><code class="language-math">a</code></p>`, "$`a`$") + + // ParseInlineDollar + test(t, `<p>$a$</p>`, `$a$`) + setting.Markdown.MathCodeBlockOptions.ParseInlineDollar = true + test(t, `<p><code class="language-math">a</code></p>`, `$a$`) + + // ParseInlineParentheses + test(t, `<p>(a)</p>`, `\(a\)`) + setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses = true + test(t, `<p><code class="language-math">a</code></p>`, `\(a\)`) + + // ParseBlockDollar + test(t, `<p>$$ +a +$$</p> +`, ` +$$ +a +$$ +`) + setting.Markdown.MathCodeBlockOptions.ParseBlockDollar = true + test(t, `<pre class="code-block is-loading"><code class="language-math display"> +a +</code></pre> +`, ` +$$ +a +$$ +`) + + // ParseBlockSquareBrackets + test(t, `<p>[ +a +]</p> +`, ` +\[ +a +\] +`) + setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets = true + test(t, `<pre class="code-block is-loading"><code class="language-math display"> +a +</code></pre> +`, ` +\[ +a +\] +`) +} diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index 7a09be8665..4eb01bcc2d 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -47,7 +47,7 @@ func TestRender_StandardLinks(t *testing.T) { func TestRender_Images(t *testing.T) { setting.AppURL = AppURL - test := func(input, expected string) { + render := func(input, expected string) { buffer, err := markdown.RenderString(markup.NewTestRenderContext(FullURL), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) @@ -59,27 +59,32 @@ func TestRender_Images(t *testing.T) { result := util.URLJoin(FullURL, url) // hint: With Markdown v2.5.2, there is a new syntax: [link](URL){:target="_blank"} , but we do not support it now - test( + render( "!["+title+"]("+url+")", `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "[["+title+"|"+url+"]]", `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) - test( + render( "[!["+title+"]("+url+")]("+href+")", `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "!["+title+"]("+url+")", `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "[["+title+"|"+url+"]]", `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) - test( + render( "[!["+title+"]("+url+")]("+href+")", `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) + + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, false)() + render( + "<a><img src='a.jpg'></a>", // by the way, empty "a" tag will be removed + `<p dir="auto"><img src="http://localhost:3000/user13/repo11/a.jpg" loading="lazy"/></p>`) } func TestTotal_RenderString(t *testing.T) { @@ -223,7 +228,7 @@ This PR has been generated by [Renovate Bot](https://github.com/renovatebot/reno <dd>This is another definition of the second term.</dd> </dl> <h3 id="user-content-footnotes">Footnotes</h3> -<p>Here is a simple footnote,<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1</a></sup> and here is a longer one.<sup id="fnref:user-content-bignote"><a href="#fn:user-content-bignote" rel="nofollow">2</a></sup></p> +<p>Here is a simple footnote,<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1 </a></sup> and here is a longer one.<sup id="fnref:user-content-bignote"><a href="#fn:user-content-bignote" rel="nofollow">2 </a></sup></p> <div> <hr/> <ol> @@ -252,7 +257,7 @@ This PR has been generated by [Renovate Bot](https://github.com/renovatebot/reno return username == "r-lyeh" }, }) - for i := 0; i < len(sameCases); i++ { + for i := range sameCases { line, err := markdown.RenderString(markup.NewTestRenderContext(localMetas), sameCases[i]) assert.NoError(t, err) assert.Equal(t, testAnswers[i], string(line)) @@ -308,12 +313,12 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) { testcase := `  ` - expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"></a> -<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"></a></p> + expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"/></a> +<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"/></a></p> ` - res, err := markdown.RenderRawString(markup.NewTestRenderContext(), testcase) + res, err := markdown.RenderString(markup.NewTestRenderContext(), testcase) assert.NoError(t, err) - assert.Equal(t, expected, res) + assert.Equal(t, expected, string(res)) } func TestRenderEmojiInLinks_Issue12331(t *testing.T) { @@ -383,18 +388,74 @@ func TestColorPreview(t *testing.T) { } } -func TestTaskList(t *testing.T) { +func TestMarkdownFrontmatter(t *testing.T) { testcases := []struct { - testcase string + name string + input string expected string }{ { + "MapInFrontmatter", + `--- +key1: val1 +key2: val2 +--- +test +`, + `<details class="frontmatter-content"><summary><span>octicon-table(12/)</span> key1, key2</summary><table> +<thead> +<tr> +<th>key1</th> +<th>key2</th> +</tr> +</thead> +<tbody> +<tr> +<td>val1</td> +<td>val2</td> +</tr> +</tbody> +</table> +</details><p>test</p> +`, + }, + + { + "ListInFrontmatter", + `--- +- item1 +- item2 +--- +test +`, + `- item1 +- item2 + +<p>test</p> +`, + }, + + { + "StringInFrontmatter", + `--- +anything +--- +test +`, + `anything + +<p>test</p> +`, + }, + + { // data-source-position should take into account YAML frontmatter. + "ListAfterFrontmatter", `--- foo: bar --- - [ ] task 1`, - `<details><summary><i class="icon table"></i></summary><table> + `<details class="frontmatter-content"><summary><span>octicon-table(12/)</span> foo</summary><table> <thead> <tr> <th>foo</th> @@ -414,9 +475,9 @@ foo: bar } for _, test := range testcases { - res, err := markdown.RenderString(markup.NewTestRenderContext(), test.testcase) - assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) - assert.Equal(t, template.HTML(test.expected), res, "Unexpected result in testcase %q", test.testcase) + res, err := markdown.RenderString(markup.NewTestRenderContext(), test.input) + assert.NoError(t, err, "Unexpected error in testcase: %q", test.name) + assert.Equal(t, test.expected, string(res), "Unexpected result in testcase %q", test.name) } } @@ -473,3 +534,16 @@ space</p> assert.NoError(t, err) assert.Equal(t, expected, string(result)) } + +func TestMarkdownLink(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + input := `<a href=foo>link1</a> +<a href='/foo'>link2</a> +<a href="#foo">link3</a>` + result, err := markdown.RenderString(markup.NewTestRenderContext("/base", localMetas), input) + assert.NoError(t, err) + assert.Equal(t, `<p><a href="/base/foo" rel="nofollow">link1</a> +<a href="/base/foo" rel="nofollow">link2</a> +<a href="#user-content-foo" rel="nofollow">link3</a></p> +`, string(result)) +} diff --git a/modules/markup/markdown/math/block_renderer.go b/modules/markup/markdown/math/block_renderer.go index c29f061882..95a336a02c 100644 --- a/modules/markup/markdown/math/block_renderer.go +++ b/modules/markup/markdown/math/block_renderer.go @@ -4,6 +4,8 @@ package math import ( + "html/template" + "code.gitea.io/gitea/modules/markup/internal" giteaUtil "code.gitea.io/gitea/modules/util" @@ -40,7 +42,7 @@ func (r *BlockRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { func (r *BlockRenderer) writeLines(w util.BufWriter, source []byte, n gast.Node) { l := n.Lines().Len() - for i := 0; i < l; i++ { + for i := range l { line := n.Lines().At(i) _, _ = w.Write(util.EscapeHTML(line.Value(source))) } @@ -49,8 +51,8 @@ func (r *BlockRenderer) writeLines(w util.BufWriter, source []byte, n gast.Node) func (r *BlockRenderer) renderBlock(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { n := node.(*Block) if entering { - code := giteaUtil.Iif(n.Inline, "", `<pre class="code-block is-loading">`) + `<code class="language-math display">` - _ = r.renderInternal.FormatWithSafeAttrs(w, code) + codeHTML := giteaUtil.Iif[template.HTML](n.Inline, "", `<pre class="code-block is-loading">`) + `<code class="language-math display">` + _, _ = w.WriteString(string(r.renderInternal.ProtectSafeAttrs(codeHTML))) r.writeLines(w, source, n) } else { _, _ = w.WriteString(`</code>` + giteaUtil.Iif(n.Inline, "", `</pre>`) + "\n") diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go index a57abe9f9b..a711d1e1cd 100644 --- a/modules/markup/markdown/math/inline_parser.go +++ b/modules/markup/markdown/math/inline_parser.go @@ -15,26 +15,26 @@ type inlineParser struct { trigger []byte endBytesSingleDollar []byte endBytesDoubleDollar []byte - endBytesBracket []byte + endBytesParentheses []byte + enableInlineDollar bool } -var defaultInlineDollarParser = &inlineParser{ - trigger: []byte{'$'}, - endBytesSingleDollar: []byte{'$'}, - endBytesDoubleDollar: []byte{'$', '$'}, -} - -func NewInlineDollarParser() parser.InlineParser { - return defaultInlineDollarParser +func NewInlineDollarParser(enableInlineDollar bool) parser.InlineParser { + return &inlineParser{ + trigger: []byte{'$'}, + endBytesSingleDollar: []byte{'$'}, + endBytesDoubleDollar: []byte{'$', '$'}, + enableInlineDollar: enableInlineDollar, + } } -var defaultInlineBracketParser = &inlineParser{ - trigger: []byte{'\\', '('}, - endBytesBracket: []byte{'\\', ')'}, +var defaultInlineParenthesesParser = &inlineParser{ + trigger: []byte{'\\', '('}, + endBytesParentheses: []byte{'\\', ')'}, } -func NewInlineBracketParser() parser.InlineParser { - return defaultInlineBracketParser +func NewInlineParenthesesParser() parser.InlineParser { + return defaultInlineParenthesesParser } // Trigger triggers this parser on $ or \ @@ -46,7 +46,7 @@ func isPunctuation(b byte) bool { return b == '.' || b == '!' || b == '?' || b == ',' || b == ';' || b == ':' } -func isBracket(b byte) bool { +func isParenthesesClose(b byte) bool { return b == ')' } @@ -70,10 +70,11 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. startMarkLen = 1 stopMark = parser.endBytesSingleDollar if len(line) > 1 { - if line[1] == '$' { + switch line[1] { + case '$': startMarkLen = 2 stopMark = parser.endBytesDoubleDollar - } else if line[1] == '`' { + case '`': pos := 1 for ; pos < len(line) && line[pos] == '`'; pos++ { } @@ -85,7 +86,11 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. } } else { startMarkLen = 2 - stopMark = parser.endBytesBracket + stopMark = parser.endBytesParentheses + } + + if line[0] == '$' && !parser.enableInlineDollar && (len(line) == 1 || line[1] != '`') { + return nil } if checkSurrounding { @@ -109,7 +114,7 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. succeedingCharacter = line[i+len(stopMark)] } // check valid ending character - isValidEndingChar := isPunctuation(succeedingCharacter) || isBracket(succeedingCharacter) || + isValidEndingChar := isPunctuation(succeedingCharacter) || isParenthesesClose(succeedingCharacter) || succeedingCharacter == ' ' || succeedingCharacter == '\n' || succeedingCharacter == 0 if checkSurrounding && !isValidEndingChar { break @@ -121,9 +126,10 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. i++ continue } - if line[i] == '{' { + switch line[i] { + case '{': depth++ - } else if line[i] == '}' { + case '}': depth-- } } diff --git a/modules/markup/markdown/math/inline_renderer.go b/modules/markup/markdown/math/inline_renderer.go index d000a7b317..eeeb60cc7e 100644 --- a/modules/markup/markdown/math/inline_renderer.go +++ b/modules/markup/markdown/math/inline_renderer.go @@ -28,7 +28,7 @@ func NewInlineRenderer(renderInternal *internal.RenderInternal) renderer.NodeRen func (r *InlineRenderer) renderInline(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { if entering { - _ = r.renderInternal.FormatWithSafeAttrs(w, `<code class="language-math">`) + _, _ = w.WriteString(string(r.renderInternal.ProtectSafeAttrs(`<code class="language-math">`))) for c := n.FirstChild(); c != nil; c = c.NextSibling() { segment := c.(*ast.Text).Segment value := util.EscapeHTML(segment.Value(source)) diff --git a/modules/markup/markdown/math/math.go b/modules/markup/markdown/math/math.go index a6ff593d62..4b74db2d76 100644 --- a/modules/markup/markdown/math/math.go +++ b/modules/markup/markdown/math/math.go @@ -14,10 +14,11 @@ import ( ) type Options struct { - Enabled bool - ParseDollarInline bool - ParseDollarBlock bool - ParseSquareBlock bool + Enabled bool + ParseInlineDollar bool // inline $$ xxx $$ text + ParseInlineParentheses bool // inline \( xxx \) text + ParseBlockDollar bool // block $$ multiple-line $$ text + ParseBlockSquareBrackets bool // block \[ multiple-line \] text } // Extension is a math extension @@ -42,16 +43,16 @@ func (e *Extension) Extend(m goldmark.Markdown) { return } - inlines := []util.PrioritizedValue{util.Prioritized(NewInlineBracketParser(), 501)} - if e.options.ParseDollarInline { - inlines = append(inlines, util.Prioritized(NewInlineDollarParser(), 502)) + var inlines []util.PrioritizedValue + if e.options.ParseInlineParentheses { + inlines = append(inlines, util.Prioritized(NewInlineParenthesesParser(), 501)) } - m.Parser().AddOptions(parser.WithInlineParsers(inlines...)) + inlines = append(inlines, util.Prioritized(NewInlineDollarParser(e.options.ParseInlineDollar), 502)) + m.Parser().AddOptions(parser.WithInlineParsers(inlines...)) m.Parser().AddOptions(parser.WithBlockParsers( - util.Prioritized(NewBlockParser(e.options.ParseDollarBlock, e.options.ParseSquareBlock), 701), + util.Prioritized(NewBlockParser(e.options.ParseBlockDollar, e.options.ParseBlockSquareBrackets), 701), )) - m.Renderer().AddOptions(renderer.WithNodeRenderers( util.Prioritized(NewBlockRenderer(e.renderInternal), 501), util.Prioritized(NewInlineRenderer(e.renderInternal), 502), diff --git a/modules/markup/markdown/meta_test.go b/modules/markup/markdown/meta_test.go index 278c33f1d2..283d289d48 100644 --- a/modules/markup/markdown/meta_test.go +++ b/modules/markup/markdown/meta_test.go @@ -51,7 +51,7 @@ func TestExtractMetadata(t *testing.T) { var meta IssueTemplate body, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) assert.NoError(t, err) - assert.Equal(t, "", body) + assert.Empty(t, body) assert.Equal(t, metaTest, meta) assert.True(t, meta.Valid()) }) @@ -60,7 +60,7 @@ func TestExtractMetadata(t *testing.T) { func TestExtractMetadataBytes(t *testing.T) { t.Run("ValidFrontAndBody", func(t *testing.T) { var meta IssueTemplate - body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta) + body, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest), &meta) assert.NoError(t, err) assert.Equal(t, bodyTest, string(body)) assert.Equal(t, metaTest, meta) @@ -69,21 +69,21 @@ func TestExtractMetadataBytes(t *testing.T) { t.Run("NoFirstSeparator", func(t *testing.T) { var meta IssueTemplate - _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", frontTest, sepTest, bodyTest)), &meta) + _, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", frontTest, sepTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoLastSeparator", func(t *testing.T) { var meta IssueTemplate - _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, bodyTest)), &meta) + _, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", sepTest, frontTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoBody", func(t *testing.T) { var meta IssueTemplate - body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta) + body, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) assert.NoError(t, err) - assert.Equal(t, "", string(body)) + assert.Empty(t, string(body)) assert.Equal(t, metaTest, meta) assert.True(t, meta.Valid()) }) diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index f4c48d1b3d..d8b1b10ce6 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -16,7 +16,6 @@ import ( // RenderConfig represents rendering configuration for this file type RenderConfig struct { Meta markup.RenderMetaMode - Icon string TOC string // "false": hide, "side"/empty: in sidebar, "main"/"true": in main view Lang string yamlNode *yaml.Node @@ -74,7 +73,7 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { type yamlRenderConfig struct { Meta *string `yaml:"meta"` - Icon *string `yaml:"details_icon"` + Icon *string `yaml:"details_icon"` // deprecated, because there is no font icon, so no custom icon TOC *string `yaml:"include_toc"` Lang *string `yaml:"lang"` } @@ -96,10 +95,6 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { rc.Meta = renderMetaModeFromString(*cfg.Gitea.Meta) } - if cfg.Gitea.Icon != nil { - rc.Icon = strings.TrimSpace(strings.ToLower(*cfg.Gitea.Icon)) - } - if cfg.Gitea.Lang != nil && *cfg.Gitea.Lang != "" { rc.Lang = *cfg.Gitea.Lang } @@ -111,7 +106,7 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { return nil } -func (rc *RenderConfig) toMetaNode() ast.Node { +func (rc *RenderConfig) toMetaNode(g *ASTTransformer) ast.Node { if rc.yamlNode == nil { return nil } @@ -119,7 +114,7 @@ func (rc *RenderConfig) toMetaNode() ast.Node { case markup.RenderMetaAsTable: return nodeToTable(rc.yamlNode) case markup.RenderMetaAsDetails: - return nodeToDetails(rc.yamlNode, rc.Icon) + return nodeToDetails(g, rc.yamlNode) default: return nil } diff --git a/modules/markup/markdown/renderconfig_test.go b/modules/markup/markdown/renderconfig_test.go index c53acdc77a..53c52177a7 100644 --- a/modules/markup/markdown/renderconfig_test.go +++ b/modules/markup/markdown/renderconfig_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -19,42 +21,36 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "empty", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", }, "", }, { "lang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "test", }, "lang: test", }, { "metatable", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", }, "gitea: table", }, { "metanone", &RenderConfig{ Meta: "none", - Icon: "table", Lang: "", }, "gitea: none", }, { "metadetails", &RenderConfig{ Meta: "details", - Icon: "table", Lang: "", }, "gitea: details", }, { "metawrong", &RenderConfig{ Meta: "details", - Icon: "table", Lang: "", }, "gitea: wrong", }, @@ -62,7 +58,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { "toc", &RenderConfig{ TOC: "true", Meta: "table", - Icon: "table", Lang: "", }, "include_toc: true", }, @@ -70,14 +65,12 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { "tocfalse", &RenderConfig{ TOC: "false", Meta: "table", - Icon: "table", Lang: "", }, "include_toc: false", }, { "toclang", &RenderConfig{ Meta: "table", - Icon: "table", TOC: "true", Lang: "testlang", }, ` @@ -88,7 +81,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` gitea: @@ -98,7 +90,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang2", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` lang: notright @@ -109,7 +100,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` gitea: @@ -121,7 +111,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Lang: "two", Meta: "table", TOC: "true", - Icon: "smiley", }, ` lang: one include_toc: true @@ -137,26 +126,14 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got := &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", } - if err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got); err != nil { - t.Errorf("RenderConfig.UnmarshalYAML() error = %v\n%q", err, tt.args) - return - } + err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got) + require.NoError(t, err) - if got.Meta != tt.expected.Meta { - t.Errorf("Meta Expected %s Got %s", tt.expected.Meta, got.Meta) - } - if got.Icon != tt.expected.Icon { - t.Errorf("Icon Expected %s Got %s", tt.expected.Icon, got.Icon) - } - if got.Lang != tt.expected.Lang { - t.Errorf("Lang Expected %s Got %s", tt.expected.Lang, got.Lang) - } - if got.TOC != tt.expected.TOC { - t.Errorf("TOC Expected %q Got %q", tt.expected.TOC, got.TOC) - } + assert.Equal(t, tt.expected.Meta, got.Meta) + assert.Equal(t, tt.expected.Lang, got.Lang) + assert.Equal(t, tt.expected.TOC, got.TOC) }) } } diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go index ea1af83a3e..a11b9d0390 100644 --- a/modules/markup/markdown/toc.go +++ b/modules/markup/markdown/toc.go @@ -4,7 +4,6 @@ package markdown import ( - "fmt" "net/url" "code.gitea.io/gitea/modules/translation" @@ -50,7 +49,7 @@ func createTOCNode(toc []Header, lang string, detailsAttrs map[string]string) as } li := ast.NewListItem(currentLevel * 2) a := ast.NewLink() - a.Destination = []byte(fmt.Sprintf("#%s", url.QueryEscape(header.ID))) + a.Destination = []byte("#" + url.QueryEscape(header.ID)) a.AppendChild(a, ast.NewString([]byte(header.Text))) li.AppendChild(li, a) ul.AppendChild(ul, li) diff --git a/modules/markup/markdown/transform_blockquote.go b/modules/markup/markdown/transform_blockquote.go index 2651d44a69..bf17f01681 100644 --- a/modules/markup/markdown/transform_blockquote.go +++ b/modules/markup/markdown/transform_blockquote.go @@ -46,7 +46,7 @@ func (g *ASTTransformer) extractBlockquoteAttentionEmphasis(firstParagraph ast.N if !ok { return "", nil } - val1 := string(node1.Text(reader.Source())) //nolint:staticcheck + val1 := string(node1.Text(reader.Source())) //nolint:staticcheck // Text is deprecated attentionType := strings.ToLower(val1) if g.attentionTypes.Contains(attentionType) { return attentionType, []ast.Node{node1} @@ -115,6 +115,9 @@ func (g *ASTTransformer) transformBlockquote(v *ast.Blockquote, reader text.Read // grab these nodes and make sure we adhere to the attention blockquote structure firstParagraph := v.FirstChild() + if firstParagraph == nil { + return ast.WalkContinue, nil + } g.applyElementDir(firstParagraph) attentionType, processedNodes := g.extractBlockquoteAttentionEmphasis(firstParagraph, reader) diff --git a/modules/markup/markdown/transform_codespan.go b/modules/markup/markdown/transform_codespan.go index bccc43aad2..c2e4295bc2 100644 --- a/modules/markup/markdown/transform_codespan.go +++ b/modules/markup/markdown/transform_codespan.go @@ -68,7 +68,7 @@ func cssColorHandler(value string) bool { } func (g *ASTTransformer) transformCodeSpan(_ *markup.RenderContext, v *ast.CodeSpan, reader text.Reader) { - colorContent := v.Text(reader.Source()) //nolint:staticcheck + colorContent := v.Text(reader.Source()) //nolint:staticcheck // Text is deprecated if cssColorHandler(string(colorContent)) { v.AppendChild(v, NewColorPreview(colorContent)) } diff --git a/modules/markup/markdown/transform_heading.go b/modules/markup/markdown/transform_heading.go index 5f8a12794d..a229a7b1a4 100644 --- a/modules/markup/markdown/transform_heading.go +++ b/modules/markup/markdown/transform_heading.go @@ -16,10 +16,10 @@ import ( func (g *ASTTransformer) transformHeading(_ *markup.RenderContext, v *ast.Heading, reader text.Reader, tocList *[]Header) { for _, attr := range v.Attributes() { if _, ok := attr.Value.([]byte); !ok { - v.SetAttribute(attr.Name, []byte(fmt.Sprintf("%v", attr.Value))) + v.SetAttribute(attr.Name, fmt.Appendf(nil, "%v", attr.Value)) } } - txt := v.Text(reader.Source()) //nolint:staticcheck + txt := v.Text(reader.Source()) //nolint:staticcheck // Text is deprecated header := Header{ Text: util.UnsafeBytesToString(txt), Level: v.Level, diff --git a/modules/markup/markdown/transform_image.go b/modules/markup/markdown/transform_image.go deleted file mode 100644 index 36512e59a8..0000000000 --- a/modules/markup/markdown/transform_image.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markdown - -import ( - "code.gitea.io/gitea/modules/markup" - - "github.com/yuin/goldmark/ast" -) - -func (g *ASTTransformer) transformImage(ctx *markup.RenderContext, v *ast.Image) { - // Images need two things: - // - // 1. Their src needs to munged to be a real value - // 2. If they're not wrapped with a link they need a link wrapper - - // Check if the destination is a real link - if len(v.Destination) > 0 && !markup.IsFullURLBytes(v.Destination) { - v.Destination = []byte(ctx.RenderHelper.ResolveLink(string(v.Destination), markup.LinkTypeMedia)) - } - - parent := v.Parent() - // Create a link around image only if parent is not already a link - if _, ok := parent.(*ast.Link); !ok && parent != nil { - next := v.NextSibling() - - // Create a link wrapper - wrap := ast.NewLink() - wrap.Destination = v.Destination - wrap.Title = v.Title - wrap.SetAttributeString("target", []byte("_blank")) - - // Duplicate the current image node - image := ast.NewImage(ast.NewLink()) - image.Destination = v.Destination - image.Title = v.Title - for _, attr := range v.Attributes() { - image.SetAttribute(attr.Name, attr.Value) - } - for child := v.FirstChild(); child != nil; { - next := child.NextSibling() - image.AppendChild(image, child) - child = next - } - - // Append our duplicate image to the wrapper link - wrap.AppendChild(wrap, image) - - // Wire in the next sibling - wrap.SetNextSibling(next) - - // Replace the current node with the wrapper link - parent.ReplaceChild(parent, v, wrap) - - // But most importantly ensure the next sibling is still on the old image too - v.SetNextSibling(next) - } -} diff --git a/modules/markup/markdown/transform_link.go b/modules/markup/markdown/transform_link.go deleted file mode 100644 index 51c2c915d8..0000000000 --- a/modules/markup/markdown/transform_link.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markdown - -import ( - "code.gitea.io/gitea/modules/markup" - - "github.com/yuin/goldmark/ast" -) - -func resolveLink(ctx *markup.RenderContext, link, userContentAnchorPrefix string) (result string, resolved bool) { - isAnchorFragment := link != "" && link[0] == '#' - if !isAnchorFragment && !markup.IsFullURLString(link) { - link, resolved = ctx.RenderHelper.ResolveLink(link, markup.LinkTypeDefault), true - } - if isAnchorFragment && userContentAnchorPrefix != "" { - link, resolved = userContentAnchorPrefix+link[1:], true - } - return link, resolved -} - -func (g *ASTTransformer) transformLink(ctx *markup.RenderContext, v *ast.Link) { - if link, resolved := resolveLink(ctx, string(v.Destination), "#user-content-"); resolved { - v.Destination = []byte(link) - } -} diff --git a/modules/markup/mdstripper/mdstripper.go b/modules/markup/mdstripper/mdstripper.go index fe0eabb473..5a6504416a 100644 --- a/modules/markup/mdstripper/mdstripper.go +++ b/modules/markup/mdstripper/mdstripper.go @@ -46,7 +46,7 @@ func (r *stripRenderer) Render(w io.Writer, source []byte, doc ast.Node) error { coalesce := prevSibIsText r.processString( w, - v.Text(source), //nolint:staticcheck + v.Text(source), //nolint:staticcheck // Text is deprecated coalesce) if v.SoftLineBreak() { r.doubleSpace(w) @@ -91,8 +91,7 @@ func (r *stripRenderer) processAutoLink(w io.Writer, link []byte) { } // Note: we're not attempting to match the URL scheme (http/https) - host := strings.ToLower(u.Host) - if host != "" && host != strings.ToLower(r.localhost.Host) { + if u.Host != "" && !strings.EqualFold(u.Host, r.localhost.Host) { // Process out of band r.links = append(r.links, linkStr) return @@ -107,11 +106,12 @@ func (r *stripRenderer) processAutoLink(w io.Writer, link []byte) { } var sep string - if parts[3] == "issues" { + switch parts[3] { + case "issues": sep = "#" - } else if parts[3] == "pulls" { + case "pulls": sep = "!" - } else { + default: // Process out of band r.links = append(r.links, linkStr) return diff --git a/modules/markup/mdstripper/mdstripper_test.go b/modules/markup/mdstripper/mdstripper_test.go index ea34df0a3b..7fb49c1e01 100644 --- a/modules/markup/mdstripper/mdstripper_test.go +++ b/modules/markup/mdstripper/mdstripper_test.go @@ -79,7 +79,7 @@ A HIDDEN ` + "`" + `GHOST` + "`" + ` IN THIS LINE. lines = append(lines, line) } } - assert.EqualValues(t, test.expectedText, lines) - assert.EqualValues(t, test.expectedLinks, links) + assert.Equal(t, test.expectedText, lines) + assert.Equal(t, test.expectedLinks, links) } } diff --git a/modules/markup/orgmode/orgmode.go b/modules/markup/orgmode/orgmode.go index c6cc334000..93c335d244 100644 --- a/modules/markup/orgmode/orgmode.go +++ b/modules/markup/orgmode/orgmode.go @@ -1,7 +1,7 @@ // Copyright 2017 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package markup +package orgmode import ( "fmt" @@ -125,29 +125,15 @@ type orgWriter struct { var _ org.Writer = (*orgWriter)(nil) -func (r *orgWriter) resolveLink(kind, link string) string { - link = strings.TrimPrefix(link, "file:") - if !strings.HasPrefix(link, "#") && // not a URL fragment - !markup.IsFullURLString(link) { - if kind == "regular" { - // orgmode reports the link kind as "regular" for "[[ImageLink.svg][The Image Desc]]" - // so we need to try to guess the link kind again here - kind = org.RegularLink{URL: link}.Kind() - } - if kind == "image" || kind == "video" { - link = r.rctx.RenderHelper.ResolveLink(link, markup.LinkTypeMedia) - } else { - link = r.rctx.RenderHelper.ResolveLink(link, markup.LinkTypeDefault) - } - } - return link +func (r *orgWriter) resolveLink(link string) string { + return strings.TrimPrefix(link, "file:") } // WriteRegularLink renders images, links or videos func (r *orgWriter) WriteRegularLink(l org.RegularLink) { - link := r.resolveLink(l.Kind(), l.URL) + link := r.resolveLink(l.URL) - printHTML := func(html string, a ...any) { + printHTML := func(html template.HTML, a ...any) { _, _ = fmt.Fprint(r, htmlutil.HTMLFormat(html, a...)) } // Inspired by https://github.com/niklasfasching/go-org/blob/6eb20dbda93cb88c3503f7508dc78cbbc639378f/org/html_writer.go#L406-L427 @@ -156,14 +142,14 @@ func (r *orgWriter) WriteRegularLink(l org.RegularLink) { if l.Description == nil { printHTML(`<img src="%s" alt="%s">`, link, link) } else { - imageSrc := r.resolveLink(l.Kind(), org.String(l.Description...)) + imageSrc := r.resolveLink(org.String(l.Description...)) printHTML(`<a href="%s"><img src="%s" alt="%s"></a>`, link, imageSrc, imageSrc) } case "video": if l.Description == nil { printHTML(`<video src="%s">%s</video>`, link, link) } else { - videoSrc := r.resolveLink(l.Kind(), org.String(l.Description...)) + videoSrc := r.resolveLink(org.String(l.Description...)) printHTML(`<a href="%s"><video src="%s">%s</video></a>`, link, videoSrc, videoSrc) } default: diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go index e3cc05b4f0..df4bb38ad1 100644 --- a/modules/markup/orgmode/orgmode_test.go +++ b/modules/markup/orgmode/orgmode_test.go @@ -1,7 +1,7 @@ // Copyright 2017 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package markup +package orgmode_test import ( "os" @@ -9,6 +9,7 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/orgmode" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" @@ -22,7 +23,7 @@ func TestMain(m *testing.M) { func TestRender_StandardLinks(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("/relative-path/media/branch/main/"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -30,37 +31,37 @@ func TestRender_StandardLinks(t *testing.T) { test("[[https://google.com/]]", `<p><a href="https://google.com/">https://google.com/</a></p>`) test("[[ImageLink.svg][The Image Desc]]", - `<p><a href="/relative-path/media/branch/main/ImageLink.svg">The Image Desc</a></p>`) + `<p><a href="ImageLink.svg">The Image Desc</a></p>`) } func TestRender_InternalLinks(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("/relative-path/src/branch/main"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } test("[[file:test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="test.org">Test</a></p>`) test("[[./test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="./test.org">Test</a></p>`) test("[[test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="test.org">Test</a></p>`) test("[[path/to/test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/path/to/test.org">Test</a></p>`) + `<p><a href="path/to/test.org">Test</a></p>`) } func TestRender_Media(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("./relative-path"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } test("[[file:../../.images/src/02/train.jpg]]", - `<p><img src=".images/src/02/train.jpg" alt=".images/src/02/train.jpg"></p>`) + `<p><img src="../../.images/src/02/train.jpg" alt="../../.images/src/02/train.jpg"></p>`) test("[[file:train.jpg]]", - `<p><img src="relative-path/train.jpg" alt="relative-path/train.jpg"></p>`) + `<p><img src="train.jpg" alt="train.jpg"></p>`) // With description. test("[[https://example.com][https://example.com/example.svg]]", @@ -91,7 +92,7 @@ func TestRender_Media(t *testing.T) { func TestRender_Source(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext(), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -103,8 +104,8 @@ func HelloWorld() { } #+end_src `, `<div class="src src-go"> -<pre><code class="chroma language-go"><span class="c1">// HelloWorld prints "Hello World" -</span><span class="c1"></span><span class="kd">func</span> <span class="nf">HelloWorld</span><span class="p">()</span> <span class="p">{</span> +<pre><code class="chroma language-go"><span class="c1">// HelloWorld prints "Hello World"</span> +<span class="kd">func</span> <span class="nf">HelloWorld</span><span class="p">()</span> <span class="p">{</span> <span class="nx">fmt</span><span class="p">.</span><span class="nf">Println</span><span class="p">(</span><span class="s">"Hello World"</span><span class="p">)</span> <span class="p">}</span></code></pre> </div>`) diff --git a/modules/markup/render.go b/modules/markup/render.go index b239e59687..79f1f473c2 100644 --- a/modules/markup/render.go +++ b/modules/markup/render.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/url" + "strconv" "strings" "time" @@ -44,9 +45,9 @@ type RenderOptions struct { MarkupType string // user&repo, format&style®exp (for external issue pattern), teams&org (for mention) - // BranchNameSubURL (for iframe&asciicast) + // RefTypeNameSubURL (for iframe&asciicast) // markupAllowShortIssuePattern - // markdownLineBreakStyle (comment, document) + // markdownNewLineHardBreak Metas map[string]string // used by external render. the router "/org/repo/render/..." will output the rendered content in a standalone page @@ -170,7 +171,7 @@ sandbox="allow-scripts" setting.AppSubURL, url.PathEscape(ctx.RenderOptions.Metas["user"]), url.PathEscape(ctx.RenderOptions.Metas["repo"]), - ctx.RenderOptions.Metas["BranchNameSubURL"], + ctx.RenderOptions.Metas["RefTypeNameSubURL"], url.PathEscape(ctx.RenderOptions.RelativePath), )) return err @@ -247,7 +248,8 @@ func Init(renderHelpFuncs *RenderHelperFuncs) { } func ComposeSimpleDocumentMetas() map[string]string { - return map[string]string{"markdownLineBreakStyle": "document"} + // TODO: there is no separate config option for "simple document" rendering, so temporarily use the same config as "repo file" + return map[string]string{"markdownNewLineHardBreak": strconv.FormatBool(setting.Markdown.RenderOptionsRepoFile.NewLineHardBreak)} } type TestRenderHelper struct { @@ -261,8 +263,14 @@ func (r *TestRenderHelper) IsCommitIDExisting(commitID string) bool { return strings.HasPrefix(commitID, "65f1bf2") //|| strings.HasPrefix(commitID, "88fc37a") } -func (r *TestRenderHelper) ResolveLink(link string, likeType LinkType) string { - return r.ctx.ResolveLinkRelative(r.BaseLink, "", link) +func (r *TestRenderHelper) ResolveLink(link, preferLinkType string) string { + linkType, link := ParseRenderedLink(link, preferLinkType) + switch linkType { + case LinkTypeRoot: + return r.ctx.ResolveLinkRoot(link) + default: + return r.ctx.ResolveLinkRelative(r.BaseLink, "", link) + } } var _ RenderHelper = (*TestRenderHelper)(nil) diff --git a/modules/markup/render_helper.go b/modules/markup/render_helper.go index 82796ef274..b16f1189c5 100644 --- a/modules/markup/render_helper.go +++ b/modules/markup/render_helper.go @@ -10,13 +10,11 @@ import ( "code.gitea.io/gitea/modules/setting" ) -type LinkType string - const ( - LinkTypeApp LinkType = "app" // the link is relative to the AppSubURL - LinkTypeDefault LinkType = "default" // the link is relative to the default base (eg: repo link, or current ref tree path) - LinkTypeMedia LinkType = "media" // the link should be used to access media files (images, videos) - LinkTypeRaw LinkType = "raw" // not really useful, mainly for environment GITEA_PREFIX_RAW for external renders + LinkTypeDefault = "" + LinkTypeRoot = "/:root" // the link is relative to the AppSubURL(ROOT_URL) + LinkTypeMedia = "/:media" // the link should be used to access media files (images, videos) + LinkTypeRaw = "/:raw" // not really useful, mainly for environment GITEA_PREFIX_RAW for external renders ) type RenderHelper interface { @@ -27,7 +25,7 @@ type RenderHelper interface { // but not make processors to guess "is it rendering a comment or a wiki?" or "does it need to check commit ID?" IsCommitIDExisting(commitID string) bool - ResolveLink(link string, likeType LinkType) string + ResolveLink(link, preferLinkType string) string } // RenderHelperFuncs is used to decouple cycle-import @@ -38,6 +36,7 @@ type RenderHelper interface { type RenderHelperFuncs struct { IsUsernameMentionable func(ctx context.Context, username string) bool RenderRepoFileCodePreview func(ctx context.Context, options RenderCodePreviewOptions) (template.HTML, error) + RenderRepoIssueIconTitle func(ctx context.Context, options RenderIssueIconTitleOptions) (template.HTML, error) } var DefaultRenderHelperFuncs *RenderHelperFuncs @@ -50,7 +49,8 @@ func (r *SimpleRenderHelper) IsCommitIDExisting(commitID string) bool { return false } -func (r *SimpleRenderHelper) ResolveLink(link string, likeType LinkType) string { +func (r *SimpleRenderHelper) ResolveLink(link, preferLinkType string) string { + _, link = ParseRenderedLink(link, preferLinkType) return resolveLinkRelative(context.Background(), setting.AppSubURL+"/", "", link, false) } diff --git a/modules/markup/render_link.go b/modules/markup/render_link.go index b2e0699681..046544ce81 100644 --- a/modules/markup/render_link.go +++ b/modules/markup/render_link.go @@ -33,10 +33,24 @@ func resolveLinkRelative(ctx context.Context, base, cur, link string, absolute b return finalLink } -func (ctx *RenderContext) ResolveLinkRelative(base, cur, link string) (finalLink string) { +func (ctx *RenderContext) ResolveLinkRelative(base, cur, link string) string { + if strings.HasPrefix(link, "/:") { + setting.PanicInDevOrTesting("invalid link %q, forgot to cut?", link) + } return resolveLinkRelative(ctx, base, cur, link, ctx.RenderOptions.UseAbsoluteLink) } -func (ctx *RenderContext) ResolveLinkApp(link string) string { +func (ctx *RenderContext) ResolveLinkRoot(link string) string { return ctx.ResolveLinkRelative(setting.AppSubURL+"/", "", link) } + +func ParseRenderedLink(s, preferLinkType string) (linkType, link string) { + if strings.HasPrefix(s, "/:") { + p := strings.IndexByte(s[1:], '/') + if p == -1 { + return s, "" + } + return s[:p+1], s[p+2:] + } + return preferLinkType, s +} diff --git a/modules/markup/render_link_test.go b/modules/markup/render_link_test.go index c904ec7f18..972e15308c 100644 --- a/modules/markup/render_link_test.go +++ b/modules/markup/render_link_test.go @@ -4,7 +4,6 @@ package markup import ( - "context" "testing" "code.gitea.io/gitea/modules/setting" @@ -13,7 +12,7 @@ import ( ) func TestResolveLinkRelative(t *testing.T) { - ctx := context.Background() + ctx := t.Context() setting.AppURL = "http://localhost:3000" assert.Equal(t, "/a", resolveLinkRelative(ctx, "/a", "", "", false)) assert.Equal(t, "/a/b", resolveLinkRelative(ctx, "/a", "b", "", false)) diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 35f90eb46c..b6e9c348b7 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -4,12 +4,12 @@ package markup import ( - "bytes" "io" "path" "strings" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" ) // Renderer defines an interface for rendering markup file to HTML @@ -37,7 +37,7 @@ type ExternalRenderer interface { // RendererContentDetector detects if the content can be rendered // by specified renderer type RendererContentDetector interface { - CanRender(filename string, input io.Reader) bool + CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool } var ( @@ -60,13 +60,9 @@ func GetRendererByFileName(filename string) Renderer { } // DetectRendererType detects the markup type of the content -func DetectRendererType(filename string, input io.Reader) string { - buf, err := io.ReadAll(input) - if err != nil { - return "" - } +func DetectRendererType(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) string { for _, renderer := range renderers { - if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, bytes.NewReader(buf)) { + if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, sniffedType, prefetchBuf) { return renderer.Name() } } diff --git a/modules/markup/renderer_test.go b/modules/markup/renderer_test.go deleted file mode 100644 index 0791081f94..0000000000 --- a/modules/markup/renderer_test.go +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markup_test diff --git a/modules/markup/sanitizer_default.go b/modules/markup/sanitizer_default.go index 5eeafe940a..0fbf0f0b24 100644 --- a/modules/markup/sanitizer_default.go +++ b/modules/markup/sanitizer_default.go @@ -4,6 +4,7 @@ package markup import ( + "html/template" "io" "net/url" "regexp" @@ -48,10 +49,12 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy { policy.AllowAttrs("class").Matching(regexp.MustCompile(`^(unchecked|checked|indeterminate)$`)).OnElements("li") // Allow 'color' and 'background-color' properties for the style attribute on text elements. - policy.AllowStyles("color", "background-color").OnElements("span", "p") + policy.AllowStyles("color", "background-color").OnElements("div", "span", "p", "tr", "th", "td") policy.AllowAttrs("src", "autoplay", "controls").OnElements("video") + policy.AllowAttrs("loading").OnElements("img") + // Allow generally safe attributes (reference: https://github.com/jch/html-pipeline) generalSafeAttrs := []string{ "abbr", "accept", "accept-charset", @@ -90,9 +93,9 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy { return policy } -// Sanitize takes a string that contains a HTML fragment or document and applies policy whitelist. -func Sanitize(s string) string { - return GetDefaultSanitizer().defaultPolicy.Sanitize(s) +// Sanitize use default sanitizer policy to sanitize a string +func Sanitize(s string) template.HTML { + return template.HTML(GetDefaultSanitizer().defaultPolicy.Sanitize(s)) } // SanitizeReader sanitizes a Reader diff --git a/modules/markup/sanitizer_default_test.go b/modules/markup/sanitizer_default_test.go index e6fbae5056..e5ba018e1b 100644 --- a/modules/markup/sanitizer_default_test.go +++ b/modules/markup/sanitizer_default_test.go @@ -62,9 +62,13 @@ func TestSanitizer(t *testing.T) { `<a href="javascript:alert('xss')">bad</a>`, `bad`, `<a href="vbscript:no">bad</a>`, `bad`, `<a href="data:1234">bad</a>`, `bad`, + + // Some classes and attributes are used by the frontend framework and will execute JS code, so make sure they are removed + `<div class="link-action" data-attr-class="foo" data-url="xxx">txt</div>`, `<div data-attr-class="foo">txt</div>`, + `<div class="form-fetch-action" data-markdown-generated-content="bar" data-global-init="a" data-global-click="b">txt</div>`, `<div data-markdown-generated-content="bar">txt</div>`, } for i := 0; i < len(testCases); i += 2 { - assert.Equal(t, testCases[i+1], Sanitize(testCases[i])) + assert.Equal(t, testCases[i+1], string(Sanitize(testCases[i]))) } } diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 230260ff94..4d2ec287a9 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -184,7 +184,7 @@ func NewCollector() Collector { Users: prometheus.NewDesc( namespace+"users", "Number of Users", - nil, nil, + []string{"state"}, nil, ), Watches: prometheus.NewDesc( namespace+"watches", @@ -373,7 +373,14 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( c.Users, prometheus.GaugeValue, - float64(stats.Counter.User), + float64(stats.Counter.UsersActive), + "active", // state label + ) + ch <- prometheus.MustNewConstMetric( + c.Users, + prometheus.GaugeValue, + float64(stats.Counter.UsersNotActive), + "inactive", // state label ) ch <- prometheus.MustNewConstMetric( c.Watches, diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index 08dbbc29a9..669222dea2 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -12,18 +12,17 @@ import ( // Downloader downloads the site repo information type Downloader interface { - SetContext(context.Context) - GetRepoInfo() (*Repository, error) - GetTopics() ([]string, error) - GetMilestones() ([]*Milestone, error) - GetReleases() ([]*Release, error) - GetLabels() ([]*Label, error) - GetIssues(page, perPage int) ([]*Issue, bool, error) - GetComments(commentable Commentable) ([]*Comment, bool, error) - GetAllComments(page, perPage int) ([]*Comment, bool, error) + GetRepoInfo(ctx context.Context) (*Repository, error) + GetTopics(ctx context.Context) ([]string, error) + GetMilestones(ctx context.Context) ([]*Milestone, error) + GetReleases(ctx context.Context) ([]*Release, error) + GetLabels(ctx context.Context) ([]*Label, error) + GetIssues(ctx context.Context, page, perPage int) ([]*Issue, bool, error) + GetComments(ctx context.Context, commentable Commentable) ([]*Comment, bool, error) + GetAllComments(ctx context.Context, page, perPage int) ([]*Comment, bool, error) SupportGetRepoComments() bool - GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) - GetReviews(reviewable Reviewable) ([]*Review, error) + GetPullRequests(ctx context.Context, page, perPage int) ([]*PullRequest, bool, error) + GetReviews(ctx context.Context, reviewable Reviewable) ([]*Review, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) } diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index e5b69331df..e488f6914f 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -13,56 +13,53 @@ type NullDownloader struct{} var _ Downloader = &NullDownloader{} -// SetContext set context -func (n NullDownloader) SetContext(_ context.Context) {} - // GetRepoInfo returns a repository information -func (n NullDownloader) GetRepoInfo() (*Repository, error) { +func (n NullDownloader) GetRepoInfo(_ context.Context) (*Repository, error) { return nil, ErrNotSupported{Entity: "RepoInfo"} } // GetTopics return repository topics -func (n NullDownloader) GetTopics() ([]string, error) { +func (n NullDownloader) GetTopics(_ context.Context) ([]string, error) { return nil, ErrNotSupported{Entity: "Topics"} } // GetMilestones returns milestones -func (n NullDownloader) GetMilestones() ([]*Milestone, error) { +func (n NullDownloader) GetMilestones(_ context.Context) ([]*Milestone, error) { return nil, ErrNotSupported{Entity: "Milestones"} } // GetReleases returns releases -func (n NullDownloader) GetReleases() ([]*Release, error) { +func (n NullDownloader) GetReleases(_ context.Context) ([]*Release, error) { return nil, ErrNotSupported{Entity: "Releases"} } // GetLabels returns labels -func (n NullDownloader) GetLabels() ([]*Label, error) { +func (n NullDownloader) GetLabels(_ context.Context) ([]*Label, error) { return nil, ErrNotSupported{Entity: "Labels"} } // GetIssues returns issues according start and limit -func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { +func (n NullDownloader) GetIssues(_ context.Context, page, perPage int) ([]*Issue, bool, error) { return nil, false, ErrNotSupported{Entity: "Issues"} } // GetComments returns comments of an issue or PR -func (n NullDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { +func (n NullDownloader) GetComments(_ context.Context, commentable Commentable) ([]*Comment, bool, error) { return nil, false, ErrNotSupported{Entity: "Comments"} } // GetAllComments returns paginated comments -func (n NullDownloader) GetAllComments(page, perPage int) ([]*Comment, bool, error) { +func (n NullDownloader) GetAllComments(_ context.Context, page, perPage int) ([]*Comment, bool, error) { return nil, false, ErrNotSupported{Entity: "AllComments"} } // GetPullRequests returns pull requests according page and perPage -func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { +func (n NullDownloader) GetPullRequests(_ context.Context, page, perPage int) ([]*PullRequest, bool, error) { return nil, false, ErrNotSupported{Entity: "PullRequests"} } // GetReviews returns pull requests review -func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { +func (n NullDownloader) GetReviews(_ context.Context, reviewable Reviewable) ([]*Review, error) { return nil, ErrNotSupported{Entity: "Reviews"} } diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go index fbfdff0315..cccab3fd7e 100644 --- a/modules/migration/pullrequest.go +++ b/modules/migration/pullrequest.go @@ -49,8 +49,8 @@ func (p *PullRequest) IsForkPullRequest() bool { return p.Head.RepoFullName() != p.Base.RepoFullName() } -// GetGitRefName returns pull request relative path to head -func (p PullRequest) GetGitRefName() string { +// GetGitHeadRefName returns pull request relative path to head +func (p PullRequest) GetGitHeadRefName() string { return fmt.Sprintf("%s%d/head", git.PullPrefix, p.Number) } diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go index 1cacf5f375..69804b7767 100644 --- a/modules/migration/retry_downloader.go +++ b/modules/migration/retry_downloader.go @@ -13,57 +13,49 @@ var _ Downloader = &RetryDownloader{} // RetryDownloader retry the downloads type RetryDownloader struct { Downloader - ctx context.Context RetryTimes int // the total execute times RetryDelay int // time to delay seconds } // NewRetryDownloader creates a retry downloader -func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader { +func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *RetryDownloader { return &RetryDownloader{ Downloader: downloader, - ctx: ctx, RetryTimes: retryTimes, RetryDelay: retryDelay, } } -func (d *RetryDownloader) retry(work func() error) error { +func (d *RetryDownloader) retry(ctx context.Context, work func(context.Context) error) error { var ( times = d.RetryTimes err error ) for ; times > 0; times-- { - if err = work(); err == nil { + if err = work(ctx); err == nil { return nil } if IsErrNotSupported(err) { return err } select { - case <-d.ctx.Done(): - return d.ctx.Err() + case <-ctx.Done(): + return ctx.Err() case <-time.After(time.Second * time.Duration(d.RetryDelay)): } } return err } -// SetContext set context -func (d *RetryDownloader) SetContext(ctx context.Context) { - d.ctx = ctx - d.Downloader.SetContext(ctx) -} - // GetRepoInfo returns a repository information with retry -func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { +func (d *RetryDownloader) GetRepoInfo(ctx context.Context) (*Repository, error) { var ( repo *Repository err error ) - err = d.retry(func() error { - repo, err = d.Downloader.GetRepoInfo() + err = d.retry(ctx, func(ctx context.Context) error { + repo, err = d.Downloader.GetRepoInfo(ctx) return err }) @@ -71,14 +63,14 @@ func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { } // GetTopics returns a repository's topics with retry -func (d *RetryDownloader) GetTopics() ([]string, error) { +func (d *RetryDownloader) GetTopics(ctx context.Context) ([]string, error) { var ( topics []string err error ) - err = d.retry(func() error { - topics, err = d.Downloader.GetTopics() + err = d.retry(ctx, func(ctx context.Context) error { + topics, err = d.Downloader.GetTopics(ctx) return err }) @@ -86,14 +78,14 @@ func (d *RetryDownloader) GetTopics() ([]string, error) { } // GetMilestones returns a repository's milestones with retry -func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) { +func (d *RetryDownloader) GetMilestones(ctx context.Context) ([]*Milestone, error) { var ( milestones []*Milestone err error ) - err = d.retry(func() error { - milestones, err = d.Downloader.GetMilestones() + err = d.retry(ctx, func(ctx context.Context) error { + milestones, err = d.Downloader.GetMilestones(ctx) return err }) @@ -101,14 +93,14 @@ func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) { } // GetReleases returns a repository's releases with retry -func (d *RetryDownloader) GetReleases() ([]*Release, error) { +func (d *RetryDownloader) GetReleases(ctx context.Context) ([]*Release, error) { var ( releases []*Release err error ) - err = d.retry(func() error { - releases, err = d.Downloader.GetReleases() + err = d.retry(ctx, func(ctx context.Context) error { + releases, err = d.Downloader.GetReleases(ctx) return err }) @@ -116,14 +108,14 @@ func (d *RetryDownloader) GetReleases() ([]*Release, error) { } // GetLabels returns a repository's labels with retry -func (d *RetryDownloader) GetLabels() ([]*Label, error) { +func (d *RetryDownloader) GetLabels(ctx context.Context) ([]*Label, error) { var ( labels []*Label err error ) - err = d.retry(func() error { - labels, err = d.Downloader.GetLabels() + err = d.retry(ctx, func(ctx context.Context) error { + labels, err = d.Downloader.GetLabels(ctx) return err }) @@ -131,15 +123,15 @@ func (d *RetryDownloader) GetLabels() ([]*Label, error) { } // GetIssues returns a repository's issues with retry -func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { +func (d *RetryDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*Issue, bool, error) { var ( issues []*Issue isEnd bool err error ) - err = d.retry(func() error { - issues, isEnd, err = d.Downloader.GetIssues(page, perPage) + err = d.retry(ctx, func(ctx context.Context) error { + issues, isEnd, err = d.Downloader.GetIssues(ctx, page, perPage) return err }) @@ -147,15 +139,15 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns a repository's comments with retry -func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { +func (d *RetryDownloader) GetComments(ctx context.Context, commentable Commentable) ([]*Comment, bool, error) { var ( comments []*Comment isEnd bool err error ) - err = d.retry(func() error { - comments, isEnd, err = d.Downloader.GetComments(commentable) + err = d.retry(ctx, func(context.Context) error { + comments, isEnd, err = d.Downloader.GetComments(ctx, commentable) return err }) @@ -163,15 +155,15 @@ func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool } // GetPullRequests returns a repository's pull requests with retry -func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { +func (d *RetryDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*PullRequest, bool, error) { var ( prs []*PullRequest err error isEnd bool ) - err = d.retry(func() error { - prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage) + err = d.retry(ctx, func(ctx context.Context) error { + prs, isEnd, err = d.Downloader.GetPullRequests(ctx, page, perPage) return err }) @@ -179,14 +171,13 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo } // GetReviews returns pull requests reviews -func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { +func (d *RetryDownloader) GetReviews(ctx context.Context, reviewable Reviewable) ([]*Review, error) { var ( reviews []*Review err error ) - - err = d.retry(func() error { - reviews, err = d.Downloader.GetReviews(reviewable) + err = d.retry(ctx, func(ctx context.Context) error { + reviews, err = d.Downloader.GetReviews(ctx, reviewable) return err }) diff --git a/modules/migration/schemas_bindata.go b/modules/migration/schemas_bindata.go index c5db3b3461..695c2c1135 100644 --- a/modules/migration/schemas_bindata.go +++ b/modules/migration/schemas_bindata.go @@ -3,6 +3,28 @@ //go:build bindata +//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas bindata.dat + package migration -//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas migration bindata.go +import ( + "io" + "io/fs" + "path" + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() fs.FS { + return assetfs.NewEmbeddedFS(bindata) +}) + +func openSchema(filename string) (io.ReadCloser, error) { + return BuiltinAssets().Open(path.Base(filename)) +} diff --git a/modules/migration/schemas_static.go b/modules/migration/schemas_static.go deleted file mode 100644 index 8a0c340a65..0000000000 --- a/modules/migration/schemas_static.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package migration - -import ( - "io" - "path" -) - -func openSchema(filename string) (io.ReadCloser, error) { - return Assets.Open(path.Base(filename)) -} diff --git a/modules/migration/uploader.go b/modules/migration/uploader.go index ff642aa4fa..65752e248e 100644 --- a/modules/migration/uploader.go +++ b/modules/migration/uploader.go @@ -4,20 +4,22 @@ package migration +import "context" + // Uploader uploads all the information of one repository type Uploader interface { MaxBatchInsertSize(tp string) int - CreateRepo(repo *Repository, opts MigrateOptions) error - CreateTopics(topic ...string) error - CreateMilestones(milestones ...*Milestone) error - CreateReleases(releases ...*Release) error - SyncTags() error - CreateLabels(labels ...*Label) error - CreateIssues(issues ...*Issue) error - CreateComments(comments ...*Comment) error - CreatePullRequests(prs ...*PullRequest) error - CreateReviews(reviews ...*Review) error + CreateRepo(ctx context.Context, repo *Repository, opts MigrateOptions) error + CreateTopics(ctx context.Context, topic ...string) error + CreateMilestones(ctx context.Context, milestones ...*Milestone) error + CreateReleases(ctx context.Context, releases ...*Release) error + SyncTags(ctx context.Context) error + CreateLabels(ctx context.Context, labels ...*Label) error + CreateIssues(ctx context.Context, issues ...*Issue) error + CreateComments(ctx context.Context, comments ...*Comment) error + CreatePullRequests(ctx context.Context, prs ...*PullRequest) error + CreateReviews(ctx context.Context, reviews ...*Review) error Rollback() error - Finish() error + Finish(ctx context.Context) error Close() } diff --git a/modules/nosql/redis_test.go b/modules/nosql/redis_test.go index 43652e314c..93276ca793 100644 --- a/modules/nosql/redis_test.go +++ b/modules/nosql/redis_test.go @@ -5,6 +5,9 @@ package nosql import ( "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestToRedisURI(t *testing.T) { @@ -26,9 +29,9 @@ func TestToRedisURI(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := ToRedisURI(tt.connection); got == nil || got.String() != tt.want { - t.Errorf(`ToRedisURI(%q) = %s, want %s`, tt.connection, got.String(), tt.want) - } + got := ToRedisURI(tt.connection) + require.NotNil(t, got) + assert.Equal(t, tt.want, got.String()) }) } } diff --git a/modules/optional/option.go b/modules/optional/option.go index af9e5ac852..cbecf86987 100644 --- a/modules/optional/option.go +++ b/modules/optional/option.go @@ -3,6 +3,14 @@ package optional +import "strconv" + +// Option is a generic type that can hold a value of type T or be empty (None). +// +// It must use the slice type to work with "chi" form values binding: +// * non-existing value are represented as an empty slice (None) +// * existing value is represented as a slice with one element (Some) +// * multiple values are represented as a slice with multiple elements (Some), the Value is the first element (not well-defined in this case) type Option[T any] []T func None[T any]() Option[T] { @@ -20,6 +28,13 @@ func FromPtr[T any](v *T) Option[T] { return Some(*v) } +func FromMapLookup[K comparable, V any](m map[K]V, k K) Option[V] { + if v, ok := m[k]; ok { + return Some(v) + } + return None[V]() +} + func FromNonDefault[T comparable](v T) Option[T] { var zero T if v == zero { @@ -43,3 +58,12 @@ func (o Option[T]) ValueOrDefault(v T) T { } return v } + +// ParseBool get the corresponding optional.Option[bool] of a string using strconv.ParseBool +func ParseBool(s string) Option[bool] { + v, e := strconv.ParseBool(s) + if e != nil { + return None[bool]() + } + return Some(v) +} diff --git a/modules/optional/option_test.go b/modules/optional/option_test.go index 203e9221e3..ea80a2e3cb 100644 --- a/modules/optional/option_test.go +++ b/modules/optional/option_test.go @@ -56,4 +56,23 @@ func TestOption(t *testing.T) { opt3 := optional.FromNonDefault(1) assert.True(t, opt3.Has()) assert.Equal(t, int(1), opt3.Value()) + + opt4 := optional.FromMapLookup(map[string]int{"a": 1}, "a") + assert.True(t, opt4.Has()) + assert.Equal(t, 1, opt4.Value()) + opt4 = optional.FromMapLookup(map[string]int{"a": 1}, "b") + assert.False(t, opt4.Has()) +} + +func Test_ParseBool(t *testing.T) { + assert.Equal(t, optional.None[bool](), optional.ParseBool("")) + assert.Equal(t, optional.None[bool](), optional.ParseBool("x")) + + assert.Equal(t, optional.Some(false), optional.ParseBool("0")) + assert.Equal(t, optional.Some(false), optional.ParseBool("f")) + assert.Equal(t, optional.Some(false), optional.ParseBool("False")) + + assert.Equal(t, optional.Some(true), optional.ParseBool("1")) + assert.Equal(t, optional.Some(true), optional.ParseBool("t")) + assert.Equal(t, optional.Some(true), optional.ParseBool("True")) } diff --git a/modules/optional/serialization_test.go b/modules/optional/serialization_test.go index 09a4bddea0..cf81a94cfc 100644 --- a/modules/optional/serialization_test.go +++ b/modules/optional/serialization_test.go @@ -4,7 +4,7 @@ package optional_test import ( - std_json "encoding/json" //nolint:depguard + std_json "encoding/json" //nolint:depguard // for testing purpose "testing" "code.gitea.io/gitea/modules/json" @@ -51,11 +51,11 @@ func TestOptionalToJson(t *testing.T) { t.Run(tc.name, func(t *testing.T) { b, err := json.Marshal(tc.obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, string(b), "gitea json module returned unexpected") + assert.Equal(t, tc.want, string(b), "gitea json module returned unexpected") b, err = std_json.Marshal(tc.obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, string(b), "std json module returned unexpected") + assert.Equal(t, tc.want, string(b), "std json module returned unexpected") }) } } @@ -89,12 +89,12 @@ func TestOptionalFromJson(t *testing.T) { var obj1 testSerializationStruct err := json.Unmarshal([]byte(tc.data), &obj1) assert.NoError(t, err) - assert.EqualValues(t, tc.want, obj1, "gitea json module returned unexpected") + assert.Equal(t, tc.want, obj1, "gitea json module returned unexpected") var obj2 testSerializationStruct err = std_json.Unmarshal([]byte(tc.data), &obj2) assert.NoError(t, err) - assert.EqualValues(t, tc.want, obj2, "std json module returned unexpected") + assert.Equal(t, tc.want, obj2, "std json module returned unexpected") }) } } @@ -135,7 +135,7 @@ optional_two_string: null t.Run(tc.name, func(t *testing.T) { b, err := yaml.Marshal(tc.obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, string(b), "yaml module returned unexpected") + assert.Equal(t, tc.want, string(b), "yaml module returned unexpected") }) } } @@ -184,7 +184,7 @@ optional_twostring: null var obj testSerializationStruct err := yaml.Unmarshal([]byte(tc.data), &obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, obj, "yaml module returned unexpected") + assert.Equal(t, tc.want, obj, "yaml module returned unexpected") }) } } diff --git a/modules/options/options_bindata.go b/modules/options/options_bindata.go index 29151cb3cb..b2321d7eb5 100644 --- a/modules/options/options_bindata.go +++ b/modules/options/options_bindata.go @@ -3,6 +3,21 @@ //go:build bindata +//go:generate go run ../../build/generate-bindata.go ../../options bindata.dat + package options -//go:generate go run ../../build/generate-bindata.go ../../options options bindata.go +import ( + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", assetfs.NewEmbeddedFS(bindata)) +}) diff --git a/modules/options/dynamic.go b/modules/options/options_dynamic.go index 085492d11c..085492d11c 100644 --- a/modules/options/dynamic.go +++ b/modules/options/options_dynamic.go diff --git a/modules/options/static.go b/modules/options/static.go deleted file mode 100644 index 72b28e990e..0000000000 --- a/modules/options/static.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package options - -import ( - "code.gitea.io/gitea/modules/assetfs" -) - -func BuiltinAssets() *assetfs.Layer { - return assetfs.Bindata("builtin(bindata)", Assets) -} diff --git a/modules/packages/conda/metadata.go b/modules/packages/conda/metadata.go index 76ba95eace..5eb2890115 100644 --- a/modules/packages/conda/metadata.go +++ b/modules/packages/conda/metadata.go @@ -17,9 +17,9 @@ import ( ) var ( - ErrInvalidStructure = util.SilentWrap{Message: "package structure is invalid", Err: util.ErrInvalidArgument} - ErrInvalidName = util.SilentWrap{Message: "package name is invalid", Err: util.ErrInvalidArgument} - ErrInvalidVersion = util.SilentWrap{Message: "package version is invalid", Err: util.ErrInvalidArgument} + ErrInvalidStructure = util.NewInvalidArgumentErrorf("package structure is invalid") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") ) const ( diff --git a/modules/packages/container/const.go b/modules/packages/container/const.go new file mode 100644 index 0000000000..6c7c9b46d1 --- /dev/null +++ b/modules/packages/container/const.go @@ -0,0 +1,11 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package container + +const ( + ContentTypeDockerDistributionManifestV2 = "application/vnd.docker.distribution.manifest.v2+json" + + ManifestFilename = "manifest.json" + UploadVersion = "_upload" +) diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go index 2a41fb9105..3ef0684d13 100644 --- a/modules/packages/container/metadata.go +++ b/modules/packages/container/metadata.go @@ -71,14 +71,34 @@ type Manifest struct { Size int64 `json:"size"` } +func IsMediaTypeValid(mt string) bool { + return strings.HasPrefix(mt, "application/vnd.docker.") || strings.HasPrefix(mt, "application/vnd.oci.") +} + +func IsMediaTypeImageManifest(mt string) bool { + return strings.EqualFold(mt, oci.MediaTypeImageManifest) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.v2+json") +} + +func IsMediaTypeImageIndex(mt string) bool { + return strings.EqualFold(mt, oci.MediaTypeImageIndex) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.list.v2+json") +} + // ParseImageConfig parses the metadata of an image config -func ParseImageConfig(mt string, r io.Reader) (*Metadata, error) { - if strings.EqualFold(mt, helm.ConfigMediaType) { +func ParseImageConfig(mediaType string, r io.Reader) (*Metadata, error) { + if strings.EqualFold(mediaType, helm.ConfigMediaType) { return parseHelmConfig(r) } // fallback to OCI Image Config - return parseOCIImageConfig(r) + // FIXME: this fallback is not right, we should strictly check the media type in the future + metadata, err := parseOCIImageConfig(r) + if err != nil { + if !IsMediaTypeImageManifest(mediaType) { + return &Metadata{Platform: "unknown/unknown"}, nil + } + return nil, err + } + return metadata, nil } func parseOCIImageConfig(r io.Reader) (*Metadata, error) { diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go index 665499b2e6..0f2d702925 100644 --- a/modules/packages/container/metadata_test.go +++ b/modules/packages/container/metadata_test.go @@ -11,6 +11,7 @@ import ( oci "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestParseImageConfig(t *testing.T) { @@ -58,4 +59,8 @@ func TestParseImageConfig(t *testing.T) { assert.ElementsMatch(t, []string{author}, metadata.Authors) assert.Equal(t, projectURL, metadata.ProjectURL) assert.Equal(t, repositoryURL, metadata.RepositoryURL) + + metadata, err = ParseImageConfig("anything-unknown", strings.NewReader("")) + require.NoError(t, err) + assert.Equal(t, &Metadata{Platform: "unknown/unknown"}, metadata) } diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index 37612556d7..57974515e2 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -28,8 +28,7 @@ func NewContentStore() *ContentStore { return contentStore } -// Get gets a package blob -func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { +func (s *ContentStore) OpenBlob(key BlobHash256Key) (storage.Object, error) { return s.store.Open(KeyToRelativePath(key)) } @@ -37,8 +36,8 @@ func (s *ContentStore) ShouldServeDirect() bool { return setting.Packages.Storage.ServeDirect() } -func (s *ContentStore) GetServeDirectURL(key BlobHash256Key, filename string, reqParams url.Values) (*url.URL, error) { - return s.store.URL(KeyToRelativePath(key), filename, reqParams) +func (s *ContentStore) GetServeDirectURL(key BlobHash256Key, filename, method string, reqParams url.Values) (*url.URL, error) { + return s.store.URL(KeyToRelativePath(key), filename, method, reqParams) } // FIXME: Workaround to be removed in v1.20 diff --git a/modules/packages/goproxy/metadata.go b/modules/packages/goproxy/metadata.go index 40f7d20508..a67b149f4d 100644 --- a/modules/packages/goproxy/metadata.go +++ b/modules/packages/goproxy/metadata.go @@ -5,7 +5,6 @@ package goproxy import ( "archive/zip" - "fmt" "io" "path" "strings" @@ -88,7 +87,7 @@ func ParsePackage(r io.ReaderAt, size int64) (*Package, error) { return nil, ErrInvalidStructure } - p.GoMod = fmt.Sprintf("module %s", p.Name) + p.GoMod = "module " + p.Name return p, nil } diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go index 4ab45edcec..0cd657cd44 100644 --- a/modules/packages/hashed_buffer.go +++ b/modules/packages/hashed_buffer.go @@ -6,6 +6,7 @@ package packages import ( "io" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util/filebuffer" ) @@ -34,11 +35,11 @@ func NewHashedBuffer() (*HashedBuffer, error) { // NewHashedBufferWithSize creates a hashed buffer with a specific memory size func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { - b, err := filebuffer.New(maxMemorySize) + tempDir, err := setting.AppDataTempDir("package-hashed-buffer").MkdirAllSub("") if err != nil { return nil, err } - + b := filebuffer.New(maxMemorySize, tempDir) hash := NewMultiHasher() combinedWriter := io.MultiWriter(b, hash) diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go index 564e782f18..5104c1fb25 100644 --- a/modules/packages/hashed_buffer_test.go +++ b/modules/packages/hashed_buffer_test.go @@ -9,10 +9,13 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestHashedBuffer(t *testing.T) { + setting.AppDataPath = t.TempDir() cases := []struct { MaxMemorySize int Data string diff --git a/modules/packages/maven/metadata.go b/modules/packages/maven/metadata.go index 42aa250718..a61a62c086 100644 --- a/modules/packages/maven/metadata.go +++ b/modules/packages/maven/metadata.go @@ -7,6 +7,7 @@ import ( "encoding/xml" "io" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/validation" "golang.org/x/net/html/charset" @@ -31,18 +32,27 @@ type Dependency struct { } type pomStruct struct { - XMLName xml.Name `xml:"project"` - GroupID string `xml:"groupId"` - ArtifactID string `xml:"artifactId"` - Version string `xml:"version"` - Name string `xml:"name"` - Description string `xml:"description"` - URL string `xml:"url"` - Licenses []struct { + XMLName xml.Name `xml:"project"` + + Parent struct { + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + } `xml:"parent"` + + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Name string `xml:"name"` + Description string `xml:"description"` + URL string `xml:"url"` + + Licenses []struct { Name string `xml:"name"` URL string `xml:"url"` Distribution string `xml:"distribution"` } `xml:"licenses>license"` + Dependencies []struct { GroupID string `xml:"groupId"` ArtifactID string `xml:"artifactId"` @@ -81,8 +91,16 @@ func ParsePackageMetaData(r io.Reader) (*Metadata, error) { }) } + pomGroupID := pom.GroupID + if pomGroupID == "" { + // the current module could inherit parent: https://maven.apache.org/pom.html#Inheritance + pomGroupID = pom.Parent.GroupID + } + if pomGroupID == "" { + return nil, util.ErrInvalidArgument + } return &Metadata{ - GroupID: pom.GroupID, + GroupID: pomGroupID, ArtifactID: pom.ArtifactID, Name: pom.Name, Description: pom.Description, diff --git a/modules/packages/maven/metadata_test.go b/modules/packages/maven/metadata_test.go index e675467730..2cff290808 100644 --- a/modules/packages/maven/metadata_test.go +++ b/modules/packages/maven/metadata_test.go @@ -7,7 +7,10 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/text/encoding/charmap" ) @@ -86,4 +89,35 @@ func TestParsePackageMetaData(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, m) }) + + t.Run("ParentInherit", func(t *testing.T) { + pom := `<?xml version="1.0"?> +<project> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.mycompany.app</groupId> + <artifactId>my-app</artifactId> + <version>1.0-SNAPSHOT</version> + </parent> + <artifactId>submodule1</artifactId> +</project> +` + m, err := ParsePackageMetaData(strings.NewReader(pom)) + require.NoError(t, err) + require.NotNil(t, m) + + assert.Equal(t, "com.mycompany.app", m.GroupID) + assert.Equal(t, "submodule1", m.ArtifactID) + }) + + t.Run("ParentInherit", func(t *testing.T) { + pom := `<?xml version="1.0"?> +<project> + <modelVersion>4.0.0</modelVersion> + <artifactId></artifactId> +</project> +` + _, err := ParsePackageMetaData(strings.NewReader(pom)) + require.ErrorIs(t, err, util.ErrInvalidArgument) + }) } diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 7d3d7cd6b5..11b5123c27 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -58,7 +58,7 @@ type PackageMetadata struct { Time map[string]time.Time `json:"time,omitempty"` Homepage string `json:"homepage,omitempty"` Keywords []string `json:"keywords,omitempty"` - Repository Repository `json:"repository,omitempty"` + Repository Repository `json:"repository"` Author User `json:"author"` ReadmeFilename string `json:"readmeFilename,omitempty"` Users map[string]bool `json:"users,omitempty"` @@ -75,12 +75,13 @@ type PackageMetadataVersion struct { Author User `json:"author"` Homepage string `json:"homepage,omitempty"` License string `json:"license,omitempty"` - Repository Repository `json:"repository,omitempty"` + Repository Repository `json:"repository"` Keywords []string `json:"keywords,omitempty"` Dependencies map[string]string `json:"dependencies,omitempty"` BundleDependencies []string `json:"bundleDependencies,omitempty"` DevDependencies map[string]string `json:"devDependencies,omitempty"` PeerDependencies map[string]string `json:"peerDependencies,omitempty"` + PeerDependenciesMeta map[string]any `json:"peerDependenciesMeta,omitempty"` Bin map[string]string `json:"bin,omitempty"` OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` Readme string `json:"readme,omitempty"` @@ -222,6 +223,7 @@ func ParsePackage(r io.Reader) (*Package, error) { BundleDependencies: meta.BundleDependencies, DevelopmentDependencies: meta.DevDependencies, PeerDependencies: meta.PeerDependencies, + PeerDependenciesMeta: meta.PeerDependenciesMeta, OptionalDependencies: meta.OptionalDependencies, Bin: meta.Bin, Readme: meta.Readme, diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index 6bb77f302b..362d0470d5 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -19,8 +19,9 @@ type Metadata struct { BundleDependencies []string `json:"bundleDependencies,omitempty"` DevelopmentDependencies map[string]string `json:"development_dependencies,omitempty"` PeerDependencies map[string]string `json:"peer_dependencies,omitempty"` + PeerDependenciesMeta map[string]any `json:"peer_dependencies_meta,omitempty"` OptionalDependencies map[string]string `json:"optional_dependencies,omitempty"` Bin map[string]string `json:"bin,omitempty"` Readme string `json:"readme,omitempty"` - Repository Repository `json:"repository,omitempty"` + Repository Repository `json:"repository"` } diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 1e98ddffde..513b4dd2b9 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -57,14 +57,25 @@ type Package struct { // Metadata represents the metadata of a Nuget package type Metadata struct { - Description string `json:"description,omitempty"` - ReleaseNotes string `json:"release_notes,omitempty"` - Readme string `json:"readme,omitempty"` - Authors string `json:"authors,omitempty"` - ProjectURL string `json:"project_url,omitempty"` - RepositoryURL string `json:"repository_url,omitempty"` - RequireLicenseAcceptance bool `json:"require_license_acceptance"` - Dependencies map[string][]Dependency `json:"dependencies,omitempty"` + Authors string `json:"authors,omitempty"` + Copyright string `json:"copyright,omitempty"` + Description string `json:"description,omitempty"` + DevelopmentDependency bool `json:"development_dependency,omitempty"` + IconURL string `json:"icon_url,omitempty"` + Language string `json:"language,omitempty"` + LicenseURL string `json:"license_url,omitempty"` + MinClientVersion string `json:"min_client_version,omitempty"` + Owners string `json:"owners,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Readme string `json:"readme,omitempty"` + ReleaseNotes string `json:"release_notes,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + RequireLicenseAcceptance bool `json:"require_license_acceptance"` + Summary string `json:"summary,omitempty"` + Tags string `json:"tags,omitempty"` + Title string `json:"title,omitempty"` + + Dependencies map[string][]Dependency `json:"dependencies,omitempty"` } // Dependency represents a dependency of a Nuget package @@ -74,24 +85,31 @@ type Dependency struct { } // https://learn.microsoft.com/en-us/nuget/reference/nuspec +// https://github.com/NuGet/NuGet.Client/blob/dev/src/NuGet.Core/NuGet.Packaging/compiler/resources/nuspec.xsd type nuspecPackage struct { Metadata struct { - ID string `xml:"id"` - Version string `xml:"version"` - Authors string `xml:"authors"` - RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + // required fields + Authors string `xml:"authors"` + Description string `xml:"description"` + ID string `xml:"id"` + Version string `xml:"version"` + + // optional fields + Copyright string `xml:"copyright"` + DevelopmentDependency bool `xml:"developmentDependency"` + IconURL string `xml:"iconUrl"` + Language string `xml:"language"` + LicenseURL string `xml:"licenseUrl"` + MinClientVersion string `xml:"minClientVersion,attr"` + Owners string `xml:"owners"` ProjectURL string `xml:"projectUrl"` - Description string `xml:"description"` - ReleaseNotes string `xml:"releaseNotes"` Readme string `xml:"readme"` - PackageTypes struct { - PackageType []struct { - Name string `xml:"name,attr"` - } `xml:"packageType"` - } `xml:"packageTypes"` - Repository struct { - URL string `xml:"url,attr"` - } `xml:"repository"` + ReleaseNotes string `xml:"releaseNotes"` + RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + Summary string `xml:"summary"` + Tags string `xml:"tags"` + Title string `xml:"title"` + Dependencies struct { Dependency []struct { ID string `xml:"id,attr"` @@ -107,6 +125,14 @@ type nuspecPackage struct { } `xml:"dependency"` } `xml:"group"` } `xml:"dependencies"` + PackageTypes struct { + PackageType []struct { + Name string `xml:"name,attr"` + } `xml:"packageType"` + } `xml:"packageTypes"` + Repository struct { + URL string `xml:"url,attr"` + } `xml:"repository"` } `xml:"metadata"` } @@ -167,13 +193,24 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { } m := &Metadata{ - Description: p.Metadata.Description, - ReleaseNotes: p.Metadata.ReleaseNotes, Authors: p.Metadata.Authors, + Copyright: p.Metadata.Copyright, + Description: p.Metadata.Description, + DevelopmentDependency: p.Metadata.DevelopmentDependency, + IconURL: p.Metadata.IconURL, + Language: p.Metadata.Language, + LicenseURL: p.Metadata.LicenseURL, + MinClientVersion: p.Metadata.MinClientVersion, + Owners: p.Metadata.Owners, ProjectURL: p.Metadata.ProjectURL, + ReleaseNotes: p.Metadata.ReleaseNotes, RepositoryURL: p.Metadata.Repository.URL, RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance, - Dependencies: make(map[string][]Dependency), + Summary: p.Metadata.Summary, + Tags: p.Metadata.Tags, + Title: p.Metadata.Title, + + Dependencies: make(map[string][]Dependency), } if p.Metadata.Readme != "" { @@ -227,13 +264,13 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { func toNormalizedVersion(v *version.Version) string { var buf bytes.Buffer segments := v.Segments64() - fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) + _, _ = fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) if len(segments) > 3 && segments[3] > 0 { - fmt.Fprintf(&buf, ".%d", segments[3]) + _, _ = fmt.Fprintf(&buf, ".%d", segments[3]) } pre := v.Prerelease() if pre != "" { - fmt.Fprint(&buf, "-", pre) + _, _ = fmt.Fprint(&buf, "-", pre) } return buf.String() } diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go index f466492f8a..90c3e8dfeb 100644 --- a/modules/packages/nuget/metadata_test.go +++ b/modules/packages/nuget/metadata_test.go @@ -12,44 +12,62 @@ import ( ) const ( - id = "System.Gitea" - semver = "1.0.1" - authors = "Gitea Authors" - projectURL = "https://gitea.io" - description = "Package Description" - releaseNotes = "Package Release Notes" - readme = "Readme" - repositoryURL = "https://gitea.io/gitea/gitea" - targetFramework = ".NETStandard2.1" - dependencyID = "System.Text.Json" - dependencyVersion = "5.0.0" + authors = "Gitea Authors" + copyright = "Package Copyright" + dependencyID = "System.Text.Json" + dependencyVersion = "5.0.0" + developmentDependency = true + description = "Package Description" + iconURL = "https://gitea.io/favicon.png" + id = "System.Gitea" + language = "Package Language" + licenseURL = "https://gitea.io/license" + minClientVersion = "1.0.0.0" + owners = "Package Owners" + projectURL = "https://gitea.io" + readme = "Readme" + releaseNotes = "Package Release Notes" + repositoryURL = "https://gitea.io/gitea/gitea" + requireLicenseAcceptance = true + tags = "tag_1 tag_2 tag_3" + targetFramework = ".NETStandard2.1" + title = "Package Title" + versionStr = "1.0.1" ) const nuspecContent = `<?xml version="1.0" encoding="utf-8"?> <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> - <metadata> - <id>` + id + `</id> - <version>` + semver + `</version> - <authors>` + authors + `</authors> - <requireLicenseAcceptance>true</requireLicenseAcceptance> - <projectUrl>` + projectURL + `</projectUrl> - <description>` + description + `</description> - <releaseNotes>` + releaseNotes + `</releaseNotes> - <repository url="` + repositoryURL + `" /> - <readme>README.md</readme> - <dependencies> - <group targetFramework="` + targetFramework + `"> - <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" /> - </group> - </dependencies> - </metadata> + <metadata minClientVersion="` + minClientVersion + `"> + <authors>` + authors + `</authors> + <copyright>` + copyright + `</copyright> + <description>` + description + `</description> + <developmentDependency>true</developmentDependency> + <iconUrl>` + iconURL + `</iconUrl> + <id>` + id + `</id> + <language>` + language + `</language> + <licenseUrl>` + licenseURL + `</licenseUrl> + <owners>` + owners + `</owners> + <projectUrl>` + projectURL + `</projectUrl> + <readme>README.md</readme> + <releaseNotes>` + releaseNotes + `</releaseNotes> + <repository url="` + repositoryURL + `" /> + <requireLicenseAcceptance>true</requireLicenseAcceptance> + <tags>` + tags + `</tags> + <title>` + title + `</title> + <version>` + versionStr + `</version> + <dependencies> + <group targetFramework="` + targetFramework + `"> + <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" /> + </group> + </dependencies> + </metadata> </package>` const symbolsNuspecContent = `<?xml version="1.0" encoding="utf-8"?> <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> <metadata> <id>` + id + `</id> - <version>` + semver + `</version> + <version>` + versionStr + `</version> <description>` + description + `</description> <packageTypes> <packageType name="SymbolsPackage" /> @@ -140,14 +158,26 @@ func TestParsePackageMetaData(t *testing.T) { assert.NotNil(t, np) assert.Equal(t, DependencyPackage, np.PackageType) - assert.Equal(t, id, np.ID) - assert.Equal(t, semver, np.Version) assert.Equal(t, authors, np.Metadata.Authors) - assert.Equal(t, projectURL, np.Metadata.ProjectURL) assert.Equal(t, description, np.Metadata.Description) - assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) + assert.Equal(t, id, np.ID) + assert.Equal(t, versionStr, np.Version) + + assert.Equal(t, copyright, np.Metadata.Copyright) + assert.Equal(t, developmentDependency, np.Metadata.DevelopmentDependency) + assert.Equal(t, iconURL, np.Metadata.IconURL) + assert.Equal(t, language, np.Metadata.Language) + assert.Equal(t, licenseURL, np.Metadata.LicenseURL) + assert.Equal(t, minClientVersion, np.Metadata.MinClientVersion) + assert.Equal(t, owners, np.Metadata.Owners) + assert.Equal(t, projectURL, np.Metadata.ProjectURL) assert.Equal(t, readme, np.Metadata.Readme) + assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL) + assert.Equal(t, requireLicenseAcceptance, np.Metadata.RequireLicenseAcceptance) + assert.Equal(t, tags, np.Metadata.Tags) + assert.Equal(t, title, np.Metadata.Title) + assert.Len(t, np.Metadata.Dependencies, 1) assert.Contains(t, np.Metadata.Dependencies, targetFramework) deps := np.Metadata.Dependencies[targetFramework] @@ -180,7 +210,7 @@ func TestParsePackageMetaData(t *testing.T) { assert.Equal(t, SymbolsPackage, np.PackageType) assert.Equal(t, id, np.ID) - assert.Equal(t, semver, np.Version) + assert.Equal(t, versionStr, np.Version) assert.Equal(t, description, np.Metadata.Description) assert.Empty(t, np.Metadata.Dependencies) }) diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go index 81bf0371a0..9c952e1f10 100644 --- a/modules/packages/nuget/symbol_extractor.go +++ b/modules/packages/nuget/symbol_extractor.go @@ -34,7 +34,7 @@ type PortablePdbList []*PortablePdb func (l PortablePdbList) Close() { for _, pdb := range l { - pdb.Content.Close() + _ = pdb.Content.Close() } } @@ -65,7 +65,7 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { buf, err := packages.CreateHashedBufferFromReader(f) - f.Close() + _ = f.Close() if err != nil { return err @@ -73,12 +73,12 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { id, err := ParseDebugHeaderID(buf) if err != nil { - buf.Close() + _ = buf.Close() return fmt.Errorf("Invalid PDB file: %w", err) } if _, err := buf.Seek(0, io.SeekStart); err != nil { - buf.Close() + _ = buf.Close() return err } diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go index fa1b80ee82..e841e377d9 100644 --- a/modules/packages/nuget/symbol_extractor_test.go +++ b/modules/packages/nuget/symbol_extractor_test.go @@ -9,6 +9,8 @@ import ( "encoding/base64" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) @@ -17,18 +19,19 @@ fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==` func TestExtractPortablePdb(t *testing.T) { + setting.AppDataPath = t.TempDir() createArchive := func(name string, content []byte) []byte { var buf bytes.Buffer archive := zip.NewWriter(&buf) w, _ := archive.Create(name) - w.Write(content) - archive.Close() + _, _ = w.Write(content) + _ = archive.Close() return buf.Bytes() } t.Run("MissingPdbFiles", func(t *testing.T) { var buf bytes.Buffer - zip.NewWriter(&buf).Close() + _ = zip.NewWriter(&buf).Close() pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len())) assert.ErrorIs(t, err, ErrMissingPdbFiles) diff --git a/modules/packages/pub/metadata.go b/modules/packages/pub/metadata.go index afb464e462..9b00472eb2 100644 --- a/modules/packages/pub/metadata.go +++ b/modules/packages/pub/metadata.go @@ -88,7 +88,7 @@ func ParsePackage(r io.Reader) (*Package, error) { if err != nil { return nil, err } - } else if strings.ToLower(hd.Name) == "readme.md" { + } else if strings.EqualFold(hd.Name, "readme.md") { data, err := io.ReadAll(tr) if err != nil { return nil, err diff --git a/modules/packages/rubygems/marshal.go b/modules/packages/rubygems/marshal.go index 4e6a5fc5f8..1505221acc 100644 --- a/modules/packages/rubygems/marshal.go +++ b/modules/packages/rubygems/marshal.go @@ -250,7 +250,7 @@ func (e *MarshalEncoder) marshalArray(arr reflect.Value) error { return err } - for i := 0; i < length; i++ { + for i := range length { if err := e.marshal(arr.Index(i).Interface()); err != nil { return err } diff --git a/modules/packages/swift/metadata.go b/modules/packages/swift/metadata.go index 24c4262ab7..85beb57607 100644 --- a/modules/packages/swift/metadata.go +++ b/modules/packages/swift/metadata.go @@ -47,7 +47,7 @@ type Metadata struct { Keywords []string `json:"keywords,omitempty"` RepositoryURL string `json:"repository_url,omitempty"` License string `json:"license,omitempty"` - Author Person `json:"author,omitempty"` + Author Person `json:"author"` Manifests map[string]*Manifest `json:"manifests,omitempty"` } diff --git a/modules/paginator/paginator.go b/modules/paginator/paginator.go index 8258d194c2..0f64e89d9a 100644 --- a/modules/paginator/paginator.go +++ b/modules/paginator/paginator.go @@ -4,6 +4,8 @@ package paginator +import "code.gitea.io/gitea/modules/util" + /* In template: @@ -32,25 +34,43 @@ Output: // Paginator represents a set of results of pagination calculations. type Paginator struct { - total int // total rows count + total int // total rows count, -1 means unknown + totalPages int // total pages count, -1 means unknown + current int // current page number + curRows int // current page rows count + pagingNum int // how many rows in one page - current int // current page number numPages int // how many pages to show on the UI } // New initialize a new pagination calculation and returns a Paginator as result. func New(total, pagingNum, current, numPages int) *Paginator { - if pagingNum <= 0 { - pagingNum = 1 + pagingNum = max(pagingNum, 1) + totalPages := util.Iif(total == -1, -1, (total+pagingNum-1)/pagingNum) + if total >= 0 { + current = min(current, totalPages) } - if current <= 0 { - current = 1 + current = max(current, 1) + return &Paginator{ + total: total, + totalPages: totalPages, + current: current, + pagingNum: pagingNum, + numPages: numPages, } - p := &Paginator{total, pagingNum, current, numPages} - if p.current > p.TotalPages() { - p.current = p.TotalPages() +} + +func (p *Paginator) SetCurRows(rows int) { + // For "unlimited paging", we need to know the rows of current page to determine if there is a next page. + // There is still an edge case: when curRows==pagingNum, then the "next page" will be an empty page. + // Ideally we should query one more row to determine if there is really a next page, but it's impossible in current framework. + p.curRows = rows + if p.total == -1 && p.current == 1 && !p.HasNext() { + // if there is only one page for the "unlimited paging", set total rows/pages count + // then the tmpl could decide to hide the nav bar. + p.total = rows + p.totalPages = util.Iif(p.total == 0, 0, 1) } - return p } // IsFirst returns true if current page is the first page. @@ -72,7 +92,10 @@ func (p *Paginator) Previous() int { // HasNext returns true if there is a next page relative to current page. func (p *Paginator) HasNext() bool { - return p.total > p.current*p.pagingNum + if p.total == -1 { + return p.curRows >= p.pagingNum + } + return p.current*p.pagingNum < p.total } func (p *Paginator) Next() int { @@ -84,10 +107,7 @@ func (p *Paginator) Next() int { // IsLast returns true if current page is the last page. func (p *Paginator) IsLast() bool { - if p.total == 0 { - return true - } - return p.total > (p.current-1)*p.pagingNum && !p.HasNext() + return !p.HasNext() } // Total returns number of total rows. @@ -97,10 +117,7 @@ func (p *Paginator) Total() int { // TotalPages returns number of total pages. func (p *Paginator) TotalPages() int { - if p.total == 0 { - return 1 - } - return (p.total + p.pagingNum - 1) / p.pagingNum + return p.totalPages } // Current returns current page number. @@ -135,10 +152,10 @@ func getMiddleIdx(numPages int) int { // If value is -1 means "..." that more pages are not showing. func (p *Paginator) Pages() []*Page { if p.numPages == 0 { - return []*Page{} - } else if p.numPages == 1 && p.TotalPages() == 1 { + return nil + } else if p.total == -1 || (p.numPages == 1 && p.TotalPages() == 1) { // Only show current page. - return []*Page{{1, true}} + return []*Page{{p.current, true}} } // Total page number is less or equal. diff --git a/modules/paginator/paginator_test.go b/modules/paginator/paginator_test.go index 8a56ee5121..ed46ecea94 100644 --- a/modules/paginator/paginator_test.go +++ b/modules/paginator/paginator_test.go @@ -76,9 +76,7 @@ func TestPaginator(t *testing.T) { t.Run("Only current page", func(t *testing.T) { p := New(0, 10, 1, 1) pages := p.Pages() - assert.Len(t, pages, 1) - assert.Equal(t, 1, pages[0].Num()) - assert.True(t, pages[0].IsCurrent()) + assert.Empty(t, pages) // no "total", so no pages p = New(1, 10, 1, 1) pages = p.Pages() diff --git a/modules/private/actions.go b/modules/private/actions.go index 311a283650..e68f2f85b0 100644 --- a/modules/private/actions.go +++ b/modules/private/actions.go @@ -17,7 +17,7 @@ type GenerateTokenRequest struct { func GenerateActionsRunnerToken(ctx context.Context, scope string) (*ResponseText, ResponseExtra) { reqURL := setting.LocalURL + "api/internal/actions/generate_actions_runner_token" - req := newInternalRequest(ctx, reqURL, "POST", GenerateTokenRequest{ + req := newInternalRequestAPI(ctx, reqURL, "POST", GenerateTokenRequest{ Scope: scope, }) diff --git a/modules/private/hook.go b/modules/private/hook.go index 745c200619..215996b9b9 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -7,9 +7,9 @@ import ( "context" "fmt" "net/url" - "time" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" ) @@ -82,29 +82,32 @@ type HookProcReceiveRefResult struct { HeadBranch string } +func newInternalRequestAPIForHooks(ctx context.Context, hookName, ownerName, repoName string, opts HookOptions) *httplib.Request { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/%s/%s/%s", hookName, url.PathEscape(ownerName), url.PathEscape(repoName)) + req := newInternalRequestAPI(ctx, reqURL, "POST", opts) + // This "timeout" applies to http.Client's timeout: A Timeout of zero means no timeout. + // This "timeout" was previously set to `time.Duration(60+len(opts.OldCommitIDs))` seconds, but it caused unnecessary timeout failures. + // It should be good enough to remove the client side timeout, only respect the "ctx" and server side timeout. + req.SetReadWriteTimeout(0) + return req +} + // HookPreReceive check whether the provided commits are allowed func HookPreReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) ResponseExtra { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - req := newInternalRequest(ctx, reqURL, "POST", opts) - req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second) + req := newInternalRequestAPIForHooks(ctx, "pre-receive", ownerName, repoName, opts) _, extra := requestJSONResp(req, &ResponseText{}) return extra } // HookPostReceive updates services and users func HookPostReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) (*HookPostReceiveResult, ResponseExtra) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - req := newInternalRequest(ctx, reqURL, "POST", opts) - req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second) + req := newInternalRequestAPIForHooks(ctx, "post-receive", ownerName, repoName, opts) return requestJSONResp(req, &HookPostReceiveResult{}) } // HookProcReceive proc-receive hook func HookProcReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) (*HookProcReceiveResult, ResponseExtra) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/proc-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - - req := newInternalRequest(ctx, reqURL, "POST", opts) - req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second) + req := newInternalRequestAPIForHooks(ctx, "proc-receive", ownerName, repoName, opts) return requestJSONResp(req, &HookProcReceiveResult{}) } @@ -115,7 +118,7 @@ func SetDefaultBranch(ctx context.Context, ownerName, repoName, branch string) R url.PathEscape(repoName), url.PathEscape(branch), ) - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") _, extra := requestJSONResp(req, &ResponseText{}) return extra } @@ -123,7 +126,7 @@ func SetDefaultBranch(ctx context.Context, ownerName, repoName, branch string) R // SSHLog sends ssh error log response func SSHLog(ctx context.Context, isErr bool, msg string) error { reqURL := setting.LocalURL + "api/internal/ssh/log" - req := newInternalRequest(ctx, reqURL, "POST", &SSHLogOption{IsError: isErr, Message: msg}) + req := newInternalRequestAPI(ctx, reqURL, "POST", &SSHLogOption{IsError: isErr, Message: msg}) _, extra := requestJSONResp(req, &ResponseText{}) return extra.Error } diff --git a/modules/private/internal.go b/modules/private/internal.go index c7e7773524..e599c6eb8e 100644 --- a/modules/private/internal.go +++ b/modules/private/internal.go @@ -6,7 +6,6 @@ package private import ( "context" "crypto/tls" - "fmt" "net" "net/http" "os" @@ -34,16 +33,20 @@ func getClientIP() string { return strings.Fields(sshConnEnv)[0] } -func newInternalRequest(ctx context.Context, url, method string, body ...any) *httplib.Request { +func NewInternalRequest(ctx context.Context, url, method string) *httplib.Request { if setting.InternalToken == "" { log.Fatal(`The INTERNAL_TOKEN setting is missing from the configuration file: %q. Ensure you are running in the correct environment or set the correct configuration file with -c.`, setting.CustomConf) } + if !strings.HasPrefix(url, setting.LocalURL) { + log.Fatal("Invalid internal request URL: %q", url) + } + req := httplib.NewRequest(url, method). SetContext(ctx). Header("X-Real-IP", getClientIP()). - Header("X-Gitea-Internal-Auth", fmt.Sprintf("Bearer %s", setting.InternalToken)). + Header("X-Gitea-Internal-Auth", "Bearer "+setting.InternalToken). SetTLSClientConfig(&tls.Config{ InsecureSkipVerify: true, ServerName: setting.Domain, @@ -82,13 +85,17 @@ Ensure you are running in the correct environment or set the correct configurati }, }) } + return req +} +func newInternalRequestAPI(ctx context.Context, url, method string, body ...any) *httplib.Request { + req := NewInternalRequest(ctx, url, method) if len(body) == 1 { req.Header("Content-Type", "application/json") jsonBytes, _ := json.Marshal(body[0]) req.Body(jsonBytes) } else if len(body) > 1 { - log.Fatal("Too many arguments for newInternalRequest") + log.Fatal("Too many arguments for newInternalRequestAPI") } req.SetTimeout(10*time.Second, 60*time.Second) diff --git a/modules/private/key.go b/modules/private/key.go index dcd1714856..114683b343 100644 --- a/modules/private/key.go +++ b/modules/private/key.go @@ -14,7 +14,7 @@ import ( func UpdatePublicKeyInRepo(ctx context.Context, keyID, repoID int64) error { // Ask for running deliver hook and test pull request tasks. reqURL := setting.LocalURL + fmt.Sprintf("api/internal/ssh/%d/update/%d", keyID, repoID) - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") _, extra := requestJSONResp(req, &ResponseText{}) return extra.Error } @@ -24,7 +24,7 @@ func UpdatePublicKeyInRepo(ctx context.Context, keyID, repoID int64) error { func AuthorizedPublicKeyByContent(ctx context.Context, content string) (*ResponseText, ResponseExtra) { // Ask for running deliver hook and test pull request tasks. reqURL := setting.LocalURL + "api/internal/ssh/authorized_keys" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") req.Param("content", content) return requestJSONResp(req, &ResponseText{}) } diff --git a/modules/private/mail.go b/modules/private/mail.go index 08de5b7e28..3904e37bea 100644 --- a/modules/private/mail.go +++ b/modules/private/mail.go @@ -23,7 +23,7 @@ type Email struct { func SendEmail(ctx context.Context, subject, message string, to []string) (*ResponseText, ResponseExtra) { reqURL := setting.LocalURL + "api/internal/mail/send" - req := newInternalRequest(ctx, reqURL, "POST", Email{ + req := newInternalRequestAPI(ctx, reqURL, "POST", Email{ Subject: subject, Message: message, To: to, diff --git a/modules/private/manager.go b/modules/private/manager.go index 6055e553bd..e3d5ad57e0 100644 --- a/modules/private/manager.go +++ b/modules/private/manager.go @@ -18,21 +18,21 @@ import ( // Shutdown calls the internal shutdown function func Shutdown(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/shutdown" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Shutting down") } // Restart calls the internal restart function func Restart(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/restart" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Restarting") } // ReloadTemplates calls the internal reload-templates function func ReloadTemplates(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/reload-templates" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Reloaded") } @@ -45,7 +45,7 @@ type FlushOptions struct { // FlushQueues calls the internal flush-queues function func FlushQueues(ctx context.Context, timeout time.Duration, nonBlocking bool) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/flush-queues" - req := newInternalRequest(ctx, reqURL, "POST", FlushOptions{Timeout: timeout, NonBlocking: nonBlocking}) + req := newInternalRequestAPI(ctx, reqURL, "POST", FlushOptions{Timeout: timeout, NonBlocking: nonBlocking}) if timeout > 0 { req.SetReadWriteTimeout(timeout + 10*time.Second) } @@ -55,28 +55,28 @@ func FlushQueues(ctx context.Context, timeout time.Duration, nonBlocking bool) R // PauseLogging pauses logging func PauseLogging(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/pause-logging" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Logging Paused") } // ResumeLogging resumes logging func ResumeLogging(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/resume-logging" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Logging Restarted") } // ReleaseReopenLogging releases and reopens logging files func ReleaseReopenLogging(ctx context.Context) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/release-and-reopen-logging" - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Logging Restarted") } // SetLogSQL sets database logging func SetLogSQL(ctx context.Context, on bool) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/set-log-sql?on=" + strconv.FormatBool(on) - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Log SQL setting set") } @@ -91,7 +91,7 @@ type LoggerOptions struct { // AddLogger adds a logger func AddLogger(ctx context.Context, logger, writer, mode string, config map[string]any) ResponseExtra { reqURL := setting.LocalURL + "api/internal/manager/add-logger" - req := newInternalRequest(ctx, reqURL, "POST", LoggerOptions{ + req := newInternalRequestAPI(ctx, reqURL, "POST", LoggerOptions{ Logger: logger, Writer: writer, Mode: mode, @@ -103,7 +103,7 @@ func AddLogger(ctx context.Context, logger, writer, mode string, config map[stri // RemoveLogger removes a logger func RemoveLogger(ctx context.Context, logger, writer string) ResponseExtra { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/remove-logger/%s/%s", url.PathEscape(logger), url.PathEscape(writer)) - req := newInternalRequest(ctx, reqURL, "POST") + req := newInternalRequestAPI(ctx, reqURL, "POST") return requestJSONClientMsg(req, "Removed") } @@ -111,7 +111,7 @@ func RemoveLogger(ctx context.Context, logger, writer string) ResponseExtra { func Processes(ctx context.Context, out io.Writer, flat, noSystem, stacktraces, json bool, cancel string) ResponseExtra { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/manager/processes?flat=%t&no-system=%t&stacktraces=%t&json=%t&cancel-pid=%s", flat, noSystem, stacktraces, json, url.QueryEscape(cancel)) - req := newInternalRequest(ctx, reqURL, "GET") + req := newInternalRequestAPI(ctx, reqURL, "GET") callback := func(resp *http.Response, extra *ResponseExtra) { _, extra.Error = io.Copy(out, resp.Body) } diff --git a/modules/private/restore_repo.go b/modules/private/restore_repo.go index 496209d3cb..9c3a008142 100644 --- a/modules/private/restore_repo.go +++ b/modules/private/restore_repo.go @@ -24,7 +24,7 @@ type RestoreParams struct { func RestoreRepo(ctx context.Context, repoDir, ownerName, repoName string, units []string, validation bool) ResponseExtra { reqURL := setting.LocalURL + "api/internal/restore_repo" - req := newInternalRequest(ctx, reqURL, "POST", RestoreParams{ + req := newInternalRequestAPI(ctx, reqURL, "POST", RestoreParams{ RepoDir: repoDir, OwnerName: ownerName, RepoName: repoName, diff --git a/modules/private/serv.go b/modules/private/serv.go index 480a446954..b1dafbd81b 100644 --- a/modules/private/serv.go +++ b/modules/private/serv.go @@ -23,7 +23,7 @@ type KeyAndOwner struct { // ServNoCommand returns information about the provided key func ServNoCommand(ctx context.Context, keyID int64) (*asymkey_model.PublicKey, *user_model.User, error) { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/serv/none/%d", keyID) - req := newInternalRequest(ctx, reqURL, "GET") + req := newInternalRequestAPI(ctx, reqURL, "GET") keyAndOwner, extra := requestJSONResp(req, &KeyAndOwner{}) if extra.HasError() { return nil, nil, extra.Error @@ -46,18 +46,16 @@ type ServCommandResults struct { } // ServCommand preps for a serv call -func ServCommand(ctx context.Context, keyID int64, ownerName, repoName string, mode perm.AccessMode, verbs ...string) (*ServCommandResults, ResponseExtra) { +func ServCommand(ctx context.Context, keyID int64, ownerName, repoName string, mode perm.AccessMode, verb, lfsVerb string) (*ServCommandResults, ResponseExtra) { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/serv/command/%d/%s/%s?mode=%d", keyID, url.PathEscape(ownerName), url.PathEscape(repoName), mode, ) - for _, verb := range verbs { - if verb != "" { - reqURL += fmt.Sprintf("&verb=%s", url.QueryEscape(verb)) - } - } - req := newInternalRequest(ctx, reqURL, "GET") + reqURL += "&verb=" + url.QueryEscape(verb) + // reqURL += "&lfs_verb=" + url.QueryEscape(lfsVerb) // TODO: actually there is no use of this parameter. In the future, the URL construction should be more flexible + _ = lfsVerb + req := newInternalRequestAPI(ctx, reqURL, "GET") return requestJSONResp(req, &ServCommandResults{}) } diff --git a/modules/process/manager_test.go b/modules/process/manager_test.go index 36b2a912ea..0d637c8acc 100644 --- a/modules/process/manager_test.go +++ b/modules/process/manager_test.go @@ -23,7 +23,7 @@ func TestGetManager(t *testing.T) { func TestManager_AddContext(t *testing.T) { pm := Manager{processMap: make(map[IDType]*process), next: 1} - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) defer cancel() p1Ctx, _, finished := pm.AddContext(ctx, "foo") @@ -42,7 +42,7 @@ func TestManager_AddContext(t *testing.T) { func TestManager_Cancel(t *testing.T) { pm := Manager{processMap: make(map[IDType]*process), next: 1} - ctx, _, finished := pm.AddContext(context.Background(), "foo") + ctx, _, finished := pm.AddContext(t.Context(), "foo") defer finished() pm.Cancel(GetPID(ctx)) @@ -54,7 +54,7 @@ func TestManager_Cancel(t *testing.T) { } finished() - ctx, cancel, finished := pm.AddContext(context.Background(), "foo") + ctx, cancel, finished := pm.AddContext(t.Context(), "foo") defer finished() cancel() @@ -70,7 +70,7 @@ func TestManager_Cancel(t *testing.T) { func TestManager_Remove(t *testing.T) { pm := Manager{processMap: make(map[IDType]*process), next: 1} - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) defer cancel() p1Ctx, _, finished := pm.AddContext(ctx, "foo") diff --git a/modules/proxyprotocol/errors.go b/modules/proxyprotocol/errors.go index 5439a86bd8..f76c82b7f6 100644 --- a/modules/proxyprotocol/errors.go +++ b/modules/proxyprotocol/errors.go @@ -20,7 +20,7 @@ type ErrBadAddressType struct { } func (e *ErrBadAddressType) Error() string { - return fmt.Sprintf("Unexpected proxy header address type: %s", e.Address) + return "Unexpected proxy header address type: " + e.Address } // ErrBadRemote is an error demonstrating a bad proxy header with bad Remote diff --git a/modules/public/public.go b/modules/public/public.go index abc6b46158..a7eace1538 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -44,7 +44,7 @@ func FileHandlerFunc() http.HandlerFunc { func parseAcceptEncoding(val string) container.Set[string] { parts := strings.Split(val, ";") types := make(container.Set[string]) - for _, v := range strings.Split(parts[0], ",") { + for v := range strings.SplitSeq(parts[0], ",") { types.Add(strings.TrimSpace(v)) } return types @@ -86,33 +86,28 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, return } - serveContent(w, req, fi, fi.ModTime(), f) + servePublicAsset(w, req, fi, fi.ModTime(), f) } -type GzipBytesProvider interface { - GzipBytes() []byte -} - -// serveContent serve http content -func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { +// servePublicAsset serve http content +func servePublicAsset(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { setWellKnownContentType(w, fi.Name()) - + httpcache.SetCacheControlInHeader(w.Header(), httpcache.CacheControlForPublicStatic()) encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) - if encodings.Contains("gzip") { - // try to provide gzip content directly from bindata (provided by vfsgenÛ°CompressedFileInfo) - if compressed, ok := fi.(GzipBytesProvider); ok { - rdGzip := bytes.NewReader(compressed.GzipBytes()) + fiEmbedded, _ := fi.(assetfs.EmbeddedFileInfo) + if encodings.Contains("gzip") && fiEmbedded != nil { + // try to provide gzip content directly from bindata + if gzipBytes, ok := fiEmbedded.GetGzipContent(); ok { + rdGzip := bytes.NewReader(gzipBytes) // all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name // then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data if w.Header().Get("Content-Type") == "" { w.Header().Set("Content-Type", "application/octet-stream") } w.Header().Set("Content-Encoding", "gzip") - httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, rdGzip) + http.ServeContent(w, req, fi.Name(), modtime, rdGzip) return } } - - httpcache.ServeContentWithCacheControl(w, req, fi.Name(), modtime, content) - return + http.ServeContent(w, req, fi.Name(), modtime, content) } diff --git a/modules/public/public_bindata.go b/modules/public/public_bindata.go index 4878f88ad1..2dcf3e72e4 100644 --- a/modules/public/public_bindata.go +++ b/modules/public/public_bindata.go @@ -5,4 +5,19 @@ package public -//go:generate go run ../../build/generate-bindata.go ../../public public bindata.go true +//go:generate go run ../../build/generate-bindata.go ../../public bindata.dat + +import ( + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", assetfs.NewEmbeddedFS(bindata)) +}) diff --git a/modules/public/serve_dynamic.go b/modules/public/public_dynamic.go index a668b17c34..a668b17c34 100644 --- a/modules/public/serve_dynamic.go +++ b/modules/public/public_dynamic.go diff --git a/modules/public/serve_static.go b/modules/public/serve_static.go deleted file mode 100644 index e79085021e..0000000000 --- a/modules/public/serve_static.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package public - -import ( - "time" - - "code.gitea.io/gitea/modules/assetfs" - "code.gitea.io/gitea/modules/timeutil" -) - -var _ GzipBytesProvider = (*vfsgenÛ°CompressedFileInfo)(nil) - -// GlobalModTime provide a global mod time for embedded asset files -func GlobalModTime(filename string) time.Time { - return timeutil.GetExecutableModTime() -} - -func BuiltinAssets() *assetfs.Layer { - return assetfs.Bindata("builtin(bindata)", Assets) -} diff --git a/modules/queue/base_levelqueue_common.go b/modules/queue/base_levelqueue_common.go index 78d3b85a8a..d37093b84d 100644 --- a/modules/queue/base_levelqueue_common.go +++ b/modules/queue/base_levelqueue_common.go @@ -83,7 +83,7 @@ func prepareLevelDB(cfg *BaseConfig) (conn string, db *leveldb.DB, err error) { } conn = cfg.ConnStr } - for i := 0; i < 10; i++ { + for range 10 { if db, err = nosql.GetManager().GetLevelDB(conn); err == nil { break } diff --git a/modules/queue/base_levelqueue_test.go b/modules/queue/base_levelqueue_test.go index b881802ca2..05d8208560 100644 --- a/modules/queue/base_levelqueue_test.go +++ b/modules/queue/base_levelqueue_test.go @@ -11,6 +11,7 @@ import ( "gitea.com/lunny/levelqueue" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/syndtr/goleveldb/leveldb" ) @@ -29,9 +30,7 @@ func TestCorruptedLevelQueue(t *testing.T) { // sometimes the levelqueue could be in a corrupted state, this test is to make sure it can recover from it dbDir := t.TempDir() + "/levelqueue-test" db, err := leveldb.OpenFile(dbDir, nil) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) defer db.Close() assert.NoError(t, db.Put([]byte("other-key"), []byte("other-value"), nil)) diff --git a/modules/queue/base_redis.go b/modules/queue/base_redis.go index a1e234943d..bea0fd7a98 100644 --- a/modules/queue/base_redis.go +++ b/modules/queue/base_redis.go @@ -29,7 +29,7 @@ func newBaseRedisGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { client := nosql.GetManager().GetRedisClient(cfg.ConnStr) var err error - for i := 0; i < 10; i++ { + for range 10 { err = client.Ping(graceful.GetManager().ShutdownContext()).Err() if err == nil { break diff --git a/modules/queue/base_redis_test.go b/modules/queue/base_redis_test.go index 19fbccbc8f..6478988d7f 100644 --- a/modules/queue/base_redis_test.go +++ b/modules/queue/base_redis_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func waitRedisReady(conn string, dur time.Duration) (ready bool) { @@ -61,9 +62,7 @@ func TestBaseRedis(t *testing.T) { return } assert.NoError(t, redisServer.Start()) - if !assert.True(t, waitRedisReady("redis://127.0.0.1:6379/0", 5*time.Second), "start redis-server") { - return - } + require.True(t, waitRedisReady("redis://127.0.0.1:6379/0", 5*time.Second), "start redis-server") } testQueueBasic(t, newBaseRedisSimple, toBaseConfig("baseRedis", setting.QueueSettings{Length: 10}), false) diff --git a/modules/queue/base_test.go b/modules/queue/base_test.go index 01b52b3c16..8e7c18d740 100644 --- a/modules/queue/base_test.go +++ b/modules/queue/base_test.go @@ -17,11 +17,11 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) q, err := newFn(cfg) assert.NoError(t, err) - ctx := context.Background() + ctx := t.Context() _ = q.RemoveAll(ctx) cnt, err := q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 0, cnt) + assert.Equal(t, 0, cnt) // push the first item err = q.PushItem(ctx, []byte("foo")) @@ -29,7 +29,7 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) cnt, err = q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 1, cnt) + assert.Equal(t, 1, cnt) // push a duplicate item err = q.PushItem(ctx, []byte("foo")) @@ -45,10 +45,10 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) has, err := q.HasItem(ctx, []byte("foo")) assert.NoError(t, err) if !isUnique { - assert.EqualValues(t, 2, cnt) + assert.Equal(t, 2, cnt) assert.False(t, has) // non-unique queues don't check for duplicates } else { - assert.EqualValues(t, 1, cnt) + assert.Equal(t, 1, cnt) assert.True(t, has) } @@ -59,18 +59,18 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) // pop the first item (and the duplicate if non-unique) it, err := q.PopItem(ctx) assert.NoError(t, err) - assert.EqualValues(t, "foo", string(it)) + assert.Equal(t, "foo", string(it)) if !isUnique { it, err = q.PopItem(ctx) assert.NoError(t, err) - assert.EqualValues(t, "foo", string(it)) + assert.Equal(t, "foo", string(it)) } // pop another item it, err = q.PopItem(ctx) assert.NoError(t, err) - assert.EqualValues(t, "bar", string(it)) + assert.Equal(t, "bar", string(it)) // pop an empty queue (timeout, cancel) ctxTimed, cancel := context.WithTimeout(ctx, 10*time.Millisecond) @@ -87,7 +87,7 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) // test blocking push if queue is full for i := 0; i < cfg.Length; i++ { - err = q.PushItem(ctx, []byte(fmt.Sprintf("item-%d", i))) + err = q.PushItem(ctx, fmt.Appendf(nil, "item-%d", i)) assert.NoError(t, err) } ctxTimed, cancel = context.WithTimeout(ctx, 10*time.Millisecond) @@ -107,13 +107,13 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) // remove all cnt, err = q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, cfg.Length, cnt) + assert.Equal(t, cfg.Length, cnt) _ = q.RemoveAll(ctx) cnt, err = q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 0, cnt) + assert.Equal(t, 0, cnt) }) } @@ -121,12 +121,12 @@ func TestBaseDummy(t *testing.T) { q, err := newBaseDummy(&BaseConfig{}, true) assert.NoError(t, err) - ctx := context.Background() + ctx := t.Context() assert.NoError(t, q.PushItem(ctx, []byte("foo"))) cnt, err := q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 0, cnt) + assert.Equal(t, 0, cnt) has, err := q.HasItem(ctx, []byte("foo")) assert.NoError(t, err) diff --git a/modules/queue/manager.go b/modules/queue/manager.go index 079e2bee7a..ae6c51872d 100644 --- a/modules/queue/manager.go +++ b/modules/queue/manager.go @@ -6,6 +6,7 @@ package queue import ( "context" "errors" + "maps" "sync" "time" @@ -70,9 +71,7 @@ func (m *Manager) ManagedQueues() map[int64]ManagedWorkerPoolQueue { defer m.mu.Unlock() queues := make(map[int64]ManagedWorkerPoolQueue, len(m.Queues)) - for k, v := range m.Queues { - queues[k] = v - } + maps.Copy(queues, m.Queues) return queues } diff --git a/modules/queue/manager_test.go b/modules/queue/manager_test.go index 15dd1b4f2f..fda498cc84 100644 --- a/modules/queue/manager_test.go +++ b/modules/queue/manager_test.go @@ -4,7 +4,6 @@ package queue import ( - "context" "path/filepath" "testing" @@ -48,7 +47,7 @@ CONN_STR = redis:// assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/common"), q.baseConfig.DataFullDir) assert.Equal(t, 100000, q.baseConfig.Length) assert.Equal(t, 20, q.batchLength) - assert.Equal(t, "", q.baseConfig.ConnStr) + assert.Empty(t, q.baseConfig.ConnStr) assert.Equal(t, "default_queue", q.baseConfig.QueueFullName) assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName) assert.NotZero(t, q.GetWorkerMaxNumber()) @@ -80,7 +79,7 @@ MAX_WORKERS = 123 assert.NoError(t, err) - q1 := createWorkerPoolQueue[string](context.Background(), "no-such", cfgProvider, nil, false) + q1 := createWorkerPoolQueue[string](t.Context(), "no-such", cfgProvider, nil, false) assert.Equal(t, "no-such", q1.GetName()) assert.Equal(t, "dummy", q1.GetType()) // no handler, so it becomes dummy assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir1"), q1.baseConfig.DataFullDir) @@ -96,13 +95,13 @@ MAX_WORKERS = 123 assert.Equal(t, "string", q1.GetItemTypeName()) qid1 := GetManager().qidCounter - q2 := createWorkerPoolQueue(context.Background(), "sub", cfgProvider, func(s ...int) (unhandled []int) { return nil }, false) + q2 := createWorkerPoolQueue(t.Context(), "sub", cfgProvider, func(s ...int) (unhandled []int) { return nil }, false) assert.Equal(t, "sub", q2.GetName()) assert.Equal(t, "level", q2.GetType()) assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir2"), q2.baseConfig.DataFullDir) assert.Equal(t, 102, q2.baseConfig.Length) assert.Equal(t, 22, q2.batchLength) - assert.Equal(t, "", q2.baseConfig.ConnStr) + assert.Empty(t, q2.baseConfig.ConnStr) assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName) assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName) assert.Equal(t, 123, q2.GetWorkerMaxNumber()) @@ -118,7 +117,7 @@ MAX_WORKERS = 123 assert.Equal(t, 120, q1.workerMaxNum) stop := runWorkerPoolQueue(q2) - assert.NoError(t, GetManager().GetManagedQueue(qid2).FlushWithContext(context.Background(), 0)) - assert.NoError(t, GetManager().FlushAll(context.Background(), 0)) + assert.NoError(t, GetManager().GetManagedQueue(qid2).FlushWithContext(t.Context(), 0)) + assert.NoError(t, GetManager().FlushAll(t.Context(), 0)) stop() } diff --git a/modules/queue/workerqueue_test.go b/modules/queue/workerqueue_test.go index c0841a1752..a6c369d5f9 100644 --- a/modules/queue/workerqueue_test.go +++ b/modules/queue/workerqueue_test.go @@ -4,7 +4,6 @@ package queue import ( - "context" "slices" "strconv" "sync" @@ -58,15 +57,15 @@ func TestWorkerPoolQueueUnhandled(t *testing.T) { testRecorder.Record("push:%v", i) assert.NoError(t, q.Push(i)) } - assert.NoError(t, q.FlushWithContext(context.Background(), 0)) + assert.NoError(t, q.FlushWithContext(t.Context(), 0)) stop() ok := true for i := 0; i < queueSetting.Length; i++ { if i%2 == 0 { - ok = ok && assert.EqualValues(t, 2, m[i], "test %s: item %d", t.Name(), i) + ok = ok && assert.Equal(t, 2, m[i], "test %s: item %d", t.Name(), i) } else { - ok = ok && assert.EqualValues(t, 1, m[i], "test %s: item %d", t.Name(), i) + ok = ok && assert.Equal(t, 1, m[i], "test %s: item %d", t.Name(), i) } } if !ok { @@ -78,17 +77,17 @@ func TestWorkerPoolQueueUnhandled(t *testing.T) { runCount := 2 // we can run these tests even hundreds times to see its stability t.Run("1/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { test(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1}) } }) t.Run("3/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { test(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1}) } }) t.Run("4/5", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { test(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5}) } }) @@ -97,17 +96,17 @@ func TestWorkerPoolQueueUnhandled(t *testing.T) { func TestWorkerPoolQueuePersistence(t *testing.T) { runCount := 2 // we can run these tests even hundreds times to see its stability t.Run("1/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1, Length: 100}) } }) t.Run("3/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1, Length: 100}) } }) t.Run("4/5", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5, Length: 100}) } }) @@ -142,7 +141,7 @@ func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSett q, _ := newWorkerPoolQueueForTest("pr_patch_checker_test", queueSetting, testHandler, true) stop := runWorkerPoolQueue(q) - for i := 0; i < testCount; i++ { + for i := range testCount { _ = q.Push("task-" + strconv.Itoa(i)) } close(startWhenAllReady) @@ -166,7 +165,7 @@ func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSett q, _ := newWorkerPoolQueueForTest("pr_patch_checker_test", queueSetting, testHandler, true) stop := runWorkerPoolQueue(q) - assert.NoError(t, q.FlushWithContext(context.Background(), 0)) + assert.NoError(t, q.FlushWithContext(t.Context(), 0)) stop() } @@ -174,7 +173,7 @@ func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSett assert.NotEmpty(t, tasksQ1) assert.NotEmpty(t, tasksQ2) - assert.EqualValues(t, testCount, len(tasksQ1)+len(tasksQ2)) + assert.Equal(t, testCount, len(tasksQ1)+len(tasksQ2)) } func TestWorkerPoolQueueActiveWorkers(t *testing.T) { @@ -187,34 +186,34 @@ func TestWorkerPoolQueueActiveWorkers(t *testing.T) { q, _ := newWorkerPoolQueueForTest("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 1, Length: 100}, handler, false) stop := runWorkerPoolQueue(q) - for i := 0; i < 5; i++ { + for i := range 5 { assert.NoError(t, q.Push(i)) } time.Sleep(50 * time.Millisecond) - assert.EqualValues(t, 1, q.GetWorkerNumber()) - assert.EqualValues(t, 1, q.GetWorkerActiveNumber()) + assert.Equal(t, 1, q.GetWorkerNumber()) + assert.Equal(t, 1, q.GetWorkerActiveNumber()) time.Sleep(500 * time.Millisecond) - assert.EqualValues(t, 1, q.GetWorkerNumber()) - assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 1, q.GetWorkerNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) time.Sleep(workerIdleDuration) - assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + assert.Equal(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working stop() q, _ = newWorkerPoolQueueForTest("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 3, Length: 100}, handler, false) stop = runWorkerPoolQueue(q) - for i := 0; i < 15; i++ { + for i := range 15 { assert.NoError(t, q.Push(i)) } time.Sleep(50 * time.Millisecond) - assert.EqualValues(t, 3, q.GetWorkerNumber()) - assert.EqualValues(t, 3, q.GetWorkerActiveNumber()) + assert.Equal(t, 3, q.GetWorkerNumber()) + assert.Equal(t, 3, q.GetWorkerActiveNumber()) time.Sleep(500 * time.Millisecond) - assert.EqualValues(t, 3, q.GetWorkerNumber()) - assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 3, q.GetWorkerNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) time.Sleep(workerIdleDuration) - assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + assert.Equal(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working stop() } @@ -241,13 +240,13 @@ func TestWorkerPoolQueueShutdown(t *testing.T) { } <-handlerCalled time.Sleep(200 * time.Millisecond) // wait for a while to make sure all workers are active - assert.EqualValues(t, 4, q.GetWorkerActiveNumber()) + assert.Equal(t, 4, q.GetWorkerActiveNumber()) stop() // stop triggers shutdown - assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) // no item was ever handled, so we still get all of them again q, _ = newWorkerPoolQueueForTest("test-workpoolqueue", qs, handler, false) - assert.EqualValues(t, 20, q.GetQueueItemNumber()) + assert.Equal(t, 20, q.GetQueueItemNumber()) } func TestWorkerPoolQueueWorkerIdleReset(t *testing.T) { @@ -275,7 +274,7 @@ func TestWorkerPoolQueueWorkerIdleReset(t *testing.T) { } q, _ = newWorkerPoolQueueForTest("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 2, Length: 100}, handler, false) stop := runWorkerPoolQueue(q) - for i := 0; i < 100; i++ { + for i := range 100 { assert.NoError(t, q.Push(i)) } time.Sleep(500 * time.Millisecond) diff --git a/modules/references/references.go b/modules/references/references.go index 6e549cb875..592bd4cbe4 100644 --- a/modules/references/references.go +++ b/modules/references/references.go @@ -32,7 +32,7 @@ var ( // issueNumericPattern matches string that references to a numeric issue, e.g. #1287 issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\'|\")([#!][0-9]+)(?:\s|$|\)|\]|\'|\"|[:;,.?!]\s|[:;,.?!]$)`) // issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234 - issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\"|\')([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$)|\"|\')`) + issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\"|\')([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$)|\"|\'|,)`) // crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository // e.g. org/repo#12345 crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+[#!][0-9]+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`) @@ -330,22 +330,22 @@ func FindAllIssueReferences(content string) []IssueReference { } // FindRenderizableReferenceNumeric returns the first unvalidated reference found in a string. -func FindRenderizableReferenceNumeric(content string, prOnly, crossLinkOnly bool) (bool, *RenderizableReference) { +func FindRenderizableReferenceNumeric(content string, prOnly, crossLinkOnly bool) *RenderizableReference { var match []int if !crossLinkOnly { match = issueNumericPattern.FindStringSubmatchIndex(content) } if match == nil { if match = crossReferenceIssueNumericPattern.FindStringSubmatchIndex(content); match == nil { - return false, nil + return nil } } r := getCrossReference(util.UnsafeStringToBytes(content), match[2], match[3], false, prOnly) if r == nil { - return false, nil + return nil } - return true, &RenderizableReference{ + return &RenderizableReference{ Issue: r.issue, Owner: r.owner, Name: r.name, @@ -372,15 +372,14 @@ func FindRenderizableCommitCrossReference(content string) (bool, *RenderizableRe } // FindRenderizableReferenceRegexp returns the first regexp unvalidated references found in a string. -func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) (bool, *RenderizableReference) { +func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) *RenderizableReference { match := pattern.FindStringSubmatchIndex(content) if len(match) < 4 { - return false, nil + return nil } action, location := findActionKeywords([]byte(content), match[2]) - - return true, &RenderizableReference{ + return &RenderizableReference{ Issue: content[match[2]:match[3]], RefLocation: &RefSpan{Start: match[0], End: match[1]}, Action: action, @@ -390,15 +389,14 @@ func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) (bo } // FindRenderizableReferenceAlphanumeric returns the first alphanumeric unvalidated references found in a string. -func FindRenderizableReferenceAlphanumeric(content string) (bool, *RenderizableReference) { +func FindRenderizableReferenceAlphanumeric(content string) *RenderizableReference { match := issueAlphanumericPattern.FindStringSubmatchIndex(content) if match == nil { - return false, nil + return nil } action, location := findActionKeywords([]byte(content), match[2]) - - return true, &RenderizableReference{ + return &RenderizableReference{ Issue: content[match[2]:match[3]], RefLocation: &RefSpan{Start: match[2], End: match[3]}, Action: action, @@ -464,11 +462,12 @@ func findAllIssueReferencesBytes(content []byte, links []string) []*rawReference continue } var sep string - if parts[3] == "issues" { + switch parts[3] { + case "issues": sep = "#" - } else if parts[3] == "pulls" { + case "pulls": sep = "!" - } else { + default: continue } // Note: closing/reopening keywords not supported with URLs diff --git a/modules/references/references_test.go b/modules/references/references_test.go index e224c919e9..a15ae99f79 100644 --- a/modules/references/references_test.go +++ b/modules/references/references_test.go @@ -46,7 +46,7 @@ owner/repo!123456789 contentBytes := []byte(test) convertFullHTMLReferencesToShortRefs(re, &contentBytes) result := string(contentBytes) - assert.EqualValues(t, expect, result) + assert.Equal(t, expect, result) } func TestFindAllIssueReferences(t *testing.T) { @@ -249,11 +249,10 @@ func TestFindAllIssueReferences(t *testing.T) { } for _, fixture := range alnumFixtures { - found, ref := FindRenderizableReferenceAlphanumeric(fixture.input) + ref := FindRenderizableReferenceAlphanumeric(fixture.input) if fixture.issue == "" { - assert.False(t, found, "Failed to parse: {%s}", fixture.input) + assert.Nil(t, ref, "Failed to parse: {%s}", fixture.input) } else { - assert.True(t, found, "Failed to parse: {%s}", fixture.input) assert.Equal(t, fixture.issue, ref.Issue, "Failed to parse: {%s}", fixture.input) assert.Equal(t, fixture.refLocation, ref.RefLocation, "Failed to parse: {%s}", fixture.input) assert.Equal(t, fixture.action, ref.Action, "Failed to parse: {%s}", fixture.input) @@ -284,9 +283,9 @@ func testFixtures(t *testing.T, fixtures []testFixture, context string) { } expref := rawToIssueReferenceList(expraw) refs := FindAllIssueReferencesMarkdown(fixture.input) - assert.EqualValues(t, expref, refs, "[%s] Failed to parse: {%s}", context, fixture.input) + assert.Equal(t, expref, refs, "[%s] Failed to parse: {%s}", context, fixture.input) rawrefs := findAllIssueReferencesMarkdown(fixture.input) - assert.EqualValues(t, expraw, rawrefs, "[%s] Failed to parse: {%s}", context, fixture.input) + assert.Equal(t, expraw, rawrefs, "[%s] Failed to parse: {%s}", context, fixture.input) } // Restore for other tests that may rely on the original value @@ -295,7 +294,7 @@ func testFixtures(t *testing.T, fixtures []testFixture, context string) { func TestFindAllMentions(t *testing.T) { res := FindAllMentionsBytes([]byte("@tasha, @mike; @lucy: @john")) - assert.EqualValues(t, []RefSpan{ + assert.Equal(t, []RefSpan{ {Start: 0, End: 6}, {Start: 8, End: 13}, {Start: 15, End: 20}, @@ -463,6 +462,7 @@ func TestRegExp_issueAlphanumericPattern(t *testing.T) { "ABC-123:", "\"ABC-123\"", "'ABC-123'", + "ABC-123, unknown PR", } falseTestCases := []string{ "RC-08", @@ -554,7 +554,7 @@ func TestParseCloseKeywords(t *testing.T) { res := pat.FindAllStringSubmatch(test.match, -1) assert.Len(t, res, 1) assert.Len(t, res[0], 2) - assert.EqualValues(t, test.expected, res[0][1]) + assert.Equal(t, test.expected, res[0][1]) } } } diff --git a/modules/regexplru/regexplru_test.go b/modules/regexplru/regexplru_test.go index 9c24b23fa9..4b539c31e9 100644 --- a/modules/regexplru/regexplru_test.go +++ b/modules/regexplru/regexplru_test.go @@ -18,9 +18,9 @@ func TestRegexpLru(t *testing.T) { assert.NoError(t, err) assert.True(t, r.MatchString("a")) - assert.EqualValues(t, 1, lruCache.Len()) + assert.Equal(t, 1, lruCache.Len()) _, err = GetCompiled("(") assert.Error(t, err) - assert.EqualValues(t, 2, lruCache.Len()) + assert.Equal(t, 2, lruCache.Len()) } diff --git a/modules/repository/branch.go b/modules/repository/branch.go index 2bf9930f19..30aa0a6e85 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -41,11 +41,12 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, if err != nil { return 0, fmt.Errorf("GetObjectFormat: %w", err) } - _, err = db.GetEngine(ctx).ID(repo.ID).Update(&repo_model.Repository{ObjectFormatName: objFmt.Name()}) - if err != nil { - return 0, fmt.Errorf("UpdateRepository: %w", err) + if objFmt.Name() != repo.ObjectFormatName { + repo.ObjectFormatName = objFmt.Name() + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "object_format_name"); err != nil { + return 0, fmt.Errorf("UpdateRepositoryColsWithAutoTime: %w", err) + } } - repo.ObjectFormatName = objFmt.Name() // keep consistent with db allBranches := container.Set[string]{} { diff --git a/modules/repository/branch_test.go b/modules/repository/branch_test.go index acf75a1ac0..ead28aa141 100644 --- a/modules/repository/branch_test.go +++ b/modules/repository/branch_test.go @@ -27,5 +27,5 @@ func TestSyncRepoBranches(t *testing.T) { assert.Equal(t, "sha1", repo.ObjectFormatName) branch, err := git_model.GetBranch(db.DefaultContext, 1, "master") assert.NoError(t, err) - assert.EqualValues(t, "master", branch.Name) + assert.Equal(t, "master", branch.Name) } diff --git a/modules/repository/commits.go b/modules/repository/commits.go index 6e4b75d5ca..878fdc1603 100644 --- a/modules/repository/commits.go +++ b/modules/repository/commits.go @@ -10,8 +10,10 @@ import ( "time" "code.gitea.io/gitea/models/avatars" + repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -43,7 +45,7 @@ func NewPushCommits() *PushCommits { } // ToAPIPayloadCommit converts a single PushCommit to an api.PayloadCommit object. -func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.User, repoPath, repoLink string, commit *PushCommit) (*api.PayloadCommit, error) { +func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.User, repo *repo_model.Repository, commit *PushCommit) (*api.PayloadCommit, error) { var err error authorUsername := "" author, ok := emailUsers[commit.AuthorEmail] @@ -70,7 +72,7 @@ func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.U committerUsername = committer.Name } - fileStatus, err := git.GetCommitFileStatus(ctx, repoPath, commit.Sha1) + fileStatus, err := git.GetCommitFileStatus(ctx, repo.RepoPath(), commit.Sha1) if err != nil { return nil, fmt.Errorf("FileStatus [commit_sha1: %s]: %w", commit.Sha1, err) } @@ -78,7 +80,7 @@ func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.U return &api.PayloadCommit{ ID: commit.Sha1, Message: commit.Message, - URL: fmt.Sprintf("%s/commit/%s", repoLink, url.PathEscape(commit.Sha1)), + URL: fmt.Sprintf("%s/commit/%s", repo.HTMLURL(), url.PathEscape(commit.Sha1)), Author: &api.PayloadUser{ Name: commit.AuthorName, Email: commit.AuthorEmail, @@ -98,14 +100,14 @@ func ToAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.U // ToAPIPayloadCommits converts a PushCommits object to api.PayloadCommit format. // It returns all converted commits and, if provided, the head commit or an error otherwise. -func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repoPath, repoLink string) ([]*api.PayloadCommit, *api.PayloadCommit, error) { +func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repo *repo_model.Repository) ([]*api.PayloadCommit, *api.PayloadCommit, error) { commits := make([]*api.PayloadCommit, len(pc.Commits)) var headCommit *api.PayloadCommit emailUsers := make(map[string]*user_model.User) for i, commit := range pc.Commits { - apiCommit, err := ToAPIPayloadCommit(ctx, emailUsers, repoPath, repoLink, commit) + apiCommit, err := ToAPIPayloadCommit(ctx, emailUsers, repo, commit) if err != nil { return nil, nil, err } @@ -117,7 +119,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repoPath, repoLi } if pc.HeadCommit != nil && headCommit == nil { var err error - headCommit, err = ToAPIPayloadCommit(ctx, emailUsers, repoPath, repoLink, pc.HeadCommit) + headCommit, err = ToAPIPayloadCommit(ctx, emailUsers, repo, pc.HeadCommit) if err != nil { return nil, nil, err } @@ -130,7 +132,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repoPath, repoLi func (pc *PushCommits) AvatarLink(ctx context.Context, email string) string { size := avatars.DefaultAvatarPixelSize * setting.Avatar.RenderedSizeFactor - v, _ := cache.GetWithContextCache(ctx, "push_commits", email, func() (string, error) { + v, _ := cache.GetWithContextCache(ctx, cachegroup.EmailAvatarLink, email, func(ctx context.Context, email string) (string, error) { u, err := user_model.GetUserByEmail(ctx, email) if err != nil { if !user_model.IsErrUserNotExist(err) { diff --git a/modules/repository/commits_test.go b/modules/repository/commits_test.go index 3afc116e68..030cd7714d 100644 --- a/modules/repository/commits_test.go +++ b/modules/repository/commits_test.go @@ -50,54 +50,54 @@ func TestPushCommits_ToAPIPayloadCommits(t *testing.T) { pushCommits.HeadCommit = &PushCommit{Sha1: "69554a6"} repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 16}) - payloadCommits, headCommit, err := pushCommits.ToAPIPayloadCommits(git.DefaultContext, repo.RepoPath(), "/user2/repo16") + payloadCommits, headCommit, err := pushCommits.ToAPIPayloadCommits(git.DefaultContext, repo) assert.NoError(t, err) assert.Len(t, payloadCommits, 3) assert.NotNil(t, headCommit) assert.Equal(t, "69554a6", payloadCommits[0].ID) assert.Equal(t, "not signed commit", payloadCommits[0].Message) - assert.Equal(t, "/user2/repo16/commit/69554a6", payloadCommits[0].URL) + assert.Equal(t, "https://try.gitea.io/user2/repo16/commit/69554a6", payloadCommits[0].URL) assert.Equal(t, "User2", payloadCommits[0].Committer.Name) assert.Equal(t, "user2", payloadCommits[0].Committer.UserName) assert.Equal(t, "User2", payloadCommits[0].Author.Name) assert.Equal(t, "user2", payloadCommits[0].Author.UserName) - assert.EqualValues(t, []string{}, payloadCommits[0].Added) - assert.EqualValues(t, []string{}, payloadCommits[0].Removed) - assert.EqualValues(t, []string{"readme.md"}, payloadCommits[0].Modified) + assert.Equal(t, []string{}, payloadCommits[0].Added) + assert.Equal(t, []string{}, payloadCommits[0].Removed) + assert.Equal(t, []string{"readme.md"}, payloadCommits[0].Modified) assert.Equal(t, "27566bd", payloadCommits[1].ID) assert.Equal(t, "good signed commit (with not yet validated email)", payloadCommits[1].Message) - assert.Equal(t, "/user2/repo16/commit/27566bd", payloadCommits[1].URL) + assert.Equal(t, "https://try.gitea.io/user2/repo16/commit/27566bd", payloadCommits[1].URL) assert.Equal(t, "User2", payloadCommits[1].Committer.Name) assert.Equal(t, "user2", payloadCommits[1].Committer.UserName) assert.Equal(t, "User2", payloadCommits[1].Author.Name) assert.Equal(t, "user2", payloadCommits[1].Author.UserName) - assert.EqualValues(t, []string{}, payloadCommits[1].Added) - assert.EqualValues(t, []string{}, payloadCommits[1].Removed) - assert.EqualValues(t, []string{"readme.md"}, payloadCommits[1].Modified) + assert.Equal(t, []string{}, payloadCommits[1].Added) + assert.Equal(t, []string{}, payloadCommits[1].Removed) + assert.Equal(t, []string{"readme.md"}, payloadCommits[1].Modified) assert.Equal(t, "5099b81", payloadCommits[2].ID) assert.Equal(t, "good signed commit", payloadCommits[2].Message) - assert.Equal(t, "/user2/repo16/commit/5099b81", payloadCommits[2].URL) + assert.Equal(t, "https://try.gitea.io/user2/repo16/commit/5099b81", payloadCommits[2].URL) assert.Equal(t, "User2", payloadCommits[2].Committer.Name) assert.Equal(t, "user2", payloadCommits[2].Committer.UserName) assert.Equal(t, "User2", payloadCommits[2].Author.Name) assert.Equal(t, "user2", payloadCommits[2].Author.UserName) - assert.EqualValues(t, []string{"readme.md"}, payloadCommits[2].Added) - assert.EqualValues(t, []string{}, payloadCommits[2].Removed) - assert.EqualValues(t, []string{}, payloadCommits[2].Modified) + assert.Equal(t, []string{"readme.md"}, payloadCommits[2].Added) + assert.Equal(t, []string{}, payloadCommits[2].Removed) + assert.Equal(t, []string{}, payloadCommits[2].Modified) assert.Equal(t, "69554a6", headCommit.ID) assert.Equal(t, "not signed commit", headCommit.Message) - assert.Equal(t, "/user2/repo16/commit/69554a6", headCommit.URL) + assert.Equal(t, "https://try.gitea.io/user2/repo16/commit/69554a6", headCommit.URL) assert.Equal(t, "User2", headCommit.Committer.Name) assert.Equal(t, "user2", headCommit.Committer.UserName) assert.Equal(t, "User2", headCommit.Author.Name) assert.Equal(t, "user2", headCommit.Author.UserName) - assert.EqualValues(t, []string{}, headCommit.Added) - assert.EqualValues(t, []string{}, headCommit.Removed) - assert.EqualValues(t, []string{"readme.md"}, headCommit.Modified) + assert.Equal(t, []string{}, headCommit.Added) + assert.Equal(t, []string{}, headCommit.Removed) + assert.Equal(t, []string{"readme.md"}, headCommit.Modified) } func TestPushCommits_AvatarLink(t *testing.T) { @@ -200,5 +200,3 @@ func TestListToPushCommits(t *testing.T) { assert.Equal(t, now, pushCommits.Commits[1].Timestamp) } } - -// TODO TestPushUpdate diff --git a/modules/repository/create.go b/modules/repository/create.go index b4f7033bd7..a75598a84b 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -7,19 +7,10 @@ import ( "context" "fmt" "os" - "path" "path/filepath" - "strings" - activities_model "code.gitea.io/gitea/models/activities" - "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" - access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" - issue_indexer "code.gitea.io/gitea/modules/indexer/issues" - "code.gitea.io/gitea/modules/log" - api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/util" ) const notRegularFileMode = os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular @@ -64,97 +55,3 @@ func UpdateRepoSize(ctx context.Context, repo *repo_model.Repository) error { return repo_model.UpdateRepoSize(ctx, repo.ID, size, lfsSize) } - -// CheckDaemonExportOK creates/removes git-daemon-export-ok for git-daemon... -func CheckDaemonExportOK(ctx context.Context, repo *repo_model.Repository) error { - if err := repo.LoadOwner(ctx); err != nil { - return err - } - - // Create/Remove git-daemon-export-ok for git-daemon... - daemonExportFile := path.Join(repo.RepoPath(), `git-daemon-export-ok`) - - isExist, err := util.IsExist(daemonExportFile) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err) - return err - } - - isPublic := !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePublic - if !isPublic && isExist { - if err = util.Remove(daemonExportFile); err != nil { - log.Error("Failed to remove %s: %v", daemonExportFile, err) - } - } else if isPublic && !isExist { - if f, err := os.Create(daemonExportFile); err != nil { - log.Error("Failed to create %s: %v", daemonExportFile, err) - } else { - f.Close() - } - } - - return nil -} - -// UpdateRepository updates a repository with db context -func UpdateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { - repo.LowerName = strings.ToLower(repo.Name) - - e := db.GetEngine(ctx) - - if _, err = e.ID(repo.ID).AllCols().Update(repo); err != nil { - return fmt.Errorf("update: %w", err) - } - - if err = UpdateRepoSize(ctx, repo); err != nil { - log.Error("Failed to update size for repository: %v", err) - } - - if visibilityChanged { - if err = repo.LoadOwner(ctx); err != nil { - return fmt.Errorf("LoadOwner: %w", err) - } - if repo.Owner.IsOrganization() { - // Organization repository need to recalculate access table when visibility is changed. - if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { - return fmt.Errorf("recalculateTeamAccesses: %w", err) - } - } - - // If repo has become private, we need to set its actions to private. - if repo.IsPrivate { - _, err = e.Where("repo_id = ?", repo.ID).Cols("is_private").Update(&activities_model.Action{ - IsPrivate: true, - }) - if err != nil { - return err - } - - if err = repo_model.ClearRepoStars(ctx, repo.ID); err != nil { - return err - } - } - - // Create/Remove git-daemon-export-ok for git-daemon... - if err := CheckDaemonExportOK(ctx, repo); err != nil { - return err - } - - forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) - if err != nil { - return fmt.Errorf("getRepositoriesByForkID: %w", err) - } - for i := range forkRepos { - forkRepos[i].IsPrivate = repo.IsPrivate || repo.Owner.Visibility == api.VisibleTypePrivate - if err = UpdateRepository(ctx, forkRepos[i], true); err != nil { - return fmt.Errorf("updateRepository[%d]: %w", forkRepos[i].ID, err) - } - } - - // If visibility is changed, we need to update the issue indexer. - // Since the data in the issue indexer have field to indicate if the repo is public or not. - issue_indexer.UpdateRepoIndexer(ctx, repo.ID) - } - - return nil -} diff --git a/modules/repository/create_test.go b/modules/repository/create_test.go index a9151482b4..b85a10adad 100644 --- a/modules/repository/create_test.go +++ b/modules/repository/create_test.go @@ -6,7 +6,6 @@ package repository import ( "testing" - activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" @@ -14,26 +13,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestUpdateRepositoryVisibilityChanged(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - - // Get sample repo and change visibility - repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 9) - assert.NoError(t, err) - repo.IsPrivate = true - - // Update it - err = UpdateRepository(db.DefaultContext, repo, true) - assert.NoError(t, err) - - // Check visibility of action has become private - act := activities_model.Action{} - _, err = db.GetEngine(db.DefaultContext).ID(3).Get(&act) - - assert.NoError(t, err) - assert.True(t, act.IsPrivate) -} - func TestGetDirectorySize(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 1) @@ -41,5 +20,5 @@ func TestGetDirectorySize(t *testing.T) { size, err := getDirectorySize(repo.RepoPath()) assert.NoError(t, err) repo.Size = 8165 // real size on the disk - assert.EqualValues(t, repo.Size, size) + assert.Equal(t, repo.Size, size) } diff --git a/modules/repository/env.go b/modules/repository/env.go index e4f32092fc..78e06f86fb 100644 --- a/modules/repository/env.go +++ b/modules/repository/env.go @@ -4,8 +4,8 @@ package repository import ( - "fmt" "os" + "strconv" "strings" repo_model "code.gitea.io/gitea/models/repo" @@ -72,9 +72,9 @@ func FullPushingEnvironment(author, committer *user_model.User, repo *repo_model EnvRepoUsername+"="+repo.OwnerName, EnvRepoIsWiki+"="+isWiki, EnvPusherName+"="+committer.Name, - EnvPusherID+"="+fmt.Sprintf("%d", committer.ID), - EnvRepoID+"="+fmt.Sprintf("%d", repo.ID), - EnvPRID+"="+fmt.Sprintf("%d", prID), + EnvPusherID+"="+strconv.FormatInt(committer.ID, 10), + EnvRepoID+"="+strconv.FormatInt(repo.ID, 10), + EnvPRID+"="+strconv.FormatInt(prID, 10), EnvAppURL+"="+setting.AppURL, "SSH_ORIGINAL_COMMAND=gitea-internal", ) diff --git a/modules/repository/init.go b/modules/repository/init.go index 5f500c5233..12e9606c74 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -11,10 +11,7 @@ import ( "strings" issues_model "code.gitea.io/gitea/models/issues" - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/label" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -81,7 +78,7 @@ func LoadRepoConfig() error { if isDir, err := util.IsDir(customPath); err != nil { return fmt.Errorf("failed to check custom %s dir: %w", t, err) } else if isDir { - if typeFiles[i].custom, err = util.StatDir(customPath); err != nil { + if typeFiles[i].custom, err = util.ListDirRecursively(customPath, &util.ListDirOptions{SkipCommonHiddenNames: true}); err != nil { return fmt.Errorf("failed to list custom %s files: %w", t, err) } } @@ -120,30 +117,6 @@ func LoadRepoConfig() error { return nil } -func CheckInitRepository(ctx context.Context, owner, name, objectFormatName string) (err error) { - // Somehow the directory could exist. - repoPath := repo_model.RepoPath(owner, name) - isExist, err := util.IsExist(repoPath) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return err - } - if isExist { - return repo_model.ErrRepoFilesAlreadyExist{ - Uname: owner, - Name: name, - } - } - - // Init git bare new repository. - if err = git.InitRepository(ctx, repoPath, true, objectFormatName); err != nil { - return fmt.Errorf("git.InitRepository: %w", err) - } else if err = CreateDelegateHooks(repoPath); err != nil { - return fmt.Errorf("createDelegateHooks: %w", err) - } - return nil -} - // InitializeLabels adds a label set to a repository using a template func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg bool) error { list, err := LoadTemplateLabelsByDisplayName(labelTemplate) @@ -152,12 +125,13 @@ func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg } labels := make([]*issues_model.Label, len(list)) - for i := 0; i < len(list); i++ { + for i := range list { labels[i] = &issues_model.Label{ - Name: list[i].Name, - Exclusive: list[i].Exclusive, - Description: list[i].Description, - Color: list[i].Color, + Name: list[i].Name, + Exclusive: list[i].Exclusive, + ExclusiveOrder: list[i].ExclusiveOrder, + Description: list[i].Description, + Color: list[i].Color, } if isOrg { labels[i].OrgID = id diff --git a/modules/repository/init_test.go b/modules/repository/init_test.go index 227efdc1db..1fa928105c 100644 --- a/modules/repository/init_test.go +++ b/modules/repository/init_test.go @@ -14,17 +14,17 @@ func TestMergeCustomLabels(t *testing.T) { all: []string{"a", "a.yaml", "a.yml"}, custom: nil, }) - assert.EqualValues(t, []string{"a.yaml"}, files, "yaml file should win") + assert.Equal(t, []string{"a.yaml"}, files, "yaml file should win") files = mergeCustomLabelFiles(optionFileList{ all: []string{"a", "a.yaml"}, custom: []string{"a"}, }) - assert.EqualValues(t, []string{"a"}, files, "custom file should win") + assert.Equal(t, []string{"a"}, files, "custom file should win") files = mergeCustomLabelFiles(optionFileList{ all: []string{"a", "a.yml", "a.yaml"}, custom: []string{"a", "a.yml"}, }) - assert.EqualValues(t, []string{"a.yml"}, files, "custom yml file should win if no yaml") + assert.Equal(t, []string{"a.yml"}, files, "custom yml file should win if no yaml") } diff --git a/modules/repository/license_test.go b/modules/repository/license_test.go index 3b0cfa1eed..d00156a496 100644 --- a/modules/repository/license_test.go +++ b/modules/repository/license_test.go @@ -31,12 +31,7 @@ func Test_getLicense(t *testing.T) { Copyright (c) 2023 Gitea -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -`, +Permission is hereby granted`, wantErr: assert.NoError, }, { @@ -53,7 +48,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI if !tt.wantErr(t, err, fmt.Sprintf("GetLicense(%v, %v)", tt.args.name, tt.args.values)) { return } - assert.Equalf(t, tt.want, string(got), "GetLicense(%v, %v)", tt.args.name, tt.args.values) + assert.Contains(t, string(got), tt.want, "GetLicense(%v, %v)", tt.args.name, tt.args.values) }) } } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 97b0343381..ad4a53b858 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -9,13 +9,10 @@ import ( "fmt" "io" "strings" - "time" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" @@ -59,118 +56,6 @@ func SyncRepoTags(ctx context.Context, repoID int64) error { return SyncReleasesWithTags(ctx, repo, gitRepo) } -// SyncReleasesWithTags synchronizes release table with repository tags -func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) - - // optimized procedure for pull-mirrors which saves a lot of time (in - // particular for repos with many tags). - if repo.IsMirror { - return pullMirrorReleaseSync(ctx, repo, gitRepo) - } - - existingRelTags := make(container.Set[string]) - opts := repo_model.FindReleasesOptions{ - IncludeDrafts: true, - IncludeTags: true, - ListOptions: db.ListOptions{PageSize: 50}, - RepoID: repo.ID, - } - for page := 1; ; page++ { - opts.Page = page - rels, err := db.Find[repo_model.Release](gitRepo.Ctx, opts) - if err != nil { - return fmt.Errorf("unable to GetReleasesByRepoID in Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) - } - if len(rels) == 0 { - break - } - for _, rel := range rels { - if rel.IsDraft { - continue - } - commitID, err := gitRepo.GetTagCommitID(rel.TagName) - if err != nil && !git.IsErrNotExist(err) { - return fmt.Errorf("unable to GetTagCommitID for %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) - } - if git.IsErrNotExist(err) || commitID != rel.Sha1 { - if err := repo_model.PushUpdateDeleteTag(ctx, repo, rel.TagName); err != nil { - return fmt.Errorf("unable to PushUpdateDeleteTag: %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) - } - } else { - existingRelTags.Add(strings.ToLower(rel.TagName)) - } - } - } - - _, err := gitRepo.WalkReferences(git.ObjectTag, 0, 0, func(sha1, refname string) error { - tagName := strings.TrimPrefix(refname, git.TagPrefix) - if existingRelTags.Contains(strings.ToLower(tagName)) { - return nil - } - - if err := PushUpdateAddTag(ctx, repo, gitRepo, tagName, sha1, refname); err != nil { - // sometimes, some tags will be sync failed. i.e. https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tag/?h=v2.6.11 - // this is a tree object, not a tag object which created before git - log.Error("unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %v", tagName, repo.ID, repo.OwnerName, repo.Name, err) - } - - return nil - }) - return err -} - -// PushUpdateAddTag must be called for any push actions to add tag -func PushUpdateAddTag(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tagName, sha1, refname string) error { - tag, err := gitRepo.GetTagWithID(sha1, tagName) - if err != nil { - return fmt.Errorf("unable to GetTag: %w", err) - } - commit, err := tag.Commit(gitRepo) - if err != nil { - return fmt.Errorf("unable to get tag Commit: %w", err) - } - - sig := tag.Tagger - if sig == nil { - sig = commit.Author - } - if sig == nil { - sig = commit.Committer - } - - var author *user_model.User - createdAt := time.Unix(1, 0) - - if sig != nil { - author, err = user_model.GetUserByEmail(ctx, sig.Email) - if err != nil && !user_model.IsErrUserNotExist(err) { - return fmt.Errorf("unable to GetUserByEmail for %q: %w", sig.Email, err) - } - createdAt = sig.When - } - - commitsCount, err := commit.CommitsCount() - if err != nil { - return fmt.Errorf("unable to get CommitsCount: %w", err) - } - - rel := repo_model.Release{ - RepoID: repo.ID, - TagName: tagName, - LowerTagName: strings.ToLower(tagName), - Sha1: commit.ID.String(), - NumCommits: commitsCount, - CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), - IsTag: true, - } - if author != nil { - rel.PublisherID = author.ID - } - - return repo_model.SaveOrUpdateTag(ctx, repo, &rel) -} - // StoreMissingLfsObjectsInRepository downloads missing LFS objects func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error { contentStore := lfs.NewContentStore() @@ -286,18 +171,19 @@ func (shortRelease) TableName() string { return "release" } -// pullMirrorReleaseSync is a pull-mirror specific tag<->release table +// SyncReleasesWithTags is a tag<->release table // synchronization which overwrites all Releases from the repository tags. This // can be relied on since a pull-mirror is always identical to its -// upstream. Hence, after each sync we want the pull-mirror release set to be +// upstream. Hence, after each sync we want the release set to be // identical to the upstream tag set. This is much more efficient for // repositories like https://github.com/vim/vim (with over 13000 tags). -func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) - tags, numTags, err := gitRepo.GetTagInfos(0, 0) +func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) + tags, _, err := gitRepo.GetTagInfos(0, 0) if err != nil { return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) } + var added, deleted, updated int err = db.WithTx(ctx, func(ctx context.Context) error { dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{ RepoID: repo.ID, @@ -318,9 +204,7 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git TagName: tag.Name, LowerTagName: strings.ToLower(tag.Name), Sha1: tag.Object.String(), - // NOTE: ignored, since NumCommits are unused - // for pull-mirrors (only relevant when - // displaying releases, IsTag: false) + // NOTE: ignored, The NumCommits value is calculated and cached on demand when the UI requires it. NumCommits: -1, CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()), IsTag: true, @@ -349,13 +233,14 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err) } } + added, deleted, updated = len(deletes), len(updates), len(inserts) return nil }) if err != nil { return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) } - log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags) + log.Trace("SyncReleasesWithTags: %d tags added, %d tags deleted, %d tags updated", added, deleted, updated) return nil } diff --git a/modules/repository/repo_test.go b/modules/repository/repo_test.go index f3e7be6d7d..f79a79ccbd 100644 --- a/modules/repository/repo_test.go +++ b/modules/repository/repo_test.go @@ -63,7 +63,7 @@ func Test_calcSync(t *testing.T) { inserts, deletes, updates := calcSync(gitTags, dbReleases) if assert.Len(t, inserts, 1, "inserts") { - assert.EqualValues(t, *gitTags[2], *inserts[0], "inserts equal") + assert.Equal(t, *gitTags[2], *inserts[0], "inserts equal") } if assert.Len(t, deletes, 1, "deletes") { @@ -71,6 +71,6 @@ func Test_calcSync(t *testing.T) { } if assert.Len(t, updates, 1, "updates") { - assert.EqualValues(t, *gitTags[1], *updates[0], "updates equal") + assert.Equal(t, *gitTags[1], *updates[0], "updates equal") } } diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 04faa9db3d..d7253d9e02 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -4,42 +4,19 @@ package repository import ( + "context" "fmt" - "os" - "path" - "path/filepath" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) -// LocalCopyPath returns the local repository temporary copy path. -func LocalCopyPath() string { - if filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { - return setting.Repository.Local.LocalCopyPath - } - return path.Join(setting.AppDataPath, setting.Repository.Local.LocalCopyPath) -} - // CreateTemporaryPath creates a temporary path -func CreateTemporaryPath(prefix string) (string, error) { - if err := os.MkdirAll(LocalCopyPath(), os.ModePerm); err != nil { - log.Error("Unable to create localcopypath directory: %s (%v)", LocalCopyPath(), err) - return "", fmt.Errorf("Failed to create localcopypath directory %s: %w", LocalCopyPath(), err) - } - basePath, err := os.MkdirTemp(LocalCopyPath(), prefix+".git") +func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { + basePath, cleanup, err := setting.AppDataTempDir("local-repo").MkdirTempRandom(prefix + ".git") if err != nil { log.Error("Unable to create temporary directory: %s-*.git (%v)", prefix, err) - return "", fmt.Errorf("Failed to create dir %s-*.git: %w", prefix, err) - } - return basePath, nil -} - -// RemoveTemporaryPath removes the temporary path -func RemoveTemporaryPath(basePath string) error { - if _, err := os.Stat(basePath); !os.IsNotExist(err) { - return util.RemoveAll(basePath) + return "", nil, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) } - return nil + return basePath, cleanup, nil } diff --git a/modules/reqctx/datastore.go b/modules/reqctx/datastore.go new file mode 100644 index 0000000000..1d4bee613f --- /dev/null +++ b/modules/reqctx/datastore.go @@ -0,0 +1,141 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package reqctx + +import ( + "context" + "io" + "maps" + "sync" + + "code.gitea.io/gitea/modules/process" +) + +type ContextDataProvider interface { + GetData() ContextData +} + +type ContextData map[string]any + +func (ds ContextData) GetData() ContextData { + return ds +} + +func (ds ContextData) MergeFrom(other ContextData) ContextData { + maps.Copy(ds, other) + return ds +} + +// RequestDataStore is a short-lived context-related object that is used to store request-specific data. +type RequestDataStore interface { + GetData() ContextData + SetContextValue(k, v any) + GetContextValue(key any) any + AddCleanUp(f func()) + AddCloser(c io.Closer) +} + +type requestDataStoreKeyType struct{} + +var RequestDataStoreKey requestDataStoreKeyType + +type requestDataStore struct { + data ContextData + + mu sync.RWMutex + values map[any]any + cleanUpFuncs []func() +} + +func (r *requestDataStore) GetContextValue(key any) any { + if key == RequestDataStoreKey { + return r + } + r.mu.RLock() + defer r.mu.RUnlock() + return r.values[key] +} + +func (r *requestDataStore) SetContextValue(k, v any) { + r.mu.Lock() + r.values[k] = v + r.mu.Unlock() +} + +// GetData and the underlying ContextData are not thread-safe, callers should ensure thread-safety. +func (r *requestDataStore) GetData() ContextData { + if r.data == nil { + r.data = make(ContextData) + } + return r.data +} + +func (r *requestDataStore) AddCleanUp(f func()) { + r.mu.Lock() + r.cleanUpFuncs = append(r.cleanUpFuncs, f) + r.mu.Unlock() +} + +func (r *requestDataStore) AddCloser(c io.Closer) { + r.AddCleanUp(func() { _ = c.Close() }) +} + +func (r *requestDataStore) cleanUp() { + for _, f := range r.cleanUpFuncs { + f() + } +} + +type RequestContext interface { + context.Context + RequestDataStore +} + +func FromContext(ctx context.Context) RequestContext { + if rc, ok := ctx.(RequestContext); ok { + return rc + } + // here we must use the current ctx and the underlying store + // the current ctx guarantees that the ctx deadline/cancellation/values are respected + // the underlying store guarantees that the request-specific data is available + if store := GetRequestDataStore(ctx); store != nil { + return &requestContext{Context: ctx, RequestDataStore: store} + } + return nil +} + +func GetRequestDataStore(ctx context.Context) RequestDataStore { + if req, ok := ctx.Value(RequestDataStoreKey).(*requestDataStore); ok { + return req + } + return nil +} + +type requestContext struct { + context.Context + RequestDataStore +} + +func (c *requestContext) Value(key any) any { + if v := c.GetContextValue(key); v != nil { + return v + } + return c.Context.Value(key) +} + +func NewRequestContext(parentCtx context.Context, profDesc string) (_ context.Context, finished func()) { + ctx, _, processFinished := process.GetManager().AddTypedContext(parentCtx, profDesc, process.RequestProcessType, true) + store := &requestDataStore{values: make(map[any]any)} + reqCtx := &requestContext{Context: ctx, RequestDataStore: store} + return reqCtx, func() { + store.cleanUp() + processFinished() + } +} + +// NewRequestContextForTest creates a new RequestContext for testing purposes +// It doesn't add the context to the process manager, nor do cleanup +func NewRequestContextForTest(parentCtx context.Context) RequestContext { + return &requestContext{Context: parentCtx, RequestDataStore: &requestDataStore{values: make(map[any]any)}} +} diff --git a/modules/secret/secret.go b/modules/secret/secret.go index e70ae1839c..af894a054c 100644 --- a/modules/secret/secret.go +++ b/modules/secret/secret.go @@ -16,6 +16,7 @@ import ( ) // AesEncrypt encrypts text and given key with AES. +// It is only internally used at the moment to use "SECRET_KEY" for some database values. func AesEncrypt(key, text []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { @@ -27,12 +28,13 @@ func AesEncrypt(key, text []byte) ([]byte, error) { if _, err = io.ReadFull(rand.Reader, iv); err != nil { return nil, fmt.Errorf("AesEncrypt unable to read IV: %w", err) } - cfb := cipher.NewCFBEncrypter(block, iv) + cfb := cipher.NewCFBEncrypter(block, iv) //nolint:staticcheck // need to migrate and refactor to a new approach cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b)) return ciphertext, nil } // AesDecrypt decrypts text and given key with AES. +// It is only internally used at the moment to use "SECRET_KEY" for some database values. func AesDecrypt(key, text []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { @@ -43,7 +45,7 @@ func AesDecrypt(key, text []byte) ([]byte, error) { } iv := text[:aes.BlockSize] text = text[aes.BlockSize:] - cfb := cipher.NewCFBDecrypter(block, iv) + cfb := cipher.NewCFBDecrypter(block, iv) //nolint:staticcheck // need to migrate and refactor to a new approach cfb.XORKeyStream(text, text) data, err := base64.StdEncoding.DecodeString(string(text)) if err != nil { diff --git a/modules/session/key.go b/modules/session/key.go new file mode 100644 index 0000000000..c3da997c67 --- /dev/null +++ b/modules/session/key.go @@ -0,0 +1,11 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package session + +const ( + KeyUID = "uid" + KeyUname = "uname" + + KeyUserHasTwoFactorAuth = "userHasTwoFactorAuth" +) diff --git a/modules/session/mem.go b/modules/session/mem.go new file mode 100644 index 0000000000..bb807bc91a --- /dev/null +++ b/modules/session/mem.go @@ -0,0 +1,68 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package session + +import ( + "bytes" + "encoding/gob" + "net/http" + + "gitea.com/go-chi/session" +) + +type mockMemRawStore struct { + s *session.MemStore +} + +var _ session.RawStore = (*mockMemRawStore)(nil) + +func (m *mockMemRawStore) Set(k, v any) error { + // We need to use gob to encode the value, to make it have the same behavior as other stores and catch abuses. + // Because gob needs to "Register" the type before it can encode it, and it's unable to decode a struct to "any" so use a map to help to decode the value. + var buf bytes.Buffer + if err := gob.NewEncoder(&buf).Encode(map[string]any{"v": v}); err != nil { + return err + } + return m.s.Set(k, buf.Bytes()) +} + +func (m *mockMemRawStore) Get(k any) (ret any) { + v, ok := m.s.Get(k).([]byte) + if !ok { + return nil + } + var w map[string]any + _ = gob.NewDecoder(bytes.NewBuffer(v)).Decode(&w) + return w["v"] +} + +func (m *mockMemRawStore) Delete(k any) error { + return m.s.Delete(k) +} + +func (m *mockMemRawStore) ID() string { + return m.s.ID() +} + +func (m *mockMemRawStore) Release() error { + return m.s.Release() +} + +func (m *mockMemRawStore) Flush() error { + return m.s.Flush() +} + +type mockMemStore struct { + *mockMemRawStore +} + +var _ Store = (*mockMemStore)(nil) + +func (m mockMemStore) Destroy(writer http.ResponseWriter, request *http.Request) error { + return nil +} + +func NewMockMemStore(sid string) Store { + return &mockMemStore{&mockMemRawStore{session.NewMemStore(sid)}} +} diff --git a/modules/session/mock.go b/modules/session/mock.go deleted file mode 100644 index 95231a3655..0000000000 --- a/modules/session/mock.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package session - -import ( - "net/http" - - "gitea.com/go-chi/session" -) - -type MockStore struct { - *session.MemStore -} - -func (m *MockStore) Destroy(writer http.ResponseWriter, request *http.Request) error { - return nil -} - -type mockStoreContextKeyStruct struct{} - -var MockStoreContextKey = mockStoreContextKeyStruct{} - -func NewMockStore(sid string) *MockStore { - return &MockStore{session.NewMemStore(sid)} -} diff --git a/modules/session/store.go b/modules/session/store.go index 09d1ef44dd..0217ed97ac 100644 --- a/modules/session/store.go +++ b/modules/session/store.go @@ -11,25 +11,25 @@ import ( "gitea.com/go-chi/session" ) -// Store represents a session store +type RawStore = session.RawStore + type Store interface { - Get(any) any - Set(any, any) error - Delete(any) error - ID() string - Release() error - Flush() error + RawStore Destroy(http.ResponseWriter, *http.Request) error } +type mockStoreContextKeyStruct struct{} + +var MockStoreContextKey = mockStoreContextKeyStruct{} + // RegenerateSession regenerates the underlying session and returns the new store func RegenerateSession(resp http.ResponseWriter, req *http.Request) (Store, error) { for _, f := range BeforeRegenerateSession { f(resp, req) } if setting.IsInTesting { - if store, ok := req.Context().Value(MockStoreContextKey).(*MockStore); ok { - return store, nil + if store := req.Context().Value(MockStoreContextKey); store != nil { + return store.(Store), nil } } return session.RegenerateSession(resp, req) @@ -37,8 +37,8 @@ func RegenerateSession(resp http.ResponseWriter, req *http.Request) (Store, erro func GetContextSession(req *http.Request) Store { if setting.IsInTesting { - if store, ok := req.Context().Value(MockStoreContextKey).(*MockStore); ok { - return store + if store := req.Context().Value(MockStoreContextKey); store != nil { + return store.(Store) } } return session.GetSession(req) diff --git a/modules/session/virtual.go b/modules/session/virtual.go index 80352b6e72..2e29b5fc6f 100644 --- a/modules/session/virtual.go +++ b/modules/session/virtual.go @@ -22,8 +22,8 @@ type VirtualSessionProvider struct { provider session.Provider } -// Init initializes the cookie session provider with given root path. -func (o *VirtualSessionProvider) Init(gclifetime int64, config string) error { +// Init initializes the cookie session provider with the given config. +func (o *VirtualSessionProvider) Init(gcLifetime int64, config string) error { var opts session.Options if err := json.Unmarshal([]byte(config), &opts); err != nil { return err @@ -52,7 +52,7 @@ func (o *VirtualSessionProvider) Init(gclifetime int64, config string) error { default: return fmt.Errorf("VirtualSessionProvider: Unknown Provider: %s", opts.Provider) } - return o.provider.Init(gclifetime, opts.ProviderConfig) + return o.provider.Init(gcLifetime, opts.ProviderConfig) } // Read returns raw session store by session ID. diff --git a/modules/setting/actions.go b/modules/setting/actions.go index 913872eaf2..8bace1f750 100644 --- a/modules/setting/actions.go +++ b/modules/setting/actions.go @@ -62,11 +62,11 @@ func (c logCompression) IsValid() bool { } func (c logCompression) IsNone() bool { - return strings.ToLower(string(c)) == "none" + return string(c) == "none" } func (c logCompression) IsZstd() bool { - return c == "" || strings.ToLower(string(c)) == "zstd" + return c == "" || string(c) == "zstd" } func loadActionsFrom(rootCfg ConfigProvider) error { diff --git a/modules/setting/actions_test.go b/modules/setting/actions_test.go index 3645a3f5da..353cc657fa 100644 --- a/modules/setting/actions_test.go +++ b/modules/setting/actions_test.go @@ -21,9 +21,9 @@ func Test_getStorageInheritNameSectionTypeForActions(t *testing.T) { assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) iniStr = ` [storage.actions_log] @@ -34,9 +34,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + assert.Equal(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) iniStr = ` [storage.actions_log] @@ -50,9 +50,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + assert.Equal(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) iniStr = ` [storage.actions_artifacts] @@ -66,9 +66,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "local", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.Equal(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) iniStr = ` [storage.actions_artifacts] @@ -82,9 +82,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "local", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.Equal(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) iniStr = `` cfg, err = NewConfigProviderFromData(iniStr) @@ -92,9 +92,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "local", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.Equal(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + assert.Equal(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) } func Test_getDefaultActionsURLForActions(t *testing.T) { @@ -175,7 +175,7 @@ DEFAULT_ACTIONS_URL = gitea if !tt.wantErr(t, loadActionsFrom(cfg)) { return } - assert.EqualValues(t, tt.wantURL, Actions.DefaultActionsURL.URL()) + assert.Equal(t, tt.wantURL, Actions.DefaultActionsURL.URL()) }) } } diff --git a/modules/setting/api.go b/modules/setting/api.go index c36f05cfd1..cdad474cb9 100644 --- a/modules/setting/api.go +++ b/modules/setting/api.go @@ -18,6 +18,7 @@ var API = struct { DefaultPagingNum int DefaultGitTreesPerPage int DefaultMaxBlobSize int64 + DefaultMaxResponseSize int64 }{ EnableSwagger: true, SwaggerURL: "", @@ -25,6 +26,7 @@ var API = struct { DefaultPagingNum: 30, DefaultGitTreesPerPage: 1000, DefaultMaxBlobSize: 10485760, + DefaultMaxResponseSize: 104857600, } func loadAPIFrom(rootCfg ConfigProvider) { diff --git a/modules/setting/attachment_test.go b/modules/setting/attachment_test.go index 3e8d2da4d9..c566dfa60c 100644 --- a/modules/setting/attachment_test.go +++ b/modules/setting/attachment_test.go @@ -25,9 +25,9 @@ MINIO_ENDPOINT = my_minio:9000 assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint) - assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint) + assert.Equal(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) { @@ -47,8 +47,8 @@ MINIO_BUCKET = gitea assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } func Test_getStorageSpecificOverridesStorage(t *testing.T) { @@ -69,8 +69,8 @@ STORAGE_TYPE = local assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } func Test_getStorageGetDefaults(t *testing.T) { @@ -80,7 +80,7 @@ func Test_getStorageGetDefaults(t *testing.T) { assert.NoError(t, loadAttachmentFrom(cfg)) // default storage is local, so bucket is empty - assert.EqualValues(t, "", Attachment.Storage.MinioConfig.Bucket) + assert.Empty(t, Attachment.Storage.MinioConfig.Bucket) } func Test_getStorageInheritNameSectionType(t *testing.T) { @@ -115,7 +115,7 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Attachment.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) } func Test_AttachmentStorage1(t *testing.T) { @@ -128,6 +128,6 @@ STORAGE_TYPE = minio assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "gitea", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } diff --git a/modules/setting/config_env.go b/modules/setting/config_env.go index dfcb7db3c8..409588dc44 100644 --- a/modules/setting/config_env.go +++ b/modules/setting/config_env.go @@ -97,7 +97,7 @@ func decodeEnvSectionKey(encoded string) (ok bool, section, key string) { // decodeEnvironmentKey decode the environment key to section and key // The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE -func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) { //nolint:unparam +func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) { if !strings.HasPrefix(envKey, prefixGitea) { return false, "", "", false } @@ -166,3 +166,25 @@ func EnvironmentToConfig(cfg ConfigProvider, envs []string) (changed bool) { } return changed } + +// InitGiteaEnvVars initializes the environment variables for gitea +func InitGiteaEnvVars() { + // Ideally Gitea should only accept the environment variables which it clearly knows instead of unsetting the ones it doesn't want, + // but the ideal behavior would be a breaking change, and it seems not bringing enough benefits to end users, + // so at the moment we could still keep "unsetting the unnecessary environments" + + // HOME is managed by Gitea, Gitea's git should use "HOME/.gitconfig". + // But git would try "XDG_CONFIG_HOME/git/config" first if "HOME/.gitconfig" does not exist, + // then our git.InitFull would still write to "XDG_CONFIG_HOME/git/config" if XDG_CONFIG_HOME is set. + _ = os.Unsetenv("XDG_CONFIG_HOME") +} + +func InitGiteaEnvVarsForTesting() { + InitGiteaEnvVars() + _ = os.Unsetenv("GIT_AUTHOR_NAME") + _ = os.Unsetenv("GIT_AUTHOR_EMAIL") + _ = os.Unsetenv("GIT_AUTHOR_DATE") + _ = os.Unsetenv("GIT_COMMITTER_NAME") + _ = os.Unsetenv("GIT_COMMITTER_EMAIL") + _ = os.Unsetenv("GIT_COMMITTER_DATE") +} diff --git a/modules/setting/config_env_test.go b/modules/setting/config_env_test.go index 7d07c479a1..7d270ac21a 100644 --- a/modules/setting/config_env_test.go +++ b/modules/setting/config_env_test.go @@ -28,8 +28,8 @@ func TestDecodeEnvSectionKey(t *testing.T) { ok, section, key = decodeEnvSectionKey("SEC") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) } func TestDecodeEnvironmentKey(t *testing.T) { @@ -38,19 +38,19 @@ func TestDecodeEnvironmentKey(t *testing.T) { ok, section, key, file := decodeEnvironmentKey(prefix, suffix, "SEC__KEY") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) assert.False(t, file) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) assert.False(t, file) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA____KEY") assert.True(t, ok) - assert.Equal(t, "", section) + assert.Empty(t, section) assert.Equal(t, "KEY", key) assert.False(t, file) @@ -64,8 +64,8 @@ func TestDecodeEnvironmentKey(t *testing.T) { // but it could be fixed in the future by adding a new suffix like "__VALUE" (no such key VALUE is used in Gitea either) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__FILE") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) assert.True(t, file) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY__FILE") @@ -73,6 +73,9 @@ func TestDecodeEnvironmentKey(t *testing.T) { assert.Equal(t, "sec", section) assert.Equal(t, "KEY", key) assert.True(t, file) + + ok, _, _, _ = decodeEnvironmentKey("PREFIX__", "", "PREFIX__SEC__KEY") + assert.True(t, ok) } func TestEnvironmentToConfig(t *testing.T) { diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index 3138f8a63e..09eaaefdaf 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -15,7 +15,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" - "gopkg.in/ini.v1" //nolint:depguard + "gopkg.in/ini.v1" //nolint:depguard // wrapper for this package ) type ConfigKey interface { @@ -26,6 +26,7 @@ type ConfigKey interface { In(defaultVal string, candidates []string) string String() string Strings(delim string) []string + Bool() (bool, error) MustString(defaultVal string) string MustBool(defaultVal ...bool) bool @@ -257,7 +258,7 @@ func (p *iniConfigProvider) Save() error { } filename := p.file if filename == "" { - return fmt.Errorf("config file path must not be empty") + return errors.New("config file path must not be empty") } if p.loadedFromEmpty { if err := os.MkdirAll(filepath.Dir(filename), os.ModePerm); err != nil { diff --git a/modules/setting/config_provider_test.go b/modules/setting/config_provider_test.go index a666d124c7..63121f0074 100644 --- a/modules/setting/config_provider_test.go +++ b/modules/setting/config_provider_test.go @@ -62,17 +62,17 @@ key = 123 // test default behavior assert.Equal(t, "123", ConfigSectionKeyString(sec, "key")) - assert.Equal(t, "", ConfigSectionKeyString(secSub, "key")) + assert.Empty(t, ConfigSectionKeyString(secSub, "key")) assert.Equal(t, "def", ConfigSectionKeyString(secSub, "key", "def")) assert.Equal(t, "123", ConfigInheritedKeyString(secSub, "key")) // Workaround for ini package's BuggyKeyOverwritten behavior - assert.Equal(t, "", ConfigSectionKeyString(sec, "empty")) - assert.Equal(t, "", ConfigSectionKeyString(secSub, "empty")) + assert.Empty(t, ConfigSectionKeyString(sec, "empty")) + assert.Empty(t, ConfigSectionKeyString(secSub, "empty")) assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("def")) assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("xyz")) - assert.Equal(t, "", ConfigSectionKeyString(sec, "empty")) + assert.Empty(t, ConfigSectionKeyString(sec, "empty")) assert.Equal(t, "def", ConfigSectionKeyString(secSub, "empty")) } diff --git a/modules/setting/cors.go b/modules/setting/cors.go index 63daaad60b..5260887d9d 100644 --- a/modules/setting/cors.go +++ b/modules/setting/cors.go @@ -5,8 +5,6 @@ package setting import ( "time" - - "code.gitea.io/gitea/modules/log" ) // CORSConfig defines CORS settings @@ -28,7 +26,4 @@ var CORSConfig = struct { func loadCorsFrom(rootCfg ConfigProvider) { mustMapSetting(rootCfg, "cors", &CORSConfig) - if CORSConfig.Enabled { - log.Info("CORS Service Enabled") - } } diff --git a/modules/setting/cron_test.go b/modules/setting/cron_test.go index 55244d7075..39a228068a 100644 --- a/modules/setting/cron_test.go +++ b/modules/setting/cron_test.go @@ -38,6 +38,6 @@ EXTEND = true _, err = getCronSettings(cfg, "test", extended) assert.NoError(t, err) assert.True(t, extended.Base) - assert.EqualValues(t, "white rabbit", extended.Second) + assert.Equal(t, "white rabbit", extended.Second) assert.True(t, extended.Extend) } diff --git a/modules/setting/git_test.go b/modules/setting/git_test.go index 441c514d8c..0d7f634abf 100644 --- a/modules/setting/git_test.go +++ b/modules/setting/git_test.go @@ -6,6 +6,8 @@ package setting import ( "testing" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" ) @@ -23,8 +25,8 @@ a.b = 1 `) assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "1", GitConfig.Options["a.b"]) - assert.EqualValues(t, "histogram", GitConfig.Options["diff.algorithm"]) + assert.Equal(t, "1", GitConfig.Options["a.b"]) + assert.Equal(t, "histogram", GitConfig.Options["diff.algorithm"]) cfg, err = NewConfigProviderFromData(` [git.config] @@ -32,24 +34,20 @@ diff.algorithm = other `) assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "other", GitConfig.Options["diff.algorithm"]) + assert.Equal(t, "other", GitConfig.Options["diff.algorithm"]) } func TestGitReflog(t *testing.T) { - oldGit := Git - oldGitConfig := GitConfig - defer func() { - Git = oldGit - GitConfig = oldGitConfig - }() + defer test.MockVariableValue(&Git) + defer test.MockVariableValue(&GitConfig) // default reflog config without legacy options cfg, err := NewConfigProviderFromData(``) assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "true", GitConfig.GetOption("core.logAllRefUpdates")) - assert.EqualValues(t, "90", GitConfig.GetOption("gc.reflogExpire")) + assert.Equal(t, "true", GitConfig.GetOption("core.logAllRefUpdates")) + assert.Equal(t, "90", GitConfig.GetOption("gc.reflogExpire")) // custom reflog config by legacy options cfg, err = NewConfigProviderFromData(` @@ -60,6 +58,6 @@ EXPIRATION = 123 assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "false", GitConfig.GetOption("core.logAllRefUpdates")) - assert.EqualValues(t, "123", GitConfig.GetOption("gc.reflogExpire")) + assert.Equal(t, "false", GitConfig.GetOption("core.logAllRefUpdates")) + assert.Equal(t, "123", GitConfig.GetOption("gc.reflogExpire")) } diff --git a/modules/setting/global_lock_test.go b/modules/setting/global_lock_test.go index 5eeb275523..5e15eb3483 100644 --- a/modules/setting/global_lock_test.go +++ b/modules/setting/global_lock_test.go @@ -16,7 +16,7 @@ func TestLoadGlobalLockConfig(t *testing.T) { assert.NoError(t, err) loadGlobalLockFrom(cfg) - assert.EqualValues(t, "memory", GlobalLock.ServiceType) + assert.Equal(t, "memory", GlobalLock.ServiceType) }) t.Run("RedisGlobalLockConfig", func(t *testing.T) { @@ -29,7 +29,7 @@ SERVICE_CONN_STR = addrs=127.0.0.1:6379 db=0 assert.NoError(t, err) loadGlobalLockFrom(cfg) - assert.EqualValues(t, "redis", GlobalLock.ServiceType) - assert.EqualValues(t, "addrs=127.0.0.1:6379 db=0", GlobalLock.ServiceConnStr) + assert.Equal(t, "redis", GlobalLock.ServiceType) + assert.Equal(t, "addrs=127.0.0.1:6379 db=0", GlobalLock.ServiceConnStr) }) } diff --git a/modules/setting/incoming_email.go b/modules/setting/incoming_email.go index bf81f292a2..4e433dde60 100644 --- a/modules/setting/incoming_email.go +++ b/modules/setting/incoming_email.go @@ -4,6 +4,7 @@ package setting import ( + "errors" "fmt" "net/mail" "strings" @@ -50,7 +51,7 @@ func checkReplyToAddress() error { } if parsed.Name != "" { - return fmt.Errorf("name must not be set") + return errors.New("name must not be set") } c := strings.Count(IncomingEmail.ReplyToAddress, IncomingEmail.TokenPlaceholder) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 18585602c3..ace7eec70e 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -31,6 +31,8 @@ var Indexer = struct { IncludePatterns []*GlobMatcher ExcludePatterns []*GlobMatcher ExcludeVendored bool + + TypeBleveMaxFuzzniess int }{ IssueType: "bleve", IssuePath: "indexers/issues.bleve", @@ -88,16 +90,17 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.ExcludeVendored = sec.Key("REPO_INDEXER_EXCLUDE_VENDORED").MustBool(true) Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024) Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second) + Indexer.TypeBleveMaxFuzzniess = sec.Key("TYPE_BLEVE_MAX_FUZZINESS").MustInt(0) } // IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing func IndexerGlobFromString(globstr string) []*GlobMatcher { extarr := make([]*GlobMatcher, 0, 10) - for _, expr := range strings.Split(strings.ToLower(globstr), ",") { + for expr := range strings.SplitSeq(strings.ToLower(globstr), ",") { expr = strings.TrimSpace(expr) if expr != "" { if g, err := GlobMatcherCompile(expr, '.', '/'); err != nil { - log.Info("Invalid glob expression '%s' (skipped): %v", expr, err) + log.Warn("Invalid glob expression '%s' (skipped): %v", expr, err) } else { extarr = append(extarr, g) } diff --git a/modules/setting/lfs_test.go b/modules/setting/lfs_test.go index d27dd7c5bf..1b829d8839 100644 --- a/modules/setting/lfs_test.go +++ b/modules/setting/lfs_test.go @@ -19,7 +19,7 @@ func Test_getStorageInheritNameSectionTypeForLFS(t *testing.T) { assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) iniStr = ` [server] @@ -54,7 +54,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) iniStr = ` [lfs] @@ -68,7 +68,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) iniStr = ` [lfs] @@ -83,7 +83,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath) } func Test_LFSStorage1(t *testing.T) { @@ -96,8 +96,8 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "gitea", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", LFS.Storage.MinioConfig.Bucket) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) } func Test_LFSClientServerConfigs(t *testing.T) { @@ -112,9 +112,9 @@ BATCH_SIZE = 0 assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, 100, LFS.MaxBatchSize) - assert.EqualValues(t, 20, LFSClient.BatchSize) - assert.EqualValues(t, 8, LFSClient.BatchOperationConcurrency) + assert.Equal(t, 100, LFS.MaxBatchSize) + assert.Equal(t, 20, LFSClient.BatchSize) + assert.Equal(t, 8, LFSClient.BatchOperationConcurrency) iniStr = ` [lfs_client] @@ -125,6 +125,6 @@ BATCH_OPERATION_CONCURRENCY = 10 assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, 50, LFSClient.BatchSize) - assert.EqualValues(t, 10, LFSClient.BatchOperationConcurrency) + assert.Equal(t, 50, LFSClient.BatchSize) + assert.Equal(t, 10, LFSClient.BatchOperationConcurrency) } diff --git a/modules/setting/log.go b/modules/setting/log.go index 50c5779994..59866c7605 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -7,7 +7,6 @@ import ( "fmt" golog "log" "os" - "path" "path/filepath" "strings" @@ -41,7 +40,7 @@ func loadLogGlobalFrom(rootCfg ConfigProvider) { Log.BufferLen = sec.Key("BUFFER_LEN").MustInt(10000) Log.Mode = sec.Key("MODE").MustString("console") - Log.RootPath = sec.Key("ROOT_PATH").MustString(path.Join(AppWorkPath, "log")) + Log.RootPath = sec.Key("ROOT_PATH").MustString(filepath.Join(AppWorkPath, "log")) if !filepath.IsAbs(Log.RootPath) { Log.RootPath = filepath.Join(AppWorkPath, Log.RootPath) } @@ -228,8 +227,8 @@ func initLoggerByName(manager *log.LoggerManager, rootCfg ConfigProvider, logger } var eventWriters []log.EventWriter - modes := strings.Split(modeVal, ",") - for _, modeName := range modes { + modes := strings.SplitSeq(modeVal, ",") + for modeName := range modes { modeName = strings.TrimSpace(modeName) if modeName == "" { continue diff --git a/modules/setting/mailer.go b/modules/setting/mailer.go index d4db55dc7b..e79ff30447 100644 --- a/modules/setting/mailer.go +++ b/modules/setting/mailer.go @@ -13,7 +13,7 @@ import ( "code.gitea.io/gitea/modules/log" - shellquote "github.com/kballard/go-shellquote" + "github.com/kballard/go-shellquote" ) // Mailer represents mail service. @@ -29,6 +29,9 @@ type Mailer struct { SubjectPrefix string `ini:"SUBJECT_PREFIX"` OverrideHeader map[string][]string `ini:"-"` + // Embed attachment images as inline base64 img src attribute + EmbedAttachmentImages bool + // SMTP sender Protocol string `ini:"PROTOCOL"` SMTPAddr string `ini:"SMTP_ADDR"` @@ -255,8 +258,6 @@ func loadMailerFrom(rootCfg ConfigProvider) { MailService.OverrideEnvelopeFrom = true MailService.EnvelopeFrom = parsed.Address } - - log.Info("Mail Service Enabled") } func loadRegisterMailFrom(rootCfg ConfigProvider) { @@ -267,7 +268,6 @@ func loadRegisterMailFrom(rootCfg ConfigProvider) { return } Service.RegisterEmailConfirm = true - log.Info("Register Mail Service Enabled") } func loadNotifyMailFrom(rootCfg ConfigProvider) { @@ -278,7 +278,6 @@ func loadNotifyMailFrom(rootCfg ConfigProvider) { return } Service.EnableNotifyMail = true - log.Info("Notify Mail Service Enabled") } func tryResolveAddr(addr string) []net.IPAddr { diff --git a/modules/setting/mailer_test.go b/modules/setting/mailer_test.go index fbabf11378..ceef35b051 100644 --- a/modules/setting/mailer_test.go +++ b/modules/setting/mailer_test.go @@ -34,8 +34,8 @@ func Test_loadMailerFrom(t *testing.T) { // Check mailer setting loadMailerFrom(cfg) - assert.EqualValues(t, kase.SMTPAddr, MailService.SMTPAddr) - assert.EqualValues(t, kase.SMTPPort, MailService.SMTPPort) + assert.Equal(t, kase.SMTPAddr, MailService.SMTPAddr) + assert.Equal(t, kase.SMTPPort, MailService.SMTPPort) }) } } diff --git a/modules/setting/markup.go b/modules/setting/markup.go index dfce8afa77..057b0650c3 100644 --- a/modules/setting/markup.go +++ b/modules/setting/markup.go @@ -8,6 +8,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" ) // ExternalMarkupRenderers represents the external markup renderers @@ -23,18 +24,33 @@ const ( RenderContentModeIframe = "iframe" ) +type MarkdownRenderOptions struct { + NewLineHardBreak bool + ShortIssuePattern bool // Actually it is a "markup" option because it is used in "post processor" +} + +type MarkdownMathCodeBlockOptions struct { + ParseInlineDollar bool + ParseInlineParentheses bool + ParseBlockDollar bool + ParseBlockSquareBrackets bool +} + // Markdown settings var Markdown = struct { - EnableHardLineBreakInComments bool - EnableHardLineBreakInDocuments bool - CustomURLSchemes []string `ini:"CUSTOM_URL_SCHEMES"` - FileExtensions []string - EnableMath bool + RenderOptionsComment MarkdownRenderOptions `ini:"-"` + RenderOptionsWiki MarkdownRenderOptions `ini:"-"` + RenderOptionsRepoFile MarkdownRenderOptions `ini:"-"` + + CustomURLSchemes []string `ini:"CUSTOM_URL_SCHEMES"` // Actually it is a "markup" option because it is used in "post processor" + FileExtensions []string + + EnableMath bool + MathCodeBlockDetection []string + MathCodeBlockOptions MarkdownMathCodeBlockOptions `ini:"-"` }{ - EnableHardLineBreakInComments: true, - EnableHardLineBreakInDocuments: false, - FileExtensions: strings.Split(".md,.markdown,.mdown,.mkd,.livemd", ","), - EnableMath: true, + FileExtensions: strings.Split(".md,.markdown,.mdown,.mkd,.livemd", ","), + EnableMath: true, } // MarkupRenderer defines the external parser configured in ini @@ -60,8 +76,58 @@ type MarkupSanitizerRule struct { func loadMarkupFrom(rootCfg ConfigProvider) { mustMapSetting(rootCfg, "markdown", &Markdown) + const none = "none" + + const renderOptionShortIssuePattern = "short-issue-pattern" + const renderOptionNewLineHardBreak = "new-line-hard-break" + cfgMarkdown := rootCfg.Section("markdown") + parseMarkdownRenderOptions := func(key string, defaults []string) (ret MarkdownRenderOptions) { + options := cfgMarkdown.Key(key).Strings(",") + options = util.IfEmpty(options, defaults) + for _, opt := range options { + switch opt { + case renderOptionShortIssuePattern: + ret.ShortIssuePattern = true + case renderOptionNewLineHardBreak: + ret.NewLineHardBreak = true + case none: + ret = MarkdownRenderOptions{} + case "": + default: + log.Error("Unknown markdown render option in %s: %s", key, opt) + } + } + return ret + } + Markdown.RenderOptionsComment = parseMarkdownRenderOptions("RENDER_OPTIONS_COMMENT", []string{renderOptionShortIssuePattern, renderOptionNewLineHardBreak}) + Markdown.RenderOptionsWiki = parseMarkdownRenderOptions("RENDER_OPTIONS_WIKI", []string{renderOptionShortIssuePattern}) + Markdown.RenderOptionsRepoFile = parseMarkdownRenderOptions("RENDER_OPTIONS_REPO_FILE", nil) + + const mathCodeInlineDollar = "inline-dollar" + const mathCodeInlineParentheses = "inline-parentheses" + const mathCodeBlockDollar = "block-dollar" + const mathCodeBlockSquareBrackets = "block-square-brackets" + Markdown.MathCodeBlockDetection = util.IfEmpty(Markdown.MathCodeBlockDetection, []string{mathCodeInlineDollar, mathCodeBlockDollar}) + Markdown.MathCodeBlockOptions = MarkdownMathCodeBlockOptions{} + for _, s := range Markdown.MathCodeBlockDetection { + switch s { + case mathCodeInlineDollar: + Markdown.MathCodeBlockOptions.ParseInlineDollar = true + case mathCodeInlineParentheses: + Markdown.MathCodeBlockOptions.ParseInlineParentheses = true + case mathCodeBlockDollar: + Markdown.MathCodeBlockOptions.ParseBlockDollar = true + case mathCodeBlockSquareBrackets: + Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets = true + case none: + Markdown.MathCodeBlockOptions = MarkdownMathCodeBlockOptions{} + case "": + default: + log.Error("Unknown math code block detection option: %s", s) + } + } - MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(5000) + MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(50000) ExternalMarkupRenderers = make([]*MarkupRenderer, 0, 10) ExternalSanitizerRules = make([]MarkupSanitizerRule, 0, 10) @@ -83,8 +149,8 @@ func loadMarkupFrom(rootCfg ConfigProvider) { func newMarkupSanitizer(name string, sec ConfigSection) { rule, ok := createMarkupSanitizerRule(name, sec) if ok { - if strings.HasPrefix(name, "sanitizer.") { - names := strings.SplitN(strings.TrimPrefix(name, "sanitizer."), ".", 2) + if after, found := strings.CutPrefix(name, "sanitizer."); found { + names := strings.SplitN(after, ".", 2) name = names[0] } for _, renderer := range ExternalMarkupRenderers { diff --git a/modules/setting/markup_test.go b/modules/setting/markup_test.go new file mode 100644 index 0000000000..c47a38ce15 --- /dev/null +++ b/modules/setting/markup_test.go @@ -0,0 +1,51 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLoadMarkup(t *testing.T) { + cfg, _ := NewConfigProviderFromData(``) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseBlockDollar: true}, Markdown.MathCodeBlockOptions) + assert.Equal(t, MarkdownRenderOptions{NewLineHardBreak: true, ShortIssuePattern: true}, Markdown.RenderOptionsComment) + assert.Equal(t, MarkdownRenderOptions{ShortIssuePattern: true}, Markdown.RenderOptionsWiki) + assert.Equal(t, MarkdownRenderOptions{}, Markdown.RenderOptionsRepoFile) + + t.Run("Math", func(t *testing.T) { + cfg, _ = NewConfigProviderFromData(` +[markdown] +MATH_CODE_BLOCK_DETECTION = none +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownMathCodeBlockOptions{}, Markdown.MathCodeBlockOptions) + + cfg, _ = NewConfigProviderFromData(` +[markdown] +MATH_CODE_BLOCK_DETECTION = inline-dollar, inline-parentheses, block-dollar, block-square-brackets +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseInlineParentheses: true, ParseBlockDollar: true, ParseBlockSquareBrackets: true}, Markdown.MathCodeBlockOptions) + }) + + t.Run("Render", func(t *testing.T) { + cfg, _ = NewConfigProviderFromData(` +[markdown] +RENDER_OPTIONS_COMMENT = none +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownRenderOptions{}, Markdown.RenderOptionsComment) + + cfg, _ = NewConfigProviderFromData(` +[markdown] +RENDER_OPTIONS_REPO_FILE = short-issue-pattern, new-line-hard-break +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownRenderOptions{NewLineHardBreak: true, ShortIssuePattern: true}, Markdown.RenderOptionsRepoFile) + }) +} diff --git a/modules/setting/mirror.go b/modules/setting/mirror.go index 3aa530a1f4..300711789d 100644 --- a/modules/setting/mirror.go +++ b/modules/setting/mirror.go @@ -48,11 +48,7 @@ func loadMirrorFrom(rootCfg ConfigProvider) { Mirror.MinInterval = 1 * time.Minute } if Mirror.DefaultInterval < Mirror.MinInterval { - if time.Hour*8 < Mirror.MinInterval { - Mirror.DefaultInterval = Mirror.MinInterval - } else { - Mirror.DefaultInterval = time.Hour * 8 - } + Mirror.DefaultInterval = max(time.Hour*8, Mirror.MinInterval) log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval, set to %s", Mirror.DefaultInterval.String()) } } diff --git a/modules/setting/oauth2.go b/modules/setting/oauth2.go index 0d3e63e0b4..1a88f3cb08 100644 --- a/modules/setting/oauth2.go +++ b/modules/setting/oauth2.go @@ -12,7 +12,7 @@ import ( "code.gitea.io/gitea/modules/log" ) -// OAuth2UsernameType is enum describing the way gitea 'name' should be generated from oauth2 data +// OAuth2UsernameType is enum describing the way gitea generates its 'username' from oauth2 data type OAuth2UsernameType string const ( diff --git a/modules/setting/oauth2_test.go b/modules/setting/oauth2_test.go index d0e5ccf13d..c6e66cad02 100644 --- a/modules/setting/oauth2_test.go +++ b/modules/setting/oauth2_test.go @@ -31,7 +31,7 @@ JWT_SECRET = BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB actual := GetGeneralTokenSigningSecret() expected, _ := generate.DecodeJwtSecretBase64("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB") assert.Len(t, actual, 32) - assert.EqualValues(t, expected, actual) + assert.Equal(t, expected, actual) } func TestGetGeneralSigningSecretSave(t *testing.T) { diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 3f618cfd64..b598424064 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -6,8 +6,6 @@ package setting import ( "fmt" "math" - "os" - "path/filepath" "github.com/dustin/go-humanize" ) @@ -15,9 +13,8 @@ import ( // Package registry settings var ( Packages = struct { - Storage *Storage - Enabled bool - ChunkedUploadPath string + Storage *Storage + Enabled bool LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 @@ -67,17 +64,6 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { return err } - Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload")) - if !filepath.IsAbs(Packages.ChunkedUploadPath) { - Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath)) - } - - if HasInstallLock(rootCfg) { - if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { - return fmt.Errorf("unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) - } - } - Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeAlpine = mustBytes(sec, "LIMIT_SIZE_ALPINE") Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH") diff --git a/modules/setting/packages_test.go b/modules/setting/packages_test.go index 87de276041..47378f35ad 100644 --- a/modules/setting/packages_test.go +++ b/modules/setting/packages_test.go @@ -41,7 +41,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) // we can also configure packages storage directly iniStr = ` @@ -53,7 +53,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) // or we can indicate the storage type in the packages section iniStr = ` @@ -68,7 +68,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) // or we can indicate the storage type and minio base path in the packages section iniStr = ` @@ -84,7 +84,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "my_packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_packages/", Packages.Storage.MinioConfig.BasePath) } func Test_PackageStorage1(t *testing.T) { @@ -109,8 +109,8 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } @@ -136,8 +136,8 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } @@ -164,8 +164,8 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "my_packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } @@ -192,7 +192,7 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "my_packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } diff --git a/modules/setting/path.go b/modules/setting/path.go index 0fdc305aa1..f51457a620 100644 --- a/modules/setting/path.go +++ b/modules/setting/path.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/tempdir" ) var ( @@ -196,3 +197,18 @@ func InitWorkPathAndCfgProvider(getEnvFn func(name string) string, args ArgWorkP CustomPath = tmpCustomPath.Value CustomConf = tmpCustomConf.Value } + +// AppDataTempDir returns a managed temporary directory for the application data. +// Using empty sub will get the managed base temp directory, and it's safe to delete it. +// Gitea only creates subdirectories under it, but not the APP_TEMP_PATH directory itself. +// * When APP_TEMP_PATH="/tmp": the managed temp directory is "/tmp/gitea-tmp" +// * When APP_TEMP_PATH is not set: the managed temp directory is "/{APP_DATA_PATH}/tmp" +func AppDataTempDir(sub string) *tempdir.TempDir { + if appTempPathInternal != "" { + return tempdir.New(appTempPathInternal, "gitea-tmp/"+sub) + } + if AppDataPath == "" { + panic("setting.AppDataPath is not set") + } + return tempdir.New(AppDataPath, "tmp/"+sub) +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index c5619d0f04..318cf41108 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -5,7 +5,6 @@ package setting import ( "os/exec" - "path" "path/filepath" "strings" @@ -63,17 +62,11 @@ var ( // Repository upload settings Upload struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int } `ini:"-"` - // Repository local settings - Local struct { - LocalCopyPath string - } `ini:"-"` - // Pull request settings PullRequest struct { WorkInProgressPrefixes []string @@ -89,6 +82,7 @@ var ( AddCoCommitterTrailers bool TestConflictingPatchesWithGitApply bool RetargetChildrenOnMerge bool + DelayCheckForInactiveDays int } `ini:"repository.pull-request"` // Issue Setting @@ -106,11 +100,13 @@ var ( SigningKey string SigningName string SigningEmail string + SigningFormat string InitialCommit []string CRUDActions []string `ini:"CRUD_ACTIONS"` Merges []string Wiki []string DefaultTrustModel string + TrustedSSHKeys []string `ini:"TRUSTED_SSH_KEYS"` } `ini:"repository.signing"` }{ DetectedCharsetsOrder: []string{ @@ -182,25 +178,16 @@ var ( // Repository upload settings Upload: struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int }{ Enabled: true, - TempPath: "data/tmp/uploads", AllowedTypes: "", FileMaxSize: 50, MaxFiles: 5, }, - // Repository local settings - Local: struct { - LocalCopyPath string - }{ - LocalCopyPath: "tmp/local-repo", - }, - // Pull request settings PullRequest: struct { WorkInProgressPrefixes []string @@ -216,6 +203,7 @@ var ( AddCoCommitterTrailers bool TestConflictingPatchesWithGitApply bool RetargetChildrenOnMerge bool + DelayCheckForInactiveDays int }{ WorkInProgressPrefixes: []string{"WIP:", "[WIP]"}, // Same as GitHub. See @@ -231,6 +219,7 @@ var ( PopulateSquashCommentWithCommitMessages: false, AddCoCommitterTrailers: true, RetargetChildrenOnMerge: true, + DelayCheckForInactiveDays: 7, }, // Issue settings @@ -255,20 +244,24 @@ var ( SigningKey string SigningName string SigningEmail string + SigningFormat string InitialCommit []string CRUDActions []string `ini:"CRUD_ACTIONS"` Merges []string Wiki []string DefaultTrustModel string + TrustedSSHKeys []string `ini:"TRUSTED_SSH_KEYS"` }{ SigningKey: "default", SigningName: "", SigningEmail: "", + SigningFormat: "openpgp", // git.SigningKeyFormatOpenPGP InitialCommit: []string{"always"}, CRUDActions: []string{"pubkey", "twofa", "parentsigned"}, Merges: []string{"pubkey", "twofa", "basesigned", "commitssigned"}, Wiki: []string{"never"}, DefaultTrustModel: "collaborator", + TrustedSSHKeys: []string{}, }, } RepoRootPath string @@ -284,7 +277,7 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { Repository.GoGetCloneURLProtocol = sec.Key("GO_GET_CLONE_URL_PROTOCOL").MustString("https") Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) Repository.DefaultBranch = sec.Key("DEFAULT_BRANCH").MustString(Repository.DefaultBranch) - RepoRootPath = sec.Key("ROOT").MustString(path.Join(AppDataPath, "gitea-repositories")) + RepoRootPath = sec.Key("ROOT").MustString(filepath.Join(AppDataPath, "gitea-repositories")) if !filepath.IsAbs(RepoRootPath) { RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath) } else { @@ -309,8 +302,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { log.Fatal("Failed to map Repository.Editor settings: %v", err) } else if err = rootCfg.Section("repository.upload").MapTo(&Repository.Upload); err != nil { log.Fatal("Failed to map Repository.Upload settings: %v", err) - } else if err = rootCfg.Section("repository.local").MapTo(&Repository.Local); err != nil { - log.Fatal("Failed to map Repository.Local settings: %v", err) } else if err = rootCfg.Section("repository.pull-request").MapTo(&Repository.PullRequest); err != nil { log.Fatal("Failed to map Repository.PullRequest settings: %v", err) } @@ -362,10 +353,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { } } - if !filepath.IsAbs(Repository.Upload.TempPath) { - Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath) - } - if err := loadRepoArchiveFrom(rootCfg); err != nil { log.Fatal("loadRepoArchiveFrom: %v", err) } diff --git a/modules/setting/repository_archive_test.go b/modules/setting/repository_archive_test.go index a0f91f0da1..d5b95272d6 100644 --- a/modules/setting/repository_archive_test.go +++ b/modules/setting/repository_archive_test.go @@ -20,7 +20,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) // we can also configure packages storage directly iniStr = ` @@ -32,7 +32,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) // or we can indicate the storage type in the packages section iniStr = ` @@ -47,7 +47,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) // or we can indicate the storage type and minio base path in the packages section iniStr = ` @@ -63,7 +63,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath) } func Test_RepoArchiveStorage(t *testing.T) { @@ -85,7 +85,7 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := RepoArchive.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) iniStr = ` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -107,5 +107,5 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage = RepoArchive.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) } diff --git a/modules/setting/security.go b/modules/setting/security.go index 3d12fcf8d9..153b6bc944 100644 --- a/modules/setting/security.go +++ b/modules/setting/security.go @@ -13,8 +13,9 @@ import ( "code.gitea.io/gitea/modules/log" ) +// Security settings + var ( - // Security settings InstallLock bool SecretKey string InternalToken string // internal access token @@ -27,7 +28,7 @@ var ( ReverseProxyTrustedProxies []string MinPasswordLength int ImportLocalPaths bool - DisableGitHooks bool + DisableGitHooks = true DisableWebhooks bool OnlyAllowPushIfGiteaEnvironmentSet bool PasswordComplexity []string @@ -38,6 +39,7 @@ var ( CSRFCookieName = "_csrf" CSRFCookieHTTPOnly = true RecordUserSignupMetadata = false + TwoFactorAuthEnforced = false ) // loadSecret load the secret from ini by uriKey or verbatimKey, only one of them could be set @@ -109,7 +111,7 @@ func loadSecurityFrom(rootCfg ConfigProvider) { if SecretKey == "" { // FIXME: https://github.com/go-gitea/gitea/issues/16832 // Until it supports rotating an existing secret key, we shouldn't move users off of the widely used default value - SecretKey = "!#@FDEWREWR&*(" //nolint:gosec + SecretKey = "!#@FDEWREWR&*(" } CookieRememberName = sec.Key("COOKIE_REMEMBER_NAME").MustString("gitea_incredible") @@ -141,6 +143,15 @@ func loadSecurityFrom(rootCfg ConfigProvider) { PasswordCheckPwn = sec.Key("PASSWORD_CHECK_PWN").MustBool(false) SuccessfulTokensCacheSize = sec.Key("SUCCESSFUL_TOKENS_CACHE_SIZE").MustInt(20) + twoFactorAuth := sec.Key("TWO_FACTOR_AUTH").String() + switch twoFactorAuth { + case "": + case "enforced": + TwoFactorAuthEnforced = true + default: + log.Fatal("Invalid two-factor auth option: %s", twoFactorAuth) + } + InternalToken = loadSecret(sec, "INTERNAL_TOKEN_URI", "INTERNAL_TOKEN") if InstallLock && InternalToken == "" { // if Gitea has been installed but the InternalToken hasn't been generated (upgrade from an old release), we should generate diff --git a/modules/setting/server.go b/modules/setting/server.go index d7a71578d4..38e166e02a 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "net" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -40,28 +41,47 @@ const ( LandingPageLogin LandingPage = "/user/login" ) +const ( + PublicURLAuto = "auto" + PublicURLLegacy = "legacy" +) + // Server settings var ( // AppURL is the Application ROOT_URL. It always has a '/' suffix // It maps to ini:"ROOT_URL" AppURL string - // AppSubURL represents the sub-url mounting point for gitea. It is either "" or starts with '/' and ends without '/', such as '/{subpath}'. + + // PublicURLDetection controls how to use the HTTP request headers to detect public URL + PublicURLDetection string + + // AppSubURL represents the sub-url mounting point for gitea, parsed from "ROOT_URL" + // It is either "" or starts with '/' and ends without '/', such as '/{sub-path}'. // This value is empty if site does not have sub-url. AppSubURL string - // UseSubURLPath makes Gitea handle requests with sub-path like "/sub-path/owner/repo/...", to make it easier to debug sub-path related problems without a reverse proxy. + + // UseSubURLPath makes Gitea handle requests with sub-path like "/sub-path/owner/repo/...", + // to make it easier to debug sub-path related problems without a reverse proxy. UseSubURLPath bool + // AppDataPath is the default path for storing data. // It maps to ini:"APP_DATA_PATH" in [server] and defaults to AppWorkPath + "/data" AppDataPath string + // LocalURL is the url for locally running applications to contact Gitea. It always has a '/' suffix // It maps to ini:"LOCAL_ROOT_URL" in [server] LocalURL string - // AssetVersion holds a opaque value that is used for cache-busting assets + + // AssetVersion holds an opaque value that is used for cache-busting assets AssetVersion string + // appTempPathInternal is the temporary path for the app, it is only an internal variable + // DO NOT use it directly, always use AppDataTempDir + appTempPathInternal string + Protocol Scheme - UseProxyProtocol bool // `ini:"USE_PROXY_PROTOCOL"` - ProxyProtocolTLSBridging bool //`ini:"PROXY_PROTOCOL_TLS_BRIDGING"` + UseProxyProtocol bool + ProxyProtocolTLSBridging bool ProxyProtocolHeaderTimeout time.Duration ProxyProtocolAcceptUnknown bool Domain string @@ -169,20 +189,25 @@ func loadServerFrom(rootCfg ConfigProvider) { HTTPAddr = sec.Key("HTTP_ADDR").MustString("0.0.0.0") HTTPPort = sec.Key("HTTP_PORT").MustString("3000") - Protocol = HTTP + // DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version + // if these are removed, the warning will not be shown + if sec.HasKey("ENABLE_ACME") { + EnableAcme = sec.Key("ENABLE_ACME").MustBool(false) + } else { + deprecatedSetting(rootCfg, "server", "ENABLE_LETSENCRYPT", "server", "ENABLE_ACME", "v1.19.0") + EnableAcme = sec.Key("ENABLE_LETSENCRYPT").MustBool(false) + } + protocolCfg := sec.Key("PROTOCOL").String() + if protocolCfg != "https" && EnableAcme { + log.Fatal("ACME could only be used with HTTPS protocol") + } + switch protocolCfg { + case "", "http": + Protocol = HTTP case "https": Protocol = HTTPS - - // DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version - // if these are removed, the warning will not be shown - if sec.HasKey("ENABLE_ACME") { - EnableAcme = sec.Key("ENABLE_ACME").MustBool(false) - } else { - deprecatedSetting(rootCfg, "server", "ENABLE_LETSENCRYPT", "server", "ENABLE_ACME", "v1.19.0") - EnableAcme = sec.Key("ENABLE_LETSENCRYPT").MustBool(false) - } if EnableAcme { AcmeURL = sec.Key("ACME_URL").MustString("") AcmeCARoot = sec.Key("ACME_CA_ROOT").MustString("") @@ -210,6 +235,9 @@ func loadServerFrom(rootCfg ConfigProvider) { deprecatedSetting(rootCfg, "server", "LETSENCRYPT_EMAIL", "server", "ACME_EMAIL", "v1.19.0") AcmeEmail = sec.Key("LETSENCRYPT_EMAIL").MustString("") } + if AcmeEmail == "" { + log.Fatal("ACME Email is not set (ACME_EMAIL).") + } } else { CertFile = sec.Key("CERT_FILE").String() KeyFile = sec.Key("KEY_FILE").String() @@ -233,7 +261,7 @@ func loadServerFrom(rootCfg ConfigProvider) { case "unix": log.Warn("unix PROTOCOL value is deprecated, please use http+unix") fallthrough - case "http+unix": + default: // "http+unix" Protocol = HTTPUnix } UnixSocketPermissionRaw := sec.Key("UNIX_SOCKET_PERMISSION").MustString("666") @@ -246,6 +274,8 @@ func loadServerFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(HTTPAddr) { HTTPAddr = filepath.Join(AppWorkPath, HTTPAddr) } + default: + log.Fatal("Invalid PROTOCOL %q", protocolCfg) } UseProxyProtocol = sec.Key("USE_PROXY_PROTOCOL").MustBool(false) ProxyProtocolTLSBridging = sec.Key("PROXY_PROTOCOL_TLS_BRIDGING").MustBool(false) @@ -259,11 +289,15 @@ func loadServerFrom(rootCfg ConfigProvider) { defaultAppURL := string(Protocol) + "://" + Domain + ":" + HTTPPort AppURL = sec.Key("ROOT_URL").MustString(defaultAppURL) + PublicURLDetection = sec.Key("PUBLIC_URL_DETECTION").MustString(PublicURLLegacy) + if PublicURLDetection != PublicURLAuto && PublicURLDetection != PublicURLLegacy { + log.Fatal("Invalid PUBLIC_URL_DETECTION value: %s", PublicURLDetection) + } // Check validity of AppURL appURL, err := url.Parse(AppURL) if err != nil { - log.Fatal("Invalid ROOT_URL '%s': %s", AppURL, err) + log.Fatal("Invalid ROOT_URL %q: %s", AppURL, err) } // Remove default ports from AppURL. // (scheme-based URL normalization, RFC 3986 section 6.2.3) @@ -299,13 +333,15 @@ func loadServerFrom(rootCfg ConfigProvider) { defaultLocalURL = AppURL case FCGIUnix: defaultLocalURL = AppURL - default: + case HTTP, HTTPS: defaultLocalURL = string(Protocol) + "://" if HTTPAddr == "0.0.0.0" { defaultLocalURL += net.JoinHostPort("localhost", HTTPPort) + "/" } else { defaultLocalURL += net.JoinHostPort(HTTPAddr, HTTPPort) + "/" } + default: + log.Fatal("Invalid PROTOCOL %q", Protocol) } LocalURL = sec.Key("LOCAL_ROOT_URL").MustString(defaultLocalURL) LocalURL = strings.TrimRight(LocalURL, "/") + "/" @@ -323,6 +359,19 @@ func loadServerFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(AppDataPath) { AppDataPath = filepath.ToSlash(filepath.Join(AppWorkPath, AppDataPath)) } + if IsInTesting && HasInstallLock(rootCfg) { + // FIXME: in testing, the "app data" directory is not correctly initialized before loading settings + if _, err := os.Stat(AppDataPath); err != nil { + _ = os.MkdirAll(AppDataPath, os.ModePerm) + } + } + + appTempPathInternal = sec.Key("APP_TEMP_PATH").String() + if appTempPathInternal != "" { + if _, err := os.Stat(appTempPathInternal); err != nil { + log.Fatal("APP_TEMP_PATH %q is not accessible: %v", appTempPathInternal, err) + } + } EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) diff --git a/modules/setting/service.go b/modules/setting/service.go index 526ad64eb4..b1b9fedd62 100644 --- a/modules/setting/service.go +++ b/modules/setting/service.go @@ -5,6 +5,7 @@ package setting import ( "regexp" + "runtime" "strings" "time" @@ -43,9 +44,11 @@ var Service = struct { ShowRegistrationButton bool EnablePasswordSignInForm bool ShowMilestonesDashboardPage bool - RequireSignInView bool + RequireSignInViewStrict bool + BlockAnonymousAccessExpensive bool EnableNotifyMail bool EnableBasicAuth bool + EnablePasskeyAuth bool EnableReverseProxyAuth bool EnableReverseProxyAuthAPI bool EnableReverseProxyAutoRegister bool @@ -96,6 +99,13 @@ var Service = struct { DisableOrganizationsPage bool `ini:"DISABLE_ORGANIZATIONS_PAGE"` DisableCodePage bool `ini:"DISABLE_CODE_PAGE"` } `ini:"service.explore"` + + QoS struct { + Enabled bool + MaxInFlightRequests int + MaxWaitingRequests int + TargetWaitTime time.Duration + } }{ AllowedUserVisibilityModesSlice: []bool{true, true, true}, } @@ -158,9 +168,21 @@ func loadServiceFrom(rootCfg ConfigProvider) { Service.EmailDomainBlockList = CompileEmailGlobList(sec, "EMAIL_DOMAIN_BLOCKLIST") Service.ShowRegistrationButton = sec.Key("SHOW_REGISTRATION_BUTTON").MustBool(!(Service.DisableRegistration || Service.AllowOnlyExternalRegistration)) Service.ShowMilestonesDashboardPage = sec.Key("SHOW_MILESTONES_DASHBOARD_PAGE").MustBool(true) - Service.RequireSignInView = sec.Key("REQUIRE_SIGNIN_VIEW").MustBool() + + // boolean values are considered as "strict" + var err error + Service.RequireSignInViewStrict, err = sec.Key("REQUIRE_SIGNIN_VIEW").Bool() + if s := sec.Key("REQUIRE_SIGNIN_VIEW").String(); err != nil && s != "" { + // non-boolean value only supports "expensive" at the moment + Service.BlockAnonymousAccessExpensive = s == "expensive" + if !Service.BlockAnonymousAccessExpensive { + log.Fatal("Invalid config option: REQUIRE_SIGNIN_VIEW = %s", s) + } + } + Service.EnableBasicAuth = sec.Key("ENABLE_BASIC_AUTHENTICATION").MustBool(true) Service.EnablePasswordSignInForm = sec.Key("ENABLE_PASSWORD_SIGNIN_FORM").MustBool(true) + Service.EnablePasskeyAuth = sec.Key("ENABLE_PASSKEY_AUTHENTICATION").MustBool(true) Service.EnableReverseProxyAuth = sec.Key("ENABLE_REVERSE_PROXY_AUTHENTICATION").MustBool() Service.EnableReverseProxyAuthAPI = sec.Key("ENABLE_REVERSE_PROXY_AUTHENTICATION_API").MustBool() Service.EnableReverseProxyAutoRegister = sec.Key("ENABLE_REVERSE_PROXY_AUTO_REGISTRATION").MustBool() @@ -241,6 +263,7 @@ func loadServiceFrom(rootCfg ConfigProvider) { mustMapSetting(rootCfg, "service.explore", &Service.Explore) loadOpenIDSetting(rootCfg) + loadQosSetting(rootCfg) } func loadOpenIDSetting(rootCfg ConfigProvider) { @@ -262,3 +285,11 @@ func loadOpenIDSetting(rootCfg ConfigProvider) { } } } + +func loadQosSetting(rootCfg ConfigProvider) { + sec := rootCfg.Section("qos") + Service.QoS.Enabled = sec.Key("ENABLED").MustBool(false) + Service.QoS.MaxInFlightRequests = sec.Key("MAX_INFLIGHT").MustInt(4 * runtime.NumCPU()) + Service.QoS.MaxWaitingRequests = sec.Key("MAX_WAITING").MustInt(100) + Service.QoS.TargetWaitTime = sec.Key("TARGET_WAIT_TIME").MustDuration(250 * time.Millisecond) +} diff --git a/modules/setting/service_test.go b/modules/setting/service_test.go index 1647bcec16..73736b793a 100644 --- a/modules/setting/service_test.go +++ b/modules/setting/service_test.go @@ -7,16 +7,14 @@ import ( "testing" "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/test" "github.com/gobwas/glob" "github.com/stretchr/testify/assert" ) func TestLoadServices(t *testing.T) { - oldService := Service - defer func() { - Service = oldService - }() + defer test.MockVariableValue(&Service)() cfg, err := NewConfigProviderFromData(` [service] @@ -48,10 +46,7 @@ EMAIL_DOMAIN_BLOCKLIST = d3, *.b } func TestLoadServiceVisibilityModes(t *testing.T) { - oldService := Service - defer func() { - Service = oldService - }() + defer test.MockVariableValue(&Service)() kases := map[string]func(){ ` @@ -130,3 +125,33 @@ ALLOWED_USER_VISIBILITY_MODES = public, limit, privated }) } } + +func TestLoadServiceRequireSignInView(t *testing.T) { + defer test.MockVariableValue(&Service)() + + cfg, err := NewConfigProviderFromData(` +[service] +`) + assert.NoError(t, err) + loadServiceFrom(cfg) + assert.False(t, Service.RequireSignInViewStrict) + assert.False(t, Service.BlockAnonymousAccessExpensive) + + cfg, err = NewConfigProviderFromData(` +[service] +REQUIRE_SIGNIN_VIEW = true +`) + assert.NoError(t, err) + loadServiceFrom(cfg) + assert.True(t, Service.RequireSignInViewStrict) + assert.False(t, Service.BlockAnonymousAccessExpensive) + + cfg, err = NewConfigProviderFromData(` +[service] +REQUIRE_SIGNIN_VIEW = expensive +`) + assert.NoError(t, err) + loadServiceFrom(cfg) + assert.False(t, Service.RequireSignInViewStrict) + assert.True(t, Service.BlockAnonymousAccessExpensive) +} diff --git a/modules/setting/session.go b/modules/setting/session.go index afe63bfdb7..19a05ce2c2 100644 --- a/modules/setting/session.go +++ b/modules/setting/session.go @@ -73,6 +73,4 @@ func loadSessionFrom(rootCfg ConfigProvider) { SessionConfig.ProviderConfig = string(shadowConfig) SessionConfig.OriginalProvider = SessionConfig.Provider SessionConfig.Provider = "VirtualSession" - - log.Info("Session Service Enabled") } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c93d199b1b..e14997801f 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -12,8 +12,8 @@ import ( "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/user" - "code.gitea.io/gitea/modules/util" ) // settings @@ -159,7 +159,7 @@ func loadRunModeFrom(rootCfg ConfigProvider) { // The following is a purposefully undocumented option. Please do not run Gitea as root. It will only cause future headaches. // Please don't use root as a bandaid to "fix" something that is broken, instead the broken thing should instead be fixed properly. unsafeAllowRunAsRoot := ConfigSectionKeyBool(rootSec, "I_AM_BEING_UNSAFE_RUNNING_AS_ROOT") - unsafeAllowRunAsRoot = unsafeAllowRunAsRoot || util.OptionalBoolParse(os.Getenv("GITEA_I_AM_BEING_UNSAFE_RUNNING_AS_ROOT")).Value() + unsafeAllowRunAsRoot = unsafeAllowRunAsRoot || optional.ParseBool(os.Getenv("GITEA_I_AM_BEING_UNSAFE_RUNNING_AS_ROOT")).Value() RunMode = os.Getenv("GITEA_RUN_MODE") if RunMode == "" { RunMode = rootSec.Key("RUN_MODE").MustString("prod") @@ -235,3 +235,9 @@ func checkOverlappedPath(name, path string) { } configuredPaths[path] = name } + +func PanicInDevOrTesting(msg string, a ...any) { + if !IsProd || IsInTesting { + panic(fmt.Sprintf(msg, a...)) + } +} diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index ea387e521f..900fc6ade2 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -4,8 +4,6 @@ package setting import ( - "os" - "path" "path/filepath" "strings" "text/template" @@ -32,8 +30,6 @@ var SSH = struct { ServerKeyExchanges []string `ini:"SSH_SERVER_KEY_EXCHANGES"` ServerMACs []string `ini:"SSH_SERVER_MACS"` ServerHostKeys []string `ini:"SSH_SERVER_HOST_KEYS"` - KeyTestPath string `ini:"SSH_KEY_TEST_PATH"` - KeygenPath string `ini:"SSH_KEYGEN_PATH"` AuthorizedKeysBackup bool `ini:"SSH_AUTHORIZED_KEYS_BACKUP"` AuthorizedPrincipalsBackup bool `ini:"SSH_AUTHORIZED_PRINCIPALS_BACKUP"` AuthorizedKeysCommandTemplate string `ini:"SSH_AUTHORIZED_KEYS_COMMAND_TEMPLATE"` @@ -55,10 +51,6 @@ var SSH = struct { StartBuiltinServer: false, Domain: "", Port: 22, - ServerCiphers: []string{"chacha20-poly1305@openssh.com", "aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "aes256-gcm@openssh.com"}, - ServerKeyExchanges: []string{"curve25519-sha256", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521", "diffie-hellman-group14-sha256", "diffie-hellman-group14-sha1"}, - ServerMACs: []string{"hmac-sha2-256-etm@openssh.com", "hmac-sha2-256", "hmac-sha1"}, - KeygenPath: "", MinimumKeySizeCheck: true, MinimumKeySizes: map[string]int{"ed25519": 256, "ed25519-sk": 256, "ecdsa": 256, "ecdsa-sk": 256, "rsa": 3071}, ServerHostKeys: []string{"ssh/gitea.rsa", "ssh/gogs.rsa"}, @@ -111,30 +103,27 @@ func loadSSHFrom(rootCfg ConfigProvider) { } homeDir = strings.ReplaceAll(homeDir, "\\", "/") - SSH.RootPath = path.Join(homeDir, ".ssh") - serverCiphers := sec.Key("SSH_SERVER_CIPHERS").Strings(",") - if len(serverCiphers) > 0 { - SSH.ServerCiphers = serverCiphers - } - serverKeyExchanges := sec.Key("SSH_SERVER_KEY_EXCHANGES").Strings(",") - if len(serverKeyExchanges) > 0 { - SSH.ServerKeyExchanges = serverKeyExchanges - } - serverMACs := sec.Key("SSH_SERVER_MACS").Strings(",") - if len(serverMACs) > 0 { - SSH.ServerMACs = serverMACs - } - SSH.KeyTestPath = os.TempDir() + SSH.RootPath = filepath.Join(homeDir, ".ssh") + if err = sec.MapTo(&SSH); err != nil { log.Fatal("Failed to map SSH settings: %v", err) } + + serverCiphers := sec.Key("SSH_SERVER_CIPHERS").Strings(",") + SSH.ServerCiphers = util.Iif(len(serverCiphers) > 0, serverCiphers, nil) + + serverKeyExchanges := sec.Key("SSH_SERVER_KEY_EXCHANGES").Strings(",") + SSH.ServerKeyExchanges = util.Iif(len(serverKeyExchanges) > 0, serverKeyExchanges, nil) + + serverMACs := sec.Key("SSH_SERVER_MACS").Strings(",") + SSH.ServerMACs = util.Iif(len(serverMACs) > 0, serverMACs, nil) + for i, key := range SSH.ServerHostKeys { if !filepath.IsAbs(key) { SSH.ServerHostKeys[i] = filepath.Join(AppDataPath, key) } } - SSH.KeygenPath = sec.Key("SSH_KEYGEN_PATH").String() SSH.Port = sec.Key("SSH_PORT").MustInt(22) SSH.ListenPort = sec.Key("SSH_LISTEN_PORT").MustInt(SSH.Port) SSH.UseProxyProtocol = sec.Key("SSH_SERVER_USE_PROXY_PROTOCOL").MustBool(false) diff --git a/modules/setting/storage.go b/modules/setting/storage.go index d3d1fb9f30..ee246158d9 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "path/filepath" + "slices" "strings" ) @@ -30,12 +31,7 @@ var storageTypes = []StorageType{ // IsValidStorageType returns true if the given storage type is valid func IsValidStorageType(storageType StorageType) bool { - for _, t := range storageTypes { - if t == storageType { - return true - } - } - return false + return slices.Contains(storageTypes, storageType) } // MinioStorageConfig represents the configuration for a minio storage @@ -162,7 +158,7 @@ const ( targetSecIsSec // target section is from the name seciont [name] ) -func getStorageSectionByType(rootCfg ConfigProvider, typ string) (ConfigSection, targetSecType, error) { //nolint:unparam +func getStorageSectionByType(rootCfg ConfigProvider, typ string) (ConfigSection, targetSecType, error) { //nolint:unparam // FIXME: targetSecType is always 0, wrong design? targetSec, err := rootCfg.GetSection(storageSectionName + "." + typ) if err != nil { if !IsValidStorageType(StorageType(typ)) { @@ -210,8 +206,8 @@ func getStorageTargetSection(rootCfg ConfigProvider, name, typ string, sec Confi targetSec, _ := rootCfg.GetSection(storageSectionName + "." + name) if targetSec != nil { targetType := targetSec.Key("STORAGE_TYPE").String() - switch { - case targetType == "": + switch targetType { + case "": if targetSec.Key("PATH").String() == "" { // both storage type and path are empty, use default return getDefaultStorageSection(rootCfg), targetSecIsDefault, nil } @@ -287,7 +283,7 @@ func getStorageForLocal(targetSec, overrideSec ConfigSection, tp targetSecType, return &storage, nil } -func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl +func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl // duplicates azure setup var storage Storage storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String()) if err := targetSec.MapTo(&storage.MinioConfig); err != nil { @@ -316,7 +312,7 @@ func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, return &storage, nil } -func getStorageForAzureBlob(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl +func getStorageForAzureBlob(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl // duplicates minio setup var storage Storage storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String()) if err := targetSec.MapTo(&storage.AzureBlobConfig); err != nil { diff --git a/modules/setting/storage_test.go b/modules/setting/storage_test.go index afff85537e..6f5a54c41c 100644 --- a/modules/setting/storage_test.go +++ b/modules/setting/storage_test.go @@ -26,16 +26,16 @@ MINIO_BUCKET = gitea-storage assert.NoError(t, err) assert.NoError(t, loadAttachmentFrom(cfg)) - assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) assert.NoError(t, loadAvatarsFrom(cfg)) - assert.EqualValues(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket) + assert.Equal(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) } func Test_getStorageUseOtherNameAsType(t *testing.T) { @@ -51,12 +51,12 @@ MINIO_BUCKET = gitea-storage assert.NoError(t, err) assert.NoError(t, loadAttachmentFrom(cfg)) - assert.EqualValues(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) } func Test_getStorageInheritStorageType(t *testing.T) { @@ -69,32 +69,32 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "gitea", Packages.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Packages.Storage.MinioConfig.Bucket) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "gitea", Actions.LogStorage.MinioConfig.Bucket) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Actions.LogStorage.MinioConfig.Bucket) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) assert.NoError(t, loadAvatarsFrom(cfg)) assert.EqualValues(t, "minio", Avatar.Storage.Type) - assert.EqualValues(t, "gitea", Avatar.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Avatar.Storage.MinioConfig.Bucket) + assert.Equal(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) assert.NoError(t, loadRepoAvatarFrom(cfg)) assert.EqualValues(t, "minio", RepoAvatar.Storage.Type) - assert.EqualValues(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket) + assert.Equal(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath) } func Test_getStorageInheritStorageTypeAzureBlob(t *testing.T) { @@ -107,32 +107,32 @@ STORAGE_TYPE = azureblob assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "azureblob", Packages.Storage.Type) - assert.EqualValues(t, "gitea", Packages.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "packages/", Packages.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Packages.Storage.AzureBlobConfig.Container) + assert.Equal(t, "packages/", Packages.Storage.AzureBlobConfig.BasePath) assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "azureblob", RepoArchive.Storage.Type) - assert.EqualValues(t, "gitea", RepoArchive.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", RepoArchive.Storage.AzureBlobConfig.Container) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "azureblob", Actions.LogStorage.Type) - assert.EqualValues(t, "gitea", Actions.LogStorage.AzureBlobConfig.Container) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Actions.LogStorage.AzureBlobConfig.Container) + assert.Equal(t, "actions_log/", Actions.LogStorage.AzureBlobConfig.BasePath) assert.EqualValues(t, "azureblob", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "gitea", Actions.ArtifactStorage.AzureBlobConfig.Container) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Actions.ArtifactStorage.AzureBlobConfig.Container) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.AzureBlobConfig.BasePath) assert.NoError(t, loadAvatarsFrom(cfg)) assert.EqualValues(t, "azureblob", Avatar.Storage.Type) - assert.EqualValues(t, "gitea", Avatar.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "avatars/", Avatar.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Avatar.Storage.AzureBlobConfig.Container) + assert.Equal(t, "avatars/", Avatar.Storage.AzureBlobConfig.BasePath) assert.NoError(t, loadRepoAvatarFrom(cfg)) assert.EqualValues(t, "azureblob", RepoAvatar.Storage.Type) - assert.EqualValues(t, "gitea", RepoAvatar.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", RepoAvatar.Storage.AzureBlobConfig.Container) + assert.Equal(t, "repo-avatars/", RepoAvatar.Storage.AzureBlobConfig.BasePath) } type testLocalStoragePathCase struct { @@ -151,7 +151,7 @@ func testLocalStoragePath(t *testing.T, appDataPath, iniStr string, cases []test assert.EqualValues(t, "local", storage.Type) assert.True(t, filepath.IsAbs(storage.Path)) - assert.EqualValues(t, filepath.Clean(c.expectedPath), filepath.Clean(storage.Path)) + assert.Equal(t, filepath.Clean(c.expectedPath), filepath.Clean(storage.Path)) } } @@ -389,8 +389,8 @@ MINIO_SECRET_ACCESS_KEY = my_secret_key assert.NoError(t, loadRepoArchiveFrom(cfg)) cp := RepoArchive.Storage.ToShadowCopy() - assert.EqualValues(t, "******", cp.MinioConfig.AccessKeyID) - assert.EqualValues(t, "******", cp.MinioConfig.SecretAccessKey) + assert.Equal(t, "******", cp.MinioConfig.AccessKeyID) + assert.Equal(t, "******", cp.MinioConfig.SecretAccessKey) } func Test_getStorageConfiguration24(t *testing.T) { @@ -445,10 +445,10 @@ MINIO_USE_SSL = true `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) assert.True(t, RepoArchive.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) } func Test_getStorageConfiguration28(t *testing.T) { @@ -462,10 +462,10 @@ MINIO_BASE_PATH = /prefix `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) assert.True(t, RepoArchive.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -476,9 +476,9 @@ MINIO_BASE_PATH = /prefix `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "127.0.0.1", RepoArchive.Storage.MinioConfig.IamEndpoint) + assert.Equal(t, "127.0.0.1", RepoArchive.Storage.MinioConfig.IamEndpoint) assert.True(t, RepoArchive.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -493,10 +493,10 @@ MINIO_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) assert.True(t, LFS.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "/lfs", LFS.Storage.MinioConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -511,10 +511,10 @@ MINIO_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) assert.True(t, LFS.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "/lfs", LFS.Storage.MinioConfig.BasePath) } func Test_getStorageConfiguration29(t *testing.T) { @@ -539,9 +539,9 @@ AZURE_BLOB_ACCOUNT_KEY = my_account_key `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) } func Test_getStorageConfiguration31(t *testing.T) { @@ -554,9 +554,9 @@ AZURE_BLOB_BASE_PATH = /prefix `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "/prefix/repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -570,9 +570,9 @@ AZURE_BLOB_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -586,7 +586,7 @@ AZURE_BLOB_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) } diff --git a/modules/setting/time.go b/modules/setting/time.go index 39acba12ef..97988211a9 100644 --- a/modules/setting/time.go +++ b/modules/setting/time.go @@ -20,7 +20,6 @@ func loadTimeFrom(rootCfg ConfigProvider) { if err != nil { log.Fatal("Load time zone failed: %v", err) } - log.Info("Default UI Location is %v", zone) } if DefaultUILocation == nil { DefaultUILocation = time.Local diff --git a/modules/setting/ui.go b/modules/setting/ui.go index db0fe9ef79..3d9c916bf7 100644 --- a/modules/setting/ui.go +++ b/modules/setting/ui.go @@ -28,6 +28,7 @@ var UI = struct { DefaultShowFullName bool DefaultTheme string Themes []string + FileIconTheme string Reactions []string ReactionsLookup container.Set[string] `ini:"-"` CustomEmojis []string @@ -63,6 +64,7 @@ var UI = struct { } `ini:"ui.admin"` User struct { RepoPagingNum int + OrgPagingNum int } `ini:"ui.user"` Meta struct { Author string @@ -83,6 +85,7 @@ var UI = struct { ReactionMaxUserNum: 10, MaxDisplayFileSize: 8388608, DefaultTheme: `gitea-auto`, + FileIconTheme: `material`, Reactions: []string{`+1`, `-1`, `laugh`, `hooray`, `confused`, `heart`, `rocket`, `eyes`}, CustomEmojis: []string{`git`, `gitea`, `codeberg`, `gitlab`, `github`, `gogs`}, CustomEmojisMap: map[string]string{"git": ":git:", "gitea": ":gitea:", "codeberg": ":codeberg:", "gitlab": ":gitlab:", "github": ":github:", "gogs": ":gogs:"}, @@ -127,8 +130,10 @@ var UI = struct { }, User: struct { RepoPagingNum int + OrgPagingNum int }{ RepoPagingNum: 15, + OrgPagingNum: 15, }, Meta: struct { Author string diff --git a/modules/ssh/init.go b/modules/ssh/init.go index 21d4f89936..cfb0d5693a 100644 --- a/modules/ssh/init.go +++ b/modules/ssh/init.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) func Init() error { @@ -23,20 +24,17 @@ func Init() error { if setting.SSH.StartBuiltinServer { Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs) - log.Info("SSH server started on %s. Cipher list (%v), key exchange algorithms (%v), MACs (%v)", + log.Info("SSH server started on %q. Ciphers: %v, key exchange algorithms: %v, MACs: %v", net.JoinHostPort(setting.SSH.ListenHost, strconv.Itoa(setting.SSH.ListenPort)), - setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs, + util.Iif[any](setting.SSH.ServerCiphers == nil, "default", setting.SSH.ServerCiphers), + util.Iif[any](setting.SSH.ServerKeyExchanges == nil, "default", setting.SSH.ServerKeyExchanges), + util.Iif[any](setting.SSH.ServerMACs == nil, "default", setting.SSH.ServerMACs), ) return nil } builtinUnused() - // FIXME: why 0o644 for a directory ..... - if err := os.MkdirAll(setting.SSH.KeyTestPath, 0o644); err != nil { - return fmt.Errorf("failed to create directory %q for ssh key test: %w", setting.SSH.KeyTestPath, err) - } - if len(setting.SSH.TrustedUserCAKeys) > 0 && setting.SSH.AuthorizedPrincipalsEnabled { caKeysFileName := setting.SSH.TrustedUserCAKeysFile caKeysFileDir := filepath.Dir(caKeysFileName) diff --git a/modules/ssh/ssh.go b/modules/ssh/ssh.go index 7479cfbd95..3fea4851c7 100644 --- a/modules/ssh/ssh.go +++ b/modules/ssh/ssh.go @@ -11,7 +11,6 @@ import ( "crypto/x509" "encoding/pem" "errors" - "fmt" "io" "net" "os" @@ -216,7 +215,7 @@ func publicKeyHandler(ctx ssh.Context, key ssh.PublicKey) bool { ctx.Permissions().Permissions = &gossh.Permissions{} setPermExt := func(keyID int64) { ctx.Permissions().Permissions.Extensions = map[string]string{ - giteaPermissionExtensionKeyID: fmt.Sprint(keyID), + giteaPermissionExtensionKeyID: strconv.FormatInt(keyID, 10), } } @@ -334,7 +333,7 @@ func sshConnectionFailed(conn net.Conn, err error) { log.Warn("Failed authentication attempt from %s", conn.RemoteAddr()) } -// Listen starts a SSH server listens on given port. +// Listen starts an SSH server listening on given port. func Listen(host string, port int, ciphers, keyExchanges, macs []string) { srv := ssh.Server{ Addr: net.JoinHostPort(host, strconv.Itoa(port)), diff --git a/modules/storage/azureblob.go b/modules/storage/azureblob.go index 96c2525b29..6860d81131 100644 --- a/modules/storage/azureblob.go +++ b/modules/storage/azureblob.go @@ -70,7 +70,7 @@ func (a *azureBlobObject) Seek(offset int64, whence int) (int64, error) { case io.SeekCurrent: offset += a.offset case io.SeekEnd: - offset = a.Size - offset + offset = a.Size + offset default: return 0, errors.New("Seek: invalid whence") } @@ -247,7 +247,7 @@ func (a *AzureBlobStorage) Delete(path string) error { } // URL gets the redirect URL to a file. The presigned link is valid for 5 minutes. -func (a *AzureBlobStorage) URL(path, name string, reqParams url.Values) (*url.URL, error) { +func (a *AzureBlobStorage) URL(path, name, _ string, reqParams url.Values) (*url.URL, error) { blobClient := a.getBlobClient(path) startTime := time.Now() diff --git a/modules/storage/azureblob_test.go b/modules/storage/azureblob_test.go index 604870cb98..b3791b4916 100644 --- a/modules/storage/azureblob_test.go +++ b/modules/storage/azureblob_test.go @@ -4,7 +4,9 @@ package storage import ( + "io" "os" + "strings" "testing" "code.gitea.io/gitea/modules/setting" @@ -31,14 +33,14 @@ func TestAzureBlobStorageIterator(t *testing.T) { func TestAzureBlobStoragePath(t *testing.T) { m := &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: ""}} - assert.Equal(t, "", m.buildAzureBlobPath("/")) - assert.Equal(t, "", m.buildAzureBlobPath(".")) + assert.Empty(t, m.buildAzureBlobPath("/")) + assert.Empty(t, m.buildAzureBlobPath(".")) assert.Equal(t, "a", m.buildAzureBlobPath("/a")) assert.Equal(t, "a/b", m.buildAzureBlobPath("/a/b/")) m = &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: "/"}} - assert.Equal(t, "", m.buildAzureBlobPath("/")) - assert.Equal(t, "", m.buildAzureBlobPath(".")) + assert.Empty(t, m.buildAzureBlobPath("/")) + assert.Empty(t, m.buildAzureBlobPath(".")) assert.Equal(t, "a", m.buildAzureBlobPath("/a")) assert.Equal(t, "a/b", m.buildAzureBlobPath("/a/b/")) @@ -54,3 +56,46 @@ func TestAzureBlobStoragePath(t *testing.T) { assert.Equal(t, "base/a", m.buildAzureBlobPath("/a")) assert.Equal(t, "base/a/b", m.buildAzureBlobPath("/a/b/")) } + +func Test_azureBlobObject(t *testing.T) { + if os.Getenv("CI") == "" { + t.Skip("azureBlobStorage not present outside of CI") + return + } + + s, err := NewStorage(setting.AzureBlobStorageType, &setting.Storage{ + AzureBlobConfig: setting.AzureBlobStorageConfig{ + // https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#ip-style-url + Endpoint: "http://devstoreaccount1.azurite.local:10000", + // https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#well-known-storage-account-and-key + AccountName: "devstoreaccount1", + AccountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + Container: "test", + }, + }) + assert.NoError(t, err) + + data := "Q2xTckt6Y1hDOWh0" + _, err = s.Save("test.txt", strings.NewReader(data), int64(len(data))) + assert.NoError(t, err) + obj, err := s.Open("test.txt") + assert.NoError(t, err) + offset, err := obj.Seek(2, io.SeekStart) + assert.NoError(t, err) + assert.EqualValues(t, 2, offset) + buf1 := make([]byte, 3) + read, err := obj.Read(buf1) + assert.NoError(t, err) + assert.Equal(t, 3, read) + assert.Equal(t, data[2:5], string(buf1)) + offset, err = obj.Seek(-5, io.SeekEnd) + assert.NoError(t, err) + assert.EqualValues(t, len(data)-5, offset) + buf2 := make([]byte, 4) + read, err = obj.Read(buf2) + assert.NoError(t, err) + assert.Equal(t, 4, read) + assert.Equal(t, data[11:15], string(buf2)) + assert.NoError(t, obj.Close()) + assert.NoError(t, s.Delete("test.txt")) +} diff --git a/modules/storage/helper.go b/modules/storage/helper.go index 9e6cceb537..f6c3d5eebb 100644 --- a/modules/storage/helper.go +++ b/modules/storage/helper.go @@ -30,7 +30,7 @@ func (s discardStorage) Delete(_ string) error { return fmt.Errorf("%s", s) } -func (s discardStorage) URL(_, _ string, _ url.Values) (*url.URL, error) { +func (s discardStorage) URL(_, _, _ string, _ url.Values) (*url.URL, error) { return nil, fmt.Errorf("%s", s) } diff --git a/modules/storage/helper_test.go b/modules/storage/helper_test.go index 62ebd8753c..3cba1e13c0 100644 --- a/modules/storage/helper_test.go +++ b/modules/storage/helper_test.go @@ -37,7 +37,7 @@ func Test_discardStorage(t *testing.T) { assert.Error(t, err, string(tt)) } { - got, err := tt.URL("path", "name", nil) + got, err := tt.URL("path", "name", "GET", nil) assert.Nil(t, got) assert.Errorf(t, err, string(tt)) } diff --git a/modules/storage/local.go b/modules/storage/local.go index 00c7f668aa..8a1776f606 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -114,7 +114,7 @@ func (l *LocalStorage) Delete(path string) error { } // URL gets the redirect URL to a file -func (l *LocalStorage) URL(path, name string, reqParams url.Values) (*url.URL, error) { +func (l *LocalStorage) URL(path, name, _ string, reqParams url.Values) (*url.URL, error) { return nil, ErrURLNotSupported } diff --git a/modules/storage/local_test.go b/modules/storage/local_test.go index e230323f67..0592fd716b 100644 --- a/modules/storage/local_test.go +++ b/modules/storage/local_test.go @@ -4,8 +4,6 @@ package storage import ( - "os" - "path/filepath" "testing" "code.gitea.io/gitea/modules/setting" @@ -50,12 +48,11 @@ func TestBuildLocalPath(t *testing.T) { t.Run(k.path, func(t *testing.T) { l := LocalStorage{dir: k.localDir} - assert.EqualValues(t, k.expected, l.buildLocalPath(k.path)) + assert.Equal(t, k.expected, l.buildLocalPath(k.path)) }) } } func TestLocalStorageIterator(t *testing.T) { - dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir") - testStorageIterator(t, setting.LocalStorageType, &setting.Storage{Path: dir}) + testStorageIterator(t, setting.LocalStorageType, &setting.Storage{Path: t.TempDir()}) } diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 6b92be61fb..01f2c16267 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -86,13 +86,14 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath) var lookup minio.BucketLookupType - if config.BucketLookUpType == "auto" || config.BucketLookUpType == "" { + switch config.BucketLookUpType { + case "auto", "": lookup = minio.BucketLookupAuto - } else if config.BucketLookUpType == "dns" { + case "dns": lookup = minio.BucketLookupDNS - } else if config.BucketLookUpType == "path" { + case "path": lookup = minio.BucketLookupPath - } else { + default: return nil, fmt.Errorf("invalid minio bucket lookup type: %s", config.BucketLookUpType) } @@ -278,7 +279,7 @@ func (m *MinioStorage) Delete(path string) error { } // URL gets the redirect URL to a file. The presigned link is valid for 5 minutes. -func (m *MinioStorage) URL(path, name string, serveDirectReqParams url.Values) (*url.URL, error) { +func (m *MinioStorage) URL(path, name, method string, serveDirectReqParams url.Values) (*url.URL, error) { // copy serveDirectReqParams reqParams, err := url.ParseQuery(serveDirectReqParams.Encode()) if err != nil { @@ -286,7 +287,12 @@ func (m *MinioStorage) URL(path, name string, serveDirectReqParams url.Values) ( } // TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we? reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"") - u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams) + expires := 5 * time.Minute + if method == http.MethodHead { + u, err := m.client.PresignedHeadObject(m.ctx, m.bucket, m.buildMinioPath(path), expires, reqParams) + return u, convertMinioErr(err) + } + u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), expires, reqParams) return u, convertMinioErr(err) } diff --git a/modules/storage/minio_test.go b/modules/storage/minio_test.go index 395da051e8..2726d765dd 100644 --- a/modules/storage/minio_test.go +++ b/modules/storage/minio_test.go @@ -34,19 +34,19 @@ func TestMinioStorageIterator(t *testing.T) { func TestMinioStoragePath(t *testing.T) { m := &MinioStorage{basePath: ""} - assert.Equal(t, "", m.buildMinioPath("/")) - assert.Equal(t, "", m.buildMinioPath(".")) + assert.Empty(t, m.buildMinioPath("/")) + assert.Empty(t, m.buildMinioPath(".")) assert.Equal(t, "a", m.buildMinioPath("/a")) assert.Equal(t, "a/b", m.buildMinioPath("/a/b/")) - assert.Equal(t, "", m.buildMinioDirPrefix("")) + assert.Empty(t, m.buildMinioDirPrefix("")) assert.Equal(t, "a/", m.buildMinioDirPrefix("/a/")) m = &MinioStorage{basePath: "/"} - assert.Equal(t, "", m.buildMinioPath("/")) - assert.Equal(t, "", m.buildMinioPath(".")) + assert.Empty(t, m.buildMinioPath("/")) + assert.Empty(t, m.buildMinioPath(".")) assert.Equal(t, "a", m.buildMinioPath("/a")) assert.Equal(t, "a/b", m.buildMinioPath("/a/b/")) - assert.Equal(t, "", m.buildMinioDirPrefix("")) + assert.Empty(t, m.buildMinioDirPrefix("")) assert.Equal(t, "a/", m.buildMinioDirPrefix("/a/")) m = &MinioStorage{basePath: "/base"} diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 750ecdfe0d..1868817c05 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -59,11 +59,15 @@ type Object interface { // ObjectStorage represents an object storage to handle a bucket and files type ObjectStorage interface { Open(path string) (Object, error) - // Save store a object, if size is unknown set -1 + + // Save store an object, if size is unknown set -1 + // NOTICE: Some storage SDK will close the Reader after saving if it is also a Closer, + // DO NOT use the reader anymore after Save, or wrap it to a non-Closer reader. Save(path string, r io.Reader, size int64) (int64, error) + Stat(path string) (os.FileInfo, error) Delete(path string) error - URL(path, name string, reqParams url.Values) (*url.URL, error) + URL(path, name, method string, reqParams url.Values) (*url.URL, error) IterateObjects(path string, iterator func(path string, obj Object) error) error } @@ -93,7 +97,7 @@ func Clean(storage ObjectStorage) error { } // SaveFrom saves data to the ObjectStorage with path p from the callback -func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) error) error { +func SaveFrom(objStorage ObjectStorage, path string, callback func(w io.Writer) error) error { pr, pw := io.Pipe() defer pr.Close() go func() { @@ -103,7 +107,7 @@ func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) err } }() - _, err := objStorage.Save(p, pr, -1) + _, err := objStorage.Save(path, pr, -1) return err } diff --git a/modules/storage/storage_test.go b/modules/storage/storage_test.go index 7edde558f3..08f274e74b 100644 --- a/modules/storage/storage_test.go +++ b/modules/storage/storage_test.go @@ -4,7 +4,7 @@ package storage import ( - "bytes" + "strings" "testing" "code.gitea.io/gitea/modules/setting" @@ -26,7 +26,7 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) { {"b/x 4.txt", "bx4"}, } for _, f := range testFiles { - _, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1) + _, err = l.Save(f[0], strings.NewReader(f[1]), -1) assert.NoError(t, err) } diff --git a/modules/structs/admin_user.go b/modules/structs/admin_user.go index f7c6d10ba0..c68b59a897 100644 --- a/modules/structs/admin_user.go +++ b/modules/structs/admin_user.go @@ -8,8 +8,11 @@ import "time" // CreateUserOption create user options type CreateUserOption struct { - SourceID int64 `json:"source_id"` + SourceID int64 `json:"source_id"` + // identifier of the user, provided by the external authenticator (if configured) + // default: empty LoginName string `json:"login_name"` + // username of the user // required: true Username string `json:"username" binding:"Required;Username;MaxSize(40)"` FullName string `json:"full_name" binding:"MaxSize(100)"` @@ -32,6 +35,8 @@ type CreateUserOption struct { type EditUserOption struct { // required: true SourceID int64 `json:"source_id"` + // identifier of the user, provided by the external authenticator (if configured) + // default: empty // required: true LoginName string `json:"login_name" binding:"Required"` // swagger:strfmt email diff --git a/modules/structs/commit_status_test.go b/modules/structs/commit_status_test.go deleted file mode 100644 index f06808534c..0000000000 --- a/modules/structs/commit_status_test.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package structs - -import ( - "testing" -) - -func TestNoBetterThan(t *testing.T) { - type args struct { - css CommitStatusState - css2 CommitStatusState - } - var unExpectedState CommitStatusState - tests := []struct { - name string - args args - want bool - }{ - { - name: "success is no better than success", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "success is no better than pending", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusPending, - }, - want: false, - }, - { - name: "success is no better than failure", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusFailure, - }, - want: false, - }, - { - name: "success is no better than error", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusError, - }, - want: false, - }, - { - name: "pending is no better than success", - args: args{ - css: CommitStatusPending, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "pending is no better than pending", - args: args{ - css: CommitStatusPending, - css2: CommitStatusPending, - }, - want: true, - }, - { - name: "pending is no better than failure", - args: args{ - css: CommitStatusPending, - css2: CommitStatusFailure, - }, - want: false, - }, - { - name: "pending is no better than error", - args: args{ - css: CommitStatusPending, - css2: CommitStatusError, - }, - want: false, - }, - { - name: "failure is no better than success", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "failure is no better than pending", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusPending, - }, - want: true, - }, - { - name: "failure is no better than failure", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusFailure, - }, - want: true, - }, - { - name: "failure is no better than error", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusError, - }, - want: false, - }, - { - name: "error is no better than success", - args: args{ - css: CommitStatusError, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "error is no better than pending", - args: args{ - css: CommitStatusError, - css2: CommitStatusPending, - }, - want: true, - }, - { - name: "error is no better than failure", - args: args{ - css: CommitStatusError, - css2: CommitStatusFailure, - }, - want: true, - }, - { - name: "error is no better than error", - args: args{ - css: CommitStatusError, - css2: CommitStatusError, - }, - want: true, - }, - { - name: "unExpectedState is no better than success", - args: args{ - css: unExpectedState, - css2: CommitStatusSuccess, - }, - want: false, - }, - { - name: "unExpectedState is no better than unExpectedState", - args: args{ - css: unExpectedState, - css2: unExpectedState, - }, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := tt.args.css.NoBetterThan(tt.args.css2) - if result != tt.want { - t.Errorf("NoBetterThan() = %v, want %v", result, tt.want) - } - }) - } -} diff --git a/modules/structs/git_blob.go b/modules/structs/git_blob.go index 96c7a271a9..643b69ed37 100644 --- a/modules/structs/git_blob.go +++ b/modules/structs/git_blob.go @@ -5,9 +5,12 @@ package structs // GitBlobResponse represents a git blob type GitBlobResponse struct { - Content string `json:"content"` - Encoding string `json:"encoding"` - URL string `json:"url"` - SHA string `json:"sha"` - Size int64 `json:"size"` + Content *string `json:"content"` + Encoding *string `json:"encoding"` + URL string `json:"url"` + SHA string `json:"sha"` + Size int64 `json:"size"` + + LfsOid *string `json:"lfs_oid,omitempty"` + LfsSize *int64 `json:"lfs_size,omitempty"` } diff --git a/modules/structs/hook.go b/modules/structs/hook.go index ce5742e5c7..ac779a5740 100644 --- a/modules/structs/hook.go +++ b/modules/structs/hook.go @@ -71,7 +71,8 @@ type PayloadUser struct { // Full name of the commit author Name string `json:"name"` // swagger:strfmt email - Email string `json:"email"` + Email string `json:"email"` + // username of the user UserName string `json:"username"` } @@ -116,14 +117,7 @@ var ( _ Payloader = &PackagePayload{} ) -// _________ __ -// \_ ___ \_______ ____ _____ _/ |_ ____ -// / \ \/\_ __ \_/ __ \\__ \\ __\/ __ \ -// \ \____| | \/\ ___/ / __ \| | \ ___/ -// \______ /|__| \___ >____ /__| \___ > -// \/ \/ \/ \/ - -// CreatePayload FIXME +// CreatePayload represents a payload information of create event. type CreatePayload struct { Sha string `json:"sha"` Ref string `json:"ref"` @@ -157,13 +151,6 @@ func ParseCreateHook(raw []byte) (*CreatePayload, error) { return hook, nil } -// ________ .__ __ -// \______ \ ____ | | _____/ |_ ____ -// | | \_/ __ \| | _/ __ \ __\/ __ \ -// | ` \ ___/| |_\ ___/| | \ ___/ -// /_______ /\___ >____/\___ >__| \___ > -// \/ \/ \/ \/ - // PusherType define the type to push type PusherType string @@ -186,13 +173,6 @@ func (p *DeletePayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } -// ___________ __ -// \_ _____/__________| | __ -// | __)/ _ \_ __ \ |/ / -// | \( <_> ) | \/ < -// \___ / \____/|__| |__|_ \ -// \/ \/ - // ForkPayload represents fork payload type ForkPayload struct { Forkee *Repository `json:"forkee"` @@ -232,13 +212,6 @@ func (p *IssueCommentPayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } -// __________ .__ -// \______ \ ____ | | ____ _____ ______ ____ -// | _// __ \| | _/ __ \\__ \ / ___// __ \ -// | | \ ___/| |_\ ___/ / __ \_\___ \\ ___/ -// |____|_ /\___ >____/\___ >____ /____ >\___ > -// \/ \/ \/ \/ \/ \/ - // HookReleaseAction defines hook release action type type HookReleaseAction string @@ -302,13 +275,6 @@ func (p *PushPayload) Branch() string { return strings.ReplaceAll(p.Ref, "refs/heads/", "") } -// .___ -// | | ______ ________ __ ____ -// | |/ ___// ___/ | \_/ __ \ -// | |\___ \ \___ \| | /\ ___/ -// |___/____ >____ >____/ \___ > -// \/ \/ \/ - // HookIssueAction FIXME type HookIssueAction string @@ -321,6 +287,8 @@ const ( HookIssueReOpened HookIssueAction = "reopened" // HookIssueEdited edited HookIssueEdited HookIssueAction = "edited" + // HookIssueDeleted is an issue action for deleting an issue + HookIssueDeleted HookIssueAction = "deleted" // HookIssueAssigned assigned HookIssueAssigned HookIssueAction = "assigned" // HookIssueUnassigned unassigned @@ -371,13 +339,6 @@ type ChangesPayload struct { Ref *ChangesFromPayload `json:"ref,omitempty"` } -// __________ .__ .__ __________ __ -// \______ \__ __| | | | \______ \ ____ ________ __ ____ _______/ |_ -// | ___/ | \ | | | | _// __ \/ ____/ | \_/ __ \ / ___/\ __\ -// | | | | / |_| |__ | | \ ___< <_| | | /\ ___/ \___ \ | | -// |____| |____/|____/____/ |____|_ /\___ >__ |____/ \___ >____ > |__| -// \/ \/ |__| \/ \/ - // PullRequestPayload represents a payload information of pull request event. type PullRequestPayload struct { Action HookIssueAction `json:"action"` @@ -402,13 +363,6 @@ type ReviewPayload struct { Content string `json:"content"` } -// __ __.__ __ .__ -// / \ / \__| | _|__| -// \ \/\/ / | |/ / | -// \ /| | <| | -// \__/\ / |__|__|_ \__| -// \/ \/ - // HookWikiAction an action that happens to a wiki page type HookWikiAction string @@ -435,13 +389,6 @@ func (p *WikiPayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } -//__________ .__ __ -//\______ \ ____ ______ ____ _____|__|/ |_ ___________ ___.__. -// | _// __ \\____ \ / _ \/ ___/ \ __\/ _ \_ __ < | | -// | | \ ___/| |_> > <_> )___ \| || | ( <_> ) | \/\___ | -// |____|_ /\___ > __/ \____/____ >__||__| \____/|__| / ____| -// \/ \/|__| \/ \/ - // HookRepoAction an action that happens to a repo type HookRepoAction string @@ -480,7 +427,7 @@ type PackagePayload struct { Action HookPackageAction `json:"action"` Repository *Repository `json:"repository"` Package *Package `json:"package"` - Organization *User `json:"organization"` + Organization *Organization `json:"organization"` Sender *User `json:"sender"` } @@ -525,3 +472,34 @@ type CommitStatusPayload struct { func (p *CommitStatusPayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } + +// WorkflowRunPayload represents a payload information of workflow run event. +type WorkflowRunPayload struct { + Action string `json:"action"` + Workflow *ActionWorkflow `json:"workflow"` + WorkflowRun *ActionWorkflowRun `json:"workflow_run"` + PullRequest *PullRequest `json:"pull_request,omitempty"` + Organization *Organization `json:"organization,omitempty"` + Repo *Repository `json:"repository"` + Sender *User `json:"sender"` +} + +// JSONPayload implements Payload +func (p *WorkflowRunPayload) JSONPayload() ([]byte, error) { + return json.MarshalIndent(p, "", " ") +} + +// WorkflowJobPayload represents a payload information of workflow job event. +type WorkflowJobPayload struct { + Action string `json:"action"` + WorkflowJob *ActionWorkflowJob `json:"workflow_job"` + PullRequest *PullRequest `json:"pull_request,omitempty"` + Organization *Organization `json:"organization,omitempty"` + Repo *Repository `json:"repository"` + Sender *User `json:"sender"` +} + +// JSONPayload implements Payload +func (p *WorkflowJobPayload) JSONPayload() ([]byte, error) { + return json.MarshalIndent(p, "", " ") +} diff --git a/modules/structs/issue.go b/modules/structs/issue.go index 3682191be5..322ac1e4ca 100644 --- a/modules/structs/issue.go +++ b/modules/structs/issue.go @@ -203,7 +203,7 @@ func (l *IssueTemplateStringSlice) UnmarshalYAML(value *yaml.Node) error { if err != nil { return err } - for _, v := range strings.Split(str, ",") { + for v := range strings.SplitSeq(str, ",") { if v = strings.TrimSpace(v); v == "" { continue } @@ -262,7 +262,13 @@ func (it IssueTemplate) Type() IssueTemplateType { // IssueMeta basic issue information // swagger:model type IssueMeta struct { - Index int64 `json:"index"` + Index int64 `json:"index"` + // owner of the issue's repo Owner string `json:"owner"` Name string `json:"repo"` } + +// LockIssueOption options to lock an issue +type LockIssueOption struct { + Reason string `json:"lock_reason"` +} diff --git a/modules/structs/issue_tracked_time.go b/modules/structs/issue_tracked_time.go index a3904af80e..befcfb323d 100644 --- a/modules/structs/issue_tracked_time.go +++ b/modules/structs/issue_tracked_time.go @@ -14,7 +14,7 @@ type AddTimeOption struct { Time int64 `json:"time" binding:"Required"` // swagger:strfmt date-time Created time.Time `json:"created"` - // User who spent the time (optional) + // username of the user who spent the time working on the issue (optional) User string `json:"user_name"` } @@ -26,7 +26,8 @@ type TrackedTime struct { // Time in seconds Time int64 `json:"time"` // deprecated (only for backwards compatibility) - UserID int64 `json:"user_id"` + UserID int64 `json:"user_id"` + // username of the user UserName string `json:"user_name"` // deprecated (only for backwards compatibility) IssueID int64 `json:"issue_id"` diff --git a/modules/structs/org.go b/modules/structs/org.go index c0a545ac1c..33b45c6344 100644 --- a/modules/structs/org.go +++ b/modules/structs/org.go @@ -15,6 +15,7 @@ type Organization struct { Location string `json:"location"` Visibility string `json:"visibility"` RepoAdminChangeTeamAccess bool `json:"repo_admin_change_team_access"` + // username of the organization // deprecated UserName string `json:"username"` } @@ -30,6 +31,7 @@ type OrganizationPermissions struct { // CreateOrgOption options for creating an organization type CreateOrgOption struct { + // username of the organization // required: true UserName string `json:"username" binding:"Required;Username;MaxSize(40)"` FullName string `json:"full_name" binding:"MaxSize(100)"` @@ -57,3 +59,12 @@ type EditOrgOption struct { Visibility string `json:"visibility" binding:"In(,public,limited,private)"` RepoAdminChangeTeamAccess *bool `json:"repo_admin_change_team_access"` } + +// RenameOrgOption options when renaming an organization +type RenameOrgOption struct { + // New username for this org. This name cannot be in use yet by any other user. + // + // required: true + // unique: true + NewName string `json:"new_name" binding:"Required"` +} diff --git a/modules/structs/package.go b/modules/structs/package.go index a9a9429de2..1973f925a5 100644 --- a/modules/structs/package.go +++ b/modules/structs/package.go @@ -23,8 +23,8 @@ type Package struct { // PackageFile represents a package file type PackageFile struct { - ID int64 `json:"id"` - Size int64 + ID int64 `json:"id"` + Size int64 `json:"size"` Name string `json:"name"` HashMD5 string `json:"md5"` HashSHA1 string `json:"sha1"` diff --git a/modules/structs/pull.go b/modules/structs/pull.go index 55831e642c..f53d6adafc 100644 --- a/modules/structs/pull.go +++ b/modules/structs/pull.go @@ -25,11 +25,13 @@ type PullRequest struct { Draft bool `json:"draft"` IsLocked bool `json:"is_locked"` Comments int `json:"comments"` + // number of review comments made on the diff of a PR review (not including comments on commits or issues in a PR) - ReviewComments int `json:"review_comments"` - Additions int `json:"additions"` - Deletions int `json:"deletions"` - ChangedFiles int `json:"changed_files"` + ReviewComments int `json:"review_comments,omitempty"` + + Additions *int `json:"additions,omitempty"` + Deletions *int `json:"deletions,omitempty"` + ChangedFiles *int `json:"changed_files,omitempty"` HTMLURL string `json:"html_url"` DiffURL string `json:"diff_url"` diff --git a/modules/structs/release.go b/modules/structs/release.go index c7378645c2..fac86ca7a2 100644 --- a/modules/structs/release.go +++ b/modules/structs/release.go @@ -33,6 +33,7 @@ type Release struct { type CreateReleaseOption struct { // required: true TagName string `json:"tag_name" binding:"Required"` + TagMessage string `json:"tag_message"` Target string `json:"target_commitish"` Title string `json:"name"` Note string `json:"body"` diff --git a/modules/structs/repo.go b/modules/structs/repo.go index fb784bd8b3..404718def0 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -48,16 +48,17 @@ type ExternalWiki struct { // Repository represents a repository type Repository struct { - ID int64 `json:"id"` - Owner *User `json:"owner"` - Name string `json:"name"` - FullName string `json:"full_name"` - Description string `json:"description"` - Empty bool `json:"empty"` - Private bool `json:"private"` - Fork bool `json:"fork"` - Template bool `json:"template"` - Parent *Repository `json:"parent"` + ID int64 `json:"id"` + Owner *User `json:"owner"` + Name string `json:"name"` + FullName string `json:"full_name"` + Description string `json:"description"` + Empty bool `json:"empty"` + Private bool `json:"private"` + Fork bool `json:"fork"` + Template bool `json:"template"` + // the original repository if this repository is a fork, otherwise null + Parent *Repository `json:"parent,omitempty"` Mirror bool `json:"mirror"` Size int `json:"size"` Language string `json:"language"` @@ -83,6 +84,7 @@ type Repository struct { Updated time.Time `json:"updated_at"` ArchivedAt time.Time `json:"archived_at"` Permissions *Permission `json:"permissions,omitempty"` + HasCode bool `json:"has_code"` HasIssues bool `json:"has_issues"` InternalTracker *InternalTracker `json:"internal_tracker,omitempty"` ExternalTracker *ExternalTracker `json:"external_tracker,omitempty"` @@ -101,6 +103,8 @@ type Repository struct { AllowSquash bool `json:"allow_squash_merge"` AllowFastForwardOnly bool `json:"allow_fast_forward_only_merge"` AllowRebaseUpdate bool `json:"allow_rebase_update"` + AllowManualMerge bool `json:"allow_manual_merge"` + AutodetectManualMerge bool `json:"autodetect_manual_merge"` DefaultDeleteBranchAfterMerge bool `json:"default_delete_branch_after_merge"` DefaultMergeStyle string `json:"default_merge_style"` DefaultAllowMaintainerEdit bool `json:"default_allow_maintainer_edit"` @@ -111,8 +115,8 @@ type Repository struct { // enum: sha1,sha256 ObjectFormatName string `json:"object_format_name"` // swagger:strfmt date-time - MirrorUpdated time.Time `json:"mirror_updated,omitempty"` - RepoTransfer *RepoTransfer `json:"repo_transfer"` + MirrorUpdated time.Time `json:"mirror_updated"` + RepoTransfer *RepoTransfer `json:"repo_transfer,omitempty"` Topics []string `json:"topics"` Licenses []string `json:"licenses"` } @@ -167,6 +171,8 @@ type EditRepoOption struct { Private *bool `json:"private,omitempty"` // either `true` to make this repository a template or `false` to make it a normal repository Template *bool `json:"template,omitempty"` + // either `true` to enable code for this repository or `false` to disable it. + HasCode *bool `json:"has_code,omitempty"` // either `true` to enable issues for this repository or `false` to disable them. HasIssues *bool `json:"has_issues,omitempty"` // set this structure to configure internal issue tracker @@ -223,15 +229,13 @@ type EditRepoOption struct { EnablePrune *bool `json:"enable_prune,omitempty"` } -// GenerateRepoOption options when creating repository using a template +// GenerateRepoOption options when creating a repository using a template // swagger:model type GenerateRepoOption struct { - // The organization or person who will own the new repository + // the organization's name or individual user's name who will own the new repository // // required: true Owner string `json:"owner"` - // Name of the repository to create - // // required: true // unique: true Name string `json:"name" binding:"Required;AlphaDashDot;MaxSize(100)"` @@ -350,14 +354,14 @@ func (gt GitServiceType) Title() string { type MigrateRepoOptions struct { // required: true CloneAddr string `json:"clone_addr" binding:"Required"` - // deprecated (only for backwards compatibility) + // deprecated (only for backwards compatibility, use repo_owner instead) RepoOwnerID int64 `json:"uid"` - // Name of User or Organisation who will own Repo after migration + // the organization's name or individual user's name who will own the migrated repository RepoOwner string `json:"repo_owner"` // required: true RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"` - // enum: git,github,gitea,gitlab,gogs,onedev,gitbucket,codebase + // enum: git,github,gitea,gitlab,gogs,onedev,gitbucket,codebase,codecommit Service string `json:"service"` AuthUsername string `json:"auth_username"` AuthPassword string `json:"auth_password"` diff --git a/modules/structs/repo_actions.go b/modules/structs/repo_actions.go index b13f344738..ac1c288270 100644 --- a/modules/structs/repo_actions.go +++ b/modules/structs/repo_actions.go @@ -32,3 +32,157 @@ type ActionTaskResponse struct { Entries []*ActionTask `json:"workflow_runs"` TotalCount int64 `json:"total_count"` } + +// CreateActionWorkflowDispatch represents the payload for triggering a workflow dispatch event +// swagger:model +type CreateActionWorkflowDispatch struct { + // required: true + // example: refs/heads/main + Ref string `json:"ref" binding:"Required"` + // required: false + Inputs map[string]string `json:"inputs,omitempty"` +} + +// ActionWorkflow represents a ActionWorkflow +type ActionWorkflow struct { + ID string `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + State string `json:"state"` + // swagger:strfmt date-time + CreatedAt time.Time `json:"created_at"` + // swagger:strfmt date-time + UpdatedAt time.Time `json:"updated_at"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + BadgeURL string `json:"badge_url"` + // swagger:strfmt date-time + DeletedAt time.Time `json:"deleted_at"` +} + +// ActionWorkflowResponse returns a ActionWorkflow +type ActionWorkflowResponse struct { + Workflows []*ActionWorkflow `json:"workflows"` + TotalCount int64 `json:"total_count"` +} + +// ActionArtifact represents a ActionArtifact +type ActionArtifact struct { + ID int64 `json:"id"` + Name string `json:"name"` + SizeInBytes int64 `json:"size_in_bytes"` + URL string `json:"url"` + ArchiveDownloadURL string `json:"archive_download_url"` + Expired bool `json:"expired"` + WorkflowRun *ActionWorkflowRun `json:"workflow_run"` + + // swagger:strfmt date-time + CreatedAt time.Time `json:"created_at"` + // swagger:strfmt date-time + UpdatedAt time.Time `json:"updated_at"` + // swagger:strfmt date-time + ExpiresAt time.Time `json:"expires_at"` +} + +// ActionWorkflowRun represents a WorkflowRun +type ActionWorkflowRun struct { + ID int64 `json:"id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + DisplayTitle string `json:"display_title"` + Path string `json:"path"` + Event string `json:"event"` + RunAttempt int64 `json:"run_attempt"` + RunNumber int64 `json:"run_number"` + RepositoryID int64 `json:"repository_id,omitempty"` + HeadSha string `json:"head_sha"` + HeadBranch string `json:"head_branch,omitempty"` + Status string `json:"status"` + Actor *User `json:"actor,omitempty"` + TriggerActor *User `json:"trigger_actor,omitempty"` + Repository *Repository `json:"repository,omitempty"` + HeadRepository *Repository `json:"head_repository,omitempty"` + Conclusion string `json:"conclusion,omitempty"` + // swagger:strfmt date-time + StartedAt time.Time `json:"started_at"` + // swagger:strfmt date-time + CompletedAt time.Time `json:"completed_at"` +} + +// ActionWorkflowRunsResponse returns ActionWorkflowRuns +type ActionWorkflowRunsResponse struct { + Entries []*ActionWorkflowRun `json:"workflow_runs"` + TotalCount int64 `json:"total_count"` +} + +// ActionWorkflowJobsResponse returns ActionWorkflowJobs +type ActionWorkflowJobsResponse struct { + Entries []*ActionWorkflowJob `json:"jobs"` + TotalCount int64 `json:"total_count"` +} + +// ActionArtifactsResponse returns ActionArtifacts +type ActionArtifactsResponse struct { + Entries []*ActionArtifact `json:"artifacts"` + TotalCount int64 `json:"total_count"` +} + +// ActionWorkflowStep represents a step of a WorkflowJob +type ActionWorkflowStep struct { + Name string `json:"name"` + Number int64 `json:"number"` + Status string `json:"status"` + Conclusion string `json:"conclusion,omitempty"` + // swagger:strfmt date-time + StartedAt time.Time `json:"started_at"` + // swagger:strfmt date-time + CompletedAt time.Time `json:"completed_at"` +} + +// ActionWorkflowJob represents a WorkflowJob +type ActionWorkflowJob struct { + ID int64 `json:"id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + RunID int64 `json:"run_id"` + RunURL string `json:"run_url"` + Name string `json:"name"` + Labels []string `json:"labels"` + RunAttempt int64 `json:"run_attempt"` + HeadSha string `json:"head_sha"` + HeadBranch string `json:"head_branch,omitempty"` + Status string `json:"status"` + Conclusion string `json:"conclusion,omitempty"` + RunnerID int64 `json:"runner_id,omitempty"` + RunnerName string `json:"runner_name,omitempty"` + Steps []*ActionWorkflowStep `json:"steps"` + // swagger:strfmt date-time + CreatedAt time.Time `json:"created_at"` + // swagger:strfmt date-time + StartedAt time.Time `json:"started_at"` + // swagger:strfmt date-time + CompletedAt time.Time `json:"completed_at"` +} + +// ActionRunnerLabel represents a Runner Label +type ActionRunnerLabel struct { + ID int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` +} + +// ActionRunner represents a Runner +type ActionRunner struct { + ID int64 `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Busy bool `json:"busy"` + Ephemeral bool `json:"ephemeral"` + Labels []*ActionRunnerLabel `json:"labels"` +} + +// ActionRunnersResponse returns Runners +type ActionRunnersResponse struct { + Entries []*ActionRunner `json:"runners"` + TotalCount int64 `json:"total_count"` +} diff --git a/modules/structs/repo_branch.go b/modules/structs/repo_branch.go index a9aa1d330a..5416f43b0d 100644 --- a/modules/structs/repo_branch.go +++ b/modules/structs/repo_branch.go @@ -133,3 +133,12 @@ type EditBranchProtectionOption struct { type UpdateBranchProtectionPriories struct { IDs []int64 `json:"ids"` } + +type MergeUpstreamRequest struct { + Branch string `json:"branch"` + FfOnly bool `json:"ff_only"` +} + +type MergeUpstreamResponse struct { + MergeStyle string `json:"merge_type"` +} diff --git a/modules/structs/repo_file.go b/modules/structs/repo_file.go index 82bde96ab6..5a86db868b 100644 --- a/modules/structs/repo_file.go +++ b/modules/structs/repo_file.go @@ -4,6 +4,8 @@ package structs +import "time" + // FileOptions options for all file APIs type FileOptions struct { // message (optional) for the commit of this file. if not supplied, a default message will be used @@ -20,6 +22,23 @@ type FileOptions struct { Signoff bool `json:"signoff"` } +type FileOptionsWithSHA struct { + FileOptions + // the blob ID (SHA) for the file that already exists, it is required for changing existing files + // required: true + SHA string `json:"sha" binding:"Required"` +} + +func (f *FileOptions) GetFileOptions() *FileOptions { + return f +} + +type FileOptionsInterface interface { + GetFileOptions() *FileOptions +} + +var _ FileOptionsInterface = (*FileOptions)(nil) + // CreateFileOptions options for creating files // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type CreateFileOptions struct { @@ -29,29 +48,16 @@ type CreateFileOptions struct { ContentBase64 string `json:"content"` } -// Branch returns branch name -func (o *CreateFileOptions) Branch() string { - return o.FileOptions.BranchName -} - // DeleteFileOptions options for deleting files (used for other File structs below) // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type DeleteFileOptions struct { - FileOptions - // sha is the SHA for the file that already exists - // required: true - SHA string `json:"sha" binding:"Required"` -} - -// Branch returns branch name -func (o *DeleteFileOptions) Branch() string { - return o.FileOptions.BranchName + FileOptionsWithSHA } // UpdateFileOptions options for updating files // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type UpdateFileOptions struct { - DeleteFileOptions + FileOptionsWithSHA // content must be base64 encoded // required: true ContentBase64 string `json:"content"` @@ -59,23 +65,21 @@ type UpdateFileOptions struct { FromPath string `json:"from_path" binding:"MaxSize(500)"` } -// Branch returns branch name -func (o *UpdateFileOptions) Branch() string { - return o.FileOptions.BranchName -} +// FIXME: there is no LastCommitID in FileOptions, actually it should be an alternative to the SHA in ChangeFileOperation // ChangeFileOperation for creating, updating or deleting a file type ChangeFileOperation struct { - // indicates what to do with the file + // indicates what to do with the file: "create" for creating a new file, "update" for updating an existing file, + // "upload" for creating or updating a file, "rename" for renaming a file, and "delete" for deleting an existing file. // required: true - // enum: create,update,delete + // enum: create,update,upload,rename,delete Operation string `json:"operation" binding:"Required"` // path to the existing or new file // required: true Path string `json:"path" binding:"Required;MaxSize(500)"` - // new or updated file content, must be base64 encoded + // new or updated file content, it must be base64 encoded ContentBase64 string `json:"content"` - // sha is the SHA for the file that already exists, required for update or delete + // the blob ID (SHA) for the file that already exists, required for changing existing files SHA string `json:"sha"` // old path of the file to move FromPath string `json:"from_path"` @@ -90,20 +94,10 @@ type ChangeFilesOptions struct { Files []*ChangeFileOperation `json:"files" binding:"Required"` } -// Branch returns branch name -func (o *ChangeFilesOptions) Branch() string { - return o.FileOptions.BranchName -} - -// FileOptionInterface provides a unified interface for the different file options -type FileOptionInterface interface { - Branch() string -} - // ApplyDiffPatchFileOptions options for applying a diff patch // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type ApplyDiffPatchFileOptions struct { - DeleteFileOptions + FileOptions // required: true Content string `json:"content"` } @@ -115,12 +109,24 @@ type FileLinksResponse struct { HTMLURL *string `json:"html"` } +type ContentsExtResponse struct { + FileContents *ContentsResponse `json:"file_contents,omitempty"` + DirContents []*ContentsResponse `json:"dir_contents,omitempty"` +} + // ContentsResponse contains information about a repo's entry's (dir, file, symlink, submodule) metadata and content type ContentsResponse struct { - Name string `json:"name"` - Path string `json:"path"` - SHA string `json:"sha"` - LastCommitSHA string `json:"last_commit_sha"` + Name string `json:"name"` + Path string `json:"path"` + SHA string `json:"sha"` + + LastCommitSHA *string `json:"last_commit_sha,omitempty"` + // swagger:strfmt date-time + LastCommitterDate *time.Time `json:"last_committer_date,omitempty"` + // swagger:strfmt date-time + LastAuthorDate *time.Time `json:"last_author_date,omitempty"` + LastCommitMessage *string `json:"last_commit_message,omitempty"` + // `type` will be `file`, `dir`, `symlink`, or `submodule` Type string `json:"type"` Size int64 `json:"size"` @@ -137,6 +143,9 @@ type ContentsResponse struct { // `submodule_git_url` is populated when `type` is `submodule`, otherwise null SubmoduleGitURL *string `json:"submodule_git_url"` Links *FileLinksResponse `json:"_links"` + + LfsOid *string `json:"lfs_oid,omitempty"` + LfsSize *int64 `json:"lfs_size,omitempty"` } // FileCommitResponse contains information generated from a Git commit for a repo's file. @@ -170,3 +179,8 @@ type FileDeleteResponse struct { Commit *FileCommitResponse `json:"commit"` Verification *PayloadCommitVerification `json:"verification"` } + +// GetFilesOptions options for retrieving metadate and content of multiple files +type GetFilesOptions struct { + Files []string `json:"files" binding:"Required"` +} diff --git a/modules/structs/repo_tag.go b/modules/structs/repo_tag.go index 5722513f4f..bb8bfd10cb 100644 --- a/modules/structs/repo_tag.go +++ b/modules/structs/repo_tag.go @@ -11,8 +11,8 @@ type Tag struct { Message string `json:"message"` ID string `json:"id"` Commit *CommitMeta `json:"commit"` - ZipballURL string `json:"zipball_url"` - TarballURL string `json:"tarball_url"` + ZipballURL string `json:"zipball_url,omitempty"` + TarballURL string `json:"tarball_url,omitempty"` } // AnnotatedTag represents an annotated tag diff --git a/modules/structs/secret.go b/modules/structs/secret.go index a0673ca08c..2afb41ec43 100644 --- a/modules/structs/secret.go +++ b/modules/structs/secret.go @@ -10,6 +10,8 @@ import "time" type Secret struct { // the secret's name Name string `json:"name"` + // the secret's description + Description string `json:"description"` // swagger:strfmt date-time Created time.Time `json:"created_at"` } @@ -21,4 +23,9 @@ type CreateOrUpdateSecretOption struct { // // required: true Data string `json:"data" binding:"Required"` + + // Description of the secret to update + // + // required: false + Description string `json:"description"` } diff --git a/modules/structs/settings.go b/modules/structs/settings.go index e48b1a493d..59176210e6 100644 --- a/modules/structs/settings.go +++ b/modules/structs/settings.go @@ -26,6 +26,7 @@ type GeneralAPISettings struct { DefaultPagingNum int `json:"default_paging_num"` DefaultGitTreesPerPage int `json:"default_git_trees_per_page"` DefaultMaxBlobSize int64 `json:"default_max_blob_size"` + DefaultMaxResponseSize int64 `json:"default_max_response_size"` } // GeneralAttachmentSettings contains global Attachment settings exposed by API diff --git a/modules/structs/status.go b/modules/structs/status.go index c1d8b902ec..a9779541ff 100644 --- a/modules/structs/status.go +++ b/modules/structs/status.go @@ -5,17 +5,19 @@ package structs import ( "time" + + "code.gitea.io/gitea/modules/commitstatus" ) // CommitStatus holds a single status of a single Commit type CommitStatus struct { - ID int64 `json:"id"` - State CommitStatusState `json:"status"` - TargetURL string `json:"target_url"` - Description string `json:"description"` - URL string `json:"url"` - Context string `json:"context"` - Creator *User `json:"creator"` + ID int64 `json:"id"` + State commitstatus.CommitStatusState `json:"status"` + TargetURL string `json:"target_url"` + Description string `json:"description"` + URL string `json:"url"` + Context string `json:"context"` + Creator *User `json:"creator"` // swagger:strfmt date-time Created time.Time `json:"created_at"` // swagger:strfmt date-time @@ -24,19 +26,19 @@ type CommitStatus struct { // CombinedStatus holds the combined state of several statuses for a single commit type CombinedStatus struct { - State CommitStatusState `json:"state"` - SHA string `json:"sha"` - TotalCount int `json:"total_count"` - Statuses []*CommitStatus `json:"statuses"` - Repository *Repository `json:"repository"` - CommitURL string `json:"commit_url"` - URL string `json:"url"` + State commitstatus.CommitStatusState `json:"state"` + SHA string `json:"sha"` + TotalCount int `json:"total_count"` + Statuses []*CommitStatus `json:"statuses"` + Repository *Repository `json:"repository"` + CommitURL string `json:"commit_url"` + URL string `json:"url"` } // CreateStatusOption holds the information needed to create a new CommitStatus for a Commit type CreateStatusOption struct { - State CommitStatusState `json:"state"` - TargetURL string `json:"target_url"` - Description string `json:"description"` - Context string `json:"context"` + State commitstatus.CommitStatusState `json:"state"` + TargetURL string `json:"target_url"` + Description string `json:"description"` + Context string `json:"context"` } diff --git a/modules/structs/user.go b/modules/structs/user.go index 5ed677f239..89349cda2c 100644 --- a/modules/structs/user.go +++ b/modules/structs/user.go @@ -15,9 +15,9 @@ import ( type User struct { // the user's id ID int64 `json:"id"` - // the user's username + // login of the user, same as `username` UserName string `json:"login"` - // the user's authentication sign-in name. + // identifier of the user, provided by the external authenticator (if configured) // default: empty LoginName string `json:"login_name"` // The ID of the user's Authentication Source @@ -35,9 +35,9 @@ type User struct { // Is the user an administrator IsAdmin bool `json:"is_admin"` // swagger:strfmt date-time - LastLogin time.Time `json:"last_login,omitempty"` + LastLogin time.Time `json:"last_login"` // swagger:strfmt date-time - Created time.Time `json:"created,omitempty"` + Created time.Time `json:"created"` // Is user restricted Restricted bool `json:"restricted"` // Is user active diff --git a/modules/structs/user_app.go b/modules/structs/user_app.go index a7d2e28b41..15811ceb66 100644 --- a/modules/structs/user_app.go +++ b/modules/structs/user_app.go @@ -11,11 +11,13 @@ import ( // AccessToken represents an API access token. // swagger:response AccessToken type AccessToken struct { - ID int64 `json:"id"` - Name string `json:"name"` - Token string `json:"sha1"` - TokenLastEight string `json:"token_last_eight"` - Scopes []string `json:"scopes"` + ID int64 `json:"id"` + Name string `json:"name"` + Token string `json:"sha1"` + TokenLastEight string `json:"token_last_eight"` + Scopes []string `json:"scopes"` + Created time.Time `json:"created_at"` + Updated time.Time `json:"last_used_at"` } // AccessTokenList represents a list of API access token. @@ -23,9 +25,11 @@ type AccessToken struct { type AccessTokenList []*AccessToken // CreateAccessTokenOption options when create access token +// swagger:model CreateAccessTokenOption type CreateAccessTokenOption struct { // required: true - Name string `json:"name" binding:"Required"` + Name string `json:"name" binding:"Required"` + // example: ["all", "read:activitypub","read:issue", "write:misc", "read:notification", "read:organization", "read:package", "read:repository", "read:user"] Scopes []string `json:"scopes"` } diff --git a/modules/structs/user_email.go b/modules/structs/user_email.go index 9319667e8f..01895a0058 100644 --- a/modules/structs/user_email.go +++ b/modules/structs/user_email.go @@ -11,6 +11,7 @@ type Email struct { Verified bool `json:"verified"` Primary bool `json:"primary"` UserID int64 `json:"user_id"` + // username of the user UserName string `json:"username"` } diff --git a/modules/structs/user_gpgkey.go b/modules/structs/user_gpgkey.go index ff9b0aea1d..deae70de33 100644 --- a/modules/structs/user_gpgkey.go +++ b/modules/structs/user_gpgkey.go @@ -21,9 +21,9 @@ type GPGKey struct { CanCertify bool `json:"can_certify"` Verified bool `json:"verified"` // swagger:strfmt date-time - Created time.Time `json:"created_at,omitempty"` + Created time.Time `json:"created_at"` // swagger:strfmt date-time - Expires time.Time `json:"expires_at,omitempty"` + Expires time.Time `json:"expires_at"` } // GPGKeyEmail an email attached to a GPGKey diff --git a/modules/structs/user_key.go b/modules/structs/user_key.go index 08eed59a89..16225a852a 100644 --- a/modules/structs/user_key.go +++ b/modules/structs/user_key.go @@ -15,7 +15,8 @@ type PublicKey struct { Title string `json:"title,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` // swagger:strfmt date-time - Created time.Time `json:"created_at,omitempty"` + Created time.Time `json:"created_at"` + Updated time.Time `json:"last_used_at"` Owner *User `json:"user,omitempty"` ReadOnly bool `json:"read_only,omitempty"` KeyType string `json:"key_type,omitempty"` diff --git a/modules/structs/variable.go b/modules/structs/variable.go index cc846cf0ec..5198937303 100644 --- a/modules/structs/variable.go +++ b/modules/structs/variable.go @@ -10,6 +10,11 @@ type CreateVariableOption struct { // // required: true Value string `json:"value" binding:"Required"` + + // Description of the variable to create + // + // required: false + Description string `json:"description"` } // UpdateVariableOption the option when updating variable @@ -21,6 +26,11 @@ type UpdateVariableOption struct { // // required: true Value string `json:"value" binding:"Required"` + + // Description of the variable to update + // + // required: false + Description string `json:"description"` } // ActionVariable return value of the query API @@ -34,4 +44,6 @@ type ActionVariable struct { Name string `json:"name"` // the value of the variable Data string `json:"data"` + // the description of the variable + Description string `json:"description"` } diff --git a/modules/svg/processor.go b/modules/svg/processor.go index 82248fb0c1..4fcb11a57d 100644 --- a/modules/svg/processor.go +++ b/modules/svg/processor.go @@ -10,7 +10,7 @@ import ( "sync" ) -type normalizeVarsStruct struct { +type globalVarsStruct struct { reXMLDoc, reComment, reAttrXMLNs, @@ -18,26 +18,23 @@ type normalizeVarsStruct struct { reAttrClassPrefix *regexp.Regexp } -var ( - normalizeVars *normalizeVarsStruct - normalizeVarsOnce sync.Once -) +var globalVars = sync.OnceValue(func() *globalVarsStruct { + return &globalVarsStruct{ + reXMLDoc: regexp.MustCompile(`(?s)<\?xml.*?>`), + reComment: regexp.MustCompile(`(?s)<!--.*?-->`), + + reAttrXMLNs: regexp.MustCompile(`(?s)\s+xmlns\s*=\s*"[^"]*"`), + reAttrSize: regexp.MustCompile(`(?s)\s+(width|height)\s*=\s*"[^"]+"`), + reAttrClassPrefix: regexp.MustCompile(`(?s)\s+class\s*=\s*"`), + } +}) // Normalize normalizes the SVG content: set default width/height, remove unnecessary tags/attributes // It's designed to work with valid SVG content. For invalid SVG content, the returned content is not guaranteed. func Normalize(data []byte, size int) []byte { - normalizeVarsOnce.Do(func() { - normalizeVars = &normalizeVarsStruct{ - reXMLDoc: regexp.MustCompile(`(?s)<\?xml.*?>`), - reComment: regexp.MustCompile(`(?s)<!--.*?-->`), - - reAttrXMLNs: regexp.MustCompile(`(?s)\s+xmlns\s*=\s*"[^"]*"`), - reAttrSize: regexp.MustCompile(`(?s)\s+(width|height)\s*=\s*"[^"]+"`), - reAttrClassPrefix: regexp.MustCompile(`(?s)\s+class\s*=\s*"`), - } - }) - data = normalizeVars.reXMLDoc.ReplaceAll(data, nil) - data = normalizeVars.reComment.ReplaceAll(data, nil) + vars := globalVars() + data = vars.reXMLDoc.ReplaceAll(data, nil) + data = vars.reComment.ReplaceAll(data, nil) data = bytes.TrimSpace(data) svgTag, svgRemaining, ok := bytes.Cut(data, []byte(">")) @@ -45,9 +42,9 @@ func Normalize(data []byte, size int) []byte { return data } normalized := bytes.Clone(svgTag) - normalized = normalizeVars.reAttrXMLNs.ReplaceAll(normalized, nil) - normalized = normalizeVars.reAttrSize.ReplaceAll(normalized, nil) - normalized = normalizeVars.reAttrClassPrefix.ReplaceAll(normalized, []byte(` class="`)) + normalized = vars.reAttrXMLNs.ReplaceAll(normalized, nil) + normalized = vars.reAttrSize.ReplaceAll(normalized, nil) + normalized = vars.reAttrClassPrefix.ReplaceAll(normalized, []byte(` class="`)) normalized = bytes.TrimSpace(normalized) normalized = fmt.Appendf(normalized, ` width="%d" height="%d"`, size, size) if !bytes.Contains(normalized, []byte(` class="`)) { diff --git a/modules/system/appstate_test.go b/modules/system/appstate_test.go index d4b9e167c2..b5c057cf88 100644 --- a/modules/system/appstate_test.go +++ b/modules/system/appstate_test.go @@ -13,9 +13,7 @@ import ( ) func TestMain(m *testing.M) { - unittest.MainTest(m, &unittest.TestOptions{ - FixtureFiles: []string{""}, // load nothing - }) + unittest.MainTest(m, &unittest.TestOptions{FixtureFiles: []string{ /* load nothing */ }}) } type testItem1 struct { @@ -36,14 +34,12 @@ func (*testItem2) Name() string { } func TestAppStateDB(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - as := &DBStore{} item1 := new(testItem1) assert.NoError(t, as.Get(db.DefaultContext, item1)) - assert.Equal(t, "", item1.Val1) - assert.EqualValues(t, 0, item1.Val2) + assert.Empty(t, item1.Val1) + assert.Equal(t, 0, item1.Val2) item1 = new(testItem1) item1.Val1 = "a" @@ -57,7 +53,7 @@ func TestAppStateDB(t *testing.T) { item1 = new(testItem1) assert.NoError(t, as.Get(db.DefaultContext, item1)) assert.Equal(t, "a", item1.Val1) - assert.EqualValues(t, 2, item1.Val2) + assert.Equal(t, 2, item1.Val2) item2 = new(testItem2) assert.NoError(t, as.Get(db.DefaultContext, item2)) diff --git a/modules/tailmsg/talimsg.go b/modules/tailmsg/talimsg.go new file mode 100644 index 0000000000..aafc98e2d2 --- /dev/null +++ b/modules/tailmsg/talimsg.go @@ -0,0 +1,73 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tailmsg + +import ( + "sync" + "time" +) + +type MsgRecord struct { + Time time.Time + Content string +} + +type MsgRecorder interface { + Record(content string) + GetRecords() []*MsgRecord +} + +type memoryMsgRecorder struct { + mu sync.RWMutex + msgs []*MsgRecord + limit int +} + +// TODO: use redis for a clustered environment + +func (m *memoryMsgRecorder) Record(content string) { + m.mu.Lock() + defer m.mu.Unlock() + m.msgs = append(m.msgs, &MsgRecord{ + Time: time.Now(), + Content: content, + }) + if len(m.msgs) > m.limit { + m.msgs = m.msgs[len(m.msgs)-m.limit:] + } +} + +func (m *memoryMsgRecorder) GetRecords() []*MsgRecord { + m.mu.RLock() + defer m.mu.RUnlock() + ret := make([]*MsgRecord, len(m.msgs)) + copy(ret, m.msgs) + return ret +} + +func NewMsgRecorder(limit int) MsgRecorder { + return &memoryMsgRecorder{ + limit: limit, + } +} + +type Manager struct { + traceRecorder MsgRecorder + logRecorder MsgRecorder +} + +func (m *Manager) GetTraceRecorder() MsgRecorder { + return m.traceRecorder +} + +func (m *Manager) GetLogRecorder() MsgRecorder { + return m.logRecorder +} + +var GetManager = sync.OnceValue(func() *Manager { + return &Manager{ + traceRecorder: NewMsgRecorder(100), + logRecorder: NewMsgRecorder(1000), + } +}) diff --git a/modules/tempdir/tempdir.go b/modules/tempdir/tempdir.go new file mode 100644 index 0000000000..22c2e4ea16 --- /dev/null +++ b/modules/tempdir/tempdir.go @@ -0,0 +1,112 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" +) + +type TempDir struct { + // base is the base directory for temporary files, it must exist before accessing and won't be created automatically. + // for example: base="/system-tmpdir", sub="gitea-tmp" + base, sub string +} + +func (td *TempDir) JoinPath(elems ...string) string { + return filepath.Join(append([]string{td.base, td.sub}, elems...)...) +} + +// MkdirAllSub works like os.MkdirAll, but the base directory must exist +func (td *TempDir) MkdirAllSub(dir string) (string, error) { + if _, err := os.Stat(td.base); err != nil { + return "", err + } + full := filepath.Join(td.base, td.sub, dir) + if err := os.MkdirAll(full, os.ModePerm); err != nil { + return "", err + } + return full, nil +} + +func (td *TempDir) prepareDirWithPattern(elems ...string) (dir, pattern string, err error) { + if _, err = os.Stat(td.base); err != nil { + return "", "", err + } + dir, pattern = filepath.Split(filepath.Join(append([]string{td.base, td.sub}, elems...)...)) + if err = os.MkdirAll(dir, os.ModePerm); err != nil { + return "", "", err + } + return dir, pattern, nil +} + +// MkdirTempRandom works like os.MkdirTemp, the last path field is the "pattern" +func (td *TempDir) MkdirTempRandom(elems ...string) (string, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return "", nil, err + } + dir, err = os.MkdirTemp(dir, pattern) + if err != nil { + return "", nil, err + } + return dir, func() { + if err := util.RemoveAll(dir); err != nil { + log.Error("Failed to remove temp directory %s: %v", dir, err) + } + }, nil +} + +// CreateTempFileRandom works like os.CreateTemp, the last path field is the "pattern" +func (td *TempDir) CreateTempFileRandom(elems ...string) (*os.File, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return nil, nil, err + } + f, err := os.CreateTemp(dir, pattern) + if err != nil { + return nil, nil, err + } + filename := f.Name() + return f, func() { + _ = f.Close() + if err := util.Remove(filename); err != nil { + log.Error("Unable to remove temporary file: %s: Error: %v", filename, err) + } + }, err +} + +func (td *TempDir) RemoveOutdated(d time.Duration) { + var remove func(path string) + remove = func(path string) { + entries, _ := os.ReadDir(path) + for _, entry := range entries { + full := filepath.Join(path, entry.Name()) + if entry.IsDir() { + remove(full) + _ = os.Remove(full) + continue + } + info, err := entry.Info() + if err == nil && time.Since(info.ModTime()) > d { + _ = os.Remove(full) + } + } + } + remove(td.JoinPath("")) +} + +// New create a new TempDir instance, "base" must be an existing directory, +// "sub" could be a multi-level directory and will be created if not exist +func New(base, sub string) *TempDir { + return &TempDir{base: base, sub: sub} +} + +func OsTempDir(sub string) *TempDir { + return New(os.TempDir(), sub) +} diff --git a/modules/tempdir/tempdir_test.go b/modules/tempdir/tempdir_test.go new file mode 100644 index 0000000000..d6afcb7bed --- /dev/null +++ b/modules/tempdir/tempdir_test.go @@ -0,0 +1,75 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestTempDir(t *testing.T) { + base := t.TempDir() + + t.Run("Create", func(t *testing.T) { + td := New(base, "sub1/sub2") // make sure the sub dir supports "/" in the path + assert.Equal(t, filepath.Join(base, "sub1", "sub2"), td.JoinPath()) + assert.Equal(t, filepath.Join(base, "sub1", "sub2/test"), td.JoinPath("test")) + + t.Run("MkdirTempRandom", func(t *testing.T) { + s, cleanup, err := td.MkdirTempRandom("foo") + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(s, filepath.Join(base, "sub1/sub2", "foo"))) + + _, err = os.Stat(s) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(s) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("CreateTempFileRandom", func(t *testing.T) { + f, cleanup, err := td.CreateTempFileRandom("foo", "bar") + filename := f.Name() + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(filename, filepath.Join(base, "sub1/sub2", "foo", "bar"))) + _, err = os.Stat(filename) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(filename) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("RemoveOutDated", func(t *testing.T) { + fa1, _, err := td.CreateTempFileRandom("dir-a", "f1") + assert.NoError(t, err) + fa2, _, err := td.CreateTempFileRandom("dir-a", "f2") + assert.NoError(t, err) + _ = os.Chtimes(fa2.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + fb1, _, err := td.CreateTempFileRandom("dir-b", "f1") + assert.NoError(t, err) + _ = os.Chtimes(fb1.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + _, _, _ = fa1.Close(), fa2.Close(), fb1.Close() + + td.RemoveOutdated(time.Minute) + + _, err = os.Stat(fa1.Name()) + assert.NoError(t, err) + _, err = os.Stat(fa2.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(fb1.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + }) + + t.Run("BaseNotExist", func(t *testing.T) { + td := New(filepath.Join(base, "not-exist"), "sub") + _, _, err := td.MkdirTempRandom("foo") + assert.ErrorIs(t, err, os.ErrNotExist) + }) +} diff --git a/modules/templates/eval/eval_test.go b/modules/templates/eval/eval_test.go index c9e514b5eb..f956f6cbdf 100644 --- a/modules/templates/eval/eval_test.go +++ b/modules/templates/eval/eval_test.go @@ -12,7 +12,7 @@ import ( ) func tokens(s string) (a []any) { - for _, v := range strings.Fields(s) { + for v := range strings.FieldsSeq(s) { a = append(a, v) } return a diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 880769dc65..e454bce4bd 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -6,9 +6,9 @@ package templates import ( "fmt" - "html" "html/template" "net/url" + "strconv" "strings" "time" @@ -37,12 +37,9 @@ func NewFuncMap() template.FuncMap { "dict": dict, // it's lowercase because this name has been widely used. Our other functions should have uppercase names. "Iif": iif, "Eval": evalTokens, - "SafeHTML": safeHTML, - "HTMLFormat": htmlutil.HTMLFormat, - "HTMLEscape": htmlEscape, + "HTMLFormat": htmlFormat, "QueryEscape": queryEscape, "QueryBuild": QueryBuild, - "JSEscape": jsEscapeSafe, "SanitizeHTML": SanitizeHTML, "URLJoin": util.URLJoin, "DotEscape": dotEscape, @@ -59,7 +56,6 @@ func NewFuncMap() template.FuncMap { // ----------------------------------------------------------------- // svg / avatar / icon / color "svg": svg.RenderHTML, - "EntryIcon": base.EntryIcon, "MigrationIcon": migrationIcon, "ActionIcon": actionIcon, "SortArrow": sortArrow, @@ -69,12 +65,12 @@ func NewFuncMap() template.FuncMap { // time / number / format "FileSize": base.FileSize, "CountFmt": countFmt, - "Sec2Time": util.SecToTime, + "Sec2Hour": util.SecToHours, "TimeEstimateString": timeEstimateString, "LoadTimes": func(startTime time.Time) string { - return fmt.Sprint(time.Since(startTime).Nanoseconds()/1e6) + "ms" + return strconv.FormatInt(time.Since(startTime).Nanoseconds()/1e6, 10) + "ms" }, // ----------------------------------------------------------------- @@ -131,15 +127,9 @@ func NewFuncMap() template.FuncMap { "EnableTimetracking": func() bool { return setting.Service.EnableTimetracking }, - "DisableGitHooks": func() bool { - return setting.DisableGitHooks - }, "DisableWebhooks": func() bool { return setting.DisableWebhooks }, - "DisableImportLocal": func() bool { - return !setting.ImportLocalPaths - }, "UserThemeName": userThemeName, "NotificationSettings": func() map[string]any { return map[string]any{ @@ -168,55 +158,28 @@ func NewFuncMap() template.FuncMap { "FilenameIsImage": filenameIsImage, "TabSizeClass": tabSizeClass, - - // for backward compatibility only, do not use them anymore - "TimeSince": timeSinceLegacy, - "TimeSinceUnix": timeSinceLegacy, - "DateTime": dateTimeLegacy, - - "RenderEmoji": renderEmojiLegacy, - "RenderLabel": renderLabelLegacy, - "RenderLabels": renderLabelsLegacy, - "RenderIssueTitle": renderIssueTitleLegacy, - - "RenderMarkdownToHtml": renderMarkdownToHtmlLegacy, - - "RenderCommitMessage": renderCommitMessageLegacy, - "RenderCommitMessageLinkSubject": renderCommitMessageLinkSubjectLegacy, - "RenderCommitBody": renderCommitBodyLegacy, - } -} - -// safeHTML render raw as HTML -func safeHTML(s any) template.HTML { - switch v := s.(type) { - case string: - return template.HTML(v) - case template.HTML: - return v } - panic(fmt.Sprintf("unexpected type %T", s)) } -// SanitizeHTML sanitizes the input by pre-defined markdown rules +// SanitizeHTML sanitizes the input by default sanitization rules. func SanitizeHTML(s string) template.HTML { - return template.HTML(markup.Sanitize(s)) + return markup.Sanitize(s) } -func htmlEscape(s any) template.HTML { +func htmlFormat(s any, args ...any) template.HTML { + if len(args) == 0 { + // to prevent developers from calling "HTMLFormat $userInput" by mistake which will lead to XSS + panic("missing arguments for HTMLFormat") + } switch v := s.(type) { case string: - return template.HTML(html.EscapeString(v)) + return htmlutil.HTMLFormat(template.HTML(v), args...) case template.HTML: - return v + return htmlutil.HTMLFormat(v, args...) } panic(fmt.Sprintf("unexpected type %T", s)) } -func jsEscapeSafe(s string) template.HTML { - return template.HTML(template.JSEscapeString(s)) -} - func queryEscape(s string) template.URL { return template.URL(url.QueryEscape(s)) } @@ -264,22 +227,42 @@ func userThemeName(user *user_model.User) string { return setting.UI.DefaultTheme } +func isQueryParamEmpty(v any) bool { + return v == nil || v == false || v == 0 || v == int64(0) || v == "" +} + // QueryBuild builds a query string from a list of key-value pairs. -// It omits the nil and empty strings, but it doesn't omit other zero values, -// because the zero value of number types may have a meaning. +// It omits the nil, false, zero int/int64 and empty string values, +// because they are default empty values for "ctx.FormXxx" calls. +// If 0 or false need to be included, use string values: "0" and "false". +// Build rules: +// * Even parameters: always build as query string: a=b&c=d +// * Odd parameters: +// * * {"/anything", param-pairs...} => "/?param-paris" +// * * {"anything?old-params", new-param-pairs...} => "anything?old-params&new-param-paris" +// * * Otherwise: {"old¶ms", new-param-pairs...} => "old¶ms&new-param-paris" +// * * Other behaviors are undefined yet. func QueryBuild(a ...any) template.URL { - var s string + var reqPath, s string + hasTrailingSep := false if len(a)%2 == 1 { if v, ok := a[0].(string); ok { - if v == "" || (v[0] != '?' && v[0] != '&') { - panic("QueryBuild: invalid argument") - } s = v } else if v, ok := a[0].(template.URL); ok { s = string(v) } else { panic("QueryBuild: invalid argument") } + hasTrailingSep = s != "&" && strings.HasSuffix(s, "&") + if strings.HasPrefix(s, "/") || strings.Contains(s, "?") { + if s1, s2, ok := strings.Cut(s, "?"); ok { + reqPath = s1 + "?" + s = s2 + } else { + reqPath += s + "?" + s = "" + } + } } for i := len(a) % 2; i < len(a); i += 2 { k, ok := a[i].(string) @@ -290,19 +273,16 @@ func QueryBuild(a ...any) template.URL { if va, ok := a[i+1].(string); ok { v = va } else if a[i+1] != nil { - v = fmt.Sprint(a[i+1]) + if !isQueryParamEmpty(a[i+1]) { + v = fmt.Sprint(a[i+1]) + } } // pos1 to pos2 is the "k=v&" part, "&" is optional pos1 := strings.Index(s, "&"+k+"=") if pos1 != -1 { pos1++ - } else { - pos1 = strings.Index(s, "?"+k+"=") - if pos1 != -1 { - pos1++ - } else if strings.HasPrefix(s, k+"=") { - pos1 = 0 - } + } else if strings.HasPrefix(s, k+"=") { + pos1 = 0 } pos2 := len(s) if pos1 == -1 { @@ -315,7 +295,7 @@ func QueryBuild(a ...any) template.URL { } if v != "" { sep := "" - hasPrefixSep := pos1 == 0 || (pos1 <= len(s) && (s[pos1-1] == '?' || s[pos1-1] == '&')) + hasPrefixSep := pos1 == 0 || (pos1 <= len(s) && s[pos1-1] == '&') if !hasPrefixSep { sep = "&" } @@ -324,14 +304,21 @@ func QueryBuild(a ...any) template.URL { s = s[:pos1] + s[pos2:] } } - if s != "" && s != "&" && s[len(s)-1] == '&' { + if s != "" && s[len(s)-1] == '&' && !hasTrailingSep { s = s[:len(s)-1] } - return template.URL(s) -} - -func panicIfDevOrTesting() { - if !setting.IsProd || setting.IsInTesting { - panic("legacy template functions are for backward compatibility only, do not use them in new code") + if reqPath != "" { + if s == "" { + s = reqPath + if s != "?" { + s = s[:len(s)-1] + } + } else { + if s[0] == '&' { + s = s[1:] + } + s = reqPath + s + } } + return template.URL(s) } diff --git a/modules/templates/helper_test.go b/modules/templates/helper_test.go index a530d484bc..7e3a952e7b 100644 --- a/modules/templates/helper_test.go +++ b/modules/templates/helper_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" - "code.gitea.io/gitea/modules/htmlutil" "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" @@ -16,7 +15,7 @@ import ( func TestSubjectBodySeparator(t *testing.T) { test := func(input, subject, body string) { - loc := mailSubjectSplit.FindIndex([]byte(input)) + loc := mailSubjectSplit.FindStringIndex(input) if loc == nil { assert.Empty(t, subject, "no subject found, but one expected") assert.Equal(t, body, input) @@ -58,10 +57,6 @@ func TestSubjectBodySeparator(t *testing.T) { "Insufficient\n--\nSeparators") } -func TestJSEscapeSafe(t *testing.T) { - assert.EqualValues(t, `\u0026\u003C\u003E\'\"`, jsEscapeSafe(`&<>'"`)) -} - func TestSanitizeHTML(t *testing.T) { assert.Equal(t, template.HTML(`<a href="/" rel="nofollow">link</a> xss <div>inline</div>`), SanitizeHTML(`<a href="/">link</a> <a href="javascript:">xss</a> <div style="dangerous">inline</div>`)) } @@ -88,7 +83,7 @@ func TestTemplateIif(t *testing.T) { func TestTemplateEscape(t *testing.T) { execTmpl := func(code string) string { tmpl := template.New("test") - tmpl.Funcs(template.FuncMap{"QueryBuild": QueryBuild, "HTMLFormat": htmlutil.HTMLFormat}) + tmpl.Funcs(template.FuncMap{"QueryBuild": QueryBuild, "HTMLFormat": htmlFormat}) template.Must(tmpl.Parse(code)) w := &strings.Builder{} assert.NoError(t, tmpl.Execute(w, nil)) @@ -118,3 +113,58 @@ func TestTemplateEscape(t *testing.T) { assert.Equal(t, `<a k="""><></a>`, actual) }) } + +func TestQueryBuild(t *testing.T) { + t.Run("construct", func(t *testing.T) { + assert.Empty(t, string(QueryBuild())) + assert.Empty(t, string(QueryBuild("a", nil, "b", false, "c", 0, "d", ""))) + assert.Equal(t, "a=1&b=true", string(QueryBuild("a", 1, "b", "true"))) + + // path with query parameters + assert.Equal(t, "/?k=1", string(QueryBuild("/", "k", 1))) + assert.Equal(t, "/", string(QueryBuild("/?k=a", "k", 0))) + + // no path but question mark with query parameters + assert.Equal(t, "?k=1", string(QueryBuild("?", "k", 1))) + assert.Equal(t, "?", string(QueryBuild("?", "k", 0))) + assert.Equal(t, "path?k=1", string(QueryBuild("path?", "k", 1))) + assert.Equal(t, "path", string(QueryBuild("path?", "k", 0))) + + // only query parameters + assert.Equal(t, "&k=1", string(QueryBuild("&", "k", 1))) + assert.Empty(t, string(QueryBuild("&", "k", 0))) + assert.Empty(t, string(QueryBuild("&k=a", "k", 0))) + assert.Empty(t, string(QueryBuild("k=a&", "k", 0))) + assert.Equal(t, "a=1&b=2", string(QueryBuild("a=1", "b", 2))) + assert.Equal(t, "&a=1&b=2", string(QueryBuild("&a=1", "b", 2))) + assert.Equal(t, "a=1&b=2&", string(QueryBuild("a=1&", "b", 2))) + }) + + t.Run("replace", func(t *testing.T) { + assert.Equal(t, "a=1&c=d&e=f", string(QueryBuild("a=b&c=d&e=f", "a", 1))) + assert.Equal(t, "a=b&c=1&e=f", string(QueryBuild("a=b&c=d&e=f", "c", 1))) + assert.Equal(t, "a=b&c=d&e=1", string(QueryBuild("a=b&c=d&e=f", "e", 1))) + assert.Equal(t, "a=b&c=d&e=f&k=1", string(QueryBuild("a=b&c=d&e=f", "k", 1))) + }) + + t.Run("replace-&", func(t *testing.T) { + assert.Equal(t, "&a=1&c=d&e=f", string(QueryBuild("&a=b&c=d&e=f", "a", 1))) + assert.Equal(t, "&a=b&c=1&e=f", string(QueryBuild("&a=b&c=d&e=f", "c", 1))) + assert.Equal(t, "&a=b&c=d&e=1", string(QueryBuild("&a=b&c=d&e=f", "e", 1))) + assert.Equal(t, "&a=b&c=d&e=f&k=1", string(QueryBuild("&a=b&c=d&e=f", "k", 1))) + }) + + t.Run("delete", func(t *testing.T) { + assert.Equal(t, "c=d&e=f", string(QueryBuild("a=b&c=d&e=f", "a", ""))) + assert.Equal(t, "a=b&e=f", string(QueryBuild("a=b&c=d&e=f", "c", ""))) + assert.Equal(t, "a=b&c=d", string(QueryBuild("a=b&c=d&e=f", "e", ""))) + assert.Equal(t, "a=b&c=d&e=f", string(QueryBuild("a=b&c=d&e=f", "k", ""))) + }) + + t.Run("delete-&", func(t *testing.T) { + assert.Equal(t, "&c=d&e=f", string(QueryBuild("&a=b&c=d&e=f", "a", ""))) + assert.Equal(t, "&a=b&e=f", string(QueryBuild("&a=b&c=d&e=f", "c", ""))) + assert.Equal(t, "&a=b&c=d", string(QueryBuild("&a=b&c=d&e=f", "e", ""))) + assert.Equal(t, "&a=b&c=d&e=f", string(QueryBuild("&a=b&c=d&e=f", "k", ""))) + }) +} diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index 529284f7e8..8073a6e5f5 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -42,7 +42,7 @@ var ( var ErrTemplateNotInitialized = errors.New("template system is not initialized, check your log for errors") -func (h *HTMLRender) HTML(w io.Writer, status int, tplName TplName, data any, ctx context.Context) error { //nolint:revive +func (h *HTMLRender) HTML(w io.Writer, status int, tplName TplName, data any, ctx context.Context) error { //nolint:revive // we don't use ctx, only pass it to the template executor name := string(tplName) if respWriter, ok := w.(http.ResponseWriter); ok { if respWriter.Header().Get("Content-Type") == "" { @@ -57,7 +57,7 @@ func (h *HTMLRender) HTML(w io.Writer, status int, tplName TplName, data any, ct return t.Execute(w, data) } -func (h *HTMLRender) TemplateLookup(name string, ctx context.Context) (TemplateExecutor, error) { //nolint:revive +func (h *HTMLRender) TemplateLookup(name string, ctx context.Context) (TemplateExecutor, error) { //nolint:revive // we don't use ctx, only pass it to the template executor tmpls := h.templates.Load() if tmpls == nil { return nil, ErrTemplateNotInitialized @@ -251,7 +251,7 @@ func extractErrorLine(code []byte, lineNum, posNum int, target string) string { b := bufio.NewReader(bytes.NewReader(code)) var line []byte var err error - for i := 0; i < lineNum; i++ { + for i := range lineNum { if line, err = b.ReadBytes('\n'); err != nil { if i == lineNum-1 && errors.Is(err, io.EOF) { err = nil diff --git a/modules/templates/htmlrenderer_test.go b/modules/templates/htmlrenderer_test.go index 2a74b74c23..e8b01c30fe 100644 --- a/modules/templates/htmlrenderer_test.go +++ b/modules/templates/htmlrenderer_test.go @@ -65,7 +65,7 @@ func TestHandleError(t *testing.T) { _, err = tmpl.Parse(s) assert.Error(t, err) msg := h(err) - assert.EqualValues(t, strings.TrimSpace(expect), strings.TrimSpace(msg)) + assert.Equal(t, strings.TrimSpace(expect), strings.TrimSpace(msg)) } test("{{", p.handleGenericTemplateError, ` @@ -102,5 +102,5 @@ god knows XXX ---------------------------------------------------------------------- ` actualMsg := p.handleExpectedEndError(errors.New("template: test:1: expected end; found XXX")) - assert.EqualValues(t, strings.TrimSpace(expectedMsg), strings.TrimSpace(actualMsg)) + assert.Equal(t, strings.TrimSpace(expectedMsg), strings.TrimSpace(actualMsg)) } diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go index ace81bf4a5..c43b760777 100644 --- a/modules/templates/mailer.go +++ b/modules/templates/mailer.go @@ -9,13 +9,20 @@ import ( "html/template" "regexp" "strings" + "sync/atomic" texttmpl "text/template" - "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) +type MailTemplates struct { + TemplateNames []string + BodyTemplates *template.Template + SubjectTemplates *texttmpl.Template +} + var mailSubjectSplit = regexp.MustCompile(`(?m)^-{3,}\s*$`) // mailSubjectTextFuncMap returns functions for injecting to text templates, it's only used for mail subject @@ -24,7 +31,7 @@ func mailSubjectTextFuncMap() texttmpl.FuncMap { "dict": dict, "Eval": evalTokens, - "EllipsisString": base.EllipsisString, + "EllipsisString": util.EllipsisDisplayString, "AppName": func() string { return setting.AppName }, @@ -52,16 +59,17 @@ func buildSubjectBodyTemplate(stpl *texttmpl.Template, btpl *template.Template, return nil } -// Mailer provides the templates required for sending notification mails. -func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { - subjectTemplates := texttmpl.New("") - bodyTemplates := template.New("") - - subjectTemplates.Funcs(mailSubjectTextFuncMap()) - bodyTemplates.Funcs(NewFuncMap()) - +// LoadMailTemplates provides the templates required for sending notification mails. +func LoadMailTemplates(ctx context.Context, loadedTemplates *atomic.Pointer[MailTemplates]) { assetFS := AssetFS() refreshTemplates := func(firstRun bool) { + var templateNames []string + subjectTemplates := texttmpl.New("") + bodyTemplates := template.New("") + + subjectTemplates.Funcs(mailSubjectTextFuncMap()) + bodyTemplates.Funcs(NewFuncMap()) + if !firstRun { log.Trace("Reloading mail templates") } @@ -81,6 +89,7 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { if firstRun { log.Trace("Adding mail template %s: %s by %s", tmplName, assetPath, layerName) } + templateNames = append(templateNames, tmplName) if err = buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content); err != nil { if firstRun { log.Fatal("Failed to parse mail template, err: %v", err) @@ -88,6 +97,12 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { log.Error("Failed to parse mail template, err: %v", err) } } + loaded := &MailTemplates{ + TemplateNames: templateNames, + BodyTemplates: bodyTemplates, + SubjectTemplates: subjectTemplates, + } + loadedTemplates.Store(loaded) } refreshTemplates(true) @@ -99,6 +114,4 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { refreshTemplates(false) }) } - - return subjectTemplates, bodyTemplates } diff --git a/modules/templates/scopedtmpl/scopedtmpl.go b/modules/templates/scopedtmpl/scopedtmpl.go index 2722ba97a2..34e8b9ad70 100644 --- a/modules/templates/scopedtmpl/scopedtmpl.go +++ b/modules/templates/scopedtmpl/scopedtmpl.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "io" + "maps" "reflect" "sync" texttemplate "text/template" @@ -40,9 +41,7 @@ func (t *ScopedTemplate) Funcs(funcMap template.FuncMap) { panic("cannot add new functions to frozen template set") } t.all.Funcs(funcMap) - for k, v := range funcMap { - t.parseFuncs[k] = v - } + maps.Copy(t.parseFuncs, funcMap) } func (t *ScopedTemplate) New(name string) *template.Template { @@ -103,31 +102,28 @@ func escapeTemplate(t *template.Template) error { return nil } -//nolint:unused type htmlTemplate struct { - escapeErr error - text *texttemplate.Template + _/*escapeErr*/ error + text *texttemplate.Template } -//nolint:unused type textTemplateCommon struct { - tmpl map[string]*template.Template // Map from name to defined templates. - muTmpl sync.RWMutex // protects tmpl - option struct { + _/*tmpl*/ map[string]*template.Template + _/*muTmpl*/ sync.RWMutex + _/*option*/ struct { missingKey int } - muFuncs sync.RWMutex // protects parseFuncs and execFuncs - parseFuncs texttemplate.FuncMap - execFuncs map[string]reflect.Value + muFuncs sync.RWMutex + _/*parseFuncs*/ texttemplate.FuncMap + execFuncs map[string]reflect.Value } -//nolint:unused type textTemplate struct { - name string + _/*name*/ string *parse.Tree *textTemplateCommon - leftDelim string - rightDelim string + _/*leftDelim*/ string + _/*rightDelim*/ string } func ptr[T, P any](ptr *P) *T { @@ -159,9 +155,7 @@ func newScopedTemplateSet(all *template.Template, name string) (*scopedTemplateS textTmplPtr.muFuncs.Lock() ts.execFuncs = map[string]reflect.Value{} - for k, v := range textTmplPtr.execFuncs { - ts.execFuncs[k] = v - } + maps.Copy(ts.execFuncs, textTmplPtr.execFuncs) textTmplPtr.muFuncs.Unlock() var collectTemplates func(nodes []parse.Node) @@ -220,9 +214,7 @@ func (ts *scopedTemplateSet) newExecutor(funcMap map[string]any) TemplateExecuto tmpl := texttemplate.New("") tmplPtr := ptr[textTemplate](tmpl) tmplPtr.execFuncs = map[string]reflect.Value{} - for k, v := range ts.execFuncs { - tmplPtr.execFuncs[k] = v - } + maps.Copy(tmplPtr.execFuncs, ts.execFuncs) if funcMap != nil { tmpl.Funcs(funcMap) } diff --git a/modules/templates/static.go b/modules/templates/static.go deleted file mode 100644 index b5a7e561ec..0000000000 --- a/modules/templates/static.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package templates - -import ( - "time" - - "code.gitea.io/gitea/modules/assetfs" - "code.gitea.io/gitea/modules/timeutil" -) - -// GlobalModTime provide a global mod time for embedded asset files -func GlobalModTime(filename string) time.Time { - return timeutil.GetExecutableModTime() -} - -func BuiltinAssets() *assetfs.Layer { - return assetfs.Bindata("builtin(bindata)", Assets) -} diff --git a/modules/templates/templates_bindata.go b/modules/templates/templates_bindata.go index 6f1d3cf539..a919591ecf 100644 --- a/modules/templates/templates_bindata.go +++ b/modules/templates/templates_bindata.go @@ -3,6 +3,21 @@ //go:build bindata +//go:generate go run ../../build/generate-bindata.go ../../templates bindata.dat + package templates -//go:generate go run ../../build/generate-bindata.go ../../templates templates bindata.go true +import ( + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", assetfs.NewEmbeddedFS(bindata)) +}) diff --git a/modules/templates/dynamic.go b/modules/templates/templates_dynamic.go index e1babd83c9..e1babd83c9 100644 --- a/modules/templates/dynamic.go +++ b/modules/templates/templates_dynamic.go diff --git a/modules/templates/util_avatar.go b/modules/templates/util_avatar.go index f7dd408ee2..ee9994ab0b 100644 --- a/modules/templates/util_avatar.go +++ b/modules/templates/util_avatar.go @@ -5,9 +5,9 @@ package templates import ( "context" - "fmt" "html" "html/template" + "strconv" activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/avatars" @@ -28,13 +28,14 @@ func NewAvatarUtils(ctx context.Context) *AvatarUtils { // AvatarHTML creates the HTML for an avatar func AvatarHTML(src string, size int, class, name string) template.HTML { - sizeStr := fmt.Sprintf(`%d`, size) + sizeStr := strconv.Itoa(size) if name == "" { name = "avatar" } - return template.HTML(`<img loading="lazy" class="` + class + `" src="` + src + `" title="` + html.EscapeString(name) + `" width="` + sizeStr + `" height="` + sizeStr + `"/>`) + // use empty alt, otherwise if the image fails to load, the width will follow the "alt" text's width + return template.HTML(`<img loading="lazy" alt class="` + class + `" src="` + src + `" title="` + html.EscapeString(name) + `" width="` + sizeStr + `" height="` + sizeStr + `"/>`) } // Avatar renders user avatars. args: user, size (int), class (string) diff --git a/modules/templates/util_date.go b/modules/templates/util_date.go index 658691ee40..fc3f3f2339 100644 --- a/modules/templates/util_date.go +++ b/modules/templates/util_date.go @@ -99,7 +99,7 @@ func dateTimeFormat(format string, datetime any) template.HTML { attrs = append(attrs, `format="datetime"`, `month="short"`, `day="numeric"`, `hour="numeric"`, `minute="numeric"`, `second="numeric"`, `data-tooltip-content`, `data-tooltip-interactive="true"`) return template.HTML(fmt.Sprintf(`<relative-time %s datetime="%s">%s</relative-time>`, strings.Join(attrs, " "), datetimeEscaped, textEscaped)) default: - panic(fmt.Sprintf("Unsupported format %s", format)) + panic("Unsupported format " + format) } } diff --git a/modules/templates/util_date_legacy.go b/modules/templates/util_date_legacy.go deleted file mode 100644 index ceefb00447..0000000000 --- a/modules/templates/util_date_legacy.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package templates - -import ( - "html/template" - - "code.gitea.io/gitea/modules/translation" -) - -func dateTimeLegacy(format string, datetime any, _ ...string) template.HTML { - panicIfDevOrTesting() - if s, ok := datetime.(string); ok { - datetime = parseLegacy(s) - } - return dateTimeFormat(format, datetime) -} - -func timeSinceLegacy(time any, _ translation.Locale) template.HTML { - panicIfDevOrTesting() - return TimeSince(time) -} diff --git a/modules/templates/util_date_test.go b/modules/templates/util_date_test.go index f3a2409a9f..2c1f2d242e 100644 --- a/modules/templates/util_date_test.go +++ b/modules/templates/util_date_test.go @@ -17,12 +17,12 @@ import ( func TestDateTime(t *testing.T) { testTz, _ := time.LoadLocation("America/New_York") defer test.MockVariableValue(&setting.DefaultUILocation, testTz)() + defer test.MockVariableValue(&setting.IsProd, true)() defer test.MockVariableValue(&setting.IsInTesting, false)() du := NewDateUtils() refTimeStr := "2018-01-01T00:00:00Z" - refDateStr := "2018-01-01" refTime, _ := time.Parse(time.RFC3339, refTimeStr) refTimeStamp := timeutil.TimeStamp(refTime.Unix()) @@ -31,18 +31,9 @@ func TestDateTime(t *testing.T) { assert.EqualValues(t, "-", du.AbsoluteShort(time.Time{})) assert.EqualValues(t, "-", du.AbsoluteShort(timeutil.TimeStamp(0))) - actual := dateTimeLegacy("short", "invalid") - assert.EqualValues(t, `-`, actual) - - actual = dateTimeLegacy("short", refTimeStr) - assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2018-01-01T00:00:00Z">2018-01-01</absolute-date>`, actual) - - actual = du.AbsoluteShort(refTime) + actual := du.AbsoluteShort(refTime) assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2018-01-01T00:00:00Z">2018-01-01</absolute-date>`, actual) - actual = dateTimeLegacy("short", refDateStr) - assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2018-01-01T00:00:00-05:00">2018-01-01</absolute-date>`, actual) - actual = du.AbsoluteShort(refTimeStamp) assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2017-12-31T19:00:00-05:00">2017-12-31</absolute-date>`, actual) @@ -53,6 +44,7 @@ func TestDateTime(t *testing.T) { func TestTimeSince(t *testing.T) { testTz, _ := time.LoadLocation("America/New_York") defer test.MockVariableValue(&setting.DefaultUILocation, testTz)() + defer test.MockVariableValue(&setting.IsProd, true)() defer test.MockVariableValue(&setting.IsInTesting, false)() du := NewDateUtils() @@ -67,6 +59,6 @@ func TestTimeSince(t *testing.T) { actual = timeSinceTo(&refTime, time.Time{}) assert.EqualValues(t, `<relative-time prefix="" tense="future" datetime="2018-01-01T00:00:00Z" data-tooltip-content data-tooltip-interactive="true">2018-01-01 00:00:00 +00:00</relative-time>`, actual) - actual = timeSinceLegacy(timeutil.TimeStampNano(refTime.UnixNano()), nil) + actual = du.TimeSince(timeutil.TimeStampNano(refTime.UnixNano())) assert.EqualValues(t, `<relative-time prefix="" tense="past" datetime="2017-12-31T19:00:00-05:00" data-tooltip-content data-tooltip-interactive="true">2017-12-31 19:00:00 -05:00</relative-time>`, actual) } diff --git a/modules/templates/util_dict.go b/modules/templates/util_dict.go index 8d6376b522..cc3018a71c 100644 --- a/modules/templates/util_dict.go +++ b/modules/templates/util_dict.go @@ -4,6 +4,7 @@ package templates import ( + "errors" "fmt" "html" "html/template" @@ -33,7 +34,7 @@ func dictMerge(base map[string]any, arg any) bool { // The dot syntax is highly discouraged because it might cause unclear key conflicts. It's always good to use explicit keys. func dict(args ...any) (map[string]any, error) { if len(args)%2 != 0 { - return nil, fmt.Errorf("invalid dict constructor syntax: must have key-value pairs") + return nil, errors.New("invalid dict constructor syntax: must have key-value pairs") } m := make(map[string]any, len(args)/2) for i := 0; i < len(args); i += 2 { diff --git a/modules/templates/util_format.go b/modules/templates/util_format.go index bee6fb7b75..3485e3251e 100644 --- a/modules/templates/util_format.go +++ b/modules/templates/util_format.go @@ -5,6 +5,7 @@ package templates import ( "fmt" + "strconv" "code.gitea.io/gitea/modules/util" ) @@ -24,7 +25,7 @@ func countFmt(data any) string { return "" } if num < 1000 { - return fmt.Sprintf("%d", num) + return strconv.FormatInt(num, 10) } else if num < 1_000_000 { num2 := float32(num) / 1000.0 return fmt.Sprintf("%.1fk", num2) diff --git a/modules/templates/util_format_test.go b/modules/templates/util_format_test.go index 8d466faff0..13a57c24e2 100644 --- a/modules/templates/util_format_test.go +++ b/modules/templates/util_format_test.go @@ -14,5 +14,5 @@ func TestCountFmt(t *testing.T) { assert.Equal(t, "1.3k", countFmt(int64(1317))) assert.Equal(t, "21.3M", countFmt(21317675)) assert.Equal(t, "45.7G", countFmt(45721317675)) - assert.Equal(t, "", countFmt("test")) + assert.Empty(t, countFmt("test")) } diff --git a/modules/templates/util_json.go b/modules/templates/util_json.go index 71a4e23d36..29a04290fa 100644 --- a/modules/templates/util_json.go +++ b/modules/templates/util_json.go @@ -9,11 +9,11 @@ import ( "code.gitea.io/gitea/modules/json" ) -type JsonUtils struct{} //nolint:revive +type JsonUtils struct{} //nolint:revive // variable naming triggers on Json, wants JSON var jsonUtils = JsonUtils{} -func NewJsonUtils() *JsonUtils { //nolint:revive +func NewJsonUtils() *JsonUtils { //nolint:revive // variable naming triggers on Json, wants JSON return &jsonUtils } diff --git a/modules/templates/util_misc.go b/modules/templates/util_misc.go index d645fa013e..cc5bf67b42 100644 --- a/modules/templates/util_misc.go +++ b/modules/templates/util_misc.go @@ -38,10 +38,11 @@ func sortArrow(normSort, revSort, urlSort string, isDefault bool) template.HTML } else { // if sort arg is in url test if it correlates with column header sort arguments // the direction of the arrow should indicate the "current sort order", up means ASC(normal), down means DESC(rev) - if urlSort == normSort { + switch urlSort { + case normSort: // the table is sorted with this header normal return svg.RenderHTML("octicon-triangle-up", 16) - } else if urlSort == revSort { + case revSort: // the table is sorted with this header reverse return svg.RenderHTML("octicon-triangle-down", 16) } @@ -150,7 +151,7 @@ func mirrorRemoteAddress(ctx context.Context, m *repo_model.Repository, remoteNa return ret } - u, err := giturl.Parse(remoteURL) + u, err := giturl.ParseGitURL(remoteURL) if err != nil { log.Error("giturl.Parse %v", err) return ret diff --git a/modules/templates/util_render.go b/modules/templates/util_render.go index 1800747f48..1056c42643 100644 --- a/modules/templates/util_render.go +++ b/modules/templates/util_render.go @@ -4,7 +4,6 @@ package templates import ( - "context" "encoding/hex" "fmt" "html/template" @@ -15,44 +14,47 @@ import ( "unicode" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/renderhelper" + "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/emoji" "code.gitea.io/gitea/modules/htmlutil" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" + "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" ) type RenderUtils struct { - ctx context.Context + ctx reqctx.RequestContext } -func NewRenderUtils(ctx context.Context) *RenderUtils { +func NewRenderUtils(ctx reqctx.RequestContext) *RenderUtils { return &RenderUtils{ctx: ctx} } // RenderCommitMessage renders commit message with XSS-safe and special links. -func (ut *RenderUtils) RenderCommitMessage(msg string, metas map[string]string) template.HTML { +func (ut *RenderUtils) RenderCommitMessage(msg string, repo *repo.Repository) template.HTML { cleanMsg := template.HTMLEscapeString(msg) - // we can safely assume that it will not return any error, since there - // shouldn't be any special HTML. - fullMessage, err := markup.PostProcessCommitMessage(markup.NewRenderContext(ut.ctx).WithMetas(metas), cleanMsg) + // we can safely assume that it will not return any error, since there shouldn't be any special HTML. + // "repo" can be nil when rendering commit messages for deleted repositories in a user's dashboard feed. + fullMessage, err := markup.PostProcessCommitMessage(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), cleanMsg) if err != nil { log.Error("PostProcessCommitMessage: %v", err) return "" } msgLines := strings.Split(strings.TrimSpace(fullMessage), "\n") if len(msgLines) == 0 { - return template.HTML("") + return "" } return renderCodeBlock(template.HTML(msgLines[0])) } // RenderCommitMessageLinkSubject renders commit message as a XSS-safe link to // the provided default url, handling for special links without email to links. -func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, metas map[string]string) template.HTML { +func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, repo *repo.Repository) template.HTML { msgLine := strings.TrimLeftFunc(msg, unicode.IsSpace) lineEnd := strings.IndexByte(msgLine, '\n') if lineEnd > 0 { @@ -63,9 +65,8 @@ func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, me return "" } - // we can safely assume that it will not return any error, since there - // shouldn't be any special HTML. - renderedMessage, err := markup.PostProcessCommitMessageSubject(markup.NewRenderContext(ut.ctx).WithMetas(metas), urlDefault, template.HTMLEscapeString(msgLine)) + // we can safely assume that it will not return any error, since there shouldn't be any special HTML. + renderedMessage, err := markup.PostProcessCommitMessageSubject(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), urlDefault, template.HTMLEscapeString(msgLine)) if err != nil { log.Error("PostProcessCommitMessageSubject: %v", err) return "" @@ -74,7 +75,7 @@ func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, me } // RenderCommitBody extracts the body of a commit message without its title. -func (ut *RenderUtils) RenderCommitBody(msg string, metas map[string]string) template.HTML { +func (ut *RenderUtils) RenderCommitBody(msg string, repo *repo.Repository) template.HTML { msgLine := strings.TrimSpace(msg) lineEnd := strings.IndexByte(msgLine, '\n') if lineEnd > 0 { @@ -87,7 +88,7 @@ func (ut *RenderUtils) RenderCommitBody(msg string, metas map[string]string) tem return "" } - renderedMessage, err := markup.PostProcessCommitMessage(markup.NewRenderContext(ut.ctx).WithMetas(metas), template.HTMLEscapeString(msgLine)) + renderedMessage, err := markup.PostProcessCommitMessage(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), template.HTMLEscapeString(msgLine)) if err != nil { log.Error("PostProcessCommitMessage: %v", err) return "" @@ -105,8 +106,8 @@ func renderCodeBlock(htmlEscapedTextToRender template.HTML) template.HTML { } // RenderIssueTitle renders issue/pull title with defined post processors -func (ut *RenderUtils) RenderIssueTitle(text string, metas map[string]string) template.HTML { - renderedText, err := markup.PostProcessIssueTitle(markup.NewRenderContext(ut.ctx).WithMetas(metas), template.HTMLEscapeString(text)) +func (ut *RenderUtils) RenderIssueTitle(text string, repo *repo.Repository) template.HTML { + renderedText, err := markup.PostProcessIssueTitle(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), template.HTMLEscapeString(text)) if err != nil { log.Error("PostProcessIssueTitle: %v", err) return "" @@ -121,8 +122,23 @@ func (ut *RenderUtils) RenderIssueSimpleTitle(text string) template.HTML { return ret } -// RenderLabel renders a label +func (ut *RenderUtils) RenderLabelWithLink(label *issues_model.Label, link any) template.HTML { + var attrHref template.HTML + switch link.(type) { + case template.URL, string: + attrHref = htmlutil.HTMLFormat(`href="%s"`, link) + default: + panic(fmt.Sprintf("unexpected type %T for link", link)) + } + return ut.renderLabelWithTag(label, "a", attrHref) +} + func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { + return ut.renderLabelWithTag(label, "span", "") +} + +// RenderLabel renders a label +func (ut *RenderUtils) renderLabelWithTag(label *issues_model.Label, tagName, tagAttrs template.HTML) template.HTML { locale := ut.ctx.Value(translation.ContextKey).(translation.Locale) var extraCSSClasses string textColor := util.ContrastColor(label.Color) @@ -136,8 +152,8 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { if labelScope == "" { // Regular label - return htmlutil.HTMLFormat(`<div class="ui label %s" style="color: %s !important; background-color: %s !important;" data-tooltip-content title="%s">%s</div>`, - extraCSSClasses, textColor, label.Color, descriptionText, ut.RenderEmoji(label.Name)) + return htmlutil.HTMLFormat(`<%s %s class="ui label %s" style="color: %s !important; background-color: %s !important;" data-tooltip-content title="%s"><span class="gt-ellipsis">%s</span></%s>`, + tagName, tagAttrs, extraCSSClasses, textColor, label.Color, descriptionText, ut.RenderEmoji(label.Name), tagName) } // Scoped label @@ -151,7 +167,7 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { // Ensure we add the same amount of contrast also near 0 and 1. darken := contrast + math.Max(luminance+contrast-1.0, 0.0) lighten := contrast + math.Max(contrast-luminance, 0.0) - // Compute factor to keep RGB values proportional. + // Compute the factor to keep RGB values proportional. darkenFactor := math.Max(luminance-darken, 0.0) / math.Max(luminance, 1.0/255.0) lightenFactor := math.Min(luminance+lighten, 1.0) / math.Max(luminance, 1.0/255.0) @@ -170,13 +186,31 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { itemColor := "#" + hex.EncodeToString(itemBytes) scopeColor := "#" + hex.EncodeToString(scopeBytes) - return htmlutil.HTMLFormat(`<span class="ui label %s scope-parent" data-tooltip-content title="%s">`+ + if label.ExclusiveOrder > 0 { + // <scope> | <label> | <order> + return htmlutil.HTMLFormat(`<%s %s class="ui label %s scope-parent" data-tooltip-content title="%s">`+ + `<div class="ui label scope-left" style="color: %s !important; background-color: %s !important">%s</div>`+ + `<div class="ui label scope-middle" style="color: %s !important; background-color: %s !important">%s</div>`+ + `<div class="ui label scope-right">%d</div>`+ + `</%s>`, + tagName, tagAttrs, + extraCSSClasses, descriptionText, + textColor, scopeColor, scopeHTML, + textColor, itemColor, itemHTML, + label.ExclusiveOrder, + tagName) + } + + // <scope> | <label> + return htmlutil.HTMLFormat(`<%s %s class="ui label %s scope-parent" data-tooltip-content title="%s">`+ `<div class="ui label scope-left" style="color: %s !important; background-color: %s !important">%s</div>`+ `<div class="ui label scope-right" style="color: %s !important; background-color: %s !important">%s</div>`+ - `</span>`, + `</%s>`, + tagName, tagAttrs, extraCSSClasses, descriptionText, textColor, scopeColor, scopeHTML, - textColor, itemColor, itemHTML) + textColor, itemColor, itemHTML, + tagName) } // RenderEmoji renders html text with emoji post processors @@ -202,7 +236,7 @@ func reactionToEmoji(reaction string) template.HTML { return template.HTML(fmt.Sprintf(`<img alt=":%s:" src="%s/assets/img/emoji/%s.png"></img>`, reaction, setting.StaticURLPrefix, url.PathEscape(reaction))) } -func (ut *RenderUtils) MarkdownToHtml(input string) template.HTML { //nolint:revive +func (ut *RenderUtils) MarkdownToHtml(input string) template.HTML { //nolint:revive // variable naming triggers on Html, wants HTML output, err := markdown.RenderString(markup.NewRenderContext(ut.ctx).WithMetas(markup.ComposeSimpleDocumentMetas()), input) if err != nil { log.Error("RenderString: %v", err) @@ -219,7 +253,8 @@ func (ut *RenderUtils) RenderLabels(labels []*issues_model.Label, repoLink strin if label == nil { continue } - htmlCode += fmt.Sprintf(`<a href="%s?labels=%d">%s</a>`, baseLink, label.ID, ut.RenderLabel(label)) + link := fmt.Sprintf("%s?labels=%d", baseLink, label.ID) + htmlCode += string(ut.RenderLabelWithLink(label, template.URL(link))) } htmlCode += "</span>" return template.HTML(htmlCode) diff --git a/modules/templates/util_render_legacy.go b/modules/templates/util_render_legacy.go deleted file mode 100644 index 994f2fa064..0000000000 --- a/modules/templates/util_render_legacy.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package templates - -import ( - "context" - "html/template" - - issues_model "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/modules/translation" -) - -func renderEmojiLegacy(ctx context.Context, text string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderEmoji(text) -} - -func renderLabelLegacy(ctx context.Context, locale translation.Locale, label *issues_model.Label) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderLabel(label) -} - -func renderLabelsLegacy(ctx context.Context, locale translation.Locale, labels []*issues_model.Label, repoLink string, issue *issues_model.Issue) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderLabels(labels, repoLink, issue) -} - -func renderMarkdownToHtmlLegacy(ctx context.Context, input string) template.HTML { //nolint:revive - panicIfDevOrTesting() - return NewRenderUtils(ctx).MarkdownToHtml(input) -} - -func renderCommitMessageLegacy(ctx context.Context, msg string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderCommitMessage(msg, metas) -} - -func renderCommitMessageLinkSubjectLegacy(ctx context.Context, msg, urlDefault string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderCommitMessageLinkSubject(msg, urlDefault, metas) -} - -func renderIssueTitleLegacy(ctx context.Context, text string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderIssueTitle(text, metas) -} - -func renderCommitBodyLegacy(ctx context.Context, msg string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(ctx).RenderCommitBody(msg, metas) -} diff --git a/modules/templates/util_render_test.go b/modules/templates/util_render_test.go index 80094ab26e..5c37f084df 100644 --- a/modules/templates/util_render_test.go +++ b/modules/templates/util_render_test.go @@ -11,10 +11,11 @@ import ( "testing" "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/reqctx" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/translation" @@ -46,19 +47,8 @@ mail@domain.com return strings.ReplaceAll(s, "<SPACE>", " ") } -var testMetas = map[string]string{ - "user": "user13", - "repo": "repo11", - "repoPath": "../../tests/gitea-repositories-meta/user13/repo11.git/", - "markdownLineBreakStyle": "comment", - "markupAllowShortIssuePattern": "true", -} - func TestMain(m *testing.M) { - unittest.InitSettings() - if err := git.InitSimple(context.Background()); err != nil { - log.Fatal("git init failed, err: %v", err) - } + setting.Markdown.RenderOptionsComment.ShortIssuePattern = true markup.Init(&markup.RenderHelperFuncs{ IsUsernameMentionable: func(ctx context.Context, username string) bool { return username == "mention-user" @@ -67,52 +57,58 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func newTestRenderUtils() *RenderUtils { - ctx := context.Background() - ctx = context.WithValue(ctx, translation.ContextKey, &translation.MockLocale{}) +func newTestRenderUtils(t *testing.T) *RenderUtils { + ctx := reqctx.NewRequestContextForTest(t.Context()) + ctx.SetContextValue(translation.ContextKey, &translation.MockLocale{}) return NewRenderUtils(ctx) } -func TestRenderCommitBody(t *testing.T) { - defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() - type args struct { - msg string +func TestRenderRepoComment(t *testing.T) { + mockRepo := &repo.Repository{ + ID: 1, OwnerName: "user13", Name: "repo11", + Owner: &user_model.User{ID: 13, Name: "user13"}, + Units: []*repo.RepoUnit{}, } - tests := []struct { - name string - args args - want template.HTML - }{ - { - name: "multiple lines", - args: args{ - msg: "first line\nsecond line", + t.Run("RenderCommitBody", func(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + type args struct { + msg string + } + tests := []struct { + name string + args args + want template.HTML + }{ + { + name: "multiple lines", + args: args{ + msg: "first line\nsecond line", + }, + want: "second line", }, - want: "second line", - }, - { - name: "multiple lines with leading newlines", - args: args{ - msg: "\n\n\n\nfirst line\nsecond line", + { + name: "multiple lines with leading newlines", + args: args{ + msg: "\n\n\n\nfirst line\nsecond line", + }, + want: "second line", }, - want: "second line", - }, - { - name: "multiple lines with trailing newlines", - args: args{ - msg: "first line\nsecond line\n\n\n", + { + name: "multiple lines with trailing newlines", + args: args{ + msg: "first line\nsecond line\n\n\n", + }, + want: "second line", }, - want: "second line", - }, - } - ut := newTestRenderUtils() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, ut.RenderCommitBody(tt.args.msg, nil), "RenderCommitBody(%v, %v)", tt.args.msg, nil) - }) - } - - expected := `/just/a/path.bin + } + ut := newTestRenderUtils(t) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, ut.RenderCommitBody(tt.args.msg, mockRepo), "RenderCommitBody(%v, %v)", tt.args.msg, nil) + }) + } + + expected := `/just/a/path.bin <a href="https://example.com/file.bin">https://example.com/file.bin</a> [local link](file.bin) [remote link](<a href="https://example.com">https://example.com</a>) @@ -122,31 +118,31 @@ func TestRenderCommitBody(t *testing.T) { ![remote image](<a href="https://example.com/image.jpg">https://example.com/image.jpg</a>) [[local image|image.jpg]] [[remote link|<a href="https://example.com/image.jpg">https://example.com/image.jpg</a>]] -<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" class="compare"><code class="nohighlight">88fc37a3c0...12fc37a3c0 (hash)</code></a> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" class="compare"><code>88fc37a3c0...12fc37a3c0 (hash)</code></a> com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare -<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" class="commit"><code class="nohighlight">88fc37a3c0</code></a> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" class="commit"><code>88fc37a3c0</code></a> com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit <span class="emoji" aria-label="thumbs up">ðŸ‘</span> <a href="mailto:mail@domain.com">mail@domain.com</a> <a href="/mention-user">@mention-user</a> test <a href="/user13/repo11/issues/123" class="ref-issue">#123</a> space` - assert.EqualValues(t, expected, string(newTestRenderUtils().RenderCommitBody(testInput(), testMetas))) -} + assert.Equal(t, expected, string(newTestRenderUtils(t).RenderCommitBody(testInput(), mockRepo))) + }) -func TestRenderCommitMessage(t *testing.T) { - expected := `space <a href="/mention-user" data-markdown-generated-content="">@mention-user</a> ` - assert.EqualValues(t, expected, newTestRenderUtils().RenderCommitMessage(testInput(), testMetas)) -} + t.Run("RenderCommitMessage", func(t *testing.T) { + expected := `space <a href="/mention-user" data-markdown-generated-content="">@mention-user</a> ` + assert.EqualValues(t, expected, newTestRenderUtils(t).RenderCommitMessage(testInput(), mockRepo)) + }) -func TestRenderCommitMessageLinkSubject(t *testing.T) { - expected := `<a href="https://example.com/link" class="muted">space </a><a href="/mention-user" data-markdown-generated-content="">@mention-user</a>` - assert.EqualValues(t, expected, newTestRenderUtils().RenderCommitMessageLinkSubject(testInput(), "https://example.com/link", testMetas)) -} + t.Run("RenderCommitMessageLinkSubject", func(t *testing.T) { + expected := `<a href="https://example.com/link" class="muted">space </a><a href="/mention-user" data-markdown-generated-content="">@mention-user</a>` + assert.EqualValues(t, expected, newTestRenderUtils(t).RenderCommitMessageLinkSubject(testInput(), "https://example.com/link", mockRepo)) + }) -func TestRenderIssueTitle(t *testing.T) { - defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() - expected := ` space @mention-user<SPACE><SPACE> + t.Run("RenderIssueTitle", func(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + expected := ` space @mention-user<SPACE><SPACE> /just/a/path.bin https://example.com/file.bin [local link](file.bin) @@ -167,8 +163,9 @@ mail@domain.com <a href="/user13/repo11/issues/123" class="ref-issue">#123</a> space<SPACE><SPACE> ` - expected = strings.ReplaceAll(expected, "<SPACE>", " ") - assert.EqualValues(t, expected, string(newTestRenderUtils().RenderIssueTitle(testInput(), testMetas))) + expected = strings.ReplaceAll(expected, "<SPACE>", " ") + assert.Equal(t, expected, string(newTestRenderUtils(t).RenderIssueTitle(testInput(), mockRepo))) + }) } func TestRenderMarkdownToHtml(t *testing.T) { @@ -194,11 +191,11 @@ com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit #123 space</p> ` - assert.Equal(t, expected, string(newTestRenderUtils().MarkdownToHtml(testInput()))) + assert.Equal(t, expected, string(newTestRenderUtils(t).MarkdownToHtml(testInput()))) } func TestRenderLabels(t *testing.T) { - ut := newTestRenderUtils() + ut := newTestRenderUtils(t) label := &issues.Label{ID: 123, Name: "label-name", Color: "label-color"} issue := &issues.Issue{} expected := `/owner/repo/issues?labels=123` @@ -208,10 +205,21 @@ func TestRenderLabels(t *testing.T) { issue = &issues.Issue{IsPull: true} expected = `/owner/repo/pulls?labels=123` assert.Contains(t, ut.RenderLabels([]*issues.Label{label}, "/owner/repo", issue), expected) + + expectedLabel := `<a href="<>" class="ui label " style="color: #fff !important; background-color: label-color !important;" data-tooltip-content title=""><span class="gt-ellipsis">label-name</span></a>` + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, "<>"))) + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, template.URL("<>")))) + + label = &issues.Label{ID: 123, Name: "</>", Exclusive: true} + expectedLabel = `<a href="" class="ui label scope-parent" data-tooltip-content title=""><div class="ui label scope-left" style="color: #fff !important; background-color: #000000 !important"><</div><div class="ui label scope-right" style="color: #fff !important; background-color: #000000 !important">></div></a>` + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, ""))) + label = &issues.Label{ID: 123, Name: "</>", Exclusive: true, ExclusiveOrder: 1} + expectedLabel = `<a href="" class="ui label scope-parent" data-tooltip-content title=""><div class="ui label scope-left" style="color: #fff !important; background-color: #000000 !important"><</div><div class="ui label scope-middle" style="color: #fff !important; background-color: #000000 !important">></div><div class="ui label scope-right">1</div></a>` + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, ""))) } func TestUserMention(t *testing.T) { markup.RenderBehaviorForTesting.DisableAdditionalAttributes = true - rendered := newTestRenderUtils().MarkdownToHtml("@no-such-user @mention-user @mention-user") - assert.EqualValues(t, `<p>@no-such-user <a href="/mention-user" rel="nofollow">@mention-user</a> <a href="/mention-user" rel="nofollow">@mention-user</a></p>`, strings.TrimSpace(string(rendered))) + rendered := newTestRenderUtils(t).MarkdownToHtml("@no-such-user @mention-user @mention-user") + assert.Equal(t, `<p>@no-such-user <a href="/mention-user" rel="nofollow">@mention-user</a> <a href="/mention-user" rel="nofollow">@mention-user</a></p>`, strings.TrimSpace(string(rendered))) } diff --git a/modules/templates/util_string.go b/modules/templates/util_string.go index 382e2de13f..683c77a870 100644 --- a/modules/templates/util_string.go +++ b/modules/templates/util_string.go @@ -8,7 +8,7 @@ import ( "html/template" "strings" - "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/util" ) type StringUtils struct{} @@ -54,7 +54,7 @@ func (su *StringUtils) Cut(s, sep string) []any { } func (su *StringUtils) EllipsisString(s string, maxLength int) string { - return base.EllipsisString(s, maxLength) + return util.EllipsisDisplayString(s, maxLength) } func (su *StringUtils) ToUpper(s string) string { diff --git a/modules/templates/util_test.go b/modules/templates/util_test.go index febaf7fa88..a6448a6ff2 100644 --- a/modules/templates/util_test.go +++ b/modules/templates/util_test.go @@ -28,7 +28,7 @@ func TestDict(t *testing.T) { for _, c := range cases { got, err := dict(c.args...) if assert.NoError(t, err) { - assert.EqualValues(t, c.want, got) + assert.Equal(t, c.want, got) } } diff --git a/modules/templates/vars/vars.go b/modules/templates/vars/vars.go index cc9d0e976f..500078d4b8 100644 --- a/modules/templates/vars/vars.go +++ b/modules/templates/vars/vars.go @@ -16,7 +16,7 @@ type ErrWrongSyntax struct { } func (err ErrWrongSyntax) Error() string { - return fmt.Sprintf("wrong syntax found in %s", err.Template) + return "wrong syntax found in " + err.Template } // ErrVarMissing represents an error that no matched variable diff --git a/modules/templates/vars/vars_test.go b/modules/templates/vars/vars_test.go index 8f421d9e4b..9b48167237 100644 --- a/modules/templates/vars/vars_test.go +++ b/modules/templates/vars/vars_test.go @@ -60,7 +60,7 @@ func TestExpandVars(t *testing.T) { for _, kase := range kases { t.Run(kase.tmpl, func(t *testing.T) { res, err := Expand(kase.tmpl, kase.data) - assert.EqualValues(t, kase.out, res) + assert.Equal(t, kase.out, res) if kase.error { assert.Error(t, err) } else { diff --git a/modules/test/logchecker.go b/modules/test/logchecker.go index 7bf234f560..829f735c7c 100644 --- a/modules/test/logchecker.go +++ b/modules/test/logchecker.go @@ -5,7 +5,7 @@ package test import ( "context" - "fmt" + "strconv" "strings" "sync" "sync/atomic" @@ -58,7 +58,7 @@ var checkerIndex int64 func NewLogChecker(namePrefix string) (logChecker *LogChecker, cancel func()) { logger := log.GetManager().GetLogger(namePrefix) newCheckerIndex := atomic.AddInt64(&checkerIndex, 1) - writerName := namePrefix + "-" + fmt.Sprint(newCheckerIndex) + writerName := namePrefix + "-" + strconv.FormatInt(newCheckerIndex, 10) lc := &LogChecker{} lc.EventWriterBaseImpl = log.NewEventWriterBase(writerName, "test-log-checker", log.WriterMode{}) diff --git a/modules/test/utils.go b/modules/test/utils.go index ec4c976388..53c6a3ed52 100644 --- a/modules/test/utils.go +++ b/modules/test/utils.go @@ -4,7 +4,6 @@ package test import ( - "fmt" "net/http" "net/http/httptest" "os" @@ -18,6 +17,7 @@ import ( // RedirectURL returns the redirect URL of a http response. // It also works for JSONRedirect: `{"redirect": "..."}` +// FIXME: it should separate the logic of checking from header and JSON body func RedirectURL(resp http.ResponseWriter) string { loc := resp.Header().Get("Location") if loc != "" { @@ -35,6 +35,15 @@ func RedirectURL(resp http.ResponseWriter) string { return "" } +func ParseJSONError(buf []byte) (ret struct { + ErrorMessage string `json:"errorMessage"` + RenderFormat string `json:"renderFormat"` +}, +) { + _ = json.Unmarshal(buf, &ret) + return ret +} + func IsNormalPageCompleted(s string) bool { return strings.Contains(s, `<footer class="page-footer"`) && strings.Contains(s, `</html>`) } @@ -57,7 +66,7 @@ func SetupGiteaRoot() string { giteaRoot = filepath.Dir(filepath.Dir(filepath.Dir(filename))) fixturesDir := filepath.Join(giteaRoot, "models", "fixtures") if exist, _ := util.IsDir(fixturesDir); !exist { - panic(fmt.Sprintf("fixtures directory not found: %s", fixturesDir)) + panic("fixtures directory not found: " + fixturesDir) } _ = os.Setenv("GITEA_ROOT", giteaRoot) return giteaRoot diff --git a/modules/testlogger/testlogger.go b/modules/testlogger/testlogger.go index 2fbfce6b03..60e281d403 100644 --- a/modules/testlogger/testlogger.go +++ b/modules/testlogger/testlogger.go @@ -4,7 +4,6 @@ package testlogger import ( - "context" "fmt" "os" "runtime" @@ -93,7 +92,7 @@ func (w *testLoggerWriterCloser) Reset() { // Printf takes a format and args and prints the string to os.Stdout func Printf(format string, args ...any) { if !log.CanColorStdout { - for i := 0; i < len(args); i++ { + for i := range args { if c, ok := args[i].(*log.ColoredValue); ok { args[i] = c.Value() } @@ -131,7 +130,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { slowFlushChecker := time.AfterFunc(TestSlowFlush, func() { Printf("+++ %s ... still flushing after %v ...\n", log.NewColoredValue(t.Name(), log.Bold, log.FgRed), TestSlowFlush) }) - if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil { + if err := queue.GetManager().FlushAll(t.Context(), -1); err != nil { t.Errorf("Flushing queues failed with error %v", err) } slowFlushChecker.Stop() diff --git a/modules/timeutil/executable.go b/modules/timeutil/executable.go deleted file mode 100644 index 57ae8b2a9d..0000000000 --- a/modules/timeutil/executable.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package timeutil - -import ( - "os" - "path/filepath" - "sync" - "time" - - "code.gitea.io/gitea/modules/log" -) - -var ( - executablModTime = time.Now() - executablModTimeOnce sync.Once -) - -// GetExecutableModTime get executable file modified time of current process. -func GetExecutableModTime() time.Time { - executablModTimeOnce.Do(func() { - exePath, err := os.Executable() - if err != nil { - log.Error("os.Executable: %v", err) - return - } - - exePath, err = filepath.Abs(exePath) - if err != nil { - log.Error("filepath.Abs: %v", err) - return - } - - exePath, err = filepath.EvalSymlinks(exePath) - if err != nil { - log.Error("filepath.EvalSymlinks: %v", err) - return - } - - st, err := os.Stat(exePath) - if err != nil { - log.Error("os.Stat: %v", err) - return - } - - executablModTime = st.ModTime() - }) - return executablModTime -} diff --git a/modules/translation/i18n/errors.go b/modules/translation/i18n/errors.go deleted file mode 100644 index 7f64ccf908..0000000000 --- a/modules/translation/i18n/errors.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package i18n - -import ( - "code.gitea.io/gitea/modules/util" -) - -var ( - ErrLocaleAlreadyExist = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist} - ErrUncertainArguments = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument} -) diff --git a/modules/translation/i18n/format.go b/modules/translation/i18n/format.go index e5e221831f..450509333f 100644 --- a/modules/translation/i18n/format.go +++ b/modules/translation/i18n/format.go @@ -4,6 +4,7 @@ package i18n import ( + "errors" "fmt" "reflect" ) @@ -30,7 +31,7 @@ func Format(format string, args ...any) (msg string, err error) { fmtArgs = append(fmtArgs, val.Index(i).Interface()) } } else { - err = ErrUncertainArguments + err = errors.New("arguments to i18n should not contain uncertain slices") break } } else { diff --git a/modules/translation/i18n/localestore.go b/modules/translation/i18n/localestore.go index b422996984..4f1ae7e13d 100644 --- a/modules/translation/i18n/localestore.go +++ b/modules/translation/i18n/localestore.go @@ -4,6 +4,7 @@ package i18n import ( + "errors" "fmt" "html/template" "slices" @@ -41,7 +42,7 @@ func NewLocaleStore() LocaleStore { // AddLocaleByIni adds locale by ini into the store func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error { if _, ok := store.localeMap[langName]; ok { - return ErrLocaleAlreadyExist + return errors.New("lang has already been added") } store.langNames = append(store.langNames, langName) diff --git a/modules/translation/translation_test.go b/modules/translation/translation_test.go index 464aa32661..87df9eb825 100644 --- a/modules/translation/translation_test.go +++ b/modules/translation/translation_test.go @@ -20,13 +20,13 @@ func TestPrettyNumber(t *testing.T) { allLangMap["id-ID"] = &LangType{Lang: "id-ID", Name: "Bahasa Indonesia"} l := NewLocale("id-ID") - assert.EqualValues(t, "1.000.000", l.PrettyNumber(1000000)) - assert.EqualValues(t, "1.000.000,1", l.PrettyNumber(1000000.1)) - assert.EqualValues(t, "1.000.000", l.PrettyNumber("1000000")) - assert.EqualValues(t, "1.000.000", l.PrettyNumber("1000000.0")) - assert.EqualValues(t, "1.000.000,1", l.PrettyNumber("1000000.1")) + assert.Equal(t, "1.000.000", l.PrettyNumber(1000000)) + assert.Equal(t, "1.000.000,1", l.PrettyNumber(1000000.1)) + assert.Equal(t, "1.000.000", l.PrettyNumber("1000000")) + assert.Equal(t, "1.000.000", l.PrettyNumber("1000000.0")) + assert.Equal(t, "1.000.000,1", l.PrettyNumber("1000000.1")) l = NewLocale("nosuch") - assert.EqualValues(t, "1,000,000", l.PrettyNumber(1000000)) - assert.EqualValues(t, "1,000,000.1", l.PrettyNumber(1000000.1)) + assert.Equal(t, "1,000,000", l.PrettyNumber(1000000)) + assert.Equal(t, "1,000,000.1", l.PrettyNumber(1000000.1)) } diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go index 8cb3d278ce..2e8d9c4a1e 100644 --- a/modules/typesniffer/typesniffer.go +++ b/modules/typesniffer/typesniffer.go @@ -6,18 +6,14 @@ package typesniffer import ( "bytes" "encoding/binary" - "fmt" - "io" "net/http" "regexp" "slices" "strings" - - "code.gitea.io/gitea/modules/util" + "sync" ) -// Use at most this many bytes to determine Content Type. -const sniffLen = 1024 +const SniffContentSize = 1024 const ( MimeTypeImageSvg = "image/svg+xml" @@ -26,22 +22,30 @@ const ( MimeTypeApplicationOctetStream = "application/octet-stream" ) -var ( - svgComment = regexp.MustCompile(`(?s)<!--.*?-->`) - svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) - svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) -) - -// SniffedType contains information about a blobs type. +var globalVars = sync.OnceValue(func() (ret struct { + svgComment, svgTagRegex, svgTagInXMLRegex *regexp.Regexp +}, +) { + ret.svgComment = regexp.MustCompile(`(?s)<!--.*?-->`) + ret.svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) + ret.svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) + return ret +}) + +// SniffedType contains information about a blob's type. type SniffedType struct { contentType string } -// IsText etects if content format is plain text. +// IsText detects if the content format is text family, including text/plain, text/html, text/css, etc. func (ct SniffedType) IsText() bool { return strings.Contains(ct.contentType, "text/") } +func (ct SniffedType) IsTextPlain() bool { + return strings.Contains(ct.contentType, "text/plain") +} + // IsImage detects if data is an image format func (ct SniffedType) IsImage() bool { return strings.Contains(ct.contentType, "image/") @@ -57,12 +61,12 @@ func (ct SniffedType) IsPDF() bool { return strings.Contains(ct.contentType, "application/pdf") } -// IsVideo detects if data is an video format +// IsVideo detects if data is a video format func (ct SniffedType) IsVideo() bool { return strings.Contains(ct.contentType, "video/") } -// IsAudio detects if data is an video format +// IsAudio detects if data is a video format func (ct SniffedType) IsAudio() bool { return strings.Contains(ct.contentType, "audio/") } @@ -103,33 +107,34 @@ func detectFileTypeBox(data []byte) (brands []string, found bool) { return brands, true } -// DetectContentType extends http.DetectContentType with more content types. Defaults to text/unknown if input is empty. +// DetectContentType extends http.DetectContentType with more content types. Defaults to text/plain if input is empty. func DetectContentType(data []byte) SniffedType { if len(data) == 0 { - return SniffedType{"text/unknown"} + return SniffedType{"text/plain"} } ct := http.DetectContentType(data) - if len(data) > sniffLen { - data = data[:sniffLen] + if len(data) > SniffContentSize { + data = data[:SniffContentSize] } + vars := globalVars() // SVG is unsupported by http.DetectContentType, https://github.com/golang/go/issues/15888 detectByHTML := strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html") detectByXML := strings.Contains(ct, "text/xml") if detectByHTML || detectByXML { - dataProcessed := svgComment.ReplaceAll(data, nil) + dataProcessed := vars.svgComment.ReplaceAll(data, nil) dataProcessed = bytes.TrimSpace(dataProcessed) - if detectByHTML && svgTagRegex.Match(dataProcessed) || - detectByXML && svgTagInXMLRegex.Match(dataProcessed) { + if detectByHTML && vars.svgTagRegex.Match(dataProcessed) || + detectByXML && vars.svgTagInXMLRegex.Match(dataProcessed) { ct = MimeTypeImageSvg } } if strings.HasPrefix(ct, "audio/") && bytes.HasPrefix(data, []byte("ID3")) { // The MP3 detection is quite inaccurate, any content with "ID3" prefix will result in "audio/mpeg". - // So remove the "ID3" prefix and detect again, if result is text, then it must be text content. + // So remove the "ID3" prefix and detect again, then if the result is "text", it must be text content. // This works especially because audio files contain many unprintable/invalid characters like `0x00` ct2 := http.DetectContentType(data[3:]) if strings.HasPrefix(ct2, "text/") { @@ -155,15 +160,3 @@ func DetectContentType(data []byte) SniffedType { } return SniffedType{ct} } - -// DetectContentTypeFromReader guesses the content type contained in the reader. -func DetectContentTypeFromReader(r io.Reader) (SniffedType, error) { - buf := make([]byte, sniffLen) - n, err := util.ReadAtMost(r, buf) - if err != nil { - return SniffedType{}, fmt.Errorf("DetectContentTypeFromReader io error: %w", err) - } - buf = buf[:n] - - return DetectContentType(buf), nil -} diff --git a/modules/typesniffer/typesniffer_test.go b/modules/typesniffer/typesniffer_test.go index 3e5db3308b..a0c824b912 100644 --- a/modules/typesniffer/typesniffer_test.go +++ b/modules/typesniffer/typesniffer_test.go @@ -4,7 +4,6 @@ package typesniffer import ( - "bytes" "encoding/base64" "encoding/hex" "strings" @@ -17,7 +16,7 @@ func TestDetectContentTypeLongerThanSniffLen(t *testing.T) { // Pre-condition: Shorter than sniffLen detects SVG. assert.Equal(t, "image/svg+xml", DetectContentType([]byte(`<!-- Comment --><svg></svg>`)).contentType) // Longer than sniffLen detects something else. - assert.NotEqual(t, "image/svg+xml", DetectContentType([]byte(`<!-- `+strings.Repeat("x", sniffLen)+` --><svg></svg>`)).contentType) + assert.NotEqual(t, "image/svg+xml", DetectContentType([]byte(`<!-- `+strings.Repeat("x", SniffContentSize)+` --><svg></svg>`)).contentType) } func TestIsTextFile(t *testing.T) { @@ -116,22 +115,13 @@ func TestIsAudio(t *testing.T) { assert.True(t, DetectContentType([]byte("ID3Toy\n====\t* hi 🌞, ..."+"🌛"[0:2])).IsText()) // test ID3 tag with incomplete UTF8 char } -func TestDetectContentTypeFromReader(t *testing.T) { - mp3, _ := base64.StdEncoding.DecodeString("SUQzBAAAAAABAFRYWFgAAAASAAADbWFqb3JfYnJhbmQAbXA0MgBUWFhYAAAAEQAAA21pbm9yX3Zl") - st, err := DetectContentTypeFromReader(bytes.NewReader(mp3)) - assert.NoError(t, err) - assert.True(t, st.IsAudio()) -} - func TestDetectContentTypeOgg(t *testing.T) { oggAudio, _ := hex.DecodeString("4f67675300020000000000000000352f0000000000007dc39163011e01766f72626973000000000244ac0000000000000071020000000000b8014f6767530000") - st, err := DetectContentTypeFromReader(bytes.NewReader(oggAudio)) - assert.NoError(t, err) + st := DetectContentType(oggAudio) assert.True(t, st.IsAudio()) oggVideo, _ := hex.DecodeString("4f676753000200000000000000007d9747ef000000009b59daf3012a807468656f7261030201001e00110001e000010e00020000001e00000001000001000001") - st, err = DetectContentTypeFromReader(bytes.NewReader(oggVideo)) - assert.NoError(t, err) + st = DetectContentType(oggVideo) assert.True(t, st.IsVideo()) } diff --git a/modules/updatechecker/update_checker.go b/modules/updatechecker/update_checker.go index 3c1e05d060..f0686c0f78 100644 --- a/modules/updatechecker/update_checker.go +++ b/modules/updatechecker/update_checker.go @@ -34,7 +34,7 @@ func GiteaUpdateChecker(httpEndpoint string) error { }, } - req, err := http.NewRequest("GET", httpEndpoint, nil) + req, err := http.NewRequest(http.MethodGet, httpEndpoint, nil) if err != nil { return err } diff --git a/modules/user/user_test.go b/modules/user/user_test.go index 372a675d34..d6b3911ca6 100644 --- a/modules/user/user_test.go +++ b/modules/user/user_test.go @@ -8,6 +8,9 @@ import ( "runtime" "strings" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func getWhoamiOutput() (string, error) { @@ -20,24 +23,19 @@ func getWhoamiOutput() (string, error) { func TestCurrentUsername(t *testing.T) { user := CurrentUsername() - if len(user) == 0 { - t.Errorf("expected non-empty user, got: %s", user) - } + require.NotEmpty(t, user) + // Windows whoami is weird, so just skip remaining tests if runtime.GOOS == "windows" { t.Skip("skipped test because of weird whoami on Windows") } whoami, err := getWhoamiOutput() - if err != nil { - t.Errorf("failed to run whoami to test current user: %f", err) - } + require.NoError(t, err) + user = CurrentUsername() - if user != whoami { - t.Errorf("expected %s as user, got: %s", whoami, user) - } + assert.Equal(t, whoami, user) + t.Setenv("USER", "spoofed") user = CurrentUsername() - if user != whoami { - t.Errorf("expected %s as user, got: %s", whoami, user) - } + assert.Equal(t, whoami, user) } diff --git a/modules/util/error.go b/modules/util/error.go index 0f3597147c..6b2721618e 100644 --- a/modules/util/error.go +++ b/modules/util/error.go @@ -10,56 +10,88 @@ import ( // Common Errors forming the base of our error system // -// Many Errors returned by Gitea can be tested against these errors -// using errors.Is. +// Many Errors returned by Gitea can be tested against these errors using "errors.Is". var ( - ErrInvalidArgument = errors.New("invalid argument") - ErrPermissionDenied = errors.New("permission denied") - ErrAlreadyExist = errors.New("resource already exists") - ErrNotExist = errors.New("resource does not exist") + ErrInvalidArgument = errors.New("invalid argument") // also implies HTTP 400 + ErrPermissionDenied = errors.New("permission denied") // also implies HTTP 403 + ErrNotExist = errors.New("resource does not exist") // also implies HTTP 404 + ErrAlreadyExist = errors.New("resource already exists") // also implies HTTP 409 + + // ErrUnprocessableContent implies HTTP 422, the syntax of the request content is correct, + // but the server is unable to process the contained instructions + ErrUnprocessableContent = errors.New("unprocessable content") ) -// SilentWrap provides a simple wrapper for a wrapped error where the wrapped error message plays no part in the error message +// errorWrapper provides a simple wrapper for a wrapped error where the wrapped error message plays no part in the error message // Especially useful for "untyped" errors created with "errors.New(…)" that can be classified as 'invalid argument', 'permission denied', 'exists already', or 'does not exist' -type SilentWrap struct { +type errorWrapper struct { Message string Err error } // Error returns the message -func (w SilentWrap) Error() string { +func (w errorWrapper) Error() string { return w.Message } // Unwrap returns the underlying error -func (w SilentWrap) Unwrap() error { +func (w errorWrapper) Unwrap() error { return w.Err } -// NewSilentWrapErrorf returns an error that formats as the given text but unwraps as the provided error -func NewSilentWrapErrorf(unwrap error, message string, args ...any) error { +type LocaleWrapper struct { + err error + TrKey string + TrArgs []any +} + +// Error returns the message +func (w LocaleWrapper) Error() string { + return w.err.Error() +} + +// Unwrap returns the underlying error +func (w LocaleWrapper) Unwrap() error { + return w.err +} + +// ErrorWrap returns an error that formats as the given text but unwraps as the provided error +func ErrorWrap(unwrap error, message string, args ...any) error { if len(args) == 0 { - return SilentWrap{Message: message, Err: unwrap} + return errorWrapper{Message: message, Err: unwrap} } - return SilentWrap{Message: fmt.Sprintf(message, args...), Err: unwrap} + return errorWrapper{Message: fmt.Sprintf(message, args...), Err: unwrap} } // NewInvalidArgumentErrorf returns an error that formats as the given text but unwraps as an ErrInvalidArgument func NewInvalidArgumentErrorf(message string, args ...any) error { - return NewSilentWrapErrorf(ErrInvalidArgument, message, args...) + return ErrorWrap(ErrInvalidArgument, message, args...) } // NewPermissionDeniedErrorf returns an error that formats as the given text but unwraps as an ErrPermissionDenied func NewPermissionDeniedErrorf(message string, args ...any) error { - return NewSilentWrapErrorf(ErrPermissionDenied, message, args...) + return ErrorWrap(ErrPermissionDenied, message, args...) } // NewAlreadyExistErrorf returns an error that formats as the given text but unwraps as an ErrAlreadyExist func NewAlreadyExistErrorf(message string, args ...any) error { - return NewSilentWrapErrorf(ErrAlreadyExist, message, args...) + return ErrorWrap(ErrAlreadyExist, message, args...) } // NewNotExistErrorf returns an error that formats as the given text but unwraps as an ErrNotExist func NewNotExistErrorf(message string, args ...any) error { - return NewSilentWrapErrorf(ErrNotExist, message, args...) + return ErrorWrap(ErrNotExist, message, args...) +} + +// ErrorWrapLocale wraps an err with a translation key and arguments +func ErrorWrapLocale(err error, trKey string, trArgs ...any) error { + return LocaleWrapper{err: err, TrKey: trKey, TrArgs: trArgs} +} + +func ErrorAsLocale(err error) *LocaleWrapper { + var e LocaleWrapper + if errors.As(err, &e) { + return &e + } + return nil } diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index 739543e297..0731ba30c8 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -7,16 +7,10 @@ import ( "bytes" "errors" "io" - "math" "os" ) -var ( - // ErrInvalidMemorySize occurs if the memory size is not in a valid range - ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") - // ErrWriteAfterRead occurs if Write is called after a read operation - ErrWriteAfterRead = errors.New("Write is unsupported after a read operation") -) +var ErrWriteAfterRead = errors.New("write is unsupported after a read operation") // occurs if Write is called after a read operation type readAtSeeker interface { io.ReadSeeker @@ -30,34 +24,17 @@ type FileBackedBuffer struct { maxMemorySize int64 size int64 buffer bytes.Buffer + tempDir string file *os.File reader readAtSeeker } // New creates a file backed buffer with a specific maximum memory size -func New(maxMemorySize int) (*FileBackedBuffer, error) { - if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 { - return nil, ErrInvalidMemorySize - } - +func New(maxMemorySize int, tempDir string) *FileBackedBuffer { return &FileBackedBuffer{ maxMemorySize: int64(maxMemorySize), - }, nil -} - -// CreateFromReader creates a file backed buffer and copies the provided reader data into it. -func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) { - b, err := New(maxMemorySize) - if err != nil { - return nil, err + tempDir: tempDir, } - - _, err = io.Copy(b, r) - if err != nil { - return nil, err - } - - return b, nil } // Write implements io.Writer @@ -73,7 +50,7 @@ func (b *FileBackedBuffer) Write(p []byte) (int, error) { n, err = b.file.Write(p) } else { if b.size+int64(len(p)) > b.maxMemorySize { - b.file, err = os.CreateTemp("", "gitea-buffer-") + b.file, err = os.CreateTemp(b.tempDir, "gitea-buffer-") if err != nil { return 0, err } @@ -148,7 +125,7 @@ func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { func (b *FileBackedBuffer) Close() error { if b.file != nil { err := b.file.Close() - os.Remove(b.file.Name()) + _ = os.Remove(b.file.Name()) b.file = nil return err } diff --git a/modules/util/filebuffer/file_backed_buffer_test.go b/modules/util/filebuffer/file_backed_buffer_test.go index 16d5a1965f..3f13c6ac7b 100644 --- a/modules/util/filebuffer/file_backed_buffer_test.go +++ b/modules/util/filebuffer/file_backed_buffer_test.go @@ -21,7 +21,8 @@ func TestFileBackedBuffer(t *testing.T) { } for _, c := range cases { - buf, err := CreateFromReader(strings.NewReader(c.Data), c.MaxMemorySize) + buf := New(c.MaxMemorySize, t.TempDir()) + _, err := io.Copy(buf, strings.NewReader(c.Data)) assert.NoError(t, err) assert.EqualValues(t, len(c.Data), buf.Size()) diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go index e732094c29..565fb7f284 100644 --- a/modules/util/legacy_test.go +++ b/modules/util/legacy_test.go @@ -17,7 +17,7 @@ import ( func TestCopyFile(t *testing.T) { testContent := []byte("hello") - tmpDir := os.TempDir() + tmpDir := t.TempDir() now := time.Now() srcFile := fmt.Sprintf("%s/copy-test-%d-src.txt", tmpDir, now.UnixMicro()) dstFile := fmt.Sprintf("%s/copy-test-%d-dst.txt", tmpDir, now.UnixMicro()) diff --git a/modules/util/map.go b/modules/util/map.go new file mode 100644 index 0000000000..f307faad1f --- /dev/null +++ b/modules/util/map.go @@ -0,0 +1,13 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +func GetMapValueOrDefault[T any](m map[string]any, key string, defaultValue T) T { + if value, ok := m[key]; ok { + if v, ok := value.(T); ok { + return v + } + } + return defaultValue +} diff --git a/modules/util/map_test.go b/modules/util/map_test.go new file mode 100644 index 0000000000..1a141cec88 --- /dev/null +++ b/modules/util/map_test.go @@ -0,0 +1,26 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetMapValueOrDefault(t *testing.T) { + testMap := map[string]any{ + "key1": "value1", + "key2": 42, + "key3": nil, + } + + assert.Equal(t, "value1", GetMapValueOrDefault(testMap, "key1", "default")) + assert.Equal(t, 42, GetMapValueOrDefault(testMap, "key2", 0)) + + assert.Equal(t, "default", GetMapValueOrDefault(testMap, "key4", "default")) + assert.Equal(t, 100, GetMapValueOrDefault(testMap, "key5", 100)) + + assert.Equal(t, "default", GetMapValueOrDefault(testMap, "key3", "default")) +} diff --git a/modules/util/paginate_test.go b/modules/util/paginate_test.go index 6e69dd19cc..3dc5095071 100644 --- a/modules/util/paginate_test.go +++ b/modules/util/paginate_test.go @@ -13,23 +13,23 @@ func TestPaginateSlice(t *testing.T) { stringSlice := []string{"a", "b", "c", "d", "e"} result, ok := PaginateSlice(stringSlice, 1, 2).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"a", "b"}, result) + assert.Equal(t, []string{"a", "b"}, result) result, ok = PaginateSlice(stringSlice, 100, 2).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{}, result) + assert.Equal(t, []string{}, result) result, ok = PaginateSlice(stringSlice, 3, 2).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"e"}, result) + assert.Equal(t, []string{"e"}, result) result, ok = PaginateSlice(stringSlice, 1, 0).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"a", "b", "c", "d", "e"}, result) + assert.Equal(t, []string{"a", "b", "c", "d", "e"}, result) result, ok = PaginateSlice(stringSlice, 1, -1).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"a", "b", "c", "d", "e"}, result) + assert.Equal(t, []string{"a", "b", "c", "d", "e"}, result) type Test struct { Val int @@ -38,9 +38,9 @@ func TestPaginateSlice(t *testing.T) { testVar := []*Test{{Val: 2}, {Val: 3}, {Val: 4}} testVar, ok = PaginateSlice(testVar, 1, 50).([]*Test) assert.True(t, ok) - assert.EqualValues(t, []*Test{{Val: 2}, {Val: 3}, {Val: 4}}, testVar) + assert.Equal(t, []*Test{{Val: 2}, {Val: 3}, {Val: 4}}, testVar) testVar, ok = PaginateSlice(testVar, 2, 2).([]*Test) assert.True(t, ok) - assert.EqualValues(t, []*Test{{Val: 4}}, testVar) + assert.Equal(t, []*Test{{Val: 4}}, testVar) } diff --git a/modules/util/path.go b/modules/util/path.go index 1272f5af2e..0e56348978 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -36,9 +36,10 @@ func PathJoinRel(elem ...string) string { elems[i] = path.Clean("/" + e) } p := path.Join(elems...) - if p == "" { + switch p { + case "": return "" - } else if p == "/" { + case "/": return "." } return p[1:] @@ -140,81 +141,51 @@ func IsExist(path string) (bool, error) { return false, err } -func statDir(dirPath, recPath string, includeDir, isDirOnly, followSymlinks bool) ([]string, error) { - dir, err := os.Open(dirPath) +func listDirRecursively(result *[]string, fsDir, recordParentPath string, opts *ListDirOptions) error { + dir, err := os.Open(fsDir) if err != nil { - return nil, err + return err } defer dir.Close() fis, err := dir.Readdir(0) if err != nil { - return nil, err + return err } - statList := make([]string, 0) for _, fi := range fis { - if CommonSkip(fi.Name()) { + if opts.SkipCommonHiddenNames && IsCommonHiddenFileName(fi.Name()) { continue } - - relPath := path.Join(recPath, fi.Name()) - curPath := path.Join(dirPath, fi.Name()) + relPath := path.Join(recordParentPath, fi.Name()) + curPath := filepath.Join(fsDir, fi.Name()) if fi.IsDir() { - if includeDir { - statList = append(statList, relPath+"/") - } - s, err := statDir(curPath, relPath, includeDir, isDirOnly, followSymlinks) - if err != nil { - return nil, err - } - statList = append(statList, s...) - } else if !isDirOnly { - statList = append(statList, relPath) - } else if followSymlinks && fi.Mode()&os.ModeSymlink != 0 { - link, err := os.Readlink(curPath) - if err != nil { - return nil, err - } - - isDir, err := IsDir(link) - if err != nil { - return nil, err + if opts.IncludeDir { + *result = append(*result, relPath+"/") } - if isDir { - if includeDir { - statList = append(statList, relPath+"/") - } - s, err := statDir(curPath, relPath, includeDir, isDirOnly, followSymlinks) - if err != nil { - return nil, err - } - statList = append(statList, s...) + if err = listDirRecursively(result, curPath, relPath, opts); err != nil { + return err } + } else { + *result = append(*result, relPath) } } - return statList, nil + return nil } -// StatDir gathers information of given directory by depth-first. -// It returns slice of file list and includes subdirectories if enabled; -// it returns error and nil slice when error occurs in underlying functions, -// or given path is not a directory or does not exist. -// +type ListDirOptions struct { + IncludeDir bool // subdirectories are also included with suffix slash + SkipCommonHiddenNames bool +} + +// ListDirRecursively gathers information of given directory by depth-first. +// The paths are always in "dir/slash/file" format (not "\\" even in Windows) // Slice does not include given path itself. -// If subdirectories is enabled, they will have suffix '/'. -func StatDir(rootPath string, includeDir ...bool) ([]string, error) { - if isDir, err := IsDir(rootPath); err != nil { +func ListDirRecursively(rootDir string, opts *ListDirOptions) (res []string, err error) { + if err = listDirRecursively(&res, rootDir, "", opts); err != nil { return nil, err - } else if !isDir { - return nil, errors.New("not a directory or does not exist: " + rootPath) - } - - isIncludeDir := false - if len(includeDir) != 0 { - isIncludeDir = includeDir[0] } - return statDir(rootPath, "", isIncludeDir, false, false) + return res, nil } func isOSWindows() bool { @@ -265,8 +236,8 @@ func HomeDir() (home string, err error) { return home, nil } -// CommonSkip will check a provided name to see if it represents file or directory that should not be watched -func CommonSkip(name string) bool { +// IsCommonHiddenFileName will check a provided name to see if it represents file or directory that should not be watched +func IsCommonHiddenFileName(name string) bool { if name == "" { return true } @@ -275,9 +246,9 @@ func CommonSkip(name string) bool { case '.': return true case 't', 'T': - return name[1:] == "humbs.db" + return name[1:] == "humbs.db" // macOS case 'd', 'D': - return name[1:] == "esktop.ini" + return name[1:] == "esktop.ini" // Windows } return false diff --git a/modules/util/path_test.go b/modules/util/path_test.go index 6a38bf4ace..79c37e55f7 100644 --- a/modules/util/path_test.go +++ b/modules/util/path_test.go @@ -5,10 +5,12 @@ package util import ( "net/url" + "os" "runtime" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestFileURLToPath(t *testing.T) { @@ -210,3 +212,21 @@ func TestCleanPath(t *testing.T) { assert.Equal(t, c.expected, FilePathJoinAbs(c.elems[0], c.elems[1:]...), "case: %v", c.elems) } } + +func TestListDirRecursively(t *testing.T) { + tmpDir := t.TempDir() + _ = os.WriteFile(tmpDir+"/.config", nil, 0o644) + _ = os.Mkdir(tmpDir+"/d1", 0o755) + _ = os.WriteFile(tmpDir+"/d1/f-d1", nil, 0o644) + _ = os.Mkdir(tmpDir+"/d1/s1", 0o755) + _ = os.WriteFile(tmpDir+"/d1/s1/f-d1s1", nil, 0o644) + _ = os.Mkdir(tmpDir+"/d2", 0o755) + + res, err := ListDirRecursively(tmpDir, &ListDirOptions{IncludeDir: true}) + require.NoError(t, err) + assert.ElementsMatch(t, []string{".config", "d1/", "d1/f-d1", "d1/s1/", "d1/s1/f-d1s1", "d2/"}, res) + + res, err = ListDirRecursively(tmpDir, &ListDirOptions{SkipCommonHiddenNames: true}) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"d1/f-d1", "d1/s1/f-d1s1"}, res) +} diff --git a/modules/util/remove.go b/modules/util/remove.go index d1e38faf5f..3db0b5a796 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -15,7 +15,7 @@ const windowsSharingViolationError syscall.Errno = 32 // Remove removes the named file or (empty) directory with at most 5 attempts. func Remove(name string) error { var err error - for i := 0; i < 5; i++ { + for range 5 { err = os.Remove(name) if err == nil { break @@ -44,7 +44,7 @@ func Remove(name string) error { // RemoveAll removes the named file or (empty) directory with at most 5 attempts. func RemoveAll(name string) error { var err error - for i := 0; i < 5; i++ { + for range 5 { err = os.RemoveAll(name) if err == nil { break @@ -73,7 +73,7 @@ func RemoveAll(name string) error { // Rename renames (moves) oldpath to newpath with at most 5 attempts. func Rename(oldpath, newpath string) error { var err error - for i := 0; i < 5; i++ { + for i := range 5 { err = os.Rename(oldpath, newpath) if err == nil { break diff --git a/modules/util/rotatingfilewriter/writer_test.go b/modules/util/rotatingfilewriter/writer_test.go index 88392797b3..f6ea1d50ae 100644 --- a/modules/util/rotatingfilewriter/writer_test.go +++ b/modules/util/rotatingfilewriter/writer_test.go @@ -23,7 +23,7 @@ func TestCompressOldFile(t *testing.T) { ng, err := os.OpenFile(nonGzip, os.O_CREATE|os.O_WRONLY, 0o660) assert.NoError(t, err) - for i := 0; i < 999; i++ { + for range 999 { f.WriteString("This is a test file\n") ng.WriteString("This is a test file\n") } diff --git a/modules/util/runtime_test.go b/modules/util/runtime_test.go index 20f9063b0b..01dd034cea 100644 --- a/modules/util/runtime_test.go +++ b/modules/util/runtime_test.go @@ -18,14 +18,14 @@ func TestCallerFuncName(t *testing.T) { func BenchmarkCallerFuncName(b *testing.B) { // BenchmarkCaller/sprintf-12 12744829 95.49 ns/op b.Run("sprintf", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { _ = fmt.Sprintf("aaaaaaaaaaaaaaaa %s %s %s", "bbbbbbbbbbbbbbbbbbb", b.Name(), "ccccccccccccccccccccc") } }) // BenchmarkCaller/caller-12 10625133 113.6 ns/op // It is almost as fast as fmt.Sprintf b.Run("caller", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { CallerFuncName(1) } }) diff --git a/modules/util/sec_to_time.go b/modules/util/sec_to_time.go index ad0fb1a68b..646f33c82a 100644 --- a/modules/util/sec_to_time.go +++ b/modules/util/sec_to_time.go @@ -8,61 +8,23 @@ import ( "strings" ) -// SecToTime converts an amount of seconds to a human-readable string. E.g. -// 66s -> 1 minute 6 seconds -// 52410s -> 14 hours 33 minutes -// 563418 -> 6 days 12 hours -// 1563418 -> 2 weeks 4 days -// 3937125s -> 1 month 2 weeks -// 45677465s -> 1 year 6 months -func SecToTime(durationVal any) string { - duration, _ := ToInt64(durationVal) +// SecToHours converts an amount of seconds to a human-readable hours string. +// This is stable for planning and managing timesheets. +// Here it only supports hours and minutes, because a work day could contain 6 or 7 or 8 hours. +// If the duration is less than 1 minute, it will be shown as seconds. +func SecToHours(durationVal any) string { + seconds, _ := ToInt64(durationVal) + hours := seconds / 3600 + minutes := (seconds / 60) % 60 formattedTime := "" - - // The following four variables are calculated by taking - // into account the previously calculated variables, this avoids - // pitfalls when using remainders. As that could lead to incorrect - // results when the calculated number equals the quotient number. - remainingDays := duration / (60 * 60 * 24) - years := remainingDays / 365 - remainingDays -= years * 365 - months := remainingDays * 12 / 365 - remainingDays -= months * 365 / 12 - weeks := remainingDays / 7 - remainingDays -= weeks * 7 - days := remainingDays - - // The following three variables are calculated without depending - // on the previous calculated variables. - hours := (duration / 3600) % 24 - minutes := (duration / 60) % 60 - seconds := duration % 60 - - // Extract only the relevant information of the time - // If the time is greater than a year, it makes no sense to display seconds. - switch { - case years > 0: - formattedTime = formatTime(years, "year", formattedTime) - formattedTime = formatTime(months, "month", formattedTime) - case months > 0: - formattedTime = formatTime(months, "month", formattedTime) - formattedTime = formatTime(weeks, "week", formattedTime) - case weeks > 0: - formattedTime = formatTime(weeks, "week", formattedTime) - formattedTime = formatTime(days, "day", formattedTime) - case days > 0: - formattedTime = formatTime(days, "day", formattedTime) - formattedTime = formatTime(hours, "hour", formattedTime) - case hours > 0: - formattedTime = formatTime(hours, "hour", formattedTime) - formattedTime = formatTime(minutes, "minute", formattedTime) - default: - formattedTime = formatTime(minutes, "minute", formattedTime) - formattedTime = formatTime(seconds, "second", formattedTime) - } + formattedTime = formatTime(hours, "hour", formattedTime) + formattedTime = formatTime(minutes, "minute", formattedTime) // The formatTime() function always appends a space at the end. This will be trimmed + if formattedTime == "" && seconds > 0 { + formattedTime = formatTime(seconds, "second", "") + } return strings.TrimRight(formattedTime, " ") } @@ -76,6 +38,5 @@ func formatTime(value int64, name, formattedTime string) string { } else if value > 1 { formattedTime = fmt.Sprintf("%s%d %ss ", formattedTime, value, name) } - return formattedTime } diff --git a/modules/util/sec_to_time_test.go b/modules/util/sec_to_time_test.go index 4d1213a52c..84e767c6e0 100644 --- a/modules/util/sec_to_time_test.go +++ b/modules/util/sec_to_time_test.go @@ -9,22 +9,20 @@ import ( "github.com/stretchr/testify/assert" ) -func TestSecToTime(t *testing.T) { +func TestSecToHours(t *testing.T) { second := int64(1) minute := 60 * second hour := 60 * minute day := 24 * hour - year := 365 * day - assert.Equal(t, "1 minute 6 seconds", SecToTime(minute+6*second)) - assert.Equal(t, "1 hour", SecToTime(hour)) - assert.Equal(t, "1 hour", SecToTime(hour+second)) - assert.Equal(t, "14 hours 33 minutes", SecToTime(14*hour+33*minute+30*second)) - assert.Equal(t, "6 days 12 hours", SecToTime(6*day+12*hour+30*minute+18*second)) - assert.Equal(t, "2 weeks 4 days", SecToTime((2*7+4)*day+2*hour+16*minute+58*second)) - assert.Equal(t, "4 weeks", SecToTime(4*7*day)) - assert.Equal(t, "4 weeks 1 day", SecToTime((4*7+1)*day)) - assert.Equal(t, "1 month 2 weeks", SecToTime((6*7+3)*day+13*hour+38*minute+45*second)) - assert.Equal(t, "11 months", SecToTime(year-25*day)) - assert.Equal(t, "1 year 5 months", SecToTime(year+163*day+10*hour+11*minute+5*second)) + assert.Equal(t, "1 minute", SecToHours(minute+6*second)) + assert.Equal(t, "1 hour", SecToHours(hour)) + assert.Equal(t, "1 hour", SecToHours(hour+second)) + assert.Equal(t, "14 hours 33 minutes", SecToHours(14*hour+33*minute+30*second)) + assert.Equal(t, "156 hours 30 minutes", SecToHours(6*day+12*hour+30*minute+18*second)) + assert.Equal(t, "98 hours 16 minutes", SecToHours(4*day+2*hour+16*minute+58*second)) + assert.Equal(t, "672 hours", SecToHours(4*7*day)) + assert.Equal(t, "1 second", SecToHours(1)) + assert.Equal(t, "2 seconds", SecToHours(2)) + assert.Empty(t, SecToHours(nil)) // old behavior, empty means no output } diff --git a/modules/util/shellquote_test.go b/modules/util/shellquote_test.go index 969998c592..4ef5ce6980 100644 --- a/modules/util/shellquote_test.go +++ b/modules/util/shellquote_test.go @@ -3,7 +3,11 @@ package util -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestShellEscape(t *testing.T) { tests := []struct { @@ -83,9 +87,7 @@ func TestShellEscape(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := ShellEscape(tt.toEscape); got != tt.want { - t.Errorf("ShellEscape(%q):\nGot: %s\nWanted: %s", tt.toEscape, got, tt.want) - } + assert.Equal(t, tt.want, ShellEscape(tt.toEscape)) }) } } diff --git a/modules/util/slice.go b/modules/util/slice.go index 9c878c24be..aaa729c1c9 100644 --- a/modules/util/slice.go +++ b/modules/util/slice.go @@ -12,8 +12,7 @@ import ( // SliceContainsString sequential searches if string exists in slice. func SliceContainsString(slice []string, target string, insensitive ...bool) bool { if len(insensitive) != 0 && insensitive[0] { - target = strings.ToLower(target) - return slices.ContainsFunc(slice, func(t string) bool { return strings.ToLower(t) == target }) + return slices.ContainsFunc(slice, func(t string) bool { return strings.EqualFold(t, target) }) } return slices.Contains(slice, target) @@ -71,3 +70,10 @@ func KeysOfMap[K comparable, V any](m map[K]V) []K { } return keys } + +func SliceNilAsEmpty[T any](a []T) []T { + if a == nil { + return []T{} + } + return a +} diff --git a/modules/util/string.go b/modules/util/string.go index cf50f591c6..b9b59df3ef 100644 --- a/modules/util/string.go +++ b/modules/util/string.go @@ -3,7 +3,10 @@ package util -import "unsafe" +import ( + "strings" + "unsafe" +) func isSnakeCaseUpper(c byte) bool { return 'A' <= c && c <= 'Z' @@ -95,3 +98,36 @@ func UnsafeBytesToString(b []byte) string { func UnsafeStringToBytes(s string) []byte { return unsafe.Slice(unsafe.StringData(s), len(s)) } + +// SplitTrimSpace splits the string at given separator and trims leading and trailing space +func SplitTrimSpace(input, sep string) []string { + input = strings.TrimSpace(input) + var stringList []string + for s := range strings.SplitSeq(input, sep) { + if s = strings.TrimSpace(s); s != "" { + stringList = append(stringList, s) + } + } + return stringList +} + +func asciiLower(b byte) byte { + if 'A' <= b && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +// AsciiEqualFold is from Golang https://cs.opensource.google/go/go/+/refs/tags/go1.24.4:src/net/http/internal/ascii/print.go +// ASCII only. In most cases for protocols, we should only use this but not [strings.EqualFold] +func AsciiEqualFold(s, t string) bool { //nolint:revive // PascalCase + if len(s) != len(t) { + return false + } + for i := 0; i < len(s); i++ { + if asciiLower(s[i]) != asciiLower(t[i]) { + return false + } + } + return true +} diff --git a/modules/util/string_test.go b/modules/util/string_test.go index 0a4a8bbcfb..ff67b5c7d4 100644 --- a/modules/util/string_test.go +++ b/modules/util/string_test.go @@ -45,3 +45,8 @@ func TestToSnakeCase(t *testing.T) { assert.Equal(t, expected, ToSnakeCase(input)) } } + +func TestSplitTrimSpace(t *testing.T) { + assert.Equal(t, []string{"a", "b", "c"}, SplitTrimSpace("a\nb\nc", "\n")) + assert.Equal(t, []string{"a", "b"}, SplitTrimSpace("\r\na\n\r\nb\n\n", "\n")) +} diff --git a/modules/util/time_str.go b/modules/util/time_str.go index 0fccfe82cc..81b132c3db 100644 --- a/modules/util/time_str.go +++ b/modules/util/time_str.go @@ -59,7 +59,7 @@ func TimeEstimateParse(timeStr string) (int64, error) { unit := timeStr[match[4]:match[5]] found := false for _, u := range timeStrGlobalVars().units { - if strings.ToLower(unit) == u.name { + if strings.EqualFold(unit, u.name) { total += amount * u.num found = true break diff --git a/modules/util/truncate.go b/modules/util/truncate.go index f2edbdc673..52534d3cac 100644 --- a/modules/util/truncate.go +++ b/modules/util/truncate.go @@ -5,6 +5,7 @@ package util import ( "strings" + "unicode" "unicode/utf8" ) @@ -14,43 +15,118 @@ const ( asciiEllipsis = "..." ) -// SplitStringAtByteN splits a string at byte n accounting for rune boundaries. (Combining characters are not accounted for.) -func SplitStringAtByteN(input string, n int) (left, right string) { - if len(input) <= n { - return input, "" +func IsLikelyEllipsisLeftPart(s string) bool { + return strings.HasSuffix(s, utf8Ellipsis) || strings.HasSuffix(s, asciiEllipsis) +} + +func ellipsisDisplayGuessWidth(r rune) int { + // To make the truncated string as long as possible, + // CJK/emoji chars are considered as 2-ASCII width but not 3-4 bytes width. + // Here we only make the best guess (better than counting them in bytes), + // it's impossible to 100% correctly determine the width of a rune without a real font and render. + // + // ATTENTION: the guessed width can't be zero, more details in ellipsisDisplayString's comment + if r <= 255 { + return 1 } - if !utf8.ValidString(input) { - if n-3 < 0 { - return input, "" + switch { + case r == '\u3000': /* ideographic (CJK) characters, still use 2 */ + return 2 + case unicode.Is(unicode.M, r), /* (Mark) */ + unicode.Is(unicode.Cf, r), /* (Other, format) */ + unicode.Is(unicode.Cs, r), /* (Other, surrogate) */ + unicode.Is(unicode.Z /* (Space) */, r): + return 1 + default: + return 2 + } +} + +// EllipsisDisplayString returns a truncated short string for display purpose. +// The length is the approximate number of ASCII-width in the string (CJK/emoji are 2-ASCII width) +// It appends "…" or "..." at the end of truncated string. +// It guarantees the length of the returned runes doesn't exceed the limit. +func EllipsisDisplayString(str string, limit int) string { + s, _, _, _ := ellipsisDisplayString(str, limit, ellipsisDisplayGuessWidth) + return s +} + +// EllipsisDisplayStringX works like EllipsisDisplayString while it also returns the right part +func EllipsisDisplayStringX(str string, limit int) (left, right string) { + return ellipsisDisplayStringX(str, limit, ellipsisDisplayGuessWidth) +} + +func ellipsisDisplayStringX(str string, limit int, widthGuess func(rune) int) (left, right string) { + left, offset, truncated, encounterInvalid := ellipsisDisplayString(str, limit, widthGuess) + if truncated { + right = str[offset:] + r, _ := utf8.DecodeRune(UnsafeStringToBytes(right)) + encounterInvalid = encounterInvalid || r == utf8.RuneError + ellipsis := utf8Ellipsis + if encounterInvalid { + ellipsis = asciiEllipsis } - return input[:n-3] + asciiEllipsis, asciiEllipsis + input[n-3:] + right = ellipsis + right } + return left, right +} - end := 0 - for end <= n-3 { - _, size := utf8.DecodeRuneInString(input[end:]) - if end+size > n-3 { +func ellipsisDisplayString(str string, limit int, widthGuess func(rune) int) (res string, offset int, truncated, encounterInvalid bool) { + if len(str) <= limit { + return str, len(str), false, false + } + + // To future maintainers: this logic must guarantee that the length of the returned runes doesn't exceed the limit, + // because the returned string will also be used as database value. UTF-8 VARCHAR(10) could store 10 rune characters, + // So each rune must be countered as at least 1 width. + // Even if there are some special Unicode characters (zero-width, combining, etc.), they should NEVER be counted as zero. + pos, used := 0, 0 + for i, r := range str { + encounterInvalid = encounterInvalid || r == utf8.RuneError + pos = i + runeWidth := widthGuess(r) + if used+runeWidth+3 > limit { break } - end += size + used += runeWidth + offset += utf8.RuneLen(r) } - return input[:end] + utf8Ellipsis, utf8Ellipsis + input[end:] + // if the remaining are fewer than 3 runes, then maybe we could add them, no need to ellipse + if len(str)-pos <= 12 { + var nextCnt, nextWidth int + for _, r := range str[pos:] { + if nextCnt >= 4 { + break + } + nextWidth += widthGuess(r) + nextCnt++ + } + if nextCnt <= 3 && used+nextWidth <= limit { + return str, len(str), false, false + } + } + if limit < 3 { + // if the limit is so small, do not add ellipsis + return str[:offset], offset, true, false + } + ellipsis := utf8Ellipsis + if encounterInvalid { + ellipsis = asciiEllipsis + } + return str[:offset] + ellipsis, offset, true, encounterInvalid } -// SplitTrimSpace splits the string at given separator and trims leading and trailing space -func SplitTrimSpace(input, sep string) []string { - // Trim initial leading & trailing space - input = strings.TrimSpace(input) - // replace CRLF with LF - input = strings.ReplaceAll(input, "\r\n", "\n") +func EllipsisTruncateRunes(str string, limit int) (left, right string) { + return ellipsisDisplayStringX(str, limit, func(r rune) int { return 1 }) +} - var stringList []string - for _, s := range strings.Split(input, sep) { - // trim leading and trailing space - stringList = append(stringList, strings.TrimSpace(s)) +// TruncateRunes returns a truncated string with given rune limit, +// it returns input string if its rune length doesn't exceed the limit. +func TruncateRunes(str string, limit int) string { + if utf8.RuneCountInString(str) < limit { + return str } - - return stringList + return string([]rune(str)[:limit]) } diff --git a/modules/util/truncate_test.go b/modules/util/truncate_test.go index dfe1230fd4..6d71f38c0c 100644 --- a/modules/util/truncate_test.go +++ b/modules/util/truncate_test.go @@ -4,43 +4,128 @@ package util import ( + "fmt" + "strconv" + "strings" "testing" "github.com/stretchr/testify/assert" ) -func TestSplitString(t *testing.T) { - type testCase struct { - input string - n int - leftSub string - ellipsis string +func TestEllipsisGuessDisplayWidth(t *testing.T) { + cases := []struct { + r string + want int + }{ + {r: "a", want: 1}, + {r: "é", want: 1}, + {r: "测", want: 2}, + {r: "âš½", want: 2}, + {r: "â˜ï¸", want: 3}, // 2 runes, it has a mark + {r: "\u200B", want: 1}, // ZWSP + {r: "\u3000", want: 2}, // ideographic space } - - test := func(tc []*testCase, f func(input string, n int) (left, right string)) { - for _, c := range tc { - l, r := f(c.input, c.n) - if c.ellipsis != "" { - assert.Equal(t, c.leftSub+c.ellipsis, l, "test split %q at %d, expected leftSub: %q", c.input, c.n, c.leftSub) - assert.Equal(t, c.ellipsis+c.input[len(c.leftSub):], r, "test split %s at %d, expected rightSub: %q", c.input, c.n, c.input[len(c.leftSub):]) - } else { - assert.Equal(t, c.leftSub, l, "test split %q at %d, expected leftSub: %q", c.input, c.n, c.leftSub) - assert.Empty(t, r, "test split %q at %d, expected rightSub: %q", c.input, c.n, "") + for _, c := range cases { + t.Run(c.r, func(t *testing.T) { + w := 0 + for _, r := range c.r { + w += ellipsisDisplayGuessWidth(r) } - } + assert.Equal(t, c.want, w, "hex=% x", []byte(c.r)) + }) } +} + +func TestEllipsisString(t *testing.T) { + cases := []struct { + limit int + + input, left, right string + }{ + {limit: 0, input: "abcde", left: "", right: "…abcde"}, + {limit: 1, input: "abcde", left: "", right: "…abcde"}, + {limit: 2, input: "abcde", left: "", right: "…abcde"}, + {limit: 3, input: "abcde", left: "…", right: "…abcde"}, + {limit: 4, input: "abcde", left: "a…", right: "…bcde"}, + {limit: 5, input: "abcde", left: "abcde", right: ""}, + {limit: 6, input: "abcde", left: "abcde", right: ""}, + {limit: 7, input: "abcde", left: "abcde", right: ""}, - tc := []*testCase{ - {"abc123xyz", 0, "", utf8Ellipsis}, - {"abc123xyz", 1, "", utf8Ellipsis}, - {"abc123xyz", 4, "a", utf8Ellipsis}, - {"å•Šbc123xyz", 4, "", utf8Ellipsis}, - {"å•Šbc123xyz", 6, "å•Š", utf8Ellipsis}, - {"å•Šbc", 5, "å•Šbc", ""}, - {"å•Šbc", 6, "å•Šbc", ""}, - {"abc\xef\x03\xfe", 3, "", asciiEllipsis}, - {"abc\xef\x03\xfe", 4, "a", asciiEllipsis}, - {"\xef\x03", 1, "\xef\x03", ""}, + // a CJK char or emoji is considered as 2-ASCII width, the ellipsis is 3-ASCII width + {limit: 0, input: "测试文本", left: "", right: "…测试文本"}, + {limit: 1, input: "测试文本", left: "", right: "…测试文本"}, + {limit: 2, input: "测试文本", left: "", right: "…测试文本"}, + {limit: 3, input: "测试文本", left: "…", right: "…测试文本"}, + {limit: 4, input: "测试文本", left: "…", right: "…测试文本"}, + {limit: 5, input: "测试文本", left: "测…", right: "…试文本"}, + {limit: 6, input: "测试文本", left: "测…", right: "…试文本"}, + {limit: 7, input: "测试文本", left: "测试…", right: "…文本"}, + {limit: 8, input: "测试文本", left: "测试文本", right: ""}, + {limit: 9, input: "测试文本", left: "测试文本", right: ""}, + + {limit: 6, input: "测试abc", left: "测…", right: "…试abc"}, + {limit: 7, input: "测试abc", left: "测试abc", right: ""}, // exactly 7-width + {limit: 8, input: "测试abc", left: "测试abc", right: ""}, + + {limit: 7, input: "测abc试啊", left: "测ab…", right: "…c试啊"}, + {limit: 8, input: "测abc试啊", left: "测abc…", right: "…试啊"}, + {limit: 9, input: "测abc试啊", left: "测abc试啊", right: ""}, // exactly 9-width + {limit: 10, input: "测abc试啊", left: "测abc试啊", right: ""}, } - test(tc, SplitStringAtByteN) + for _, c := range cases { + t.Run(fmt.Sprintf("%s(%d)", c.input, c.limit), func(t *testing.T) { + left, right := EllipsisDisplayStringX(c.input, c.limit) + assert.Equal(t, c.left, left, "left") + assert.Equal(t, c.right, right, "right") + }) + } + + t.Run("LongInput", func(t *testing.T) { + left, right := EllipsisDisplayStringX(strings.Repeat("abc", 240), 90) + assert.Equal(t, strings.Repeat("abc", 29)+"…", left) + assert.Equal(t, "…"+strings.Repeat("abc", 211), right) + }) + + t.Run("InvalidUtf8", func(t *testing.T) { + invalidCases := []struct { + limit int + left, right string + }{ + {limit: 0, left: "", right: "...\xef\x03\xfe\xef\x03\xfe"}, + {limit: 1, left: "", right: "...\xef\x03\xfe\xef\x03\xfe"}, + {limit: 2, left: "", right: "...\xef\x03\xfe\xef\x03\xfe"}, + {limit: 3, left: "...", right: "...\xef\x03\xfe\xef\x03\xfe"}, + {limit: 4, left: "...", right: "...\xef\x03\xfe\xef\x03\xfe"}, + {limit: 5, left: "\xef\x03\xfe...", right: "...\xef\x03\xfe"}, + {limit: 6, left: "\xef\x03\xfe\xef\x03\xfe", right: ""}, + {limit: 7, left: "\xef\x03\xfe\xef\x03\xfe", right: ""}, + } + for _, c := range invalidCases { + t.Run(strconv.Itoa(c.limit), func(t *testing.T) { + left, right := EllipsisDisplayStringX("\xef\x03\xfe\xef\x03\xfe", c.limit) + assert.Equal(t, c.left, left, "left") + assert.Equal(t, c.right, right, "right") + }) + } + }) + + t.Run("IsLikelyEllipsisLeftPart", func(t *testing.T) { + assert.True(t, IsLikelyEllipsisLeftPart("abcde…")) + assert.True(t, IsLikelyEllipsisLeftPart("abcde...")) + }) +} + +func TestTruncateRunes(t *testing.T) { + assert.Empty(t, TruncateRunes("", 0)) + assert.Empty(t, TruncateRunes("", 1)) + + assert.Empty(t, TruncateRunes("ab", 0)) + assert.Equal(t, "a", TruncateRunes("ab", 1)) + assert.Equal(t, "ab", TruncateRunes("ab", 2)) + assert.Equal(t, "ab", TruncateRunes("ab", 3)) + + assert.Empty(t, TruncateRunes("测试", 0)) + assert.Equal(t, "测", TruncateRunes("测试", 1)) + assert.Equal(t, "测试", TruncateRunes("测试", 2)) + assert.Equal(t, "测试", TruncateRunes("测试", 3)) } diff --git a/modules/util/util.go b/modules/util/util.go index 1fb4cb21cb..dd8e073888 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -11,21 +11,10 @@ import ( "strconv" "strings" - "code.gitea.io/gitea/modules/optional" - "golang.org/x/text/cases" "golang.org/x/text/language" ) -// OptionalBoolParse get the corresponding optional.Option[bool] of a string using strconv.ParseBool -func OptionalBoolParse(s string) optional.Option[bool] { - v, e := strconv.ParseBool(s) - if e != nil { - return optional.None[bool]() - } - return optional.Some(v) -} - // IsEmptyString checks if the provided string is empty func IsEmptyString(s string) bool { return len(strings.TrimSpace(s)) == 0 @@ -230,6 +219,13 @@ func IfZero[T comparable](v, def T) T { return v } +func IfEmpty[T any](v, def []T) []T { + if len(v) == 0 { + return def + } + return v +} + // OptionalArg helps the "optional argument" in Golang: // // func foo(optArg ...int) { return OptionalArg(optArg) } diff --git a/modules/util/util_test.go b/modules/util/util_test.go index 52b05acc5b..fe4125cdb5 100644 --- a/modules/util/util_test.go +++ b/modules/util/util_test.go @@ -8,8 +8,6 @@ import ( "strings" "testing" - "code.gitea.io/gitea/modules/optional" - "github.com/stretchr/testify/assert" ) @@ -175,19 +173,6 @@ func Test_RandomBytes(t *testing.T) { assert.NotEqual(t, bytes3, bytes4) } -func TestOptionalBoolParse(t *testing.T) { - assert.Equal(t, optional.None[bool](), OptionalBoolParse("")) - assert.Equal(t, optional.None[bool](), OptionalBoolParse("x")) - - assert.Equal(t, optional.Some(false), OptionalBoolParse("0")) - assert.Equal(t, optional.Some(false), OptionalBoolParse("f")) - assert.Equal(t, optional.Some(false), OptionalBoolParse("False")) - - assert.Equal(t, optional.Some(true), OptionalBoolParse("1")) - assert.Equal(t, optional.Some(true), OptionalBoolParse("t")) - assert.Equal(t, optional.Some(true), OptionalBoolParse("True")) -} - // Test case for any function which accepts and returns a single string. type StringTest struct { in, out string @@ -215,7 +200,7 @@ func TestToUpperASCII(t *testing.T) { func BenchmarkToUpper(b *testing.B) { for _, tc := range upperTests { b.Run(tc.in, func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { ToUpperASCII(tc.in) } }) diff --git a/modules/validation/binding_test.go b/modules/validation/binding_test.go index 28d0f57b5c..0cd328f312 100644 --- a/modules/validation/binding_test.go +++ b/modules/validation/binding_test.go @@ -47,7 +47,7 @@ func performValidationTest(t *testing.T, testCase validationTestCase) { assert.Equal(t, testCase.expectedErrors, actual) }) - req, err := http.NewRequest("POST", testRoute, nil) + req, err := http.NewRequest(http.MethodPost, testRoute, nil) if err != nil { panic(err) } diff --git a/modules/validation/glob_pattern_test.go b/modules/validation/glob_pattern_test.go index 1bf622e61d..7f3e609acf 100644 --- a/modules/validation/glob_pattern_test.go +++ b/modules/validation/glob_pattern_test.go @@ -19,39 +19,39 @@ func getGlobPatternErrorString(pattern string) string { return "" } -var globValidationTestCases = []validationTestCase{ - { - description: "Empty glob pattern", - data: TestForm{ - GlobPattern: "", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Valid glob", - data: TestForm{ - GlobPattern: "{master,release*}", - }, - expectedErrors: binding.Errors{}, - }, +func Test_GlobPatternValidation(t *testing.T) { + AddBindingRules() - { - description: "Invalid glob", - data: TestForm{ - GlobPattern: "[a-", + globValidationTestCases := []validationTestCase{ + { + description: "Empty glob pattern", + data: TestForm{ + GlobPattern: "", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"GlobPattern"}, - Classification: ErrGlobPattern, - Message: getGlobPatternErrorString("[a-"), + { + description: "Valid glob", + data: TestForm{ + GlobPattern: "{master,release*}", }, + expectedErrors: binding.Errors{}, }, - }, -} -func Test_GlobPatternValidation(t *testing.T) { - AddBindingRules() + { + description: "Invalid glob", + data: TestForm{ + GlobPattern: "[a-", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"GlobPattern"}, + Classification: ErrGlobPattern, + Message: getGlobPatternErrorString("[a-"), + }, + }, + }, + } for _, testCase := range globValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/validation/helpers.go b/modules/validation/helpers.go index f6e00f3887..ba383ba195 100644 --- a/modules/validation/helpers.go +++ b/modules/validation/helpers.go @@ -7,14 +7,28 @@ import ( "net" "net/url" "regexp" + "slices" "strings" + "sync" "code.gitea.io/gitea/modules/setting" "github.com/gobwas/glob" ) -var externalTrackerRegex = regexp.MustCompile(`({?)(?:user|repo|index)+?(}?)`) +type globalVarsStruct struct { + externalTrackerRegex *regexp.Regexp + validUsernamePattern *regexp.Regexp + invalidUsernamePattern *regexp.Regexp +} + +var globalVars = sync.OnceValue(func() *globalVarsStruct { + return &globalVarsStruct{ + externalTrackerRegex: regexp.MustCompile(`({?)(?:user|repo|index)+?(}?)`), + validUsernamePattern: regexp.MustCompile(`^[\da-zA-Z][-.\w]*$`), + invalidUsernamePattern: regexp.MustCompile(`[-._]{2,}|[-._]$`), // No consecutive or trailing non-alphanumeric chars + } +}) func isLoopbackIP(ip string) bool { return net.ParseIP(ip).IsLoopback() @@ -42,12 +56,7 @@ func IsValidSiteURL(uri string) bool { return false } - for _, scheme := range setting.Service.ValidSiteURLSchemes { - if scheme == u.Scheme { - return true - } - } - return false + return slices.Contains(setting.Service.ValidSiteURLSchemes, u.Scheme) } // IsEmailDomainListed checks whether the domain of an email address @@ -105,9 +114,9 @@ func IsValidExternalTrackerURLFormat(uri string) bool { if !IsValidExternalURL(uri) { return false } - + vars := globalVars() // check for typoed variables like /{index/ or /[repo} - for _, match := range externalTrackerRegex.FindAllStringSubmatch(uri, -1) { + for _, match := range vars.externalTrackerRegex.FindAllStringSubmatch(uri, -1) { if (match[1] == "{" || match[2] == "}") && (match[1] != "{" || match[2] != "}") { return false } @@ -116,14 +125,10 @@ func IsValidExternalTrackerURLFormat(uri string) bool { return true } -var ( - validUsernamePattern = regexp.MustCompile(`^[\da-zA-Z][-.\w]*$`) - invalidUsernamePattern = regexp.MustCompile(`[-._]{2,}|[-._]$`) // No consecutive or trailing non-alphanumeric chars -) - // IsValidUsername checks if username is valid func IsValidUsername(name string) bool { // It is difficult to find a single pattern that is both readable and effective, // but it's easier to use positive and negative checks. - return validUsernamePattern.MatchString(name) && !invalidUsernamePattern.MatchString(name) + vars := globalVars() + return vars.validUsernamePattern.MatchString(name) && !vars.invalidUsernamePattern.MatchString(name) } diff --git a/modules/validation/helpers_test.go b/modules/validation/helpers_test.go index 52f383f698..6a982965f6 100644 --- a/modules/validation/helpers_test.go +++ b/modules/validation/helpers_test.go @@ -7,6 +7,7 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" ) @@ -47,7 +48,7 @@ func Test_IsValidURL(t *testing.T) { } func Test_IsValidExternalURL(t *testing.T) { - setting.AppURL = "https://try.gitea.io/" + defer test.MockVariableValue(&setting.AppURL, "https://try.gitea.io/")() cases := []struct { description string @@ -89,7 +90,7 @@ func Test_IsValidExternalURL(t *testing.T) { } func Test_IsValidExternalTrackerURLFormat(t *testing.T) { - setting.AppURL = "https://try.gitea.io/" + defer test.MockVariableValue(&setting.AppURL, "https://try.gitea.io/")() cases := []struct { description string diff --git a/modules/validation/refname_test.go b/modules/validation/refname_test.go index 3af7387c47..93703761cf 100644 --- a/modules/validation/refname_test.go +++ b/modules/validation/refname_test.go @@ -9,253 +9,252 @@ import ( "gitea.com/go-chi/binding" ) -var gitRefNameValidationTestCases = []validationTestCase{ - { - description: "Reference name contains only characters", - data: TestForm{ - BranchName: "test", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Reference name contains single slash", - data: TestForm{ - BranchName: "feature/test", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Reference name has allowed special characters", - data: TestForm{ - BranchName: "debian/1%1.6.0-2", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Reference name contains backslash", - data: TestForm{ - BranchName: "feature\\test", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", +func Test_GitRefNameValidation(t *testing.T) { + AddBindingRules() + gitRefNameValidationTestCases := []validationTestCase{ + { + description: "Reference name contains only characters", + data: TestForm{ + BranchName: "test", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Reference name starts with dot", - data: TestForm{ - BranchName: ".test", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name contains single slash", + data: TestForm{ + BranchName: "feature/test", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Reference name ends with dot", - data: TestForm{ - BranchName: "test.", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has allowed special characters", + data: TestForm{ + BranchName: "debian/1%1.6.0-2", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Reference name starts with slash", - data: TestForm{ - BranchName: "/test", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name contains backslash", + data: TestForm{ + BranchName: "feature\\test", }, - }, - }, - { - description: "Reference name ends with slash", - data: TestForm{ - BranchName: "test/", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name ends with .lock", - data: TestForm{ - BranchName: "test.lock", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name starts with dot", + data: TestForm{ + BranchName: ".test", }, - }, - }, - { - description: "Reference name contains multiple consecutive dots", - data: TestForm{ - BranchName: "te..st", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name contains multiple consecutive slashes", - data: TestForm{ - BranchName: "te//st", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name ends with dot", + data: TestForm{ + BranchName: "test.", }, - }, - }, - { - description: "Reference name is single @", - data: TestForm{ - BranchName: "@", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has @{", - data: TestForm{ - BranchName: "branch@{", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name starts with slash", + data: TestForm{ + BranchName: "/test", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character ~", - data: TestForm{ - BranchName: "~debian/1%1.6.0-2", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name ends with slash", + data: TestForm{ + BranchName: "test/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character *", - data: TestForm{ - BranchName: "*debian/1%1.6.0-2", + { + description: "Reference name ends with .lock", + data: TestForm{ + BranchName: "test.lock", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name contains multiple consecutive dots", + data: TestForm{ + BranchName: "te..st", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character ?", - data: TestForm{ - BranchName: "?debian/1%1.6.0-2", + { + description: "Reference name contains multiple consecutive slashes", + data: TestForm{ + BranchName: "te//st", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name is single @", + data: TestForm{ + BranchName: "@", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character ^", - data: TestForm{ - BranchName: "^debian/1%1.6.0-2", + { + description: "Reference name has @{", + data: TestForm{ + BranchName: "branch@{", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has unallowed special character ~", + data: TestForm{ + BranchName: "~debian/1%1.6.0-2", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character :", - data: TestForm{ - BranchName: "debian:jessie", + { + description: "Reference name has unallowed special character *", + data: TestForm{ + BranchName: "*debian/1%1.6.0-2", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has unallowed special character ?", + data: TestForm{ + BranchName: "?debian/1%1.6.0-2", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character (whitespace)", - data: TestForm{ - BranchName: "debian jessie", + { + description: "Reference name has unallowed special character ^", + data: TestForm{ + BranchName: "^debian/1%1.6.0-2", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has unallowed special character :", + data: TestForm{ + BranchName: "debian:jessie", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, - { - description: "Reference name has unallowed special character [", - data: TestForm{ - BranchName: "debian[jessie", + { + description: "Reference name has unallowed special character (whitespace)", + data: TestForm{ + BranchName: "debian jessie", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"BranchName"}, - Classification: ErrGitRefName, - Message: "GitRefName", + { + description: "Reference name has unallowed special character [", + data: TestForm{ + BranchName: "debian[jessie", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"BranchName"}, + Classification: ErrGitRefName, + Message: "GitRefName", + }, }, }, - }, -} - -func Test_GitRefNameValidation(t *testing.T) { - AddBindingRules() + } for _, testCase := range gitRefNameValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/validation/regex_pattern_test.go b/modules/validation/regex_pattern_test.go index efcb276734..80790a23b1 100644 --- a/modules/validation/regex_pattern_test.go +++ b/modules/validation/regex_pattern_test.go @@ -17,39 +17,39 @@ func getRegexPatternErrorString(pattern string) string { return "" } -var regexValidationTestCases = []validationTestCase{ - { - description: "Empty regex pattern", - data: TestForm{ - RegexPattern: "", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Valid regex", - data: TestForm{ - RegexPattern: `(\d{1,3})+`, - }, - expectedErrors: binding.Errors{}, - }, +func Test_RegexPatternValidation(t *testing.T) { + AddBindingRules() - { - description: "Invalid regex", - data: TestForm{ - RegexPattern: "[a-", + regexValidationTestCases := []validationTestCase{ + { + description: "Empty regex pattern", + data: TestForm{ + RegexPattern: "", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"RegexPattern"}, - Classification: ErrRegexPattern, - Message: getRegexPatternErrorString("[a-"), + { + description: "Valid regex", + data: TestForm{ + RegexPattern: `(\d{1,3})+`, }, + expectedErrors: binding.Errors{}, }, - }, -} -func Test_RegexPatternValidation(t *testing.T) { - AddBindingRules() + { + description: "Invalid regex", + data: TestForm{ + RegexPattern: "[a-", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"RegexPattern"}, + Classification: ErrRegexPattern, + Message: getRegexPatternErrorString("[a-"), + }, + }, + }, + } for _, testCase := range regexValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/validation/validurl_test.go b/modules/validation/validurl_test.go index 39f7fa5d65..ce4898b271 100644 --- a/modules/validation/validurl_test.go +++ b/modules/validation/validurl_test.go @@ -9,98 +9,98 @@ import ( "gitea.com/go-chi/binding" ) -var urlValidationTestCases = []validationTestCase{ - { - description: "Empty URL", - data: TestForm{ - URL: "", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL without port", - data: TestForm{ - URL: "http://test.lan/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with port", - data: TestForm{ - URL: "http://test.lan:3000/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with IPv6 address without port", - data: TestForm{ - URL: "http://[::1]/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with IPv6 address with port", - data: TestForm{ - URL: "http://[::1]:3000/", +func Test_ValidURLValidation(t *testing.T) { + AddBindingRules() + + urlValidationTestCases := []validationTestCase{ + { + description: "Empty URL", + data: TestForm{ + URL: "", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{}, - }, - { - description: "Invalid URL", - data: TestForm{ - URL: "http//test.lan/", + { + description: "URL without port", + data: TestForm{ + URL: "http://test.lan/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URL"}, - Classification: binding.ERR_URL, - Message: "Url", + { + description: "URL with port", + data: TestForm{ + URL: "http://test.lan:3000/", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid schema", - data: TestForm{ - URL: "ftp://test.lan/", + { + description: "URL with IPv6 address without port", + data: TestForm{ + URL: "http://[::1]/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URL"}, - Classification: binding.ERR_URL, - Message: "Url", + { + description: "URL with IPv6 address with port", + data: TestForm{ + URL: "http://[::1]:3000/", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid port", - data: TestForm{ - URL: "http://test.lan:3x4/", + { + description: "Invalid URL", + data: TestForm{ + URL: "http//test.lan/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URL"}, + Classification: binding.ERR_URL, + Message: "Url", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URL"}, - Classification: binding.ERR_URL, - Message: "Url", + { + description: "Invalid schema", + data: TestForm{ + URL: "ftp://test.lan/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URL"}, + Classification: binding.ERR_URL, + Message: "Url", + }, }, }, - }, - { - description: "Invalid port with IPv6 address", - data: TestForm{ - URL: "http://[::1]:3x4/", + { + description: "Invalid port", + data: TestForm{ + URL: "http://test.lan:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URL"}, + Classification: binding.ERR_URL, + Message: "Url", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URL"}, - Classification: binding.ERR_URL, - Message: "Url", + { + description: "Invalid port with IPv6 address", + data: TestForm{ + URL: "http://[::1]:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URL"}, + Classification: binding.ERR_URL, + Message: "Url", + }, }, }, - }, -} - -func Test_ValidURLValidation(t *testing.T) { - AddBindingRules() + } for _, testCase := range urlValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/validation/validurllist_test.go b/modules/validation/validurllist_test.go index c6f862a962..cccc570a1a 100644 --- a/modules/validation/validurllist_test.go +++ b/modules/validation/validurllist_test.go @@ -9,145 +9,145 @@ import ( "gitea.com/go-chi/binding" ) -// This is a copy of all the URL tests cases, plus additional ones to -// account for multiple URLs -var urlListValidationTestCases = []validationTestCase{ - { - description: "Empty URL", - data: TestForm{ - URLs: "", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL without port", - data: TestForm{ - URLs: "http://test.lan/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with port", - data: TestForm{ - URLs: "http://test.lan:3000/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with IPv6 address without port", - data: TestForm{ - URLs: "http://[::1]/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "URL with IPv6 address with port", - data: TestForm{ - URLs: "http://[::1]:3000/", - }, - expectedErrors: binding.Errors{}, - }, - { - description: "Invalid URL", - data: TestForm{ - URLs: "http//test.lan/", - }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http//test.lan/", +func Test_ValidURLListValidation(t *testing.T) { + AddBindingRules() + + // This is a copy of all the URL tests cases, plus additional ones to + // account for multiple URLs + urlListValidationTestCases := []validationTestCase{ + { + description: "Empty URL", + data: TestForm{ + URLs: "", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid schema", - data: TestForm{ - URLs: "ftp://test.lan/", + { + description: "URL without port", + data: TestForm{ + URLs: "http://test.lan/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "ftp://test.lan/", + { + description: "URL with port", + data: TestForm{ + URLs: "http://test.lan:3000/", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid port", - data: TestForm{ - URLs: "http://test.lan:3x4/", + { + description: "URL with IPv6 address without port", + data: TestForm{ + URLs: "http://[::1]/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http://test.lan:3x4/", + { + description: "URL with IPv6 address with port", + data: TestForm{ + URLs: "http://[::1]:3000/", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "Invalid port with IPv6 address", - data: TestForm{ - URLs: "http://[::1]:3x4/", + { + description: "Invalid URL", + data: TestForm{ + URLs: "http//test.lan/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http//test.lan/", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http://[::1]:3x4/", + { + description: "Invalid schema", + data: TestForm{ + URLs: "ftp://test.lan/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "ftp://test.lan/", + }, }, }, - }, - { - description: "Multi URLs", - data: TestForm{ - URLs: "http://test.lan:3000/\nhttp://test.local/", + { + description: "Invalid port", + data: TestForm{ + URLs: "http://test.lan:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http://test.lan:3x4/", + }, + }, }, - expectedErrors: binding.Errors{}, - }, - { - description: "Multi URLs with newline", - data: TestForm{ - URLs: "http://test.lan:3000/\nhttp://test.local/\n", + { + description: "Invalid port with IPv6 address", + data: TestForm{ + URLs: "http://[::1]:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http://[::1]:3x4/", + }, + }, }, - expectedErrors: binding.Errors{}, - }, - { - description: "List with invalid entry", - data: TestForm{ - URLs: "http://test.lan:3000/\nhttp://[::1]:3x4/", + { + description: "Multi URLs", + data: TestForm{ + URLs: "http://test.lan:3000/\nhttp://test.local/", + }, + expectedErrors: binding.Errors{}, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http://[::1]:3x4/", + { + description: "Multi URLs with newline", + data: TestForm{ + URLs: "http://test.lan:3000/\nhttp://test.local/\n", }, + expectedErrors: binding.Errors{}, }, - }, - { - description: "List with two invalid entries", - data: TestForm{ - URLs: "ftp://test.lan:3000/\nhttp://[::1]:3x4/\n", + { + description: "List with invalid entry", + data: TestForm{ + URLs: "http://test.lan:3000/\nhttp://[::1]:3x4/", + }, + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http://[::1]:3x4/", + }, + }, }, - expectedErrors: binding.Errors{ - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "ftp://test.lan:3000/", + { + description: "List with two invalid entries", + data: TestForm{ + URLs: "ftp://test.lan:3000/\nhttp://[::1]:3x4/\n", }, - binding.Error{ - FieldNames: []string{"URLs"}, - Classification: binding.ERR_URL, - Message: "http://[::1]:3x4/", + expectedErrors: binding.Errors{ + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "ftp://test.lan:3000/", + }, + binding.Error{ + FieldNames: []string{"URLs"}, + Classification: binding.ERR_URL, + Message: "http://[::1]:3x4/", + }, }, }, - }, -} - -func Test_ValidURLListValidation(t *testing.T) { - AddBindingRules() + } for _, testCase := range urlListValidationTestCases { t.Run(testCase.description, func(t *testing.T) { diff --git a/modules/web/handler.go b/modules/web/handler.go index 1812c664b3..42a649714d 100644 --- a/modules/web/handler.go +++ b/modules/web/handler.go @@ -4,7 +4,6 @@ package web import ( - goctx "context" "fmt" "net/http" "reflect" @@ -51,7 +50,6 @@ func (r *responseWriter) WriteHeader(statusCode int) { var ( httpReqType = reflect.TypeOf((*http.Request)(nil)) respWriterType = reflect.TypeOf((*http.ResponseWriter)(nil)).Elem() - cancelFuncType = reflect.TypeOf((*goctx.CancelFunc)(nil)).Elem() ) // preCheckHandler checks whether the handler is valid, developers could get first-time feedback, all mistakes could be found at startup @@ -65,11 +63,8 @@ func preCheckHandler(fn reflect.Value, argsIn []reflect.Value) { if !hasStatusProvider { panic(fmt.Sprintf("handler should have at least one ResponseStatusProvider argument, but got %s", fn.Type())) } - if fn.Type().NumOut() != 0 && fn.Type().NumIn() != 1 { - panic(fmt.Sprintf("handler should have no return value or only one argument, but got %s", fn.Type())) - } - if fn.Type().NumOut() == 1 && fn.Type().Out(0) != cancelFuncType { - panic(fmt.Sprintf("handler should return a cancel function, but got %s", fn.Type())) + if fn.Type().NumOut() != 0 { + panic(fmt.Sprintf("handler should have no return value other than registered ones, but got %s", fn.Type())) } } @@ -105,16 +100,10 @@ func prepareHandleArgsIn(resp http.ResponseWriter, req *http.Request, fn reflect return argsIn } -func handleResponse(fn reflect.Value, ret []reflect.Value) goctx.CancelFunc { - if len(ret) == 1 { - if cancelFunc, ok := ret[0].Interface().(goctx.CancelFunc); ok { - return cancelFunc - } - panic(fmt.Sprintf("unsupported return type: %s", ret[0].Type())) - } else if len(ret) > 1 { +func handleResponse(fn reflect.Value, ret []reflect.Value) { + if len(ret) != 0 { panic(fmt.Sprintf("unsupported return values: %s", fn.Type())) } - return nil } func hasResponseBeenWritten(argsIn []reflect.Value) bool { @@ -132,7 +121,7 @@ func wrapHandlerProvider[T http.Handler](hp func(next http.Handler) T, funcInfo return func(next http.Handler) http.Handler { h := hp(next) // this handle could be dynamically generated, so we can't use it for debug info return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - routing.UpdateFuncInfo(req.Context(), funcInfo) + defer routing.RecordFuncInfo(req.Context(), funcInfo)() h.ServeHTTP(resp, req) }) } @@ -168,14 +157,11 @@ func toHandlerProvider(handler any) func(next http.Handler) http.Handler { return // it's doing pre-check, just return } - routing.UpdateFuncInfo(req.Context(), funcInfo) + defer routing.RecordFuncInfo(req.Context(), funcInfo)() ret := fn.Call(argsIn) - // handle the return value, and defer the cancel function if there is one - cancelFunc := handleResponse(fn, ret) - if cancelFunc != nil { - defer cancelFunc() - } + // handle the return value (no-op at the moment) + handleResponse(fn, ret) // if the response has not been written, call the next handler if next != nil && !hasResponseBeenWritten(argsIn) { diff --git a/modules/web/middleware/binding.go b/modules/web/middleware/binding.go index 43e1bbc70e..ee4eca976e 100644 --- a/modules/web/middleware/binding.go +++ b/modules/web/middleware/binding.go @@ -50,7 +50,7 @@ func AssignForm(form any, data map[string]any) { } func getRuleBody(field reflect.StructField, prefix string) string { - for _, rule := range strings.Split(field.Tag.Get("binding"), ";") { + for rule := range strings.SplitSeq(field.Tag.Get("binding"), ";") { if strings.HasPrefix(rule, prefix) { return rule[len(prefix) : len(rule)-1] } @@ -78,7 +78,7 @@ func GetInclude(field reflect.StructField) string { return getRuleBody(field, "Include(") } -// Validate validate TODO: +// Validate validate func Validate(errs binding.Errors, data map[string]any, f Form, l translation.Locale) binding.Errors { if errs.Len() == 0 { return errs diff --git a/modules/web/middleware/data.go b/modules/web/middleware/data.go index 08d83f94be..a47da0f836 100644 --- a/modules/web/middleware/data.go +++ b/modules/web/middleware/data.go @@ -7,46 +7,21 @@ import ( "context" "time" + "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/setting" ) -// ContextDataStore represents a data store -type ContextDataStore interface { - GetData() ContextData -} - -type ContextData map[string]any - -func (ds ContextData) GetData() ContextData { - return ds -} - -func (ds ContextData) MergeFrom(other ContextData) ContextData { - for k, v := range other { - ds[k] = v - } - return ds -} - const ContextDataKeySignedUser = "SignedUser" -type contextDataKeyType struct{} - -var contextDataKey contextDataKeyType - -func WithContextData(c context.Context) context.Context { - return context.WithValue(c, contextDataKey, make(ContextData, 10)) -} - -func GetContextData(c context.Context) ContextData { - if ds, ok := c.Value(contextDataKey).(ContextData); ok { - return ds +func GetContextData(c context.Context) reqctx.ContextData { + if rc := reqctx.GetRequestDataStore(c); rc != nil { + return rc.GetData() } return nil } -func CommonTemplateContextData() ContextData { - return ContextData{ +func CommonTemplateContextData() reqctx.ContextData { + return reqctx.ContextData{ "IsLandingPageOrganizations": setting.LandingPageURL == setting.LandingPageOrganizations, "ShowRegistrationButton": setting.Service.ShowRegistrationButton, diff --git a/modules/web/middleware/flash.go b/modules/web/middleware/flash.go index 88da2049a4..0e848c7902 100644 --- a/modules/web/middleware/flash.go +++ b/modules/web/middleware/flash.go @@ -6,12 +6,15 @@ package middleware import ( "fmt" "html/template" + "net/http" "net/url" + + "code.gitea.io/gitea/modules/reqctx" ) // Flash represents a one time data transfer between two requests. type Flash struct { - DataStore ContextDataStore + DataStore reqctx.RequestDataStore url.Values ErrorMsg, WarningMsg, InfoMsg, SuccessMsg string } @@ -63,3 +66,27 @@ func (f *Flash) Success(msg any, current ...bool) { f.SuccessMsg = flashMsgStringOrHTML(msg) f.set("success", f.SuccessMsg, current...) } + +func ParseCookieFlashMessage(val string) *Flash { + if vals, _ := url.ParseQuery(val); len(vals) > 0 { + return &Flash{ + Values: vals, + ErrorMsg: vals.Get("error"), + SuccessMsg: vals.Get("success"), + InfoMsg: vals.Get("info"), + WarningMsg: vals.Get("warning"), + } + } + return nil +} + +func GetSiteCookieFlashMessage(dataStore reqctx.RequestDataStore, req *http.Request, cookieName string) (string, *Flash) { + // Get the last flash message from cookie + lastFlashCookie := GetSiteCookie(req, cookieName) + lastFlashMsg := ParseCookieFlashMessage(lastFlashCookie) + if lastFlashMsg != nil { + lastFlashMsg.DataStore = dataStore + return lastFlashCookie, lastFlashMsg + } + return lastFlashCookie, nil +} diff --git a/modules/web/middleware/request.go b/modules/web/middleware/request.go deleted file mode 100644 index 0bb155df70..0000000000 --- a/modules/web/middleware/request.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package middleware - -import ( - "net/http" - "strings" -) - -// IsAPIPath returns true if the specified URL is an API path -func IsAPIPath(req *http.Request) bool { - return strings.HasPrefix(req.URL.Path, "/api/") -} diff --git a/modules/web/route_test.go b/modules/web/route_test.go deleted file mode 100644 index 6e4c309293..0000000000 --- a/modules/web/route_test.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package web - -import ( - "bytes" - "net/http" - "net/http/httptest" - "strconv" - "testing" - - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/test" - - chi "github.com/go-chi/chi/v5" - "github.com/stretchr/testify/assert" -) - -func TestRoute1(t *testing.T) { - buff := bytes.NewBufferString("") - recorder := httptest.NewRecorder() - recorder.Body = buff - - r := NewRouter() - r.Get("/{username}/{reponame}/{type:issues|pulls}", func(resp http.ResponseWriter, req *http.Request) { - username := chi.URLParam(req, "username") - assert.EqualValues(t, "gitea", username) - reponame := chi.URLParam(req, "reponame") - assert.EqualValues(t, "gitea", reponame) - tp := chi.URLParam(req, "type") - assert.EqualValues(t, "issues", tp) - }) - - req, err := http.NewRequest("GET", "http://localhost:8000/gitea/gitea/issues", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code) -} - -func TestRoute2(t *testing.T) { - buff := bytes.NewBufferString("") - recorder := httptest.NewRecorder() - recorder.Body = buff - - hit := -1 - - r := NewRouter() - r.Group("/{username}/{reponame}", func() { - r.Group("", func() { - r.Get("/{type:issues|pulls}", func(resp http.ResponseWriter, req *http.Request) { - username := chi.URLParam(req, "username") - assert.EqualValues(t, "gitea", username) - reponame := chi.URLParam(req, "reponame") - assert.EqualValues(t, "gitea", reponame) - tp := chi.URLParam(req, "type") - assert.EqualValues(t, "issues", tp) - hit = 0 - }) - - r.Get("/{type:issues|pulls}/{index}", func(resp http.ResponseWriter, req *http.Request) { - username := chi.URLParam(req, "username") - assert.EqualValues(t, "gitea", username) - reponame := chi.URLParam(req, "reponame") - assert.EqualValues(t, "gitea", reponame) - tp := chi.URLParam(req, "type") - assert.EqualValues(t, "issues", tp) - index := chi.URLParam(req, "index") - assert.EqualValues(t, "1", index) - hit = 1 - }) - }, func(resp http.ResponseWriter, req *http.Request) { - if stop, err := strconv.Atoi(req.FormValue("stop")); err == nil { - hit = stop - resp.WriteHeader(http.StatusOK) - } - }) - - r.Group("/issues/{index}", func() { - r.Get("/view", func(resp http.ResponseWriter, req *http.Request) { - username := chi.URLParam(req, "username") - assert.EqualValues(t, "gitea", username) - reponame := chi.URLParam(req, "reponame") - assert.EqualValues(t, "gitea", reponame) - index := chi.URLParam(req, "index") - assert.EqualValues(t, "1", index) - hit = 2 - }) - }) - }) - - req, err := http.NewRequest("GET", "http://localhost:8000/gitea/gitea/issues", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code) - assert.EqualValues(t, 0, hit) - - req, err = http.NewRequest("GET", "http://localhost:8000/gitea/gitea/issues/1", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code) - assert.EqualValues(t, 1, hit) - - req, err = http.NewRequest("GET", "http://localhost:8000/gitea/gitea/issues/1?stop=100", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code) - assert.EqualValues(t, 100, hit) - - req, err = http.NewRequest("GET", "http://localhost:8000/gitea/gitea/issues/1/view", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code) - assert.EqualValues(t, 2, hit) -} - -func TestRoute3(t *testing.T) { - buff := bytes.NewBufferString("") - recorder := httptest.NewRecorder() - recorder.Body = buff - - hit := -1 - - m := NewRouter() - r := NewRouter() - r.Mount("/api/v1", m) - - m.Group("/repos", func() { - m.Group("/{username}/{reponame}", func() { - m.Group("/branch_protections", func() { - m.Get("", func(resp http.ResponseWriter, req *http.Request) { - hit = 0 - }) - m.Post("", func(resp http.ResponseWriter, req *http.Request) { - hit = 1 - }) - m.Group("/{name}", func() { - m.Get("", func(resp http.ResponseWriter, req *http.Request) { - hit = 2 - }) - m.Patch("", func(resp http.ResponseWriter, req *http.Request) { - hit = 3 - }) - m.Delete("", func(resp http.ResponseWriter, req *http.Request) { - hit = 4 - }) - }) - }) - }) - }) - - req, err := http.NewRequest("GET", "http://localhost:8000/api/v1/repos/gitea/gitea/branch_protections", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code) - assert.EqualValues(t, 0, hit) - - req, err = http.NewRequest("POST", "http://localhost:8000/api/v1/repos/gitea/gitea/branch_protections", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code, http.StatusOK) - assert.EqualValues(t, 1, hit) - - req, err = http.NewRequest("GET", "http://localhost:8000/api/v1/repos/gitea/gitea/branch_protections/master", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code) - assert.EqualValues(t, 2, hit) - - req, err = http.NewRequest("PATCH", "http://localhost:8000/api/v1/repos/gitea/gitea/branch_protections/master", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code) - assert.EqualValues(t, 3, hit) - - req, err = http.NewRequest("DELETE", "http://localhost:8000/api/v1/repos/gitea/gitea/branch_protections/master", nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.EqualValues(t, http.StatusOK, recorder.Code) - assert.EqualValues(t, 4, hit) -} - -func TestRouteNormalizePath(t *testing.T) { - type paths struct { - EscapedPath, RawPath, Path string - } - testPath := func(reqPath string, expectedPaths paths) { - recorder := httptest.NewRecorder() - recorder.Body = bytes.NewBuffer(nil) - - actualPaths := paths{EscapedPath: "(none)", RawPath: "(none)", Path: "(none)"} - r := NewRouter() - r.Get("/*", func(resp http.ResponseWriter, req *http.Request) { - actualPaths.EscapedPath = req.URL.EscapedPath() - actualPaths.RawPath = req.URL.RawPath - actualPaths.Path = req.URL.Path - }) - - req, err := http.NewRequest("GET", reqPath, nil) - assert.NoError(t, err) - r.ServeHTTP(recorder, req) - assert.Equal(t, expectedPaths, actualPaths, "req path = %q", reqPath) - } - - // RawPath could be empty if the EscapedPath is the same as escape(Path) and it is already normalized - testPath("/", paths{EscapedPath: "/", RawPath: "", Path: "/"}) - testPath("//", paths{EscapedPath: "/", RawPath: "/", Path: "/"}) - testPath("/%2f", paths{EscapedPath: "/%2f", RawPath: "/%2f", Path: "//"}) - testPath("///a//b/", paths{EscapedPath: "/a/b", RawPath: "/a/b", Path: "/a/b"}) - - defer test.MockVariableValue(&setting.UseSubURLPath, true)() - defer test.MockVariableValue(&setting.AppSubURL, "/sub-path")() - testPath("/", paths{EscapedPath: "(none)", RawPath: "(none)", Path: "(none)"}) // 404 - testPath("/sub-path", paths{EscapedPath: "/", RawPath: "/", Path: "/"}) - testPath("/sub-path/", paths{EscapedPath: "/", RawPath: "/", Path: "/"}) - testPath("/sub-path//a/b///", paths{EscapedPath: "/a/b", RawPath: "/a/b", Path: "/a/b"}) - testPath("/sub-path/%2f/", paths{EscapedPath: "/%2f", RawPath: "/%2f", Path: "//"}) - // "/v2" is special for OCI container registry, it should always be in the root of the site - testPath("/v2", paths{EscapedPath: "/v2", RawPath: "/v2", Path: "/v2"}) - testPath("/v2/", paths{EscapedPath: "/v2", RawPath: "/v2", Path: "/v2"}) - testPath("/v2/%2f", paths{EscapedPath: "/v2/%2f", RawPath: "/v2/%2f", Path: "/v2//"}) -} diff --git a/modules/web/routemock_test.go b/modules/web/routemock_test.go index 89cfaacdd1..a0949bf622 100644 --- a/modules/web/routemock_test.go +++ b/modules/web/routemock_test.go @@ -30,13 +30,13 @@ func TestRouteMock(t *testing.T) { // normal request recorder := httptest.NewRecorder() - req, err := http.NewRequest("GET", "http://localhost:8000/foo", nil) + req, err := http.NewRequest(http.MethodGet, "http://localhost:8000/foo", nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Len(t, recorder.Header(), 3) - assert.EqualValues(t, "m1", recorder.Header().Get("X-Test-Middleware1")) - assert.EqualValues(t, "m2", recorder.Header().Get("X-Test-Middleware2")) - assert.EqualValues(t, "h", recorder.Header().Get("X-Test-Handler")) + assert.Equal(t, "m1", recorder.Header().Get("X-Test-Middleware1")) + assert.Equal(t, "m2", recorder.Header().Get("X-Test-Middleware2")) + assert.Equal(t, "h", recorder.Header().Get("X-Test-Handler")) RouteMockReset() // mock at "mock-point" @@ -45,12 +45,12 @@ func TestRouteMock(t *testing.T) { resp.WriteHeader(http.StatusOK) }) recorder = httptest.NewRecorder() - req, err = http.NewRequest("GET", "http://localhost:8000/foo", nil) + req, err = http.NewRequest(http.MethodGet, "http://localhost:8000/foo", nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Len(t, recorder.Header(), 2) - assert.EqualValues(t, "m1", recorder.Header().Get("X-Test-Middleware1")) - assert.EqualValues(t, "a", recorder.Header().Get("X-Test-MockPoint")) + assert.Equal(t, "m1", recorder.Header().Get("X-Test-Middleware1")) + assert.Equal(t, "a", recorder.Header().Get("X-Test-MockPoint")) RouteMockReset() // mock at MockAfterMiddlewares @@ -59,12 +59,12 @@ func TestRouteMock(t *testing.T) { resp.WriteHeader(http.StatusOK) }) recorder = httptest.NewRecorder() - req, err = http.NewRequest("GET", "http://localhost:8000/foo", nil) + req, err = http.NewRequest(http.MethodGet, "http://localhost:8000/foo", nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Len(t, recorder.Header(), 3) - assert.EqualValues(t, "m1", recorder.Header().Get("X-Test-Middleware1")) - assert.EqualValues(t, "m2", recorder.Header().Get("X-Test-Middleware2")) - assert.EqualValues(t, "b", recorder.Header().Get("X-Test-MockPoint")) + assert.Equal(t, "m1", recorder.Header().Get("X-Test-Middleware1")) + assert.Equal(t, "m2", recorder.Header().Get("X-Test-Middleware2")) + assert.Equal(t, "b", recorder.Header().Get("X-Test-MockPoint")) RouteMockReset() } diff --git a/modules/web/route.go b/modules/web/router.go index 787521dfb0..5812ff69d4 100644 --- a/modules/web/route.go +++ b/modules/web/router.go @@ -10,6 +10,7 @@ import ( "strings" "code.gitea.io/gitea/modules/htmlutil" + "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web/middleware" @@ -29,18 +30,18 @@ func Bind[T any](_ T) http.HandlerFunc { } // SetForm set the form object -func SetForm(dataStore middleware.ContextDataStore, obj any) { +func SetForm(dataStore reqctx.ContextDataProvider, obj any) { dataStore.GetData()["__form"] = obj } // GetForm returns the validate form information -func GetForm(dataStore middleware.ContextDataStore) any { +func GetForm(dataStore reqctx.RequestDataStore) any { return dataStore.GetData()["__form"] } // Router defines a route based on chi's router type Router struct { - chiRouter chi.Router + chiRouter *chi.Mux curGroupPrefix string curMiddlewares []any } @@ -92,16 +93,21 @@ func isNilOrFuncNil(v any) bool { return r.Kind() == reflect.Func && r.IsNil() } -func (r *Router) wrapMiddlewareAndHandler(h []any) ([]func(http.Handler) http.Handler, http.HandlerFunc) { - handlerProviders := make([]func(http.Handler) http.Handler, 0, len(r.curMiddlewares)+len(h)+1) - for _, m := range r.curMiddlewares { +func wrapMiddlewareAndHandler(curMiddlewares, h []any) ([]func(http.Handler) http.Handler, http.HandlerFunc) { + handlerProviders := make([]func(http.Handler) http.Handler, 0, len(curMiddlewares)+len(h)+1) + for _, m := range curMiddlewares { if !isNilOrFuncNil(m) { handlerProviders = append(handlerProviders, toHandlerProvider(m)) } } - for _, m := range h { + if len(h) == 0 { + panic("no endpoint handler provided") + } + for i, m := range h { if !isNilOrFuncNil(m) { handlerProviders = append(handlerProviders, toHandlerProvider(m)) + } else if i == len(h)-1 { + panic("endpoint handler can't be nil") } } middlewares := handlerProviders[:len(handlerProviders)-1] @@ -116,11 +122,11 @@ func (r *Router) wrapMiddlewareAndHandler(h []any) ([]func(http.Handler) http.Ha // Methods adds the same handlers for multiple http "methods" (separated by ","). // If any method is invalid, the lower level router will panic. func (r *Router) Methods(methods, pattern string, h ...any) { - middlewares, handlerFunc := r.wrapMiddlewareAndHandler(h) + middlewares, handlerFunc := wrapMiddlewareAndHandler(r.curMiddlewares, h) fullPattern := r.getPattern(pattern) if strings.Contains(methods, ",") { - methods := strings.Split(methods, ",") - for _, method := range methods { + methods := strings.SplitSeq(methods, ",") + for method := range methods { r.chiRouter.With(middlewares...).Method(strings.TrimSpace(method), fullPattern, handlerFunc) } } else { @@ -136,7 +142,7 @@ func (r *Router) Mount(pattern string, subRouter *Router) { // Any delegate requests for all methods func (r *Router) Any(pattern string, h ...any) { - middlewares, handlerFunc := r.wrapMiddlewareAndHandler(h) + middlewares, handlerFunc := wrapMiddlewareAndHandler(r.curMiddlewares, h) r.chiRouter.With(middlewares...).HandleFunc(r.getPattern(pattern), handlerFunc) } @@ -242,39 +248,11 @@ func (r *Router) Combo(pattern string, h ...any) *Combo { return &Combo{r, pattern, h} } -// Combo represents a tiny group routes with same pattern -type Combo struct { - r *Router - pattern string - h []any -} - -// Get delegates Get method -func (c *Combo) Get(h ...any) *Combo { - c.r.Get(c.pattern, append(c.h, h...)...) - return c -} - -// Post delegates Post method -func (c *Combo) Post(h ...any) *Combo { - c.r.Post(c.pattern, append(c.h, h...)...) - return c -} - -// Delete delegates Delete method -func (c *Combo) Delete(h ...any) *Combo { - c.r.Delete(c.pattern, append(c.h, h...)...) - return c -} - -// Put delegates Put method -func (c *Combo) Put(h ...any) *Combo { - c.r.Put(c.pattern, append(c.h, h...)...) - return c -} - -// Patch delegates Patch method -func (c *Combo) Patch(h ...any) *Combo { - c.r.Patch(c.pattern, append(c.h, h...)...) - return c +// PathGroup creates a group of paths which could be matched by regexp. +// It is only designed to resolve some special cases which chi router can't handle. +// For most cases, it shouldn't be used because it needs to iterate all rules to find the matched one (inefficient). +func (r *Router) PathGroup(pattern string, fn func(g *RouterPathGroup), h ...any) { + g := &RouterPathGroup{r: r, pathParam: "*"} + fn(g) + r.Any(pattern, append(h, g.ServeHTTP)...) } diff --git a/modules/web/router_combo.go b/modules/web/router_combo.go new file mode 100644 index 0000000000..4478689027 --- /dev/null +++ b/modules/web/router_combo.go @@ -0,0 +1,41 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package web + +// Combo represents a tiny group routes with same pattern +type Combo struct { + r *Router + pattern string + h []any +} + +// Get delegates Get method +func (c *Combo) Get(h ...any) *Combo { + c.r.Get(c.pattern, append(c.h, h...)...) + return c +} + +// Post delegates Post method +func (c *Combo) Post(h ...any) *Combo { + c.r.Post(c.pattern, append(c.h, h...)...) + return c +} + +// Delete delegates Delete method +func (c *Combo) Delete(h ...any) *Combo { + c.r.Delete(c.pattern, append(c.h, h...)...) + return c +} + +// Put delegates Put method +func (c *Combo) Put(h ...any) *Combo { + c.r.Put(c.pattern, append(c.h, h...)...) + return c +} + +// Patch delegates Patch method +func (c *Combo) Patch(h ...any) *Combo { + c.r.Patch(c.pattern, append(c.h, h...)...) + return c +} diff --git a/modules/web/router_path.go b/modules/web/router_path.go new file mode 100644 index 0000000000..64154c34a5 --- /dev/null +++ b/modules/web/router_path.go @@ -0,0 +1,169 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package web + +import ( + "net/http" + "regexp" + "slices" + "strings" + + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/util" + + "github.com/go-chi/chi/v5" +) + +type RouterPathGroup struct { + r *Router + pathParam string + matchers []*routerPathMatcher +} + +func (g *RouterPathGroup) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + chiCtx := chi.RouteContext(req.Context()) + path := chiCtx.URLParam(g.pathParam) + for _, m := range g.matchers { + if m.matchPath(chiCtx, path) { + chiCtx.RoutePatterns = append(chiCtx.RoutePatterns, m.pattern) + handler := m.handlerFunc + for i := len(m.middlewares) - 1; i >= 0; i-- { + handler = m.middlewares[i](handler).ServeHTTP + } + handler(resp, req) + return + } + } + g.r.chiRouter.NotFoundHandler().ServeHTTP(resp, req) +} + +type RouterPathGroupPattern struct { + pattern string + re *regexp.Regexp + params []routerPathParam + middlewares []any +} + +// MatchPath matches the request method, and uses regexp to match the path. +// The pattern uses "<...>" to define path parameters, for example, "/<name>" (different from chi router) +// It is only designed to resolve some special cases that chi router can't handle. +// For most cases, it shouldn't be used because it needs to iterate all rules to find the matched one (inefficient). +func (g *RouterPathGroup) MatchPath(methods, pattern string, h ...any) { + g.MatchPattern(methods, g.PatternRegexp(pattern), h...) +} + +func (g *RouterPathGroup) MatchPattern(methods string, pattern *RouterPathGroupPattern, h ...any) { + g.matchers = append(g.matchers, newRouterPathMatcher(methods, pattern, h...)) +} + +type routerPathParam struct { + name string + captureGroup int +} + +type routerPathMatcher struct { + methods container.Set[string] + pattern string + re *regexp.Regexp + params []routerPathParam + middlewares []func(http.Handler) http.Handler + handlerFunc http.HandlerFunc +} + +func (p *routerPathMatcher) matchPath(chiCtx *chi.Context, path string) bool { + if !p.methods.Contains(chiCtx.RouteMethod) { + return false + } + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + pathMatches := p.re.FindStringSubmatchIndex(path) // Golang regexp match pairs [start, end, start, end, ...] + if pathMatches == nil { + return false + } + var paramMatches [][]int + for i := 2; i < len(pathMatches); { + paramMatches = append(paramMatches, []int{pathMatches[i], pathMatches[i+1]}) + pmIdx := len(paramMatches) - 1 + end := pathMatches[i+1] + i += 2 + for ; i < len(pathMatches); i += 2 { + if pathMatches[i] >= end { + break + } + paramMatches[pmIdx] = append(paramMatches[pmIdx], pathMatches[i], pathMatches[i+1]) + } + } + for i, pm := range paramMatches { + groupIdx := p.params[i].captureGroup * 2 + chiCtx.URLParams.Add(p.params[i].name, path[pm[groupIdx]:pm[groupIdx+1]]) + } + return true +} + +func isValidMethod(name string) bool { + switch name { + case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodHead, http.MethodOptions, http.MethodConnect, http.MethodTrace: + return true + } + return false +} + +func newRouterPathMatcher(methods string, patternRegexp *RouterPathGroupPattern, h ...any) *routerPathMatcher { + middlewares, handlerFunc := wrapMiddlewareAndHandler(patternRegexp.middlewares, h) + p := &routerPathMatcher{methods: make(container.Set[string]), middlewares: middlewares, handlerFunc: handlerFunc} + for method := range strings.SplitSeq(methods, ",") { + method = strings.TrimSpace(method) + if !isValidMethod(method) { + panic("invalid HTTP method: " + method) + } + p.methods.Add(method) + } + p.pattern, p.re, p.params = patternRegexp.pattern, patternRegexp.re, patternRegexp.params + return p +} + +func patternRegexp(pattern string, h ...any) *RouterPathGroupPattern { + p := &RouterPathGroupPattern{middlewares: slices.Clone(h)} + re := []byte{'^'} + lastEnd := 0 + for lastEnd < len(pattern) { + start := strings.IndexByte(pattern[lastEnd:], '<') + if start == -1 { + re = append(re, regexp.QuoteMeta(pattern[lastEnd:])...) + break + } + end := strings.IndexByte(pattern[lastEnd+start:], '>') + if end == -1 { + panic("invalid pattern: " + pattern) + } + re = append(re, regexp.QuoteMeta(pattern[lastEnd:lastEnd+start])...) + partName, partExp, _ := strings.Cut(pattern[lastEnd+start+1:lastEnd+start+end], ":") + lastEnd += start + end + 1 + + // TODO: it could support to specify a "capture group" for the name, for example: "/<name[2]:(\d)-(\d)>" + // it is not used so no need to implement it now + param := routerPathParam{} + if partExp == "*" { + re = append(re, "(.*?)/?"...) + if lastEnd < len(pattern) && pattern[lastEnd] == '/' { + lastEnd++ // the "*" pattern is able to handle the last slash, so skip it + } + } else { + partExp = util.IfZero(partExp, "[^/]+") + re = append(re, '(') + re = append(re, partExp...) + re = append(re, ')') + } + param.name = partName + p.params = append(p.params, param) + } + re = append(re, '$') + p.pattern, p.re = pattern, regexp.MustCompile(string(re)) + return p +} + +func (g *RouterPathGroup) PatternRegexp(pattern string, h ...any) *RouterPathGroupPattern { + return patternRegexp(pattern, h...) +} diff --git a/modules/web/router_test.go b/modules/web/router_test.go new file mode 100644 index 0000000000..f216aa6180 --- /dev/null +++ b/modules/web/router_test.go @@ -0,0 +1,275 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package web + +import ( + "bytes" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/util" + + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/assert" +) + +func chiURLParamsToMap(chiCtx *chi.Context) map[string]string { + pathParams := chiCtx.URLParams + m := make(map[string]string, len(pathParams.Keys)) + for i, key := range pathParams.Keys { + if key == "*" && pathParams.Values[i] == "" { + continue // chi router will add an empty "*" key if there is a "Mount" + } + m[key] = pathParams.Values[i] + } + return util.Iif(len(m) == 0, nil, m) +} + +func TestPathProcessor(t *testing.T) { + testProcess := func(pattern, uri string, expectedPathParams map[string]string) { + chiCtx := chi.NewRouteContext() + chiCtx.RouteMethod = "GET" + p := newRouterPathMatcher("GET", patternRegexp(pattern), http.NotFound) + assert.True(t, p.matchPath(chiCtx, uri), "use pattern %s to process uri %s", pattern, uri) + assert.Equal(t, expectedPathParams, chiURLParamsToMap(chiCtx), "use pattern %s to process uri %s", pattern, uri) + } + + // the "<...>" is intentionally designed to distinguish from chi's path parameters, because: + // 1. their behaviors are totally different, we do not want to mislead developers + // 2. we can write regexp in "<name:\w{3,4}>" easily and parse it easily + testProcess("/<p1>/<p2>", "/a/b", map[string]string{"p1": "a", "p2": "b"}) + testProcess("/<p1:*>", "", map[string]string{"p1": ""}) // this is a special case, because chi router could use empty path + testProcess("/<p1:*>", "/", map[string]string{"p1": ""}) + testProcess("/<p1:*>/<p2>", "/a", map[string]string{"p1": "", "p2": "a"}) + testProcess("/<p1:*>/<p2>", "/a/b", map[string]string{"p1": "a", "p2": "b"}) + testProcess("/<p1:*>/<p2>", "/a/b/c", map[string]string{"p1": "a/b", "p2": "c"}) +} + +func TestRouter(t *testing.T) { + buff := &bytes.Buffer{} + recorder := httptest.NewRecorder() + recorder.Body = buff + + type resultStruct struct { + method string + pathParams map[string]string + handlerMarks []string + chiRoutePattern *string + } + + var res resultStruct + h := func(optMark ...string) func(resp http.ResponseWriter, req *http.Request) { + mark := util.OptionalArg(optMark, "") + return func(resp http.ResponseWriter, req *http.Request) { + chiCtx := chi.RouteContext(req.Context()) + res.method = req.Method + res.pathParams = chiURLParamsToMap(chiCtx) + res.chiRoutePattern = util.ToPointer(chiCtx.RoutePattern()) + if mark != "" { + res.handlerMarks = append(res.handlerMarks, mark) + } + } + } + + stopMark := func(optMark ...string) func(resp http.ResponseWriter, req *http.Request) { + mark := util.OptionalArg(optMark, "") + return func(resp http.ResponseWriter, req *http.Request) { + if stop := req.FormValue("stop"); stop != "" && (mark == "" || mark == stop) { + h(stop)(resp, req) + resp.WriteHeader(http.StatusOK) + } else if mark != "" { + res.handlerMarks = append(res.handlerMarks, mark) + } + } + } + + r := NewRouter() + r.NotFound(h("not-found:/")) + r.Get("/{username}/{reponame}/{type:issues|pulls}", h("list-issues-a")) // this one will never be called + r.Group("/{username}/{reponame}", func() { + r.Get("/{type:issues|pulls}", h("list-issues-b")) + r.Group("", func() { + r.Get("/{type:issues|pulls}/{index}", h("view-issue")) + }, stopMark()) + r.Group("/issues/{index}", func() { + r.Post("/update", h("update-issue")) + }) + }) + + m := NewRouter() + m.NotFound(h("not-found:/api/v1")) + r.Mount("/api/v1", m) + m.Group("/repos", func() { + m.Group("/{username}/{reponame}", func() { + m.Group("/branches", func() { + m.Get("", h()) + m.Post("", h()) + m.Group("/{name}", func() { + m.Get("", h()) + m.Patch("", h()) + m.Delete("", h()) + }) + m.PathGroup("/*", func(g *RouterPathGroup) { + g.MatchPattern("GET", g.PatternRegexp(`/<dir:*>/<file:[a-z]{1,2}>`, stopMark("s2")), stopMark("s3"), h("match-path")) + }, stopMark("s1")) + }) + }) + }) + + testRoute := func(t *testing.T, methodPath string, expected resultStruct) { + t.Run(methodPath, func(t *testing.T) { + res = resultStruct{} + methodPathFields := strings.Fields(methodPath) + req, err := http.NewRequest(methodPathFields[0], methodPathFields[1], nil) + assert.NoError(t, err) + r.ServeHTTP(recorder, req) + if expected.chiRoutePattern == nil { + res.chiRoutePattern = nil + } + assert.Equal(t, expected, res) + }) + } + + t.Run("RootRouter", func(t *testing.T) { + testRoute(t, "GET /the-user/the-repo/other", resultStruct{ + method: "GET", + handlerMarks: []string{"not-found:/"}, + chiRoutePattern: util.ToPointer(""), + }) + testRoute(t, "GET /the-user/the-repo/pulls", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "pulls"}, + handlerMarks: []string{"list-issues-b"}, + }) + testRoute(t, "GET /the-user/the-repo/issues/123", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, + handlerMarks: []string{"view-issue"}, + chiRoutePattern: util.ToPointer("/{username}/{reponame}/{type:issues|pulls}/{index}"), + }) + testRoute(t, "GET /the-user/the-repo/issues/123?stop=hijack", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, + handlerMarks: []string{"hijack"}, + }) + testRoute(t, "POST /the-user/the-repo/issues/123/update", resultStruct{ + method: "POST", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "index": "123"}, + handlerMarks: []string{"update-issue"}, + }) + }) + + t.Run("Sub Router", func(t *testing.T) { + testRoute(t, "GET /api/v1/other", resultStruct{ + method: "GET", + handlerMarks: []string{"not-found:/api/v1"}, + }) + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo"}, + }) + + testRoute(t, "POST /api/v1/repos/the-user/the-repo/branches", resultStruct{ + method: "POST", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo"}, + }) + + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/master", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "name": "master"}, + }) + + testRoute(t, "PATCH /api/v1/repos/the-user/the-repo/branches/master", resultStruct{ + method: "PATCH", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "name": "master"}, + }) + + testRoute(t, "DELETE /api/v1/repos/the-user/the-repo/branches/master", resultStruct{ + method: "DELETE", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "name": "master"}, + }) + }) + + t.Run("MatchPath", func(t *testing.T) { + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3", "match-path"}, + }) + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1%2fd2/fn", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1%2fd2/fn", "dir": "d1%2fd2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3", "match-path"}, + }) + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/000", resultStruct{ + method: "GET", + pathParams: map[string]string{"reponame": "the-repo", "username": "the-user", "*": "d1/d2/000"}, + handlerMarks: []string{"s1", "not-found:/api/v1"}, + }) + + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s1", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn"}, + handlerMarks: []string{"s1"}, + }) + + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s2", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2"}, + }) + + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s3", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3"}, + chiRoutePattern: util.ToPointer("/api/v1/repos/{username}/{reponame}/branches/<dir:*>/<file:[a-z]{1,2}>"), + }) + }) +} + +func TestRouteNormalizePath(t *testing.T) { + type paths struct { + EscapedPath, RawPath, Path string + } + testPath := func(reqPath string, expectedPaths paths) { + recorder := httptest.NewRecorder() + recorder.Body = bytes.NewBuffer(nil) + + actualPaths := paths{EscapedPath: "(none)", RawPath: "(none)", Path: "(none)"} + r := NewRouter() + r.Get("/*", func(resp http.ResponseWriter, req *http.Request) { + actualPaths.EscapedPath = req.URL.EscapedPath() + actualPaths.RawPath = req.URL.RawPath + actualPaths.Path = req.URL.Path + }) + + req, err := http.NewRequest(http.MethodGet, reqPath, nil) + assert.NoError(t, err) + r.ServeHTTP(recorder, req) + assert.Equal(t, expectedPaths, actualPaths, "req path = %q", reqPath) + } + + // RawPath could be empty if the EscapedPath is the same as escape(Path) and it is already normalized + testPath("/", paths{EscapedPath: "/", RawPath: "", Path: "/"}) + testPath("//", paths{EscapedPath: "/", RawPath: "/", Path: "/"}) + testPath("/%2f", paths{EscapedPath: "/%2f", RawPath: "/%2f", Path: "//"}) + testPath("///a//b/", paths{EscapedPath: "/a/b", RawPath: "/a/b", Path: "/a/b"}) + + defer test.MockVariableValue(&setting.UseSubURLPath, true)() + defer test.MockVariableValue(&setting.AppSubURL, "/sub-path")() + testPath("/", paths{EscapedPath: "(none)", RawPath: "(none)", Path: "(none)"}) // 404 + testPath("/sub-path", paths{EscapedPath: "/", RawPath: "/", Path: "/"}) + testPath("/sub-path/", paths{EscapedPath: "/", RawPath: "/", Path: "/"}) + testPath("/sub-path//a/b///", paths{EscapedPath: "/a/b", RawPath: "/a/b", Path: "/a/b"}) + testPath("/sub-path/%2f/", paths{EscapedPath: "/%2f", RawPath: "/%2f", Path: "//"}) + // "/v2" is special for OCI container registry, it should always be in the root of the site + testPath("/v2", paths{EscapedPath: "/v2", RawPath: "/v2", Path: "/v2"}) + testPath("/v2/", paths{EscapedPath: "/v2", RawPath: "/v2", Path: "/v2"}) + testPath("/v2/%2f", paths{EscapedPath: "/v2/%2f", RawPath: "/v2/%2f", Path: "/v2//"}) +} diff --git a/modules/web/routing/context.go b/modules/web/routing/context.go index c5e85a415b..d3eb98f83d 100644 --- a/modules/web/routing/context.go +++ b/modules/web/routing/context.go @@ -6,22 +6,29 @@ package routing import ( "context" "net/http" + + "code.gitea.io/gitea/modules/gtprof" + "code.gitea.io/gitea/modules/reqctx" ) type contextKeyType struct{} var contextKey contextKeyType -// UpdateFuncInfo updates a context's func info -func UpdateFuncInfo(ctx context.Context, funcInfo *FuncInfo) { - record, ok := ctx.Value(contextKey).(*requestRecord) - if !ok { - return +// RecordFuncInfo records a func info into context +func RecordFuncInfo(ctx context.Context, funcInfo *FuncInfo) (end func()) { + end = func() {} + if reqCtx := reqctx.FromContext(ctx); reqCtx != nil { + var traceSpan *gtprof.TraceSpan + traceSpan, end = gtprof.GetTracer().StartInContext(reqCtx, "http.func") + traceSpan.SetAttributeString("func", funcInfo.shortName) } - - record.lock.Lock() - record.funcInfo = funcInfo - record.lock.Unlock() + if record, ok := ctx.Value(contextKey).(*requestRecord); ok { + record.lock.Lock() + record.funcInfo = funcInfo + record.lock.Unlock() + } + return end } // MarkLongPolling marks the request is a long-polling request, and the logger may output different message for it diff --git a/modules/web/routing/funcinfo_test.go b/modules/web/routing/funcinfo_test.go index 2ab5960373..974af58931 100644 --- a/modules/web/routing/funcinfo_test.go +++ b/modules/web/routing/funcinfo_test.go @@ -6,6 +6,8 @@ package routing import ( "fmt" "testing" + + "github.com/stretchr/testify/assert" ) func Test_shortenFilename(t *testing.T) { @@ -37,9 +39,8 @@ func Test_shortenFilename(t *testing.T) { } for _, tt := range tests { t.Run(fmt.Sprintf("shortenFilename('%s')", tt.filename), func(t *testing.T) { - if gotShort := shortenFilename(tt.filename, tt.fallback); gotShort != tt.expected { - t.Errorf("shortenFilename('%s'), expect '%s', but get '%s'", tt.filename, tt.expected, gotShort) - } + gotShort := shortenFilename(tt.filename, tt.fallback) + assert.Equal(t, tt.expected, gotShort) }) } } @@ -72,9 +73,8 @@ func Test_trimAnonymousFunctionSuffix(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := trimAnonymousFunctionSuffix(tt.name); got != tt.want { - t.Errorf("trimAnonymousFunctionSuffix() = %v, want %v", got, tt.want) - } + got := trimAnonymousFunctionSuffix(tt.name) + assert.Equal(t, tt.want, got) }) } } diff --git a/modules/web/routing/logger.go b/modules/web/routing/logger.go index 5f3a7592af..3bca9b3420 100644 --- a/modules/web/routing/logger.go +++ b/modules/web/routing/logger.go @@ -35,6 +35,19 @@ var ( ) func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { + const callerName = "HTTPRequest" + logTrace := func(fmt string, args ...any) { + logger.Log(2, &log.Event{Level: log.TRACE, Caller: callerName}, fmt, args...) + } + logInfo := func(fmt string, args ...any) { + logger.Log(2, &log.Event{Level: log.INFO, Caller: callerName}, fmt, args...) + } + logWarn := func(fmt string, args ...any) { + logger.Log(2, &log.Event{Level: log.WARN, Caller: callerName}, fmt, args...) + } + logError := func(fmt string, args ...any) { + logger.Log(2, &log.Event{Level: log.ERROR, Caller: callerName}, fmt, args...) + } return func(trigger Event, record *requestRecord) { if trigger == StartEvent { if !logger.LevelEnabled(log.TRACE) { @@ -44,7 +57,7 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { } // when a request starts, we have no information about the handler function information, we only have the request path req := record.request - logger.Trace("router: %s %v %s for %s", startMessage, log.ColoredMethod(req.Method), req.RequestURI, req.RemoteAddr) + logTrace("router: %s %v %s for %s", startMessage, log.ColoredMethod(req.Method), req.RequestURI, req.RemoteAddr) return } @@ -60,9 +73,9 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { if trigger == StillExecutingEvent { message := slowMessage - logf := logger.Warn + logf := logWarn if isLongPolling { - logf = logger.Info + logf = logInfo message = pollingMessage } logf("router: %s %v %s for %s, elapsed %v @ %s", @@ -75,7 +88,7 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { } if panicErr != nil { - logger.Warn("router: %s %v %s for %s, panic in %v @ %s, err=%v", + logWarn("router: %s %v %s for %s, panic in %v @ %s, err=%v", failedMessage, log.ColoredMethod(req.Method), req.RequestURI, req.RemoteAddr, log.ColoredTime(time.Since(record.startTime)), @@ -89,13 +102,16 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { if v, ok := record.responseWriter.(types.ResponseStatusProvider); ok { status = v.WrittenStatus() } - logf := logger.Info - if strings.HasPrefix(req.RequestURI, "/assets/") { - logf = logger.Trace + logf := logInfo + // lower the log level for some specific requests, in most cases these logs are not useful + if strings.HasPrefix(req.RequestURI, "/assets/") /* static assets */ || + req.RequestURI == "/user/events" /* Server-Sent Events (SSE) handler */ || + req.RequestURI == "/api/actions/runner.v1.RunnerService/FetchTask" /* Actions Runner polling */ { + logf = logTrace } message := completedMessage if isUnknownHandler { - logf = logger.Error + logf = logError message = unknownHandlerMessage } diff --git a/modules/webhook/events.go b/modules/webhook/events.go new file mode 100644 index 0000000000..f4dfff0294 --- /dev/null +++ b/modules/webhook/events.go @@ -0,0 +1,20 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package webhook + +type HookEvents map[HookEventType]bool + +func (he HookEvents) Get(evt HookEventType) bool { + return he[evt] +} + +// HookEvent represents events that will delivery hook. +type HookEvent struct { + PushOnly bool `json:"push_only"` + SendEverything bool `json:"send_everything"` + ChooseEvents bool `json:"choose_events"` + BranchFilter string `json:"branch_filter"` + + HookEvents `json:"events"` +} diff --git a/modules/webhook/structs.go b/modules/webhook/structs.go deleted file mode 100644 index 927a91a74c..0000000000 --- a/modules/webhook/structs.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package webhook - -// HookEvents is a set of web hook events -type HookEvents struct { - Create bool `json:"create"` - Delete bool `json:"delete"` - Fork bool `json:"fork"` - Issues bool `json:"issues"` - IssueAssign bool `json:"issue_assign"` - IssueLabel bool `json:"issue_label"` - IssueMilestone bool `json:"issue_milestone"` - IssueComment bool `json:"issue_comment"` - Push bool `json:"push"` - PullRequest bool `json:"pull_request"` - PullRequestAssign bool `json:"pull_request_assign"` - PullRequestLabel bool `json:"pull_request_label"` - PullRequestMilestone bool `json:"pull_request_milestone"` - PullRequestComment bool `json:"pull_request_comment"` - PullRequestReview bool `json:"pull_request_review"` - PullRequestSync bool `json:"pull_request_sync"` - PullRequestReviewRequest bool `json:"pull_request_review_request"` - Wiki bool `json:"wiki"` - Repository bool `json:"repository"` - Release bool `json:"release"` - Package bool `json:"package"` -} - -// HookEvent represents events that will delivery hook. -type HookEvent struct { - PushOnly bool `json:"push_only"` - SendEverything bool `json:"send_everything"` - ChooseEvents bool `json:"choose_events"` - BranchFilter string `json:"branch_filter"` - - HookEvents `json:"events"` -} diff --git a/modules/webhook/type.go b/modules/webhook/type.go index fbec889272..89c6a4bfe5 100644 --- a/modules/webhook/type.go +++ b/modules/webhook/type.go @@ -31,21 +31,51 @@ const ( HookEventRepository HookEventType = "repository" HookEventRelease HookEventType = "release" HookEventPackage HookEventType = "package" - HookEventSchedule HookEventType = "schedule" HookEventStatus HookEventType = "status" + // once a new event added here, please also added to AllEvents() function + + // FIXME: This event should be a group of pull_request_review_xxx events + HookEventPullRequestReview HookEventType = "pull_request_review" + // Actions event only + HookEventSchedule HookEventType = "schedule" + HookEventWorkflowRun HookEventType = "workflow_run" + HookEventWorkflowJob HookEventType = "workflow_job" ) +func AllEvents() []HookEventType { + return []HookEventType{ + HookEventCreate, + HookEventDelete, + HookEventFork, + HookEventPush, + HookEventIssues, + HookEventIssueAssign, + HookEventIssueLabel, + HookEventIssueMilestone, + HookEventIssueComment, + HookEventPullRequest, + HookEventPullRequestAssign, + HookEventPullRequestLabel, + HookEventPullRequestMilestone, + HookEventPullRequestComment, + HookEventPullRequestReviewApproved, + HookEventPullRequestReviewRejected, + HookEventPullRequestReviewComment, + HookEventPullRequestSync, + HookEventPullRequestReviewRequest, + HookEventWiki, + HookEventRepository, + HookEventRelease, + HookEventPackage, + HookEventStatus, + HookEventWorkflowRun, + HookEventWorkflowJob, + } +} + // Event returns the HookEventType as an event string func (h HookEventType) Event() string { switch h { - case HookEventCreate: - return "create" - case HookEventDelete: - return "delete" - case HookEventFork: - return "fork" - case HookEventPush: - return "push" case HookEventIssues, HookEventIssueAssign, HookEventIssueLabel, HookEventIssueMilestone: return "issues" case HookEventPullRequest, HookEventPullRequestAssign, HookEventPullRequestLabel, HookEventPullRequestMilestone, @@ -59,14 +89,13 @@ func (h HookEventType) Event() string { return "pull_request_rejected" case HookEventPullRequestReviewComment: return "pull_request_comment" - case HookEventWiki: - return "wiki" - case HookEventRepository: - return "repository" - case HookEventRelease: - return "release" + default: + return string(h) } - return "" +} + +func (h HookEventType) IsPullRequest() bool { + return h.Event() == "pull_request" } // HookType is the type of a webhook diff --git a/modules/zstd/zstd_test.go b/modules/zstd/zstd_test.go index c3ca8e78f7..7fd30484ca 100644 --- a/modules/zstd/zstd_test.go +++ b/modules/zstd/zstd_test.go @@ -16,7 +16,7 @@ import ( ) func TestWriterReader(t *testing.T) { - testData := prepareTestData(t, 20_000_000) + testData := prepareTestData(t, 1_000_000) result := bytes.NewBuffer(nil) @@ -64,7 +64,7 @@ func TestWriterReader(t *testing.T) { } func TestSeekableWriterReader(t *testing.T) { - testData := prepareTestData(t, 20_000_000) + testData := prepareTestData(t, 2_000_000) result := bytes.NewBuffer(nil) @@ -109,7 +109,7 @@ func TestSeekableWriterReader(t *testing.T) { reader, err := NewSeekableReader(assertReader) require.NoError(t, err) - _, err = reader.Seek(10_000_000, io.SeekStart) + _, err = reader.Seek(1_000_000, io.SeekStart) require.NoError(t, err) data := make([]byte, 1000) @@ -117,7 +117,7 @@ func TestSeekableWriterReader(t *testing.T) { require.NoError(t, err) require.NoError(t, reader.Close()) - assert.Equal(t, testData[10_000_000:10_000_000+1000], data) + assert.Equal(t, testData[1_000_000:1_000_000+1000], data) // Should seek 3 times, // the first two times are for getting the index, |