diff options
Diffstat (limited to 'modules')
418 files changed, 6116 insertions, 4405 deletions
diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 0d2b0dd919..27bcafa649 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -6,6 +6,7 @@ package actions import ( "bytes" "io" + "slices" "strings" "code.gitea.io/gitea/modules/git" @@ -43,21 +44,23 @@ func IsWorkflow(path string) bool { return strings.HasPrefix(path, ".gitea/workflows") || strings.HasPrefix(path, ".github/workflows") } -func ListWorkflows(commit *git.Commit) (git.Entries, error) { - tree, err := commit.SubTree(".gitea/workflows") +func ListWorkflows(commit *git.Commit) (string, git.Entries, error) { + rpath := ".gitea/workflows" + tree, err := commit.SubTree(rpath) if _, ok := err.(git.ErrNotExist); ok { - tree, err = commit.SubTree(".github/workflows") + rpath = ".github/workflows" + tree, err = commit.SubTree(rpath) } if _, ok := err.(git.ErrNotExist); ok { - return nil, nil + return "", nil, nil } if err != nil { - return nil, err + return "", nil, err } entries, err := tree.ListEntriesRecursiveFast() if err != nil { - return nil, err + return "", nil, err } ret := make(git.Entries, 0, len(entries)) @@ -66,7 +69,7 @@ func ListWorkflows(commit *git.Commit) (git.Entries, error) { ret = append(ret, entry) } } - return ret, nil + return rpath, ret, nil } func GetContentFromEntry(entry *git.TreeEntry) ([]byte, error) { @@ -102,7 +105,7 @@ func DetectWorkflows( payload api.Payloader, detectSchedule bool, ) ([]*DetectedWorkflow, []*DetectedWorkflow, error) { - entries, err := ListWorkflows(commit) + _, entries, err := ListWorkflows(commit) if err != nil { return nil, nil, err } @@ -147,7 +150,7 @@ func DetectWorkflows( } func DetectScheduledWorkflows(gitRepo *git.Repository, commit *git.Commit) ([]*DetectedWorkflow, error) { - entries, err := ListWorkflows(commit) + _, entries, err := ListWorkflows(commit) if err != nil { return nil, err } @@ -243,6 +246,10 @@ func detectMatched(gitRepo *git.Repository, commit *git.Commit, triggedEvent web webhook_module.HookEventPackage: return matchPackageEvent(payload.(*api.PackagePayload), evt) + case // workflow_run + webhook_module.HookEventWorkflowRun: + return matchWorkflowRunEvent(payload.(*api.WorkflowRunPayload), evt) + default: log.Warn("unsupported event %q", triggedEvent) return false @@ -311,6 +318,10 @@ func matchPushEvent(commit *git.Commit, pushPayload *api.PushPayload, evt *jobpa matchTimes++ } case "paths": + if refName.IsTag() { + matchTimes++ + break + } filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err) @@ -324,6 +335,10 @@ func matchPushEvent(commit *git.Commit, pushPayload *api.PushPayload, evt *jobpa } } case "paths-ignore": + if refName.IsTag() { + matchTimes++ + break + } filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err) @@ -463,7 +478,7 @@ func matchPullRequestEvent(gitRepo *git.Repository, commit *git.Commit, prPayloa matchTimes++ } case "paths": - filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.Base.Ref) + filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err) } else { @@ -476,7 +491,7 @@ func matchPullRequestEvent(gitRepo *git.Repository, commit *git.Commit, prPayloa } } case "paths-ignore": - filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.Base.Ref) + filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase) if err != nil { log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err) } else { @@ -554,21 +569,12 @@ func matchPullRequestReviewEvent(prPayload *api.PullRequestPayload, evt *jobpars actions = append(actions, "submitted", "edited") } - matched := false for _, val := range vals { - for _, action := range actions { - if glob.MustCompile(val, '/').Match(action) { - matched = true - break - } - } - if matched { + if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) { + matchTimes++ break } } - if matched { - matchTimes++ - } default: log.Warn("pull request review event unsupported condition %q", cond) } @@ -603,21 +609,12 @@ func matchPullRequestReviewCommentEvent(prPayload *api.PullRequestPayload, evt * actions = append(actions, "created", "edited") } - matched := false for _, val := range vals { - for _, action := range actions { - if glob.MustCompile(val, '/').Match(action) { - matched = true - break - } - } - if matched { + if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) { + matchTimes++ break } } - if matched { - matchTimes++ - } default: log.Warn("pull request review comment event unsupported condition %q", cond) } @@ -698,3 +695,53 @@ func matchPackageEvent(payload *api.PackagePayload, evt *jobparser.Event) bool { } return matchTimes == len(evt.Acts()) } + +func matchWorkflowRunEvent(payload *api.WorkflowRunPayload, evt *jobparser.Event) bool { + // with no special filter parameters + if len(evt.Acts()) == 0 { + return true + } + + matchTimes := 0 + // all acts conditions should be satisfied + for cond, vals := range evt.Acts() { + switch cond { + case "types": + action := payload.Action + for _, val := range vals { + if glob.MustCompile(val, '/').Match(action) { + matchTimes++ + break + } + } + case "workflows": + workflow := payload.Workflow + patterns, err := workflowpattern.CompilePatterns(vals...) + if err != nil { + break + } + if !workflowpattern.Skip(patterns, []string{workflow.Name}, &workflowpattern.EmptyTraceWriter{}) { + matchTimes++ + } + case "branches": + patterns, err := workflowpattern.CompilePatterns(vals...) + if err != nil { + break + } + if !workflowpattern.Skip(patterns, []string{payload.WorkflowRun.HeadBranch}, &workflowpattern.EmptyTraceWriter{}) { + matchTimes++ + } + case "branches-ignore": + patterns, err := workflowpattern.CompilePatterns(vals...) + if err != nil { + break + } + if !workflowpattern.Filter(patterns, []string{payload.WorkflowRun.HeadBranch}, &workflowpattern.EmptyTraceWriter{}) { + matchTimes++ + } + default: + log.Warn("workflow run event unsupported condition %q", cond) + } + } + return matchTimes == len(evt.Acts()) +} diff --git a/modules/actions/workflows_test.go b/modules/actions/workflows_test.go index c8e1e553fe..e23431651d 100644 --- a/modules/actions/workflows_test.go +++ b/modules/actions/workflows_test.go @@ -125,6 +125,24 @@ func TestDetectMatched(t *testing.T) { yamlOn: "on: schedule", expected: true, }, + { + desc: "push to tag matches workflow with paths condition (should skip paths check)", + triggedEvent: webhook_module.HookEventPush, + payload: &api.PushPayload{ + Ref: "refs/tags/v1.0.0", + Before: "0000000", + Commits: []*api.PayloadCommit{ + { + ID: "abcdef123456", + Added: []string{"src/main.go"}, + Message: "Release v1.0.0", + }, + }, + }, + commit: nil, + yamlOn: "on:\n push:\n paths:\n - src/**", + expected: true, + }, } for _, tc := range testCases { diff --git a/modules/assetfs/embed.go b/modules/assetfs/embed.go new file mode 100644 index 0000000000..95176372d1 --- /dev/null +++ b/modules/assetfs/embed.go @@ -0,0 +1,375 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "bytes" + "compress/gzip" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" +) + +type EmbeddedFile interface { + io.ReadSeeker + fs.ReadDirFile + ReadDir(n int) ([]fs.DirEntry, error) +} + +type EmbeddedFileInfo interface { + fs.FileInfo + fs.DirEntry + GetGzipContent() ([]byte, bool) +} + +type decompressor interface { + io.Reader + Close() error + Reset(io.Reader) error +} + +type embeddedFileInfo struct { + fs *embeddedFS + fullName string + data []byte + + BaseName string `json:"n"` + OriginSize int64 `json:"s,omitempty"` + DataBegin int64 `json:"b,omitempty"` + DataLen int64 `json:"l,omitempty"` + Children []*embeddedFileInfo `json:"c,omitempty"` +} + +func (fi *embeddedFileInfo) GetGzipContent() ([]byte, bool) { + // when generating the bindata, if the compressed data equals or is larger than the original data, we store the original data + if fi.DataLen == fi.OriginSize { + return nil, false + } + return fi.data, true +} + +type EmbeddedFileBase struct { + info *embeddedFileInfo + dataReader io.ReadSeeker + seekPos int64 +} + +func (f *EmbeddedFileBase) ReadDir(n int) ([]fs.DirEntry, error) { + // this method is used to satisfy the "func (f ioFile) ReadDir(...)" in httpfs + l, err := f.info.fs.ReadDir(f.info.fullName) + if err != nil { + return nil, err + } + if n < 0 || n > len(l) { + return l, nil + } + return l[:n], nil +} + +type EmbeddedOriginFile struct { + EmbeddedFileBase +} + +type EmbeddedCompressedFile struct { + EmbeddedFileBase + decompressor decompressor + decompressorPos int64 +} + +type embeddedFS struct { + meta func() *EmbeddedMeta + + files map[string]*embeddedFileInfo + filesMu sync.RWMutex + + data []byte +} + +type EmbeddedMeta struct { + Root *embeddedFileInfo +} + +func NewEmbeddedFS(data []byte) fs.ReadDirFS { + efs := &embeddedFS{data: data, files: make(map[string]*embeddedFileInfo)} + efs.meta = sync.OnceValue(func() *EmbeddedMeta { + var meta EmbeddedMeta + p := bytes.LastIndexByte(data, '\n') + if p < 0 { + return &meta + } + if err := json.Unmarshal(data[p+1:], &meta); err != nil { + panic("embedded file is not valid") + } + return &meta + }) + return efs +} + +var _ fs.ReadDirFS = (*embeddedFS)(nil) + +func (e *embeddedFS) ReadDir(name string) (l []fs.DirEntry, err error) { + fi, err := e.getFileInfo(name) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fs.ErrNotExist + } + l = make([]fs.DirEntry, len(fi.Children)) + for i, child := range fi.Children { + l[i], err = e.getFileInfo(name + "/" + child.BaseName) + if err != nil { + return nil, err + } + } + return l, nil +} + +func (e *embeddedFS) getFileInfo(fullName string) (*embeddedFileInfo, error) { + // no need to do heavy "path.Clean()" because we don't want to support "foo/../bar" or absolute paths + fullName = strings.TrimPrefix(fullName, "./") + if fullName == "" { + fullName = "." + } + + e.filesMu.RLock() + fi := e.files[fullName] + e.filesMu.RUnlock() + if fi != nil { + return fi, nil + } + + fields := strings.Split(fullName, "/") + fi = e.meta().Root + if fullName != "." { + found := true + for _, field := range fields { + for _, child := range fi.Children { + if found = child.BaseName == field; found { + fi = child + break + } + } + if !found { + return nil, fs.ErrNotExist + } + } + } + + e.filesMu.Lock() + defer e.filesMu.Unlock() + if fi != nil { + fi.fs = e + fi.fullName = fullName + fi.data = e.data[fi.DataBegin : fi.DataBegin+fi.DataLen] + e.files[fullName] = fi // do not cache nil, otherwise keeping accessing random non-existing file will cause OOM + return fi, nil + } + return nil, fs.ErrNotExist +} + +func (e *embeddedFS) Open(name string) (fs.File, error) { + info, err := e.getFileInfo(name) + if err != nil { + return nil, err + } + base := EmbeddedFileBase{info: info} + base.dataReader = bytes.NewReader(base.info.data) + if info.DataLen != info.OriginSize { + decomp, err := gzip.NewReader(base.dataReader) + if err != nil { + return nil, err + } + return &EmbeddedCompressedFile{EmbeddedFileBase: base, decompressor: decomp}, nil + } + return &EmbeddedOriginFile{base}, nil +} + +var ( + _ EmbeddedFileInfo = (*embeddedFileInfo)(nil) + _ EmbeddedFile = (*EmbeddedOriginFile)(nil) + _ EmbeddedFile = (*EmbeddedCompressedFile)(nil) +) + +func (f *EmbeddedOriginFile) Read(p []byte) (n int, err error) { + return f.dataReader.Read(p) +} + +func (f *EmbeddedCompressedFile) Read(p []byte) (n int, err error) { + if f.decompressorPos > f.seekPos { + if err = f.decompressor.Reset(bytes.NewReader(f.info.data)); err != nil { + return 0, err + } + f.decompressorPos = 0 + } + if f.decompressorPos < f.seekPos { + if _, err = io.CopyN(io.Discard, f.decompressor, f.seekPos-f.decompressorPos); err != nil { + return 0, err + } + f.decompressorPos = f.seekPos + } + n, err = f.decompressor.Read(p) + f.decompressorPos += int64(n) + f.seekPos = f.decompressorPos + return n, err +} + +func (f *EmbeddedFileBase) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + f.seekPos = offset + case io.SeekCurrent: + f.seekPos += offset + case io.SeekEnd: + f.seekPos = f.info.OriginSize + offset + } + return f.seekPos, nil +} + +func (f *EmbeddedFileBase) Stat() (fs.FileInfo, error) { + return f.info, nil +} + +func (f *EmbeddedOriginFile) Close() error { + return nil +} + +func (f *EmbeddedCompressedFile) Close() error { + return f.decompressor.Close() +} + +func (fi *embeddedFileInfo) Name() string { + return fi.BaseName +} + +func (fi *embeddedFileInfo) Size() int64 { + return fi.OriginSize +} + +func (fi *embeddedFileInfo) Mode() fs.FileMode { + return util.Iif(fi.IsDir(), fs.ModeDir|0o555, 0o444) +} + +func (fi *embeddedFileInfo) ModTime() time.Time { + return getExecutableModTime() +} + +func (fi *embeddedFileInfo) IsDir() bool { + return fi.Children != nil +} + +func (fi *embeddedFileInfo) Sys() any { + return nil +} + +func (fi *embeddedFileInfo) Type() fs.FileMode { + return util.Iif(fi.IsDir(), fs.ModeDir, 0) +} + +func (fi *embeddedFileInfo) Info() (fs.FileInfo, error) { + return fi, nil +} + +// getExecutableModTime returns the modification time of the executable file. +// In bindata, we can't use the ModTime of the files because we need to make the build reproducible +var getExecutableModTime = sync.OnceValue(func() (modTime time.Time) { + exePath, err := os.Executable() + if err != nil { + return modTime + } + exePath, err = filepath.Abs(exePath) + if err != nil { + return modTime + } + exePath, err = filepath.EvalSymlinks(exePath) + if err != nil { + return modTime + } + st, err := os.Stat(exePath) + if err != nil { + return modTime + } + return st.ModTime() +}) + +func GenerateEmbedBindata(fsRootPath, outputFile string) error { + output, err := os.OpenFile(outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer output.Close() + + meta := &EmbeddedMeta{} + meta.Root = &embeddedFileInfo{} + var outputOffset int64 + var embedFiles func(parent *embeddedFileInfo, fsPath, embedPath string) error + embedFiles = func(parent *embeddedFileInfo, fsPath, embedPath string) error { + dirEntries, err := os.ReadDir(fsPath) + if err != nil { + return err + } + for _, dirEntry := range dirEntries { + if err != nil { + return err + } + if dirEntry.IsDir() { + child := &embeddedFileInfo{ + BaseName: dirEntry.Name(), + Children: []*embeddedFileInfo{}, // non-nil means it's a directory + } + parent.Children = append(parent.Children, child) + if err = embedFiles(child, filepath.Join(fsPath, dirEntry.Name()), path.Join(embedPath, dirEntry.Name())); err != nil { + return err + } + } else { + data, err := os.ReadFile(filepath.Join(fsPath, dirEntry.Name())) + if err != nil { + return err + } + var compressed bytes.Buffer + gz, _ := gzip.NewWriterLevel(&compressed, gzip.BestCompression) + if _, err = gz.Write(data); err != nil { + return err + } + if err = gz.Close(); err != nil { + return err + } + + // only use the compressed data if it is smaller than the original data + outputBytes := util.Iif(len(compressed.Bytes()) < len(data), compressed.Bytes(), data) + child := &embeddedFileInfo{ + BaseName: dirEntry.Name(), + OriginSize: int64(len(data)), + DataBegin: outputOffset, + DataLen: int64(len(outputBytes)), + } + if _, err = output.Write(outputBytes); err != nil { + return err + } + outputOffset += child.DataLen + parent.Children = append(parent.Children, child) + } + } + return nil + } + + if err = embedFiles(meta.Root, fsRootPath, ""); err != nil { + return err + } + jsonBuf, err := json.Marshal(meta) // can't use json.NewEncoder here because it writes extra EOL + if err != nil { + return err + } + _, _ = output.Write([]byte{'\n'}) + _, err = output.Write(jsonBuf) + return err +} diff --git a/modules/assetfs/embed_test.go b/modules/assetfs/embed_test.go new file mode 100644 index 0000000000..06598da4c4 --- /dev/null +++ b/modules/assetfs/embed_test.go @@ -0,0 +1,98 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package assetfs + +import ( + "bytes" + "io/fs" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEmbed(t *testing.T) { + tmpDir := t.TempDir() + tmpDataDir := tmpDir + "/data" + _ = os.MkdirAll(tmpDataDir+"/foo/bar", 0o755) + _ = os.WriteFile(tmpDataDir+"/a.txt", []byte("a"), 0o644) + _ = os.WriteFile(tmpDataDir+"/foo/bar/b.txt", bytes.Repeat([]byte("a"), 1000), 0o644) + _ = os.WriteFile(tmpDataDir+"/foo/c.txt", []byte("c"), 0o644) + require.NoError(t, GenerateEmbedBindata(tmpDataDir, tmpDir+"/out.dat")) + + data, err := os.ReadFile(tmpDir + "/out.dat") + require.NoError(t, err) + efs := NewEmbeddedFS(data) + + // test a non-existing file + _, err = fs.ReadFile(efs, "not exist") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // test a normal file (no compression) + content, err := fs.ReadFile(efs, "a.txt") + require.NoError(t, err) + assert.Equal(t, "a", string(content)) + fi, err := fs.Stat(efs, "a.txt") + require.NoError(t, err) + _, ok := fi.(EmbeddedFileInfo).GetGzipContent() + assert.False(t, ok) + + // test a compressed file + content, err = fs.ReadFile(efs, "foo/bar/b.txt") + require.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte("a"), 1000), content) + fi, err = fs.Stat(efs, "foo/bar/b.txt") + require.NoError(t, err) + assert.False(t, fi.Mode().IsDir()) + assert.True(t, fi.Mode().IsRegular()) + gzipContent, ok := fi.(EmbeddedFileInfo).GetGzipContent() + assert.True(t, ok) + assert.Greater(t, len(gzipContent), 1) + assert.Less(t, len(gzipContent), 1000) + + // test list root directory + entries, err := fs.ReadDir(efs, ".") + require.NoError(t, err) + assert.Len(t, entries, 2) + assert.Equal(t, "a.txt", entries[0].Name()) + assert.False(t, entries[0].IsDir()) + + // test list subdirectory + entries, err = fs.ReadDir(efs, "foo") + require.NoError(t, err) + require.Len(t, entries, 2) + assert.Equal(t, "bar", entries[0].Name()) + assert.True(t, entries[0].IsDir()) + assert.Equal(t, "c.txt", entries[1].Name()) + assert.False(t, entries[1].IsDir()) + + // test directory mode + fi, err = fs.Stat(efs, "foo") + require.NoError(t, err) + assert.True(t, fi.IsDir()) + assert.True(t, fi.Mode().IsDir()) + assert.False(t, fi.Mode().IsRegular()) + + // test httpfs + hfs := http.FS(efs) + hf, err := hfs.Open("foo/bar/b.txt") + require.NoError(t, err) + hi, err := hf.Stat() + require.NoError(t, err) + fiEmbedded, ok := hi.(EmbeddedFileInfo) + require.True(t, ok) + gzipContent, ok = fiEmbedded.GetGzipContent() + assert.True(t, ok) + assert.Greater(t, len(gzipContent), 1) + assert.Less(t, len(gzipContent), 1000) + + // test httpfs directory listing + hf, err = hfs.Open("foo") + require.NoError(t, err) + dirs, err := hf.Readdir(1) + require.NoError(t, err) + assert.Len(t, dirs, 1) +} diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go index 4f3811ba2b..ce55475bd9 100644 --- a/modules/assetfs/layered.go +++ b/modules/assetfs/layered.go @@ -52,8 +52,8 @@ func Local(name, base string, sub ...string) *Layer { } // Bindata returns a new Layer with the given name, it serves files from the given bindata asset. -func Bindata(name string, fs http.FileSystem) *Layer { - return &Layer{name: name, fs: fs} +func Bindata(name string, fs fs.FS) *Layer { + return &Layer{name: name, fs: http.FS(fs)} } // LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers. diff --git a/modules/assetfs/layered_test.go b/modules/assetfs/layered_test.go index 03a3ae0d7c..b549815ea5 100644 --- a/modules/assetfs/layered_test.go +++ b/modules/assetfs/layered_test.go @@ -52,7 +52,7 @@ func TestLayered(t *testing.T) { assert.NoError(t, err) bs, err := io.ReadAll(f) assert.NoError(t, err) - assert.EqualValues(t, "f1", string(bs)) + assert.Equal(t, "f1", string(bs)) _ = f.Close() assertRead := func(expected string, expectedErr error, elems ...string) { @@ -76,27 +76,27 @@ func TestLayered(t *testing.T) { files, err := assets.ListFiles(".", true) assert.NoError(t, err) - assert.EqualValues(t, []string{"f1", "f2", "fa"}, files) + assert.Equal(t, []string{"f1", "f2", "fa"}, files) files, err = assets.ListFiles(".", false) assert.NoError(t, err) - assert.EqualValues(t, []string{"d1", "d2", "da"}, files) + assert.Equal(t, []string{"d1", "d2", "da"}, files) files, err = assets.ListFiles(".") assert.NoError(t, err) - assert.EqualValues(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files) + assert.Equal(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files) files, err = assets.ListAllFiles(".", true) assert.NoError(t, err) - assert.EqualValues(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files) + assert.Equal(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files) files, err = assets.ListAllFiles(".", false) assert.NoError(t, err) - assert.EqualValues(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files) + assert.Equal(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files) files, err = assets.ListAllFiles(".") assert.NoError(t, err) - assert.EqualValues(t, []string{ + assert.Equal(t, []string{ "d1", "d1/f", "d2", "d2/f", "da", "da/f", "da/sub1", "da/sub2", @@ -104,6 +104,6 @@ func TestLayered(t *testing.T) { }, files) assert.Empty(t, assets.GetFileLayerName("no-such")) - assert.EqualValues(t, "l1", assets.GetFileLayerName("f1")) - assert.EqualValues(t, "l2", assets.GetFileLayerName("f2")) + assert.Equal(t, "l1", assets.GetFileLayerName("f1")) + assert.Equal(t, "l2", assets.GetFileLayerName("f2")) } diff --git a/modules/auth/httpauth/httpauth.go b/modules/auth/httpauth/httpauth.go new file mode 100644 index 0000000000..7f1f1ee152 --- /dev/null +++ b/modules/auth/httpauth/httpauth.go @@ -0,0 +1,47 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httpauth + +import ( + "encoding/base64" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +type BasicAuth struct { + Username, Password string +} + +type BearerToken struct { + Token string +} + +type ParsedAuthorizationHeader struct { + BasicAuth *BasicAuth + BearerToken *BearerToken +} + +func ParseAuthorizationHeader(header string) (ret ParsedAuthorizationHeader, _ bool) { + parts := strings.Fields(header) + if len(parts) != 2 { + return ret, false + } + if util.AsciiEqualFold(parts[0], "basic") { + s, err := base64.StdEncoding.DecodeString(parts[1]) + if err != nil { + return ret, false + } + u, p, ok := strings.Cut(string(s), ":") + if !ok { + return ret, false + } + ret.BasicAuth = &BasicAuth{Username: u, Password: p} + return ret, true + } else if util.AsciiEqualFold(parts[0], "token") || util.AsciiEqualFold(parts[0], "bearer") { + ret.BearerToken = &BearerToken{Token: parts[1]} + return ret, true + } + return ret, false +} diff --git a/modules/auth/httpauth/httpauth_test.go b/modules/auth/httpauth/httpauth_test.go new file mode 100644 index 0000000000..087b86917f --- /dev/null +++ b/modules/auth/httpauth/httpauth_test.go @@ -0,0 +1,43 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httpauth + +import ( + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseAuthorizationHeader(t *testing.T) { + type parsed = ParsedAuthorizationHeader + type basic = BasicAuth + type bearer = BearerToken + cases := []struct { + headerValue string + expected parsed + ok bool + }{ + {"", parsed{}, false}, + {"?", parsed{}, false}, + {"foo", parsed{}, false}, + {"any value", parsed{}, false}, + + {"Basic ?", parsed{}, false}, + {"Basic " + base64.StdEncoding.EncodeToString([]byte("foo")), parsed{}, false}, + {"Basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true}, + {"basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true}, + + {"token value", parsed{BearerToken: &bearer{"value"}}, true}, + {"Token value", parsed{BearerToken: &bearer{"value"}}, true}, + {"bearer value", parsed{BearerToken: &bearer{"value"}}, true}, + {"Bearer value", parsed{BearerToken: &bearer{"value"}}, true}, + {"Bearer wrong value", parsed{}, false}, + } + for _, c := range cases { + ret, ok := ParseAuthorizationHeader(c.headerValue) + assert.Equal(t, c.ok, ok, "header %q", c.headerValue) + assert.Equal(t, c.expected, ret, "header %q", c.headerValue) + } +} diff --git a/modules/auth/openid/discovery_cache_test.go b/modules/auth/openid/discovery_cache_test.go index 7d4b27c5df..f3d7dd226e 100644 --- a/modules/auth/openid/discovery_cache_test.go +++ b/modules/auth/openid/discovery_cache_test.go @@ -26,7 +26,8 @@ func (s *testDiscoveredInfo) OpLocalID() string { } func TestTimedDiscoveryCache(t *testing.T) { - dc := newTimedDiscoveryCache(1 * time.Second) + ttl := 50 * time.Millisecond + dc := newTimedDiscoveryCache(ttl) // Put some initial values dc.Put("foo", &testDiscoveredInfo{}) // openid.opEndpoint: "a", openid.opLocalID: "b", openid.claimedID: "c"}) @@ -41,8 +42,8 @@ func TestTimedDiscoveryCache(t *testing.T) { // Attempt to get a non-existent value assert.Nil(t, dc.Get("bar")) - // Sleep one second and try retrieve again - time.Sleep(1 * time.Second) + // Sleep for a while and try to retrieve again + time.Sleep(ttl * 3 / 2) assert.Nil(t, dc.Get("foo")) } diff --git a/modules/auth/password/hash/common.go b/modules/auth/password/hash/common.go index 487c0738f4..d5e2c34314 100644 --- a/modules/auth/password/hash/common.go +++ b/modules/auth/password/hash/common.go @@ -18,7 +18,7 @@ func parseIntParam(value, param, algorithmName, config string, previousErr error return parsed, previousErr // <- Keep the previous error as this function should still return an error once everything has been checked if any call failed } -func parseUIntParam(value, param, algorithmName, config string, previousErr error) (uint64, error) { //nolint:unparam +func parseUIntParam(value, param, algorithmName, config string, previousErr error) (uint64, error) { //nolint:unparam // algorithmName is always argon2 parsed, err := strconv.ParseUint(value, 10, 64) if err != nil { log.Error("invalid integer for %s representation in %s hash spec %s", param, algorithmName, config) diff --git a/modules/auth/password/password.go b/modules/auth/password/password.go index c66b62937f..a1e101dd62 100644 --- a/modules/auth/password/password.go +++ b/modules/auth/password/password.go @@ -101,7 +101,7 @@ func Generate(n int) (string, error) { buffer := make([]byte, n) maxInt := big.NewInt(int64(len(validChars))) for { - for j := 0; j < n; j++ { + for j := range n { rnd, err := rand.Int(rand.Reader, maxInt) if err != nil { return "", err diff --git a/modules/auth/password/password_test.go b/modules/auth/password/password_test.go index 6c35dc86bd..0fea593c85 100644 --- a/modules/auth/password/password_test.go +++ b/modules/auth/password/password_test.go @@ -50,7 +50,7 @@ func TestComplexity_Generate(t *testing.T) { test := func(t *testing.T, modes []string) { testComplextity(modes) - for i := 0; i < maxCount; i++ { + for range maxCount { pwd, err := Generate(pwdLen) assert.NoError(t, err) assert.Len(t, pwd, pwdLen) diff --git a/modules/auth/password/pwn/pwn.go b/modules/auth/password/pwn/pwn.go index f77ce9f40b..99a6ca6cea 100644 --- a/modules/auth/password/pwn/pwn.go +++ b/modules/auth/password/pwn/pwn.go @@ -101,7 +101,7 @@ func (c *Client) CheckPassword(pw string, padding bool) (int, error) { } defer resp.Body.Close() - for _, pair := range strings.Split(string(body), "\n") { + for pair := range strings.SplitSeq(string(body), "\n") { parts := strings.Split(pair, ":") if len(parts) != 2 { continue diff --git a/modules/avatar/avatar_test.go b/modules/avatar/avatar_test.go index a721c77868..a5a1a7c1b0 100644 --- a/modules/avatar/avatar_test.go +++ b/modules/avatar/avatar_test.go @@ -94,8 +94,8 @@ func Test_ProcessAvatarImage(t *testing.T) { assert.NotEqual(t, origin, result) decoded, err := png.Decode(bytes.NewReader(result)) assert.NoError(t, err) - assert.EqualValues(t, scaledSize, decoded.Bounds().Max.X) - assert.EqualValues(t, scaledSize, decoded.Bounds().Max.Y) + assert.Equal(t, scaledSize, decoded.Bounds().Max.X) + assert.Equal(t, scaledSize, decoded.Bounds().Max.Y) // if origin image is smaller than the default size, use the origin image origin = newImgData(1) diff --git a/modules/avatar/hash_test.go b/modules/avatar/hash_test.go index 1b8249c696..c518144b47 100644 --- a/modules/avatar/hash_test.go +++ b/modules/avatar/hash_test.go @@ -19,8 +19,8 @@ func Test_HashAvatar(t *testing.T) { var buff bytes.Buffer png.Encode(&buff, myImage) - assert.EqualValues(t, "9ddb5bac41d57e72aa876321d0c09d71090c05f94bc625303801be2f3240d2cb", avatar.HashAvatar(1, buff.Bytes())) - assert.EqualValues(t, "9a5d44e5d637b9582a976676e8f3de1dccd877c2fe3e66ca3fab1629f2f47609", avatar.HashAvatar(8, buff.Bytes())) - assert.EqualValues(t, "ed7399158672088770de6f5211ce15528ebd675e92fc4fc060c025f4b2794ccb", avatar.HashAvatar(1024, buff.Bytes())) - assert.EqualValues(t, "161178642c7d59eb25a61dddced5e6b66eae1c70880d5f148b1b497b767e72d9", avatar.HashAvatar(1024, []byte{})) + assert.Equal(t, "9ddb5bac41d57e72aa876321d0c09d71090c05f94bc625303801be2f3240d2cb", avatar.HashAvatar(1, buff.Bytes())) + assert.Equal(t, "9a5d44e5d637b9582a976676e8f3de1dccd877c2fe3e66ca3fab1629f2f47609", avatar.HashAvatar(8, buff.Bytes())) + assert.Equal(t, "ed7399158672088770de6f5211ce15528ebd675e92fc4fc060c025f4b2794ccb", avatar.HashAvatar(1024, buff.Bytes())) + assert.Equal(t, "161178642c7d59eb25a61dddced5e6b66eae1c70880d5f148b1b497b767e72d9", avatar.HashAvatar(1024, []byte{})) } diff --git a/modules/avatar/identicon/block.go b/modules/avatar/identicon/block.go index cb1803a231..fc8ce90212 100644 --- a/modules/avatar/identicon/block.go +++ b/modules/avatar/identicon/block.go @@ -24,8 +24,8 @@ func drawBlock(img *image.Paletted, x, y, size, angle int, points []int) { rotate(points, m, m, angle) } - for i := 0; i < size; i++ { - for j := 0; j < size; j++ { + for i := range size { + for j := range size { if pointInPolygon(i, j, points) { img.SetColorIndex(x+i, y+j, 1) } diff --git a/modules/avatar/identicon/identicon.go b/modules/avatar/identicon/identicon.go index 63926d5f19..ee92416a53 100644 --- a/modules/avatar/identicon/identicon.go +++ b/modules/avatar/identicon/identicon.go @@ -8,6 +8,7 @@ package identicon import ( "crypto/sha256" + "errors" "fmt" "image" "image/color" @@ -29,7 +30,7 @@ type Identicon struct { // fore all possible foreground colors. only one foreground color will be picked randomly for one image func New(size int, back color.Color, fore ...color.Color) (*Identicon, error) { if len(fore) == 0 { - return nil, fmt.Errorf("foreground is not set") + return nil, errors.New("foreground is not set") } if size < minImageSize { @@ -133,7 +134,7 @@ func drawBlocks(p *image.Paletted, size int, c, b1, b2 blockFunc, b1Angle, b2Ang // then we make it left-right mirror, so we didn't draw 3/6/9 before for x := 0; x < size/2; x++ { - for y := 0; y < size; y++ { + for y := range size { p.SetColorIndex(size-x, y, p.ColorIndexAt(x, y)) } } diff --git a/modules/badge/badge.go b/modules/badge/badge.go index b30d0b4729..d2e9bd9d1b 100644 --- a/modules/badge/badge.go +++ b/modules/badge/badge.go @@ -4,6 +4,10 @@ package badge import ( + "strings" + "sync" + "unicode" + actions_model "code.gitea.io/gitea/models/actions" ) @@ -11,94 +15,115 @@ import ( // We use 10x scale to calculate more precisely // Then scale down to normal size in tmpl file -type Label struct { - text string - width int -} - -func (l Label) Text() string { - return l.text -} - -func (l Label) Width() int { - return l.width -} - -func (l Label) TextLength() int { - return int(float64(l.width-defaultOffset) * 9.5) -} - -func (l Label) X() int { - return l.width*5 + 10 -} - -type Message struct { +type Text struct { text string width int x int } -func (m Message) Text() string { - return m.text +func (t Text) Text() string { + return t.text } -func (m Message) Width() int { - return m.width +func (t Text) Width() int { + return t.width } -func (m Message) X() int { - return m.x +func (t Text) X() int { + return t.x } -func (m Message) TextLength() int { - return int(float64(m.width-defaultOffset) * 9.5) +func (t Text) TextLength() int { + return int(float64(t.width-defaultOffset) * 10) } type Badge struct { - Color string - FontSize int - Label Label - Message Message + IDPrefix string + FontFamily string + Color string + FontSize int + Label Text + Message Text } func (b Badge) Width() int { return b.Label.width + b.Message.width } +// Style follows https://shields.io/badges const ( - defaultOffset = 9 - defaultFontSize = 11 - DefaultColor = "#9f9f9f" // Grey - defaultFontWidth = 7 // approximate speculation + StyleFlat = "flat" + StyleFlatSquare = "flat-square" ) -var StatusColorMap = map[actions_model.Status]string{ - actions_model.StatusSuccess: "#4c1", // Green - actions_model.StatusSkipped: "#dfb317", // Yellow - actions_model.StatusUnknown: "#97ca00", // Light Green - actions_model.StatusFailure: "#e05d44", // Red - actions_model.StatusCancelled: "#fe7d37", // Orange - actions_model.StatusWaiting: "#dfb317", // Yellow - actions_model.StatusRunning: "#dfb317", // Yellow - actions_model.StatusBlocked: "#dfb317", // Yellow -} +const ( + defaultOffset = 10 + defaultFontSize = 11 + DefaultColor = "#9f9f9f" // Grey + DefaultFontFamily = "DejaVu Sans,Verdana,Geneva,sans-serif" + DefaultStyle = StyleFlat +) + +var GlobalVars = sync.OnceValue(func() (ret struct { + StatusColorMap map[actions_model.Status]string + DejaVuGlyphWidthData map[rune]uint8 + AllStyles []string +}, +) { + ret.StatusColorMap = map[actions_model.Status]string{ + actions_model.StatusSuccess: "#4c1", // Green + actions_model.StatusSkipped: "#dfb317", // Yellow + actions_model.StatusUnknown: "#97ca00", // Light Green + actions_model.StatusFailure: "#e05d44", // Red + actions_model.StatusCancelled: "#fe7d37", // Orange + actions_model.StatusWaiting: "#dfb317", // Yellow + actions_model.StatusRunning: "#dfb317", // Yellow + actions_model.StatusBlocked: "#dfb317", // Yellow + } + ret.DejaVuGlyphWidthData = dejaVuGlyphWidthDataFunc() + ret.AllStyles = []string{StyleFlat, StyleFlatSquare} + return ret +}) // GenerateBadge generates badge with given template func GenerateBadge(label, message, color string) Badge { - lw := defaultFontWidth*len(label) + defaultOffset - mw := defaultFontWidth*len(message) + defaultOffset - x := lw*10 + mw*5 - 10 + lw := calculateTextWidth(label) + defaultOffset + mw := calculateTextWidth(message) + defaultOffset + + lx := lw * 5 + mx := lw*10 + mw*5 - 10 return Badge{ - Label: Label{ + FontFamily: DefaultFontFamily, + Label: Text{ text: label, width: lw, + x: lx, }, - Message: Message{ + Message: Text{ text: message, width: mw, - x: x, + x: mx, }, FontSize: defaultFontSize * 10, Color: color, } } + +func calculateTextWidth(text string) int { + width := 0 + widthData := GlobalVars().DejaVuGlyphWidthData + for _, char := range strings.TrimSpace(text) { + charWidth, ok := widthData[char] + if !ok { + // use the width of 'm' in case of missing glyph width data for a printable character + if unicode.IsPrint(char) { + charWidth = widthData['m'] + } else { + charWidth = 0 + } + } + width += int(charWidth) + } + + return width +} diff --git a/modules/badge/badge_glyph_width.go b/modules/badge/badge_glyph_width.go new file mode 100644 index 0000000000..0d950c5a70 --- /dev/null +++ b/modules/badge/badge_glyph_width.go @@ -0,0 +1,206 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package badge + +// DejaVuGlyphWidthData is generated by `sfnt.Face.GlyphAdvance(nil, <rune>, 11, font.HintingNone)` with DejaVu Sans +// v2.37 (https://github.com/dejavu-fonts/dejavu-fonts/releases/download/version_2_37/dejavu-sans-ttf-2.37.zip). +// +// Fonts defined in "DefaultFontFamily" all have similar widths (including "DejaVu Sans"), +// and these widths are fixed and don't seem to change. +// +// A devtest page "/devtest/badge-actions-svg" could be used to check the rendered images. + +func dejaVuGlyphWidthDataFunc() map[rune]uint8 { + return map[rune]uint8{ + 32: 3, + 33: 4, + 34: 5, + 35: 9, + 36: 7, + 37: 10, + 38: 9, + 39: 3, + 40: 4, + 41: 4, + 42: 6, + 43: 9, + 44: 3, + 45: 4, + 46: 3, + 47: 4, + 48: 7, + 49: 7, + 50: 7, + 51: 7, + 52: 7, + 53: 7, + 54: 7, + 55: 7, + 56: 7, + 57: 7, + 58: 4, + 59: 4, + 60: 9, + 61: 9, + 62: 9, + 63: 6, + 64: 11, + 65: 8, + 66: 8, + 67: 8, + 68: 8, + 69: 7, + 70: 6, + 71: 9, + 72: 8, + 73: 3, + 74: 3, + 75: 7, + 76: 6, + 77: 9, + 78: 8, + 79: 9, + 80: 7, + 81: 9, + 82: 8, + 83: 7, + 84: 7, + 85: 8, + 86: 8, + 87: 11, + 88: 8, + 89: 7, + 90: 8, + 91: 4, + 92: 4, + 93: 4, + 94: 9, + 95: 6, + 96: 6, + 97: 7, + 98: 7, + 99: 6, + 100: 7, + 101: 7, + 102: 4, + 103: 7, + 104: 7, + 105: 3, + 106: 3, + 107: 6, + 108: 3, + 109: 11, + 110: 7, + 111: 7, + 112: 7, + 113: 7, + 114: 5, + 115: 6, + 116: 4, + 117: 7, + 118: 7, + 119: 9, + 120: 7, + 121: 7, + 122: 6, + 123: 7, + 124: 4, + 125: 7, + 126: 9, + 161: 4, + 162: 7, + 163: 7, + 164: 7, + 165: 7, + 166: 4, + 167: 6, + 168: 6, + 169: 11, + 170: 5, + 171: 7, + 172: 9, + 174: 11, + 175: 6, + 176: 6, + 177: 9, + 178: 4, + 179: 4, + 180: 6, + 181: 7, + 182: 7, + 183: 3, + 184: 6, + 185: 4, + 186: 5, + 187: 7, + 188: 11, + 189: 11, + 190: 11, + 191: 6, + 192: 8, + 193: 8, + 194: 8, + 195: 8, + 196: 8, + 197: 8, + 198: 11, + 199: 8, + 200: 7, + 201: 7, + 202: 7, + 203: 7, + 204: 3, + 205: 3, + 206: 3, + 207: 3, + 208: 9, + 209: 8, + 210: 9, + 211: 9, + 212: 9, + 213: 9, + 214: 9, + 215: 9, + 216: 9, + 217: 8, + 218: 8, + 219: 8, + 220: 8, + 221: 7, + 222: 7, + 223: 7, + 224: 7, + 225: 7, + 226: 7, + 227: 7, + 228: 7, + 229: 7, + 230: 11, + 231: 6, + 232: 7, + 233: 7, + 234: 7, + 235: 7, + 236: 3, + 237: 3, + 238: 3, + 239: 3, + 240: 7, + 241: 7, + 242: 7, + 243: 7, + 244: 7, + 245: 7, + 246: 7, + 247: 9, + 248: 7, + 249: 7, + 250: 7, + 251: 7, + 252: 7, + 253: 7, + 254: 7, + 255: 7, + } +} diff --git a/modules/base/tool.go b/modules/base/tool.go index 02ca85569e..ed94575e74 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -8,13 +8,10 @@ import ( "crypto/sha1" "crypto/sha256" "crypto/subtle" - "encoding/base64" "encoding/hex" - "errors" "fmt" "hash" "strconv" - "strings" "time" "code.gitea.io/gitea/modules/setting" @@ -36,19 +33,6 @@ func ShortSha(sha1 string) string { return util.TruncateRunes(sha1, 10) } -// BasicAuthDecode decode basic auth string -func BasicAuthDecode(encoded string) (string, string, error) { - s, err := base64.StdEncoding.DecodeString(encoded) - if err != nil { - return "", "", err - } - - if username, password, ok := strings.Cut(string(s), ":"); ok { - return username, password, nil - } - return "", "", errors.New("invalid basic authentication") -} - // VerifyTimeLimitCode verify time limit code func VerifyTimeLimitCode(now time.Time, data string, minutes int, code string) bool { if len(code) <= 18 { diff --git a/modules/base/tool_test.go b/modules/base/tool_test.go index 7cebedb073..b7365e40c4 100644 --- a/modules/base/tool_test.go +++ b/modules/base/tool_test.go @@ -26,25 +26,6 @@ func TestShortSha(t *testing.T) { assert.Equal(t, "veryverylo", ShortSha("veryverylong")) } -func TestBasicAuthDecode(t *testing.T) { - _, _, err := BasicAuthDecode("?") - assert.Equal(t, "illegal base64 data at input byte 0", err.Error()) - - user, pass, err := BasicAuthDecode("Zm9vOmJhcg==") - assert.NoError(t, err) - assert.Equal(t, "foo", user) - assert.Equal(t, "bar", pass) - - _, _, err = BasicAuthDecode("aW52YWxpZA==") - assert.Error(t, err) - - _, _, err = BasicAuthDecode("invalid") - assert.Error(t, err) - - _, _, err = BasicAuthDecode("YWxpY2U=") // "alice", no colon - assert.Error(t, err) -} - func TestVerifyTimeLimitCode(t *testing.T) { defer test.MockVariableValue(&setting.InstallLock, true)() initGeneralSecret := func(secret string) { diff --git a/modules/cache/cache.go b/modules/cache/cache.go index f7828e3cae..039caa9fbc 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -4,6 +4,8 @@ package cache import ( + "encoding/hex" + "errors" "fmt" "strconv" "time" @@ -22,7 +24,7 @@ func Init() error { if err != nil { return err } - for i := 0; i < 10; i++ { + for range 10 { if err = c.Ping(); err == nil { break } @@ -48,10 +50,10 @@ const ( // returns func Test() (time.Duration, error) { if defaultCache == nil { - return 0, fmt.Errorf("default cache not initialized") + return 0, errors.New("default cache not initialized") } - testData := fmt.Sprintf("%x", make([]byte, 500)) + testData := hex.EncodeToString(make([]byte, 500)) start := time.Now() @@ -63,10 +65,10 @@ func Test() (time.Duration, error) { } testVal, hit := defaultCache.Get(testCacheKey) if !hit { - return 0, fmt.Errorf("expect cache hit but got none") + return 0, errors.New("expect cache hit but got none") } if testVal != testData { - return 0, fmt.Errorf("expect cache to return same value as stored but got other") + return 0, errors.New("expect cache to return same value as stored but got other") } return time.Since(start), nil diff --git a/modules/cache/cache_redis.go b/modules/cache/cache_redis.go index c5b52a2086..7473c938af 100644 --- a/modules/cache/cache_redis.go +++ b/modules/cache/cache_redis.go @@ -11,7 +11,7 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/nosql" - "gitea.com/go-chi/cache" //nolint:depguard + "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here "github.com/redis/go-redis/v9" ) diff --git a/modules/cache/cache_test.go b/modules/cache/cache_test.go index 5408020306..d6ea2032ee 100644 --- a/modules/cache/cache_test.go +++ b/modules/cache/cache_test.go @@ -4,7 +4,7 @@ package cache import ( - "fmt" + "errors" "testing" "time" @@ -57,22 +57,22 @@ func TestGetString(t *testing.T) { createTestCache() data, err := GetString("key", func() (string, error) { - return "", fmt.Errorf("some error") + return "", errors.New("some error") }) assert.Error(t, err) - assert.Equal(t, "", data) + assert.Empty(t, data) data, err = GetString("key", func() (string, error) { return "", nil }) assert.NoError(t, err) - assert.Equal(t, "", data) + assert.Empty(t, data) data, err = GetString("key", func() (string, error) { return "some data", nil }) assert.NoError(t, err) - assert.Equal(t, "", data) + assert.Empty(t, data) Remove("key") data, err = GetString("key", func() (string, error) { @@ -82,7 +82,7 @@ func TestGetString(t *testing.T) { assert.Equal(t, "some data", data) data, err = GetString("key", func() (string, error) { - return "", fmt.Errorf("some error") + return "", errors.New("some error") }) assert.NoError(t, err) assert.Equal(t, "some data", data) @@ -93,7 +93,7 @@ func TestGetInt64(t *testing.T) { createTestCache() data, err := GetInt64("key", func() (int64, error) { - return 0, fmt.Errorf("some error") + return 0, errors.New("some error") }) assert.Error(t, err) assert.EqualValues(t, 0, data) @@ -118,7 +118,7 @@ func TestGetInt64(t *testing.T) { assert.EqualValues(t, 100, data) data, err = GetInt64("key", func() (int64, error) { - return 0, fmt.Errorf("some error") + return 0, errors.New("some error") }) assert.NoError(t, err) assert.EqualValues(t, 100, data) diff --git a/modules/cache/cache_twoqueue.go b/modules/cache/cache_twoqueue.go index 1eda2debc4..c8db686e57 100644 --- a/modules/cache/cache_twoqueue.go +++ b/modules/cache/cache_twoqueue.go @@ -10,7 +10,7 @@ import ( "code.gitea.io/gitea/modules/json" - mc "gitea.com/go-chi/cache" //nolint:depguard + mc "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here lru "github.com/hashicorp/golang-lru/v2" ) diff --git a/modules/cache/context.go b/modules/cache/context.go index 484cee659a..23f7c23a52 100644 --- a/modules/cache/context.go +++ b/modules/cache/context.go @@ -5,176 +5,39 @@ package cache import ( "context" - "sync" "time" - - "code.gitea.io/gitea/modules/log" ) -// cacheContext is a context that can be used to cache data in a request level context -// This is useful for caching data that is expensive to calculate and is likely to be -// used multiple times in a request. -type cacheContext struct { - data map[any]map[any]any - lock sync.RWMutex - created time.Time - discard bool -} - -func (cc *cacheContext) Get(tp, key any) any { - cc.lock.RLock() - defer cc.lock.RUnlock() - return cc.data[tp][key] -} - -func (cc *cacheContext) Put(tp, key, value any) { - cc.lock.Lock() - defer cc.lock.Unlock() - - if cc.discard { - return - } - - d := cc.data[tp] - if d == nil { - d = make(map[any]any) - cc.data[tp] = d - } - d[key] = value -} - -func (cc *cacheContext) Delete(tp, key any) { - cc.lock.Lock() - defer cc.lock.Unlock() - delete(cc.data[tp], key) -} - -func (cc *cacheContext) Discard() { - cc.lock.Lock() - defer cc.lock.Unlock() - cc.data = nil - cc.discard = true -} - -func (cc *cacheContext) isDiscard() bool { - cc.lock.RLock() - defer cc.lock.RUnlock() - return cc.discard -} - -// cacheContextLifetime is the max lifetime of cacheContext. -// Since cacheContext is used to cache data in a request level context, 5 minutes is enough. -// If a cacheContext is used more than 5 minutes, it's probably misuse. -const cacheContextLifetime = 5 * time.Minute - -var timeNow = time.Now +type cacheContextKeyType struct{} -func (cc *cacheContext) Expired() bool { - return timeNow().Sub(cc.created) > cacheContextLifetime -} - -var cacheContextKey = struct{}{} - -/* -Since there are both WithCacheContext and WithNoCacheContext, -it may be confusing when there is nesting. - -Some cases to explain the design: - -When: -- A, B or C means a cache context. -- A', B' or C' means a discard cache context. -- ctx means context.Backgrand(). -- A(ctx) means a cache context with ctx as the parent context. -- B(A(ctx)) means a cache context with A(ctx) as the parent context. -- With is alias of WithCacheContext. -- WithNo is alias of WithNoCacheContext. +var cacheContextKey = cacheContextKeyType{} -So: -- With(ctx) -> A(ctx) -- With(With(ctx)) -> A(ctx), not B(A(ctx)), always reuse parent cache context if possible. -- With(With(With(ctx))) -> A(ctx), not C(B(A(ctx))), ditto. -- WithNo(ctx) -> ctx, not A'(ctx), don't create new cache context if we don't have to. -- WithNo(With(ctx)) -> A'(ctx) -- WithNo(WithNo(With(ctx))) -> A'(ctx), not B'(A'(ctx)), don't create new cache context if we don't have to. -- With(WithNo(With(ctx))) -> B(A'(ctx)), not A(ctx), never reuse a discard cache context. -- WithNo(With(WithNo(With(ctx)))) -> B'(A'(ctx)) -- With(WithNo(With(WithNo(With(ctx))))) -> C(B'(A'(ctx))), so there's always only one not-discard cache context. -*/ +// contextCacheLifetime is the max lifetime of context cache. +// Since context cache is used to cache data in a request level context, 5 minutes is enough. +// If a context cache is used more than 5 minutes, it's probably abused. +const contextCacheLifetime = 5 * time.Minute func WithCacheContext(ctx context.Context) context.Context { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if !c.isDiscard() { - // reuse parent context - return ctx - } - } - // FIXME: review the use of this nolint directive - return context.WithValue(ctx, cacheContextKey, &cacheContext{ //nolint:staticcheck - data: make(map[any]map[any]any), - created: timeNow(), - }) -} - -func WithNoCacheContext(ctx context.Context) context.Context { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - // The caller want to run long-life tasks, but the parent context is a cache context. - // So we should disable and clean the cache data, or it will be kept in memory for a long time. - c.Discard() + if c := GetContextCache(ctx); c != nil { return ctx } - - return ctx -} - -func GetContextData(ctx context.Context, tp, key any) any { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if c.Expired() { - // The warning means that the cache context is misused for long-life task, - // it can be resolved with WithNoCacheContext(ctx). - log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c) - return nil - } - return c.Get(tp, key) - } - return nil + return context.WithValue(ctx, cacheContextKey, NewEphemeralCache(contextCacheLifetime)) } -func SetContextData(ctx context.Context, tp, key, value any) { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if c.Expired() { - // The warning means that the cache context is misused for long-life task, - // it can be resolved with WithNoCacheContext(ctx). - log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c) - return - } - c.Put(tp, key, value) - return - } -} - -func RemoveContextData(ctx context.Context, tp, key any) { - if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { - if c.Expired() { - // The warning means that the cache context is misused for long-life task, - // it can be resolved with WithNoCacheContext(ctx). - log.Warn("cache context is expired, is highly likely to be misused for long-life tasks: %v", c) - return - } - c.Delete(tp, key) - } +func GetContextCache(ctx context.Context) *EphemeralCache { + c, _ := ctx.Value(cacheContextKey).(*EphemeralCache) + return c } // GetWithContextCache returns the cache value of the given key in the given context. -func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error) { - v := GetContextData(ctx, cacheGroupKey, cacheTargetID) - if vv, ok := v.(T); ok { - return vv, nil - } - t, err := f() - if err != nil { - return t, err - } - SetContextData(ctx, cacheGroupKey, cacheTargetID, t) - return t, nil +// FIXME: in some cases, the "context cache" should not be used, because it has uncontrollable behaviors +// For example, these calls: +// * GetWithContextCache(TargetID) -> OtherCodeCreateModel(TargetID) -> GetWithContextCache(TargetID) +// Will cause the second call is not able to get the correct created target. +// UNLESS it is certain that the target won't be changed during the request, DO NOT use it. +func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) { + if c := GetContextCache(ctx); c != nil { + return GetWithEphemeralCache(ctx, c, groupKey, targetKey, f) + } + return f(ctx, targetKey) } diff --git a/modules/cache/context_test.go b/modules/cache/context_test.go index cfc186a7bd..8371c2b908 100644 --- a/modules/cache/context_test.go +++ b/modules/cache/context_test.go @@ -4,74 +4,47 @@ package cache import ( + "context" "testing" "time" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" ) func TestWithCacheContext(t *testing.T) { ctx := WithCacheContext(t.Context()) - - v := GetContextData(ctx, "empty_field", "my_config1") + c := GetContextCache(ctx) + v, _ := c.Get("empty_field", "my_config1") assert.Nil(t, v) const field = "system_setting" - v = GetContextData(ctx, field, "my_config1") + v, _ = c.Get(field, "my_config1") assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") + c.Put(field, "my_config1", 1) + v, _ = c.Get(field, "my_config1") assert.NotNil(t, v) - assert.EqualValues(t, 1, v.(int)) + assert.Equal(t, 1, v.(int)) - RemoveContextData(ctx, field, "my_config1") - RemoveContextData(ctx, field, "my_config2") // remove a non-exist key + c.Delete(field, "my_config1") + c.Delete(field, "my_config2") // remove a non-exist key - v = GetContextData(ctx, field, "my_config1") + v, _ = c.Get(field, "my_config1") assert.Nil(t, v) - vInt, err := GetWithContextCache(ctx, field, "my_config1", func() (int, error) { + vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) { return 1, nil }) assert.NoError(t, err) - assert.EqualValues(t, 1, vInt) + assert.Equal(t, 1, vInt) - v = GetContextData(ctx, field, "my_config1") + v, _ = c.Get(field, "my_config1") assert.EqualValues(t, 1, v) - now := timeNow - defer func() { - timeNow = now - }() - timeNow = func() time.Time { - return now().Add(5 * time.Minute) - } - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) -} - -func TestWithNoCacheContext(t *testing.T) { - ctx := t.Context() - - const field = "system_setting" - - v := GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) // still no cache - - ctx = WithCacheContext(ctx) - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") - assert.NotNil(t, v) - - ctx = WithNoCacheContext(ctx) - v = GetContextData(ctx, field, "my_config1") + defer test.MockVariableValue(&timeNow, func() time.Time { + return time.Now().Add(5 * time.Minute) + })() + v, _ = c.Get(field, "my_config1") assert.Nil(t, v) - SetContextData(ctx, field, "my_config1", 1) - v = GetContextData(ctx, field, "my_config1") - assert.Nil(t, v) // still no cache } diff --git a/modules/cache/ephemeral.go b/modules/cache/ephemeral.go new file mode 100644 index 0000000000..6996010ac4 --- /dev/null +++ b/modules/cache/ephemeral.go @@ -0,0 +1,90 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cache + +import ( + "context" + "sync" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" +) + +// EphemeralCache is a cache that can be used to store data in a request level context +// This is useful for caching data that is expensive to calculate and is likely to be +// used multiple times in a request. +type EphemeralCache struct { + data map[any]map[any]any + lock sync.RWMutex + created time.Time + checkLifeTime time.Duration +} + +var timeNow = time.Now + +func NewEphemeralCache(checkLifeTime ...time.Duration) *EphemeralCache { + return &EphemeralCache{ + data: make(map[any]map[any]any), + created: timeNow(), + checkLifeTime: util.OptionalArg(checkLifeTime, 0), + } +} + +func (cc *EphemeralCache) checkExceededLifeTime(tp, key any) bool { + if cc.checkLifeTime > 0 && timeNow().Sub(cc.created) > cc.checkLifeTime { + log.Warn("EphemeralCache is expired, is highly likely to be abused for long-life tasks: %v, %v", tp, key) + return true + } + return false +} + +func (cc *EphemeralCache) Get(tp, key any) (any, bool) { + if cc.checkExceededLifeTime(tp, key) { + return nil, false + } + cc.lock.RLock() + defer cc.lock.RUnlock() + ret, ok := cc.data[tp][key] + return ret, ok +} + +func (cc *EphemeralCache) Put(tp, key, value any) { + if cc.checkExceededLifeTime(tp, key) { + return + } + + cc.lock.Lock() + defer cc.lock.Unlock() + + d := cc.data[tp] + if d == nil { + d = make(map[any]any) + cc.data[tp] = d + } + d[key] = value +} + +func (cc *EphemeralCache) Delete(tp, key any) { + if cc.checkExceededLifeTime(tp, key) { + return + } + + cc.lock.Lock() + defer cc.lock.Unlock() + delete(cc.data[tp], key) +} + +func GetWithEphemeralCache[T, K any](ctx context.Context, c *EphemeralCache, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) { + v, has := c.Get(groupKey, targetKey) + if vv, ok := v.(T); has && ok { + return vv, nil + } + t, err := f(ctx, targetKey) + if err != nil { + return t, err + } + c.Put(groupKey, targetKey, t) + return t, nil +} diff --git a/modules/cache/string_cache.go b/modules/cache/string_cache.go index 4f659616f5..3562b7a926 100644 --- a/modules/cache/string_cache.go +++ b/modules/cache/string_cache.go @@ -11,7 +11,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - chi_cache "gitea.com/go-chi/cache" //nolint:depguard + chi_cache "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here ) type GetJSONError struct { diff --git a/modules/cachegroup/cachegroup.go b/modules/cachegroup/cachegroup.go new file mode 100644 index 0000000000..06085f860f --- /dev/null +++ b/modules/cachegroup/cachegroup.go @@ -0,0 +1,12 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cachegroup + +const ( + User = "user" + EmailAvatarLink = "email_avatar_link" + UserEmailAddresses = "user_email_addresses" + GPGKeyWithSubKeys = "gpg_key_with_subkeys" + RepoUserPermission = "repo_user_permission" +) diff --git a/modules/charset/ambiguous_gen_test.go b/modules/charset/ambiguous_gen_test.go index 221c27d0e1..d3be0b1a13 100644 --- a/modules/charset/ambiguous_gen_test.go +++ b/modules/charset/ambiguous_gen_test.go @@ -14,7 +14,7 @@ import ( func TestAmbiguousCharacters(t *testing.T) { for locale, ambiguous := range AmbiguousCharacters { assert.Equal(t, locale, ambiguous.Locale) - assert.Equal(t, len(ambiguous.Confusable), len(ambiguous.With)) + assert.Len(t, ambiguous.With, len(ambiguous.Confusable)) assert.True(t, sort.SliceIsSorted(ambiguous.Confusable, func(i, j int) bool { return ambiguous.Confusable[i] < ambiguous.Confusable[j] })) diff --git a/modules/charset/charset.go b/modules/charset/charset.go index 1855446a98..597ce5120c 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -164,7 +164,7 @@ func DetectEncoding(content []byte) (string, error) { } times := 1024 / len(content) detectContent = make([]byte, 0, times*len(content)) - for i := 0; i < times; i++ { + for range times { detectContent = append(detectContent, content...) } } else { diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index 19b1303365..cd2e3b9aaa 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -242,7 +242,7 @@ func stringMustEndWith(t *testing.T, expected, value string) { func TestToUTF8WithFallbackReader(t *testing.T) { resetDefaultCharsetsOrder() - for testLen := 0; testLen < 2048; testLen++ { + for testLen := range 2048 { pattern := " test { () }\n" input := "" for len(input) < testLen { @@ -252,7 +252,7 @@ func TestToUTF8WithFallbackReader(t *testing.T) { input += "// Выключаем" rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)), ConvertOpts{}) r, _ := io.ReadAll(rd) - assert.EqualValuesf(t, input, string(r), "testing string len=%d", testLen) + assert.Equalf(t, input, string(r), "testing string len=%d", testLen) } truncatedOneByteExtension := failFastBytes diff --git a/modules/structs/commit_status.go b/modules/commitstatus/commit_status.go index dc880ef5eb..a0ab4e7186 100644 --- a/modules/structs/commit_status.go +++ b/modules/commitstatus/commit_status.go @@ -1,11 +1,11 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package structs +package commitstatus // CommitStatusState holds the state of a CommitStatus -// It can be "pending", "success", "error" and "failure" -type CommitStatusState string +// swagger:enum CommitStatusState +type CommitStatusState string //nolint:revive // export stutter const ( // CommitStatusPending is for when the CommitStatus is Pending @@ -18,35 +18,14 @@ const ( CommitStatusFailure CommitStatusState = "failure" // CommitStatusWarning is for when the CommitStatus is Warning CommitStatusWarning CommitStatusState = "warning" + // CommitStatusSkipped is for when CommitStatus is Skipped + CommitStatusSkipped CommitStatusState = "skipped" ) -var commitStatusPriorities = map[CommitStatusState]int{ - CommitStatusError: 0, - CommitStatusFailure: 1, - CommitStatusWarning: 2, - CommitStatusPending: 3, - CommitStatusSuccess: 4, -} - func (css CommitStatusState) String() string { return string(css) } -// NoBetterThan returns true if this State is no better than the given State -// This function only handles the states defined in CommitStatusPriorities -func (css CommitStatusState) NoBetterThan(css2 CommitStatusState) bool { - // NoBetterThan only handles the 5 states above - if _, exist := commitStatusPriorities[css]; !exist { - return false - } - - if _, exist := commitStatusPriorities[css2]; !exist { - return false - } - - return commitStatusPriorities[css] <= commitStatusPriorities[css2] -} - // IsPending represents if commit status state is pending func (css CommitStatusState) IsPending() bool { return css == CommitStatusPending @@ -71,3 +50,32 @@ func (css CommitStatusState) IsFailure() bool { func (css CommitStatusState) IsWarning() bool { return css == CommitStatusWarning } + +// IsSkipped represents if commit status state is skipped +func (css CommitStatusState) IsSkipped() bool { + return css == CommitStatusSkipped +} + +type CommitStatusStates []CommitStatusState //nolint:revive // export stutter + +// According to https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#get-the-combined-status-for-a-specific-reference +// > Additionally, a combined state is returned. The state is one of: +// > failure if any of the contexts report as error or failure +// > pending if there are no statuses or a context is pending +// > success if the latest status for all contexts is success +func (css CommitStatusStates) Combine() CommitStatusState { + successCnt := 0 + for _, state := range css { + switch { + case state.IsError() || state.IsFailure(): + return CommitStatusFailure + case state.IsPending(): + case state.IsSuccess() || state.IsWarning() || state.IsSkipped(): + successCnt++ + } + } + if successCnt > 0 && successCnt == len(css) { + return CommitStatusSuccess + } + return CommitStatusPending +} diff --git a/modules/commitstatus/commit_status_test.go b/modules/commitstatus/commit_status_test.go new file mode 100644 index 0000000000..10d8f20aa4 --- /dev/null +++ b/modules/commitstatus/commit_status_test.go @@ -0,0 +1,201 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package commitstatus + +import "testing" + +func TestCombine(t *testing.T) { + tests := []struct { + name string + states CommitStatusStates + expected CommitStatusState + }{ + // 0 states + { + name: "empty", + states: CommitStatusStates{}, + expected: CommitStatusPending, + }, + // 1 state + { + name: "pending", + states: CommitStatusStates{CommitStatusPending}, + expected: CommitStatusPending, + }, + { + name: "success", + states: CommitStatusStates{CommitStatusSuccess}, + expected: CommitStatusSuccess, + }, + { + name: "error", + states: CommitStatusStates{CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "failure", + states: CommitStatusStates{CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "warning", + states: CommitStatusStates{CommitStatusWarning}, + expected: CommitStatusSuccess, + }, + // 2 states + { + name: "pending and success", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess}, + expected: CommitStatusPending, + }, + { + name: "pending and error", + states: CommitStatusStates{CommitStatusPending, CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "pending and failure", + states: CommitStatusStates{CommitStatusPending, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "pending and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusWarning}, + expected: CommitStatusPending, + }, + { + name: "success and error", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "success and failure", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "success and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusWarning}, + expected: CommitStatusSuccess, + }, + { + name: "error and failure", + states: CommitStatusStates{CommitStatusError, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "error and warning", + states: CommitStatusStates{CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "failure and warning", + states: CommitStatusStates{CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + // 3 states + { + name: "pending, success and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusWarning}, + expected: CommitStatusPending, + }, + { + name: "pending, success and error", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError}, + expected: CommitStatusFailure, + }, + { + name: "pending, success and failure", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "pending, error and failure", + states: CommitStatusStates{CommitStatusPending, CommitStatusError, CommitStatusFailure}, + expected: CommitStatusFailure, + }, + { + name: "success, error and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "success, failure and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "error, failure and warning", + states: CommitStatusStates{CommitStatusError, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "success, warning and skipped", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusWarning, CommitStatusSkipped}, + expected: CommitStatusSuccess, + }, + // All success + { + name: "all success", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusSuccess}, + expected: CommitStatusSuccess, + }, + // All pending + { + name: "all pending", + states: CommitStatusStates{CommitStatusPending, CommitStatusPending, CommitStatusPending}, + expected: CommitStatusPending, + }, + { + name: "all skipped", + states: CommitStatusStates{CommitStatusSkipped, CommitStatusSkipped, CommitStatusSkipped}, + expected: CommitStatusSuccess, + }, + // 4 states + { + name: "pending, success, error and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "pending, success, failure and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "pending, error, failure and warning", + states: CommitStatusStates{CommitStatusPending, CommitStatusError, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "success, error, failure and warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusError, CommitStatusFailure, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "mixed states", + states: CommitStatusStates{CommitStatusPending, CommitStatusSuccess, CommitStatusError, CommitStatusWarning}, + expected: CommitStatusFailure, + }, + { + name: "mixed states with all success", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusPending, CommitStatusWarning}, + expected: CommitStatusPending, + }, + { + name: "all success with warning", + states: CommitStatusStates{CommitStatusSuccess, CommitStatusSuccess, CommitStatusSuccess, CommitStatusWarning}, + expected: CommitStatusSuccess, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.states.Combine() + if result != tt.expected { + t.Errorf("expected %v, got %v", tt.expected, result) + } + }) + } +} diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 25945113a7..be9fc5f823 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -99,10 +99,10 @@ j, ,\x20 for n, c := range cases { rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(decodeSlashes(t, c.csv))) assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err) - assert.EqualValues(t, c.expectedDelimiter, rd.Comma, "case %d: delimiter should be '%c', got '%c'", n, c.expectedDelimiter, rd.Comma) + assert.Equal(t, c.expectedDelimiter, rd.Comma, "case %d: delimiter should be '%c', got '%c'", n, c.expectedDelimiter, rd.Comma) rows, err := rd.ReadAll() assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err) - assert.EqualValues(t, c.expectedRows, rows, "case %d: rows should be equal", n) + assert.Equal(t, c.expectedRows, rows, "case %d: rows should be equal", n) } } @@ -231,7 +231,7 @@ John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimiters`, for n, c := range cases { delimiter := determineDelimiter(markup.NewRenderContext(t.Context()).WithRelativePath(c.filename), []byte(decodeSlashes(t, c.csv))) - assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -296,7 +296,7 @@ abc | |123 for n, c := range cases { modifiedText := removeQuotedString(decodeSlashes(t, c.text)) - assert.EqualValues(t, c.expectedText, modifiedText, "case %d: modified text should be equal", n) + assert.Equal(t, c.expectedText, modifiedText, "case %d: modified text should be equal", n) } } @@ -451,7 +451,7 @@ jkl`, for n, c := range cases { delimiter := guessDelimiter([]byte(decodeSlashes(t, c.csv))) - assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -543,7 +543,7 @@ a|"he said, ""here I am"""`, for n, c := range cases { delimiter := guessFromBeforeAfterQuotes([]byte(decodeSlashes(t, c.csv))) - assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + assert.Equal(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -579,7 +579,7 @@ func TestFormatError(t *testing.T) { assert.Error(t, err, "case %d: expected an error to be returned", n) } else { assert.NoError(t, err, "case %d: no error was expected, got error: %v", n, err) - assert.EqualValues(t, c.expectedMessage, message, "case %d: messages should be equal, expected '%s' got '%s'", n, c.expectedMessage, message) + assert.Equal(t, c.expectedMessage, message, "case %d: messages should be equal, expected '%s' got '%s'", n, c.expectedMessage, message) } } } diff --git a/modules/dump/dumper_test.go b/modules/dump/dumper_test.go index 2db3a598a4..8f06c1851d 100644 --- a/modules/dump/dumper_test.go +++ b/modules/dump/dumper_test.go @@ -103,11 +103,11 @@ func TestDumper(t *testing.T) { d.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1")) err := d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")}) assert.NoError(t, err) - assert.EqualValues(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added)) + assert.Equal(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added)) tw = &testWriter{} d = &Dumper{Writer: tw} err = d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil) assert.NoError(t, err) - assert.EqualValues(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added)) + assert.Equal(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added)) } diff --git a/modules/fileicon/basic.go b/modules/fileicon/basic.go index 040a8e87de..9c513ccbd9 100644 --- a/modules/fileicon/basic.go +++ b/modules/fileicon/basic.go @@ -6,22 +6,26 @@ package fileicon import ( "html/template" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/svg" + "code.gitea.io/gitea/modules/util" ) -func BasicThemeIcon(entry *git.TreeEntry) template.HTML { +func BasicEntryIconName(entry *EntryInfo) string { svgName := "octicon-file" switch { - case entry.IsLink(): + case entry.EntryMode.IsLink(): svgName = "octicon-file-symlink-file" - if te, err := entry.FollowLink(); err == nil && te.IsDir() { + if entry.SymlinkToMode.IsDir() { svgName = "octicon-file-directory-symlink" } - case entry.IsDir(): - svgName = "octicon-file-directory-fill" - case entry.IsSubModule(): + case entry.EntryMode.IsDir(): + svgName = util.Iif(entry.IsOpen, "octicon-file-directory-open-fill", "octicon-file-directory-fill") + case entry.EntryMode.IsSubModule(): svgName = "octicon-file-submodule" } - return svg.RenderHTML(svgName) + return svgName +} + +func BasicEntryIconHTML(entry *EntryInfo) template.HTML { + return svg.RenderHTML(BasicEntryIconName(entry)) } diff --git a/modules/fileicon/entry.go b/modules/fileicon/entry.go new file mode 100644 index 0000000000..0326c2bfa8 --- /dev/null +++ b/modules/fileicon/entry.go @@ -0,0 +1,31 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import "code.gitea.io/gitea/modules/git" + +type EntryInfo struct { + BaseName string + EntryMode git.EntryMode + SymlinkToMode git.EntryMode + IsOpen bool +} + +func EntryInfoFromGitTreeEntry(commit *git.Commit, fullPath string, gitEntry *git.TreeEntry) *EntryInfo { + ret := &EntryInfo{BaseName: gitEntry.Name(), EntryMode: gitEntry.Mode()} + if gitEntry.IsLink() { + if res, err := git.EntryFollowLink(commit, fullPath, gitEntry); err == nil && res.TargetEntry.IsDir() { + ret.SymlinkToMode = res.TargetEntry.Mode() + } + } + return ret +} + +func EntryInfoFolder() *EntryInfo { + return &EntryInfo{EntryMode: git.EntryModeTree} +} + +func EntryInfoFolderOpen() *EntryInfo { + return &EntryInfo{EntryMode: git.EntryModeTree, IsOpen: true} +} diff --git a/modules/fileicon/material.go b/modules/fileicon/material.go index aa31cd8d7c..5361592d8a 100644 --- a/modules/fileicon/material.go +++ b/modules/fileicon/material.go @@ -5,16 +5,15 @@ package fileicon import ( "html/template" - "path" "strings" "sync" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" - "code.gitea.io/gitea/modules/reqctx" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/svg" + "code.gitea.io/gitea/modules/util" ) type materialIconRulesData struct { @@ -62,13 +61,7 @@ func (m *MaterialIconProvider) loadData() { log.Debug("Loaded material icon rules and SVG images") } -func (m *MaterialIconProvider) renderFileIconSVG(ctx reqctx.RequestContext, name, svg, extraClass string) template.HTML { - data := ctx.GetData() - renderedSVGs, _ := data["_RenderedSVGs"].(map[string]bool) - if renderedSVGs == nil { - renderedSVGs = make(map[string]bool) - data["_RenderedSVGs"] = renderedSVGs - } +func (m *MaterialIconProvider) renderFileIconSVG(p *RenderedIconPool, name, svg, extraClass string) template.HTML { // This part is a bit hacky, but it works really well. It should be safe to do so because all SVG icons are generated by us. // Will try to refactor this in the future. if !strings.HasPrefix(svg, "<svg") { @@ -76,46 +69,51 @@ func (m *MaterialIconProvider) renderFileIconSVG(ctx reqctx.RequestContext, name } svgID := "svg-mfi-" + name svgCommonAttrs := `class="svg git-entry-icon ` + extraClass + `" width="16" height="16" aria-hidden="true"` - posOuterBefore := strings.IndexByte(svg, '>') - if renderedSVGs[svgID] && posOuterBefore != -1 { - return template.HTML(`<svg ` + svgCommonAttrs + `><use xlink:href="#` + svgID + `"></use></svg>`) + svgHTML := template.HTML(`<svg id="` + svgID + `" ` + svgCommonAttrs + svg[4:]) + if p == nil { + return svgHTML + } + if p.IconSVGs[svgID] == "" { + p.IconSVGs[svgID] = svgHTML } - svg = `<svg id="` + svgID + `" ` + svgCommonAttrs + svg[4:] - renderedSVGs[svgID] = true - return template.HTML(svg) + return template.HTML(`<svg ` + svgCommonAttrs + `><use xlink:href="#` + svgID + `"></use></svg>`) } -func (m *MaterialIconProvider) FileIcon(ctx reqctx.RequestContext, entry *git.TreeEntry) template.HTML { +func (m *MaterialIconProvider) EntryIconHTML(p *RenderedIconPool, entry *EntryInfo) template.HTML { if m.rules == nil { - return BasicThemeIcon(entry) + return BasicEntryIconHTML(entry) } - if entry.IsLink() { - if te, err := entry.FollowLink(); err == nil && te.IsDir() { + if entry.EntryMode.IsLink() { + if entry.SymlinkToMode.IsDir() { // keep the old "octicon-xxx" class name to make some "theme plugin selector" could still work return svg.RenderHTML("material-folder-symlink", 16, "octicon-file-directory-symlink") } return svg.RenderHTML("octicon-file-symlink-file") // TODO: find some better icons for them } - name := m.findIconNameByGit(entry) - if name == "folder" { - // the material icon pack's "folder" icon doesn't look good, so use our built-in one - // keep the old "octicon-xxx" class name to make some "theme plugin selector" could still work - return svg.RenderHTML("material-folder-generic", 16, "octicon-file-directory-fill") - } - if iconSVG, ok := m.svgs[name]; ok && iconSVG != "" { - // keep the old "octicon-xxx" class name to make some "theme plugin selector" could still work - extraClass := "octicon-file" - switch { - case entry.IsDir(): - extraClass = "octicon-file-directory-fill" - case entry.IsSubModule(): - extraClass = "octicon-file-submodule" + name := m.FindIconName(entry) + iconSVG := m.svgs[name] + if iconSVG == "" { + name = "file" + if entry.EntryMode.IsDir() { + name = util.Iif(entry.IsOpen, "folder-open", "folder") + } + iconSVG = m.svgs[name] + if iconSVG == "" { + setting.PanicInDevOrTesting("missing file icon for %s", name) } - return m.renderFileIconSVG(ctx, name, iconSVG, extraClass) } - return svg.RenderHTML("octicon-file") + + // keep the old "octicon-xxx" class name to make some "theme plugin selector" could still work + extraClass := "octicon-file" + switch { + case entry.EntryMode.IsDir(): + extraClass = BasicEntryIconName(entry) + case entry.EntryMode.IsSubModule(): + extraClass = "octicon-file-submodule" + } + return m.renderFileIconSVG(p, name, iconSVG, extraClass) } func (m *MaterialIconProvider) findIconNameWithLangID(s string) string { @@ -130,13 +128,17 @@ func (m *MaterialIconProvider) findIconNameWithLangID(s string) string { return "" } -func (m *MaterialIconProvider) FindIconName(name string, isDir bool) string { - fileNameLower := strings.ToLower(path.Base(name)) - if isDir { +func (m *MaterialIconProvider) FindIconName(entry *EntryInfo) string { + if entry.EntryMode.IsSubModule() { + return "folder-git" + } + + fileNameLower := strings.ToLower(entry.BaseName) + if entry.EntryMode.IsDir() { if s, ok := m.rules.FolderNames[fileNameLower]; ok { return s } - return "folder" + return util.Iif(entry.IsOpen, "folder-open", "folder") } if s, ok := m.rules.FileNames[fileNameLower]; ok { @@ -158,10 +160,3 @@ func (m *MaterialIconProvider) FindIconName(name string, isDir bool) string { return "file" } - -func (m *MaterialIconProvider) findIconNameByGit(entry *git.TreeEntry) string { - if entry.IsSubModule() { - return "folder-git" - } - return m.FindIconName(entry.Name(), entry.IsDir()) -} diff --git a/modules/fileicon/material_test.go b/modules/fileicon/material_test.go index f36385aaf3..d2a769eaac 100644 --- a/modules/fileicon/material_test.go +++ b/modules/fileicon/material_test.go @@ -8,6 +8,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/fileicon" + "code.gitea.io/gitea/modules/git" "github.com/stretchr/testify/assert" ) @@ -19,8 +20,8 @@ func TestMain(m *testing.M) { func TestFindIconName(t *testing.T) { unittest.PrepareTestEnv(t) p := fileicon.DefaultMaterialIconProvider() - assert.Equal(t, "php", p.FindIconName("foo.php", false)) - assert.Equal(t, "php", p.FindIconName("foo.PHP", false)) - assert.Equal(t, "javascript", p.FindIconName("foo.js", false)) - assert.Equal(t, "visualstudio", p.FindIconName("foo.vba", false)) + assert.Equal(t, "php", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.php", EntryMode: git.EntryModeBlob})) + assert.Equal(t, "php", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.PHP", EntryMode: git.EntryModeBlob})) + assert.Equal(t, "javascript", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.js", EntryMode: git.EntryModeBlob})) + assert.Equal(t, "visualstudio", p.FindIconName(&fileicon.EntryInfo{BaseName: "foo.vba", EntryMode: git.EntryModeBlob})) } diff --git a/modules/fileicon/render.go b/modules/fileicon/render.go new file mode 100644 index 0000000000..8ed86b9ac0 --- /dev/null +++ b/modules/fileicon/render.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package fileicon + +import ( + "html/template" + "strings" + + "code.gitea.io/gitea/modules/setting" +) + +type RenderedIconPool struct { + IconSVGs map[string]template.HTML +} + +func NewRenderedIconPool() *RenderedIconPool { + return &RenderedIconPool{ + IconSVGs: make(map[string]template.HTML), + } +} + +func (p *RenderedIconPool) RenderToHTML() template.HTML { + if len(p.IconSVGs) == 0 { + return "" + } + sb := &strings.Builder{} + sb.WriteString(`<div class=tw-hidden>`) + for _, icon := range p.IconSVGs { + sb.WriteString(string(icon)) + } + sb.WriteString(`</div>`) + return template.HTML(sb.String()) +} + +func RenderEntryIconHTML(renderedIconPool *RenderedIconPool, entry *EntryInfo) template.HTML { + if setting.UI.FileIconTheme == "material" { + return DefaultMaterialIconProvider().EntryIconHTML(renderedIconPool, entry) + } + return BasicEntryIconHTML(entry) +} diff --git a/modules/git/attribute.go b/modules/git/attribute.go deleted file mode 100644 index 4dfa510369..0000000000 --- a/modules/git/attribute.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "code.gitea.io/gitea/modules/optional" -) - -const ( - AttributeLinguistVendored = "linguist-vendored" - AttributeLinguistGenerated = "linguist-generated" - AttributeLinguistDocumentation = "linguist-documentation" - AttributeLinguistDetectable = "linguist-detectable" - AttributeLinguistLanguage = "linguist-language" - AttributeGitlabLanguage = "gitlab-language" -) - -// true if "set"/"true", false if "unset"/"false", none otherwise -func AttributeToBool(attr map[string]string, name string) optional.Option[bool] { - switch attr[name] { - case "set", "true": - return optional.Some(true) - case "unset", "false": - return optional.Some(false) - } - return optional.None[bool]() -} - -func AttributeToString(attr map[string]string, name string) optional.Option[string] { - if value, has := attr[name]; has && value != "unspecified" { - return optional.Some(value) - } - return optional.None[string]() -} diff --git a/modules/git/attribute/attribute.go b/modules/git/attribute/attribute.go new file mode 100644 index 0000000000..adf323ef41 --- /dev/null +++ b/modules/git/attribute/attribute.go @@ -0,0 +1,114 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "strings" + + "code.gitea.io/gitea/modules/optional" +) + +type Attribute string + +const ( + LinguistVendored = "linguist-vendored" + LinguistGenerated = "linguist-generated" + LinguistDocumentation = "linguist-documentation" + LinguistDetectable = "linguist-detectable" + LinguistLanguage = "linguist-language" + GitlabLanguage = "gitlab-language" + Lockable = "lockable" + Filter = "filter" +) + +var LinguistAttributes = []string{ + LinguistVendored, + LinguistGenerated, + LinguistDocumentation, + LinguistDetectable, + LinguistLanguage, + GitlabLanguage, +} + +func (a Attribute) IsUnspecified() bool { + return a == "" || a == "unspecified" +} + +func (a Attribute) ToString() optional.Option[string] { + if !a.IsUnspecified() { + return optional.Some(string(a)) + } + return optional.None[string]() +} + +// ToBool converts the attribute value to optional boolean: true if "set"/"true", false if "unset"/"false", none otherwise +func (a Attribute) ToBool() optional.Option[bool] { + switch a { + case "set", "true": + return optional.Some(true) + case "unset", "false": + return optional.Some(false) + } + return optional.None[bool]() +} + +type Attributes struct { + m map[string]Attribute +} + +func NewAttributes() *Attributes { + return &Attributes{m: make(map[string]Attribute)} +} + +func (attrs *Attributes) Get(name string) Attribute { + if value, has := attrs.m[name]; has { + return value + } + return "" +} + +func (attrs *Attributes) GetVendored() optional.Option[bool] { + return attrs.Get(LinguistVendored).ToBool() +} + +func (attrs *Attributes) GetGenerated() optional.Option[bool] { + return attrs.Get(LinguistGenerated).ToBool() +} + +func (attrs *Attributes) GetDocumentation() optional.Option[bool] { + return attrs.Get(LinguistDocumentation).ToBool() +} + +func (attrs *Attributes) GetDetectable() optional.Option[bool] { + return attrs.Get(LinguistDetectable).ToBool() +} + +func (attrs *Attributes) GetLinguistLanguage() optional.Option[string] { + return attrs.Get(LinguistLanguage).ToString() +} + +func (attrs *Attributes) GetGitlabLanguage() optional.Option[string] { + attrStr := attrs.Get(GitlabLanguage).ToString() + if attrStr.Has() { + raw := attrStr.Value() + // gitlab-language may have additional parameters after the language + // ignore them and just use the main language + // https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type + if idx := strings.IndexByte(raw, '?'); idx >= 0 { + return optional.Some(raw[:idx]) + } + } + return attrStr +} + +func (attrs *Attributes) GetLanguage() optional.Option[string] { + // prefer linguist-language over gitlab-language + // if linguist-language is not set, use gitlab-language + // if both are not set, return none + language := attrs.GetLinguistLanguage() + if language.Value() == "" { + language = attrs.GetGitlabLanguage() + } + return language +} diff --git a/modules/git/attribute/attribute_test.go b/modules/git/attribute/attribute_test.go new file mode 100644 index 0000000000..dadb5582a3 --- /dev/null +++ b/modules/git/attribute/attribute_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Attribute(t *testing.T) { + assert.Empty(t, Attribute("").ToString().Value()) + assert.Empty(t, Attribute("unspecified").ToString().Value()) + assert.Equal(t, "python", Attribute("python").ToString().Value()) + assert.Equal(t, "Java", Attribute("Java").ToString().Value()) + + attributes := Attributes{ + m: map[string]Attribute{ + LinguistGenerated: "true", + LinguistDocumentation: "false", + LinguistDetectable: "set", + LinguistLanguage: "Python", + GitlabLanguage: "Java", + "filter": "unspecified", + "test": "", + }, + } + + assert.Empty(t, attributes.Get("test").ToString().Value()) + assert.Empty(t, attributes.Get("filter").ToString().Value()) + assert.Equal(t, "Python", attributes.Get(LinguistLanguage).ToString().Value()) + assert.Equal(t, "Java", attributes.Get(GitlabLanguage).ToString().Value()) + assert.True(t, attributes.Get(LinguistGenerated).ToBool().Value()) + assert.False(t, attributes.Get(LinguistDocumentation).ToBool().Value()) + assert.True(t, attributes.Get(LinguistDetectable).ToBool().Value()) +} diff --git a/modules/git/attribute/batch.go b/modules/git/attribute/batch.go new file mode 100644 index 0000000000..4e31fda575 --- /dev/null +++ b/modules/git/attribute/batch.go @@ -0,0 +1,216 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" +) + +// BatchChecker provides a reader for check-attribute content that can be long running +type BatchChecker struct { + attributesNum int + repo *git.Repository + stdinWriter *os.File + stdOut *nulSeparatedAttributeWriter + ctx context.Context + cancel context.CancelFunc + cmd *git.Command +} + +// NewBatchChecker creates a check attribute reader for the current repository and provided commit ID +// If treeish is empty, then it will use current working directory, otherwise it will use the provided treeish on the bare repo +func NewBatchChecker(repo *git.Repository, treeish string, attributes []string) (checker *BatchChecker, returnedErr error) { + ctx, cancel := context.WithCancel(repo.Ctx) + defer func() { + if returnedErr != nil { + cancel() + } + }() + + cmd, envs, cleanup, err := checkAttrCommand(repo, treeish, nil, attributes) + if err != nil { + return nil, err + } + defer func() { + if returnedErr != nil { + cleanup() + } + }() + + cmd.AddArguments("--stdin") + + checker = &BatchChecker{ + attributesNum: len(attributes), + repo: repo, + ctx: ctx, + cmd: cmd, + cancel: func() { + cancel() + cleanup() + }, + } + + stdinReader, stdinWriter, err := os.Pipe() + if err != nil { + return nil, err + } + checker.stdinWriter = stdinWriter + + lw := new(nulSeparatedAttributeWriter) + lw.attributes = make(chan attributeTriple, len(attributes)) + lw.closed = make(chan struct{}) + checker.stdOut = lw + + go func() { + defer func() { + _ = stdinReader.Close() + _ = lw.Close() + }() + stdErr := new(bytes.Buffer) + err := cmd.Run(ctx, &git.RunOpts{ + Env: envs, + Dir: repo.Path, + Stdin: stdinReader, + Stdout: lw, + Stderr: stdErr, + }) + + if err != nil && !git.IsErrCanceledOrKilled(err) { + log.Error("Attribute checker for commit %s exits with error: %v", treeish, err) + } + checker.cancel() + }() + + return checker, nil +} + +// CheckPath check attr for given path +func (c *BatchChecker) CheckPath(path string) (rs *Attributes, err error) { + defer func() { + if err != nil && err != c.ctx.Err() { + log.Error("Unexpected error when checking path %s in %s, error: %v", path, filepath.Base(c.repo.Path), err) + } + }() + + select { + case <-c.ctx.Done(): + return nil, c.ctx.Err() + default: + } + + if _, err = c.stdinWriter.Write([]byte(path + "\x00")); err != nil { + defer c.Close() + return nil, err + } + + reportTimeout := func() error { + stdOutClosed := false + select { + case <-c.stdOut.closed: + stdOutClosed = true + default: + } + debugMsg := fmt.Sprintf("check path %q in repo %q", path, filepath.Base(c.repo.Path)) + debugMsg += fmt.Sprintf(", stdOut: tmp=%q, pos=%d, closed=%v", string(c.stdOut.tmp), c.stdOut.pos, stdOutClosed) + if c.cmd != nil { + debugMsg += fmt.Sprintf(", process state: %q", c.cmd.ProcessState()) + } + _ = c.Close() + return fmt.Errorf("CheckPath timeout: %s", debugMsg) + } + + rs = NewAttributes() + for i := 0; i < c.attributesNum; i++ { + select { + case <-time.After(5 * time.Second): + // there is no "hang" problem now. This code is just used to catch other potential problems. + return nil, reportTimeout() + case attr, ok := <-c.stdOut.ReadAttribute(): + if !ok { + return nil, c.ctx.Err() + } + rs.m[attr.Attribute] = Attribute(attr.Value) + case <-c.ctx.Done(): + return nil, c.ctx.Err() + } + } + return rs, nil +} + +func (c *BatchChecker) Close() error { + c.cancel() + err := c.stdinWriter.Close() + return err +} + +type attributeTriple struct { + Filename string + Attribute string + Value string +} + +type nulSeparatedAttributeWriter struct { + tmp []byte + attributes chan attributeTriple + closed chan struct{} + working attributeTriple + pos int +} + +func (wr *nulSeparatedAttributeWriter) Write(p []byte) (n int, err error) { + l, read := len(p), 0 + + nulIdx := bytes.IndexByte(p, '\x00') + for nulIdx >= 0 { + wr.tmp = append(wr.tmp, p[:nulIdx]...) + switch wr.pos { + case 0: + wr.working = attributeTriple{ + Filename: string(wr.tmp), + } + case 1: + wr.working.Attribute = string(wr.tmp) + case 2: + wr.working.Value = string(wr.tmp) + } + wr.tmp = wr.tmp[:0] + wr.pos++ + if wr.pos > 2 { + wr.attributes <- wr.working + wr.pos = 0 + } + read += nulIdx + 1 + if l > read { + p = p[nulIdx+1:] + nulIdx = bytes.IndexByte(p, '\x00') + } else { + return l, nil + } + } + wr.tmp = append(wr.tmp, p...) + return l, nil +} + +func (wr *nulSeparatedAttributeWriter) ReadAttribute() <-chan attributeTriple { + return wr.attributes +} + +func (wr *nulSeparatedAttributeWriter) Close() error { + select { + case <-wr.closed: + return nil + default: + } + close(wr.attributes) + close(wr.closed) + return nil +} diff --git a/modules/git/attribute/batch_test.go b/modules/git/attribute/batch_test.go new file mode 100644 index 0000000000..30a3d805fe --- /dev/null +++ b/modules/git/attribute/batch_test.go @@ -0,0 +1,172 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "path/filepath" + "testing" + "time" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { + wr := &nulSeparatedAttributeWriter{ + attributes: make(chan attributeTriple, 5), + } + + testStr := ".gitignore\"\n\x00linguist-vendored\x00unspecified\x00" + + n, err := wr.Write([]byte(testStr)) + + assert.Len(t, testStr, n) + assert.NoError(t, err) + select { + case attr := <-wr.ReadAttribute(): + assert.Equal(t, ".gitignore\"\n", attr.Filename) + assert.Equal(t, LinguistVendored, attr.Attribute) + assert.Equal(t, "unspecified", attr.Value) + case <-time.After(100 * time.Millisecond): + assert.FailNow(t, "took too long to read an attribute from the list") + } + // Write a second attribute again + n, err = wr.Write([]byte(testStr)) + + assert.Len(t, testStr, n) + assert.NoError(t, err) + + select { + case attr := <-wr.ReadAttribute(): + assert.Equal(t, ".gitignore\"\n", attr.Filename) + assert.Equal(t, LinguistVendored, attr.Attribute) + assert.Equal(t, "unspecified", attr.Value) + case <-time.After(100 * time.Millisecond): + assert.FailNow(t, "took too long to read an attribute from the list") + } + + // Write a partial attribute + _, err = wr.Write([]byte("incomplete-file")) + assert.NoError(t, err) + _, err = wr.Write([]byte("name\x00")) + assert.NoError(t, err) + + select { + case <-wr.ReadAttribute(): + assert.FailNow(t, "There should not be an attribute ready to read") + case <-time.After(100 * time.Millisecond): + } + _, err = wr.Write([]byte("attribute\x00")) + assert.NoError(t, err) + select { + case <-wr.ReadAttribute(): + assert.FailNow(t, "There should not be an attribute ready to read") + case <-time.After(100 * time.Millisecond): + } + + _, err = wr.Write([]byte("value\x00")) + assert.NoError(t, err) + + attr := <-wr.ReadAttribute() + assert.Equal(t, "incomplete-filename", attr.Filename) + assert.Equal(t, "attribute", attr.Attribute) + assert.Equal(t, "value", attr.Value) + + _, err = wr.Write([]byte("shouldbe.vendor\x00linguist-vendored\x00set\x00shouldbe.vendor\x00linguist-generated\x00unspecified\x00shouldbe.vendor\x00linguist-language\x00unspecified\x00")) + assert.NoError(t, err) + attr = <-wr.ReadAttribute() + assert.NoError(t, err) + assert.Equal(t, attributeTriple{ + Filename: "shouldbe.vendor", + Attribute: LinguistVendored, + Value: "set", + }, attr) + attr = <-wr.ReadAttribute() + assert.NoError(t, err) + assert.Equal(t, attributeTriple{ + Filename: "shouldbe.vendor", + Attribute: LinguistGenerated, + Value: "unspecified", + }, attr) + attr = <-wr.ReadAttribute() + assert.NoError(t, err) + assert.Equal(t, attributeTriple{ + Filename: "shouldbe.vendor", + Attribute: LinguistLanguage, + Value: "unspecified", + }, attr) +} + +func expectedAttrs() *Attributes { + return &Attributes{ + m: map[string]Attribute{ + LinguistGenerated: "unspecified", + LinguistDetectable: "unspecified", + LinguistDocumentation: "unspecified", + LinguistVendored: "unspecified", + LinguistLanguage: "Python", + GitlabLanguage: "unspecified", + }, + } +} + +func Test_BatchChecker(t *testing.T) { + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + require.NoError(t, err) + defer gitRepo.Close() + + commitID := "8fee858da5796dfb37704761701bb8e800ad9ef3" + + t.Run("Create index file to run git check-attr", func(t *testing.T) { + defer test.MockVariableValue(&git.DefaultFeatures().SupportCheckAttrOnBare, false)() + checker, err := NewBatchChecker(gitRepo, commitID, LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) + + // run git check-attr on work tree + t.Run("Run git check-attr on git work tree", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "test-repo") + err := git.Clone(t.Context(), repoPath, dir, git.CloneRepoOptions{ + Shared: true, + Branch: "master", + }) + assert.NoError(t, err) + + tempRepo, err := git.OpenRepository(t.Context(), dir) + assert.NoError(t, err) + defer tempRepo.Close() + + checker, err := NewBatchChecker(tempRepo, "", LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) + + if !git.DefaultFeatures().SupportCheckAttrOnBare { + t.Skip("git version 2.40 is required to support run check-attr on bare repo") + return + } + + t.Run("Run git check-attr in bare repository", func(t *testing.T) { + checker, err := NewBatchChecker(gitRepo, commitID, LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) +} diff --git a/modules/git/attribute/checker.go b/modules/git/attribute/checker.go new file mode 100644 index 0000000000..167b31416e --- /dev/null +++ b/modules/git/attribute/checker.go @@ -0,0 +1,101 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + + "code.gitea.io/gitea/modules/git" +) + +func checkAttrCommand(gitRepo *git.Repository, treeish string, filenames, attributes []string) (*git.Command, []string, func(), error) { + cancel := func() {} + envs := []string{"GIT_FLUSH=1"} + cmd := git.NewCommand("check-attr", "-z") + if len(attributes) == 0 { + cmd.AddArguments("--all") + } + + // there is treeish, read from bare repo or temp index created by "read-tree" + if treeish != "" { + if git.DefaultFeatures().SupportCheckAttrOnBare { + cmd.AddArguments("--source") + cmd.AddDynamicArguments(treeish) + } else { + indexFilename, worktree, deleteTemporaryFile, err := gitRepo.ReadTreeToTemporaryIndex(treeish) + if err != nil { + return nil, nil, nil, err + } + + cmd.AddArguments("--cached") + envs = append(envs, + "GIT_INDEX_FILE="+indexFilename, + "GIT_WORK_TREE="+worktree, + ) + cancel = deleteTemporaryFile + } + } else { + // Read from existing index, in cases where the repo is bare and has an index, + // or the work tree contains unstaged changes that shouldn't affect the attribute check. + // It is caller's responsibility to add changed ".gitattributes" into the index if they want to respect the new changes. + cmd.AddArguments("--cached") + } + + cmd.AddDynamicArguments(attributes...) + if len(filenames) > 0 { + cmd.AddDashesAndList(filenames...) + } + return cmd, envs, cancel, nil +} + +type CheckAttributeOpts struct { + Filenames []string + Attributes []string +} + +// CheckAttributes return the attributes of the given filenames and attributes in the given treeish. +// If treeish is empty, then it will use current working directory, otherwise it will use the provided treeish on the bare repo +func CheckAttributes(ctx context.Context, gitRepo *git.Repository, treeish string, opts CheckAttributeOpts) (map[string]*Attributes, error) { + cmd, envs, cancel, err := checkAttrCommand(gitRepo, treeish, opts.Filenames, opts.Attributes) + if err != nil { + return nil, err + } + defer cancel() + + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + if err := cmd.Run(ctx, &git.RunOpts{ + Env: append(os.Environ(), envs...), + Dir: gitRepo.Path, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + return nil, fmt.Errorf("failed to run check-attr: %w\n%s\n%s", err, stdOut.String(), stdErr.String()) + } + + fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) + if len(fields)%3 != 1 { + return nil, errors.New("wrong number of fields in return from check-attr") + } + + attributesMap := make(map[string]*Attributes) + for i := 0; i < (len(fields) / 3); i++ { + filename := string(fields[3*i]) + attribute := string(fields[3*i+1]) + info := string(fields[3*i+2]) + attribute2info, ok := attributesMap[filename] + if !ok { + attribute2info = NewAttributes() + attributesMap[filename] = attribute2info + } + attribute2info.m[attribute] = Attribute(info) + } + + return attributesMap, nil +} diff --git a/modules/git/attribute/checker_test.go b/modules/git/attribute/checker_test.go new file mode 100644 index 0000000000..67fbda8918 --- /dev/null +++ b/modules/git/attribute/checker_test.go @@ -0,0 +1,84 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_Checker(t *testing.T) { + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + require.NoError(t, err) + defer gitRepo.Close() + + commitID := "8fee858da5796dfb37704761701bb8e800ad9ef3" + + t.Run("Create index file to run git check-attr", func(t *testing.T) { + defer test.MockVariableValue(&git.DefaultFeatures().SupportCheckAttrOnBare, false)() + attrs, err := CheckAttributes(t.Context(), gitRepo, commitID, CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + // run git check-attr on work tree + t.Run("Run git check-attr on git work tree", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "test-repo") + err := git.Clone(t.Context(), repoPath, dir, git.CloneRepoOptions{ + Shared: true, + Branch: "master", + }) + assert.NoError(t, err) + + tempRepo, err := git.OpenRepository(t.Context(), dir) + assert.NoError(t, err) + defer tempRepo.Close() + + attrs, err := CheckAttributes(t.Context(), tempRepo, "", CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + t.Run("Run git check-attr in bare repository using index", func(t *testing.T) { + attrs, err := CheckAttributes(t.Context(), gitRepo, "", CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + if !git.DefaultFeatures().SupportCheckAttrOnBare { + t.Skip("git version 2.40 is required to support run check-attr on bare repo without using index") + return + } + + t.Run("Run git check-attr in bare repository", func(t *testing.T) { + attrs, err := CheckAttributes(t.Context(), gitRepo, commitID, CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) +} diff --git a/modules/git/attribute/main_test.go b/modules/git/attribute/main_test.go new file mode 100644 index 0000000000..df8241bfb0 --- /dev/null +++ b/modules/git/attribute/main_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "context" + "fmt" + "os" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func testRun(m *testing.M) error { + gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + if err != nil { + return fmt.Errorf("unable to create temp dir: %w", err) + } + defer util.RemoveAll(gitHomePath) + setting.Git.HomePath = gitHomePath + + if err = git.InitFull(context.Background()); err != nil { + return fmt.Errorf("failed to call Init: %w", err) + } + + exitCode := m.Run() + if exitCode != 0 { + return fmt.Errorf("run test failed, ExitCode=%d", exitCode) + } + return nil +} + +func TestMain(m *testing.M) { + if err := testRun(m); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Test failed: %v", err) + os.Exit(1) + } +} diff --git a/modules/git/batch.go b/modules/git/batch.go index 3ec4f1ddcc..f9e1748b54 100644 --- a/modules/git/batch.go +++ b/modules/git/batch.go @@ -14,25 +14,26 @@ type Batch struct { Writer WriteCloserError } -func (repo *Repository) NewBatch(ctx context.Context) (*Batch, error) { +// NewBatch creates a new batch for the given repository, the Close must be invoked before release the batch +func NewBatch(ctx context.Context, repoPath string) (*Batch, error) { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repo.Path); err != nil { + if err := ensureValidGitRepository(ctx, repoPath); err != nil { return nil, err } var batch Batch - batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repo.Path) + batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repoPath) return &batch, nil } -func (repo *Repository) NewBatchCheck(ctx context.Context) (*Batch, error) { +func NewBatchCheck(ctx context.Context, repoPath string) (*Batch, error) { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repo.Path); err != nil { + if err := ensureValidGitRepository(ctx, repoPath); err != nil { return nil, err } var check Batch - check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repo.Path) + check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repoPath) return &check, nil } diff --git a/modules/git/blame.go b/modules/git/blame.go index d1d732c716..659dec34a1 100644 --- a/modules/git/blame.go +++ b/modules/git/blame.go @@ -11,7 +11,7 @@ import ( "os" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/setting" ) // BlamePart represents block of blame - continuous lines with one sha @@ -29,12 +29,13 @@ type BlameReader struct { bufferedReader *bufio.Reader done chan error lastSha *string - ignoreRevsFile *string + ignoreRevsFile string objectFormat ObjectFormat + cleanupFuncs []func() } func (r *BlameReader) UsesIgnoreRevs() bool { - return r.ignoreRevsFile != nil + return r.ignoreRevsFile != "" } // NextPart returns next part of blame (sequential code lines with the same commit) @@ -122,36 +123,45 @@ func (r *BlameReader) Close() error { r.bufferedReader = nil _ = r.reader.Close() _ = r.output.Close() - if r.ignoreRevsFile != nil { - _ = util.Remove(*r.ignoreRevsFile) + for _, cleanup := range r.cleanupFuncs { + if cleanup != nil { + cleanup() + } } return err } // CreateBlameReader creates reader for given repository, commit and file -func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (*BlameReader, error) { - var ignoreRevsFile *string - if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { - ignoreRevsFile = tryCreateBlameIgnoreRevsFile(commit) - } +func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (rd *BlameReader, err error) { + var ignoreRevsFileName string + var ignoreRevsFileCleanup func() + defer func() { + if err != nil && ignoreRevsFileCleanup != nil { + ignoreRevsFileCleanup() + } + }() cmd := NewCommandNoGlobals("blame", "--porcelain") - if ignoreRevsFile != nil { - // Possible improvement: use --ignore-revs-file /dev/stdin on unix - // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. - cmd.AddOptionValues("--ignore-revs-file", *ignoreRevsFile) + + if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { + ignoreRevsFileName, ignoreRevsFileCleanup, err = tryCreateBlameIgnoreRevsFile(commit) + if err != nil && !IsErrNotExist(err) { + return nil, err + } + if ignoreRevsFileName != "" { + // Possible improvement: use --ignore-revs-file /dev/stdin on unix + // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. + cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName) + } } + cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file) + + done := make(chan error, 1) reader, stdout, err := os.Pipe() if err != nil { - if ignoreRevsFile != nil { - _ = util.Remove(*ignoreRevsFile) - } return nil, err } - - done := make(chan error, 1) - go func() { stderr := bytes.Buffer{} // TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close" @@ -169,40 +179,40 @@ func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath }() bufferedReader := bufio.NewReader(reader) - return &BlameReader{ output: stdout, reader: reader, bufferedReader: bufferedReader, done: done, - ignoreRevsFile: ignoreRevsFile, + ignoreRevsFile: ignoreRevsFileName, objectFormat: objectFormat, + cleanupFuncs: []func(){ignoreRevsFileCleanup}, }, nil } -func tryCreateBlameIgnoreRevsFile(commit *Commit) *string { +func tryCreateBlameIgnoreRevsFile(commit *Commit) (string, func(), error) { entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs") if err != nil { - return nil + return "", nil, err } r, err := entry.Blob().DataAsync() if err != nil { - return nil + return "", nil, err } defer r.Close() - f, err := os.CreateTemp("", "gitea_git-blame-ignore-revs") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs") if err != nil { - return nil + return "", nil, err } - + filename := f.Name() _, err = io.Copy(f, r) _ = f.Close() if err != nil { - _ = util.Remove(f.Name()) - return nil + cleanup() + return "", nil, err } - return util.ToPointer(f.Name()) + return filename, cleanup, nil } diff --git a/modules/git/blame_sha256_test.go b/modules/git/blame_sha256_test.go index 99c23429e2..c0a97bed3b 100644 --- a/modules/git/blame_sha256_test.go +++ b/modules/git/blame_sha256_test.go @@ -7,10 +7,13 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutputSha256(t *testing.T) { + setting.AppDataPath = t.TempDir() ctx, cancel := context.WithCancel(t.Context()) defer cancel() diff --git a/modules/git/blame_test.go b/modules/git/blame_test.go index 36b5fb9349..809d6fbcf7 100644 --- a/modules/git/blame_test.go +++ b/modules/git/blame_test.go @@ -7,10 +7,13 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutput(t *testing.T) { + setting.AppDataPath = t.TempDir() ctx, cancel := context.WithCancel(t.Context()) defer cancel() diff --git a/modules/git/blob.go b/modules/git/blob.go index b7857dbbc6..40d8f44e79 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "errors" "io" + "strings" "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" @@ -21,17 +22,22 @@ func (b *Blob) Name() string { return b.name } -// GetBlobContent Gets the limited content of the blob as raw text -func (b *Blob) GetBlobContent(limit int64) (string, error) { +// GetBlobBytes Gets the limited content of the blob +func (b *Blob) GetBlobBytes(limit int64) ([]byte, error) { if limit <= 0 { - return "", nil + return nil, nil } dataRc, err := b.DataAsync() if err != nil { - return "", err + return nil, err } defer dataRc.Close() - buf, err := util.ReadWithLimit(dataRc, int(limit)) + return util.ReadWithLimit(dataRc, int(limit)) +} + +// GetBlobContent Gets the limited content of the blob as raw text +func (b *Blob) GetBlobContent(limit int64) (string, error) { + buf, err := b.GetBlobBytes(limit) return string(buf), err } @@ -63,42 +69,44 @@ func (b *Blob) GetBlobLineCount(w io.Writer) (int, error) { } } -// GetBlobContentBase64 Reads the content of the blob with a base64 encode and returns the encoded string -func (b *Blob) GetBlobContentBase64() (string, error) { +// GetBlobContentBase64 Reads the content of the blob with a base64 encoding and returns the encoded string +func (b *Blob) GetBlobContentBase64(originContent *strings.Builder) (string, error) { dataRc, err := b.DataAsync() if err != nil { return "", err } defer dataRc.Close() - pr, pw := io.Pipe() - encoder := base64.NewEncoder(base64.StdEncoding, pw) - - go func() { - _, err := io.Copy(encoder, dataRc) - _ = encoder.Close() - - if err != nil { - _ = pw.CloseWithError(err) - } else { - _ = pw.Close() + base64buf := &strings.Builder{} + encoder := base64.NewEncoder(base64.StdEncoding, base64buf) + buf := make([]byte, 32*1024) +loop: + for { + n, err := dataRc.Read(buf) + if n > 0 { + if originContent != nil { + _, _ = originContent.Write(buf[:n]) + } + if _, err := encoder.Write(buf[:n]); err != nil { + return "", err + } + } + switch { + case errors.Is(err, io.EOF): + break loop + case err != nil: + return "", err } - }() - - out, err := io.ReadAll(pr) - if err != nil { - return "", err } - return string(out), nil + _ = encoder.Close() + return base64buf.String(), nil } // GuessContentType guesses the content type of the blob. func (b *Blob) GuessContentType() (typesniffer.SniffedType, error) { - r, err := b.DataAsync() + buf, err := b.GetBlobBytes(typesniffer.SniffContentSize) if err != nil { return typesniffer.SniffedType{}, err } - defer r.Close() - - return typesniffer.DetectContentTypeFromReader(r) + return typesniffer.DetectContentType(buf), nil } diff --git a/modules/git/cmdverb.go b/modules/git/cmdverb.go new file mode 100644 index 0000000000..3d6f4ae0c6 --- /dev/null +++ b/modules/git/cmdverb.go @@ -0,0 +1,36 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +const ( + CmdVerbUploadPack = "git-upload-pack" + CmdVerbUploadArchive = "git-upload-archive" + CmdVerbReceivePack = "git-receive-pack" + CmdVerbLfsAuthenticate = "git-lfs-authenticate" + CmdVerbLfsTransfer = "git-lfs-transfer" + + CmdSubVerbLfsUpload = "upload" + CmdSubVerbLfsDownload = "download" +) + +func IsAllowedVerbForServe(verb string) bool { + switch verb { + case CmdVerbUploadPack, + CmdVerbUploadArchive, + CmdVerbReceivePack, + CmdVerbLfsAuthenticate, + CmdVerbLfsTransfer: + return true + } + return false +} + +func IsAllowedVerbForServeLfs(verb string) bool { + switch verb { + case CmdVerbLfsAuthenticate, + CmdVerbLfsTransfer: + return true + } + return false +} diff --git a/modules/git/command.go b/modules/git/command.go index d85a91804a..22f1d02339 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -47,6 +47,7 @@ type Command struct { globalArgsLength int brokenArgs []string cmd *exec.Cmd // for debug purpose only + configArgs []string } func logArgSanitize(arg string) string { @@ -80,6 +81,13 @@ func (c *Command) LogString() string { return strings.Join(a, " ") } +func (c *Command) ProcessState() string { + if c.cmd == nil { + return "" + } + return c.cmd.ProcessState.String() +} + // NewCommand creates and returns a new Git Command based on given command and arguments. // Each argument should be safe to be trusted. User-provided arguments should be passed to AddDynamicArguments instead. func NewCommand(args ...internal.CmdArg) *Command { @@ -189,6 +197,16 @@ func (c *Command) AddDashesAndList(list ...string) *Command { return c } +func (c *Command) AddConfig(key, value string) *Command { + kv := key + "=" + value + if !isSafeArgumentValue(kv) { + c.brokenArgs = append(c.brokenArgs, key) + } else { + c.configArgs = append(c.configArgs, "-c", kv) + } + return c +} + // ToTrustedCmdArgs converts a list of strings (trusted as argument) to TrustedCmdArgs // In most cases, it shouldn't be used. Use NewCommand().AddXxx() function instead func ToTrustedCmdArgs(args []string) TrustedCmdArgs { @@ -314,7 +332,7 @@ func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { startTime := time.Now() - cmd := exec.CommandContext(ctx, c.prog, c.args...) + cmd := exec.CommandContext(ctx, c.prog, append(c.configArgs, c.args...)...) c.cmd = cmd // for debug purpose only if opts.Env == nil { cmd.Env = os.Environ() @@ -350,9 +368,10 @@ func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { // We need to check if the context is canceled by the program on Windows. // This is because Windows does not have signal checking when terminating the process. // It always returns exit code 1, unlike Linux, which has many exit codes for signals. + // `err.Error()` returns "exit status 1" when using the `git check-attr` command after the context is canceled. if runtime.GOOS == "windows" && err != nil && - err.Error() == "" && + (err.Error() == "" || err.Error() == "exit status 1") && cmd.ProcessState.ExitCode() == 1 && ctx.Err() == context.Canceled { return ctx.Err() diff --git a/modules/git/command_test.go b/modules/git/command_test.go index 005a760ed1..eb112707e7 100644 --- a/modules/git/command_test.go +++ b/modules/git/command_test.go @@ -54,8 +54,8 @@ func TestGitArgument(t *testing.T) { func TestCommandString(t *testing.T) { cmd := NewCommandNoGlobals("a", "-m msg", "it's a test", `say "hello"`) - assert.EqualValues(t, cmd.prog+` a "-m msg" "it's a test" "say \"hello\""`, cmd.LogString()) + assert.Equal(t, cmd.prog+` a "-m msg" "it's a test" "say \"hello\""`, cmd.LogString()) cmd = NewCommandNoGlobals("url: https://a:b@c/", "/root/dir-a/dir-b") - assert.EqualValues(t, cmd.prog+` "url: https://sanitized-credential@c/" .../dir-a/dir-b`, cmd.LogString()) + assert.Equal(t, cmd.prog+` "url: https://sanitized-credential@c/" .../dir-a/dir-b`, cmd.LogString()) } diff --git a/modules/git/commit.go b/modules/git/commit.go index 3e790e89d9..aae40c575b 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -20,10 +20,11 @@ import ( // Commit represents a git commit. type Commit struct { - Tree - ID ObjectID // The ID of this commit object - Author *Signature - Committer *Signature + Tree // FIXME: bad design, this field can be nil if the commit is from "last commit cache" + + ID ObjectID + Author *Signature // never nil + Committer *Signature // never nil CommitMessage string Signature *CommitSignature @@ -34,7 +35,7 @@ type Commit struct { // CommitSignature represents a git commit signature part. type CommitSignature struct { Signature string - Payload string // TODO check if can be reconstruct from the rest of commit information to not have duplicate data + Payload string } // Message returns the commit message. Same as retrieving CommitMessage directly. @@ -166,6 +167,8 @@ type CommitsCountOptions struct { Not string Revision []string RelPath []string + Since string + Until string } // CommitsCount returns number of total commits of until given revision. @@ -199,8 +202,8 @@ func (c *Commit) CommitsCount() (int64, error) { } // CommitsByRange returns the specific page commits before current revision, every page's number default by CommitsRangeSize -func (c *Commit) CommitsByRange(page, pageSize int, not string) ([]*Commit, error) { - return c.repo.commitsByRange(c.ID, page, pageSize, not) +func (c *Commit) CommitsByRange(page, pageSize int, not, since, until string) ([]*Commit, error) { + return c.repo.commitsByRangeWithTime(c.ID, page, pageSize, not, since, until) } // CommitsBefore returns all the commits before current revision @@ -275,8 +278,8 @@ func NewSearchCommitsOptions(searchString string, forAllRefs bool) SearchCommits var keywords, authors, committers []string var after, before string - fields := strings.Fields(searchString) - for _, k := range fields { + fields := strings.FieldsSeq(searchString) + for k := range fields { switch { case strings.HasPrefix(k, "author:"): authors = append(authors, strings.TrimPrefix(k, "author:")) diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index 7a6af0410b..1b45fc8a6c 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -7,8 +7,7 @@ package git import ( "context" - "fmt" - "io" + "maps" "path" "sort" @@ -40,9 +39,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath return nil, nil, err } - for pth, found := range commits { - revs[pth] = found - } + maps.Copy(revs, commits) } } else { sort.Strings(entryPaths) @@ -124,48 +121,25 @@ func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, return nil, err } - batchStdinWriter, batchReader, cancel, err := commit.repo.CatFileBatch(ctx) - if err != nil { - return nil, err - } - defer cancel() - commitsMap := map[string]*Commit{} commitsMap[commit.ID.String()] = commit commitCommits := map[string]*Commit{} for path, commitID := range revs { - c, ok := commitsMap[commitID] - if ok { - commitCommits[path] = c + if len(commitID) == 0 { continue } - if len(commitID) == 0 { + c, ok := commitsMap[commitID] + if ok { + commitCommits[path] = c continue } - _, err := batchStdinWriter.Write([]byte(commitID + "\n")) - if err != nil { - return nil, err - } - _, typ, size, err := ReadBatchLine(batchReader) + c, err := commit.repo.GetCommit(commitID) // Ensure the commit exists in the repository if err != nil { return nil, err } - if typ != "commit" { - if err := DiscardFull(batchReader, size+1); err != nil { - return nil, err - } - return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID) - } - c, err = CommitFromReader(commit.repo, MustIDFromString(commitID), io.LimitReader(batchReader, size)) - if err != nil { - return nil, err - } - if _, err := batchReader.Discard(1); err != nil { - return nil, err - } commitCommits[path] = c } diff --git a/modules/git/commit_reader.go b/modules/git/commit_reader.go index 228bbaf314..eb8f4c6322 100644 --- a/modules/git/commit_reader.go +++ b/modules/git/commit_reader.go @@ -6,10 +6,44 @@ package git import ( "bufio" "bytes" + "fmt" "io" - "strings" ) +const ( + commitHeaderGpgsig = "gpgsig" + commitHeaderGpgsigSha256 = "gpgsig-sha256" +) + +func assignCommitFields(gitRepo *Repository, commit *Commit, headerKey string, headerValue []byte) error { + if len(headerValue) > 0 && headerValue[len(headerValue)-1] == '\n' { + headerValue = headerValue[:len(headerValue)-1] // remove trailing newline + } + switch headerKey { + case "tree": + objID, err := NewIDFromString(string(headerValue)) + if err != nil { + return fmt.Errorf("invalid tree ID %q: %w", string(headerValue), err) + } + commit.Tree = *NewTree(gitRepo, objID) + case "parent": + objID, err := NewIDFromString(string(headerValue)) + if err != nil { + return fmt.Errorf("invalid parent ID %q: %w", string(headerValue), err) + } + commit.Parents = append(commit.Parents, objID) + case "author": + commit.Author.Decode(headerValue) + case "committer": + commit.Committer.Decode(headerValue) + case commitHeaderGpgsig, commitHeaderGpgsigSha256: + // if there are duplicate "gpgsig" and "gpgsig-sha256" headers, then the signature must have already been invalid + // so we don't need to handle duplicate headers here + commit.Signature = &CommitSignature{Signature: string(headerValue)} + } + return nil +} + // CommitFromReader will generate a Commit from a provided reader // We need this to interpret commits from cat-file or cat-file --batch // @@ -21,90 +55,46 @@ func CommitFromReader(gitRepo *Repository, objectID ObjectID, reader io.Reader) Committer: &Signature{}, } - payloadSB := new(strings.Builder) - signatureSB := new(strings.Builder) - messageSB := new(strings.Builder) - message := false - pgpsig := false - - bufReader, ok := reader.(*bufio.Reader) - if !ok { - bufReader = bufio.NewReader(reader) - } - -readLoop: + bufReader := bufio.NewReader(reader) + inHeader := true + var payloadSB, messageSB bytes.Buffer + var headerKey string + var headerValue []byte for { line, err := bufReader.ReadBytes('\n') - if err != nil { - if err == io.EOF { - if message { - _, _ = messageSB.Write(line) - } - _, _ = payloadSB.Write(line) - break readLoop - } - return nil, err + if err != nil && err != io.EOF { + return nil, fmt.Errorf("unable to read commit %q: %w", objectID.String(), err) } - if pgpsig { - if len(line) > 0 && line[0] == ' ' { - _, _ = signatureSB.Write(line[1:]) - continue - } - pgpsig = false + if len(line) == 0 { + break } - if !message { - // This is probably not correct but is copied from go-gits interpretation... - trimmed := bytes.TrimSpace(line) - if len(trimmed) == 0 { - message = true - _, _ = payloadSB.Write(line) - continue - } - - split := bytes.SplitN(trimmed, []byte{' '}, 2) - var data []byte - if len(split) > 1 { - data = split[1] + if inHeader { + inHeader = !(len(line) == 1 && line[0] == '\n') // still in header if line is not just a newline + k, v, _ := bytes.Cut(line, []byte{' '}) + if len(k) != 0 || !inHeader { + if headerKey != "" { + if err = assignCommitFields(gitRepo, commit, headerKey, headerValue); err != nil { + return nil, fmt.Errorf("unable to parse commit %q: %w", objectID.String(), err) + } + } + headerKey = string(k) // it also resets the headerValue to empty string if not inHeader + headerValue = v + } else { + headerValue = append(headerValue, v...) } - - switch string(split[0]) { - case "tree": - commit.Tree = *NewTree(gitRepo, MustIDFromString(string(data))) + if headerKey != commitHeaderGpgsig && headerKey != commitHeaderGpgsigSha256 { _, _ = payloadSB.Write(line) - case "parent": - commit.Parents = append(commit.Parents, MustIDFromString(string(data))) - _, _ = payloadSB.Write(line) - case "author": - commit.Author = &Signature{} - commit.Author.Decode(data) - _, _ = payloadSB.Write(line) - case "committer": - commit.Committer = &Signature{} - commit.Committer.Decode(data) - _, _ = payloadSB.Write(line) - case "encoding": - _, _ = payloadSB.Write(line) - case "gpgsig": - fallthrough - case "gpgsig-sha256": // FIXME: no intertop, so only 1 exists at present. - _, _ = signatureSB.Write(data) - _ = signatureSB.WriteByte('\n') - pgpsig = true } } else { _, _ = messageSB.Write(line) _, _ = payloadSB.Write(line) } } + commit.CommitMessage = messageSB.String() - commit.Signature = &CommitSignature{ - Signature: signatureSB.String(), - Payload: payloadSB.String(), - } - if len(commit.Signature.Signature) == 0 { - commit.Signature = nil + if commit.Signature != nil { + commit.Signature.Payload = payloadSB.String() } - return commit, nil } diff --git a/modules/git/commit_sha256_test.go b/modules/git/commit_sha256_test.go index f6ca83c9ed..97ccecdacc 100644 --- a/modules/git/commit_sha256_test.go +++ b/modules/git/commit_sha256_test.go @@ -60,8 +60,7 @@ func TestGetFullCommitIDErrorSha256(t *testing.T) { } func TestCommitFromReaderSha256(t *testing.T) { - commitString := `9433b2a62b964c17a4485ae180f45f595d3e69d31b786087775e28c6b6399df0 commit 1114 -tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e + commitString := `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e parent 26e9ccc29fad747e9c5d9f4c9ddeb7eff61cc45ef6a8dc258cbeb181afc055e8 author Adam Majer <amajer@suse.de> 1698676906 +0100 committer Adam Majer <amajer@suse.de> 1698676906 +0100 @@ -97,7 +96,7 @@ signed commit` assert.NoError(t, err) require.NotNil(t, commitFromReader) assert.EqualValues(t, sha, commitFromReader.ID) - assert.EqualValues(t, `-----BEGIN PGP SIGNATURE----- + assert.Equal(t, `-----BEGIN PGP SIGNATURE----- iQIrBAABCgAtFiEES+fB08xlgTrzSdQvhkUIsBsmec8FAmU/wKoPHGFtYWplckBz dXNlLmRlAAoJEIZFCLAbJnnP4s4PQIJATa++WPzR6/H4etT7bsOGoMyguEJYyWOd @@ -112,21 +111,20 @@ VAEUo6ecdDxSpyt2naeg9pKus/BRi7P6g4B1hkk/zZstUX/QP4IQuAJbXjkvsC+X HKRr3NlRM/DygzTyj0gN74uoa0goCIbyAQhiT42nm0cuhM7uN/W0ayrlZjGF1cbR 8NCJUL2Nwj0ywKIavC99Ipkb8AsFwpVT6U6effs6 =xybZ ------END PGP SIGNATURE----- -`, commitFromReader.Signature.Signature) - assert.EqualValues(t, `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e +-----END PGP SIGNATURE-----`, commitFromReader.Signature.Signature) + assert.Equal(t, `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e parent 26e9ccc29fad747e9c5d9f4c9ddeb7eff61cc45ef6a8dc258cbeb181afc055e8 author Adam Majer <amajer@suse.de> 1698676906 +0100 committer Adam Majer <amajer@suse.de> 1698676906 +0100 signed commit`, commitFromReader.Signature.Payload) - assert.EqualValues(t, "Adam Majer <amajer@suse.de>", commitFromReader.Author.String()) + assert.Equal(t, "Adam Majer <amajer@suse.de>", commitFromReader.Author.String()) commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n")) assert.NoError(t, err) commitFromReader.CommitMessage += "\n\n" commitFromReader.Signature.Payload += "\n\n" - assert.EqualValues(t, commitFromReader, commitFromReader2) + assert.Equal(t, commitFromReader, commitFromReader2) } func TestHasPreviousCommitSha256(t *testing.T) { diff --git a/modules/git/commit_submodule_file.go b/modules/git/commit_submodule_file.go index 729401f752..5def80f3bd 100644 --- a/modules/git/commit_submodule_file.go +++ b/modules/git/commit_submodule_file.go @@ -6,17 +6,18 @@ package git import ( "context" + "strings" giturl "code.gitea.io/gitea/modules/git/url" ) // CommitSubmoduleFile represents a file with submodule type. type CommitSubmoduleFile struct { - refURL string - parsedURL *giturl.RepositoryURL - parsed bool - refID string - repoLink string + refURL string + refID string + + parsed bool + targetRepoLink string } // NewCommitSubmoduleFile create a new submodule file @@ -35,20 +36,27 @@ func (sf *CommitSubmoduleFile) SubmoduleWebLink(ctx context.Context, optCommitID } if !sf.parsed { sf.parsed = true - parsedURL, err := giturl.ParseRepositoryURL(ctx, sf.refURL) - if err != nil { - return nil + if strings.HasPrefix(sf.refURL, "../") { + // FIXME: when handling relative path, this logic is not right. It needs to: + // 1. Remember the submodule's full path and its commit's repo home link + // 2. Resolve the relative path: targetRepoLink = path.Join(repoHomeLink, path.Dir(submoduleFullPath), refURL) + // Not an easy task and need to refactor related code a lot. + sf.targetRepoLink = sf.refURL + } else { + parsedURL, err := giturl.ParseRepositoryURL(ctx, sf.refURL) + if err != nil { + return nil + } + sf.targetRepoLink = giturl.MakeRepositoryWebLink(parsedURL) } - sf.parsedURL = parsedURL - sf.repoLink = giturl.MakeRepositoryWebLink(sf.parsedURL) } var commitLink string if len(optCommitID) == 2 { - commitLink = sf.repoLink + "/compare/" + optCommitID[0] + "..." + optCommitID[1] + commitLink = sf.targetRepoLink + "/compare/" + optCommitID[0] + "..." + optCommitID[1] } else if len(optCommitID) == 1 { - commitLink = sf.repoLink + "/tree/" + optCommitID[0] + commitLink = sf.targetRepoLink + "/tree/" + optCommitID[0] } else { - commitLink = sf.repoLink + "/tree/" + sf.refID + commitLink = sf.targetRepoLink + "/tree/" + sf.refID } - return &SubmoduleWebLink{RepoWebLink: sf.repoLink, CommitWebLink: commitLink} + return &SubmoduleWebLink{RepoWebLink: sf.targetRepoLink, CommitWebLink: commitLink} } diff --git a/modules/git/commit_submodule_file_test.go b/modules/git/commit_submodule_file_test.go index 6581fa8712..103e55e920 100644 --- a/modules/git/commit_submodule_file_test.go +++ b/modules/git/commit_submodule_file_test.go @@ -10,20 +10,29 @@ import ( ) func TestCommitSubmoduleLink(t *testing.T) { - sf := NewCommitSubmoduleFile("git@github.com:user/repo.git", "aaaa") + wl := (*CommitSubmoduleFile)(nil).SubmoduleWebLink(t.Context()) + assert.Nil(t, wl) - wl := sf.SubmoduleWebLink(t.Context()) - assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) - assert.Equal(t, "https://github.com/user/repo/tree/aaaa", wl.CommitWebLink) + t.Run("GitHubRepo", func(t *testing.T) { + sf := NewCommitSubmoduleFile("git@github.com:user/repo.git", "aaaa") - wl = sf.SubmoduleWebLink(t.Context(), "1111") - assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) - assert.Equal(t, "https://github.com/user/repo/tree/1111", wl.CommitWebLink) + wl := sf.SubmoduleWebLink(t.Context()) + assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) + assert.Equal(t, "https://github.com/user/repo/tree/aaaa", wl.CommitWebLink) - wl = sf.SubmoduleWebLink(t.Context(), "1111", "2222") - assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) - assert.Equal(t, "https://github.com/user/repo/compare/1111...2222", wl.CommitWebLink) + wl = sf.SubmoduleWebLink(t.Context(), "1111") + assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) + assert.Equal(t, "https://github.com/user/repo/tree/1111", wl.CommitWebLink) - wl = (*CommitSubmoduleFile)(nil).SubmoduleWebLink(t.Context()) - assert.Nil(t, wl) + wl = sf.SubmoduleWebLink(t.Context(), "1111", "2222") + assert.Equal(t, "https://github.com/user/repo", wl.RepoWebLink) + assert.Equal(t, "https://github.com/user/repo/compare/1111...2222", wl.CommitWebLink) + }) + + t.Run("RelativePath", func(t *testing.T) { + sf := NewCommitSubmoduleFile("../../user/repo", "aaaa") + wl := sf.SubmoduleWebLink(t.Context()) + assert.Equal(t, "../../user/repo", wl.RepoWebLink) + assert.Equal(t, "../../user/repo/tree/aaaa", wl.CommitWebLink) + }) } diff --git a/modules/git/commit_test.go b/modules/git/commit_test.go index 5319e09bb7..81fb91dfc6 100644 --- a/modules/git/commit_test.go +++ b/modules/git/commit_test.go @@ -59,8 +59,7 @@ func TestGetFullCommitIDError(t *testing.T) { } func TestCommitFromReader(t *testing.T) { - commitString := `feaf4ba6bc635fec442f46ddd4512416ec43c2c2 commit 1074 -tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 + commitString := `tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 parent 37991dec2c8e592043f47155ce4808d4580f9123 author silverwind <me@silverwind.io> 1563741793 +0200 committer silverwind <me@silverwind.io> 1563741793 +0200 @@ -93,7 +92,7 @@ empty commit` assert.NoError(t, err) require.NotNil(t, commitFromReader) assert.EqualValues(t, sha, commitFromReader.ID) - assert.EqualValues(t, `-----BEGIN PGP SIGNATURE----- + assert.Equal(t, `-----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEEWPb2jX6FS2mqyJRQLmK0HJOGlEMFAl00zmEACgkQLmK0HJOG lEMDFBAAhQKKqLD1VICygJMEB8t1gBmNLgvziOLfpX4KPWdPtBk3v/QJ7OrfMrVK @@ -108,26 +107,24 @@ sD53z/f0J+We4VZjY+pidvA9BGZPFVdR3wd3xGs8/oH6UWaLJAMGkLG6dDb3qDLm mfeFhT57UbE4qukTDIQ0Y0WM40UYRTakRaDY7ubhXgLgx09Cnp9XTVMsHgT6j9/i 1pxsB104XLWjQHTjr1JtiaBQEwFh9r2OKTcpvaLcbNtYpo7CzOs= =FRsO ------END PGP SIGNATURE----- -`, commitFromReader.Signature.Signature) - assert.EqualValues(t, `tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 +-----END PGP SIGNATURE-----`, commitFromReader.Signature.Signature) + assert.Equal(t, `tree f1a6cb52b2d16773290cefe49ad0684b50a4f930 parent 37991dec2c8e592043f47155ce4808d4580f9123 author silverwind <me@silverwind.io> 1563741793 +0200 committer silverwind <me@silverwind.io> 1563741793 +0200 empty commit`, commitFromReader.Signature.Payload) - assert.EqualValues(t, "silverwind <me@silverwind.io>", commitFromReader.Author.String()) + assert.Equal(t, "silverwind <me@silverwind.io>", commitFromReader.Author.String()) commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n")) assert.NoError(t, err) commitFromReader.CommitMessage += "\n\n" commitFromReader.Signature.Payload += "\n\n" - assert.EqualValues(t, commitFromReader, commitFromReader2) + assert.Equal(t, commitFromReader, commitFromReader2) } func TestCommitWithEncodingFromReader(t *testing.T) { - commitString := `feaf4ba6bc635fec442f46ddd4512416ec43c2c2 commit 1074 -tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 + commitString := `tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 parent 47b24e7ab977ed31c5a39989d570847d6d0052af author KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 committer KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 @@ -159,7 +156,7 @@ ISO-8859-1` assert.NoError(t, err) require.NotNil(t, commitFromReader) assert.EqualValues(t, sha, commitFromReader.ID) - assert.EqualValues(t, `-----BEGIN PGP SIGNATURE----- + assert.Equal(t, `-----BEGIN PGP SIGNATURE----- iQGzBAABCgAdFiEE9HRrbqvYxPT8PXbefPSEkrowAa8FAmYGg7IACgkQfPSEkrow Aa9olwv+P0HhtCM6CRvlUmPaqswRsDPNR4i66xyXGiSxdI9V5oJL7HLiQIM7KrFR @@ -172,22 +169,21 @@ SONRzusmu5n3DgV956REL7x62h7JuqmBz/12HZkr0z0zgXkcZ04q08pSJATX5N1F yN+tWxTsWg+zhDk96d5Esdo9JMjcFvPv0eioo30GAERaz1hoD7zCMT4jgUFTQwgz jw4YcO5u =r3UU ------END PGP SIGNATURE----- -`, commitFromReader.Signature.Signature) - assert.EqualValues(t, `tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 +-----END PGP SIGNATURE-----`, commitFromReader.Signature.Signature) + assert.Equal(t, `tree ca3fad42080dd1a6d291b75acdfc46e5b9b307e5 parent 47b24e7ab977ed31c5a39989d570847d6d0052af author KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 committer KN4CK3R <admin@oldschoolhack.me> 1711702962 +0100 encoding ISO-8859-1 ISO-8859-1`, commitFromReader.Signature.Payload) - assert.EqualValues(t, "KN4CK3R <admin@oldschoolhack.me>", commitFromReader.Author.String()) + assert.Equal(t, "KN4CK3R <admin@oldschoolhack.me>", commitFromReader.Author.String()) commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n")) assert.NoError(t, err) commitFromReader.CommitMessage += "\n\n" commitFromReader.Signature.Payload += "\n\n" - assert.EqualValues(t, commitFromReader, commitFromReader2) + assert.Equal(t, commitFromReader, commitFromReader2) } func TestHasPreviousCommit(t *testing.T) { @@ -351,10 +347,10 @@ func Test_GetCommitBranchStart(t *testing.T) { defer repo.Close() commit, err := repo.GetBranchCommit("branch1") assert.NoError(t, err) - assert.EqualValues(t, "2839944139e0de9737a044f78b0e4b40d989a9e3", commit.ID.String()) + assert.Equal(t, "2839944139e0de9737a044f78b0e4b40d989a9e3", commit.ID.String()) startCommitID, err := repo.GetCommitBranchStart(os.Environ(), "branch1", commit.ID.String()) assert.NoError(t, err) assert.NotEmpty(t, startCommitID) - assert.EqualValues(t, "95bb4d39648ee7e325106df01a621c530863a653", startCommitID) + assert.Equal(t, "95bb4d39648ee7e325106df01a621c530863a653", startCommitID) } diff --git a/modules/git/diff.go b/modules/git/diff.go index c4df6b8063..35d115be0e 100644 --- a/modules/git/diff.go +++ b/modules/git/diff.go @@ -99,9 +99,9 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff return nil } -// ParseDiffHunkString parse the diffhunk content and return -func ParseDiffHunkString(diffhunk string) (leftLine, leftHunk, rightLine, righHunk int) { - ss := strings.Split(diffhunk, "@@") +// ParseDiffHunkString parse the diff hunk content and return +func ParseDiffHunkString(diffHunk string) (leftLine, leftHunk, rightLine, rightHunk int) { + ss := strings.Split(diffHunk, "@@") ranges := strings.Split(ss[1][1:], " ") leftRange := strings.Split(ranges[0], ",") leftLine, _ = strconv.Atoi(leftRange[0][1:]) @@ -112,14 +112,19 @@ func ParseDiffHunkString(diffhunk string) (leftLine, leftHunk, rightLine, righHu rightRange := strings.Split(ranges[1], ",") rightLine, _ = strconv.Atoi(rightRange[0]) if len(rightRange) > 1 { - righHunk, _ = strconv.Atoi(rightRange[1]) + rightHunk, _ = strconv.Atoi(rightRange[1]) } } else { - log.Debug("Parse line number failed: %v", diffhunk) + log.Debug("Parse line number failed: %v", diffHunk) rightLine = leftLine - righHunk = leftHunk + rightHunk = leftHunk } - return leftLine, leftHunk, rightLine, righHunk + if rightLine == 0 { + // FIXME: GIT-DIFF-CUT-BUG search this tag to see details + // this is only a hacky patch, the rightLine&rightHunk might still be incorrect in some cases. + rightLine++ + } + return leftLine, leftHunk, rightLine, rightHunk } // Example: @@ -1,8 +1,9 @@ => [..., 1, 8, 1, 9] @@ -270,6 +275,12 @@ func CutDiffAroundLine(originalDiff io.Reader, line int64, old bool, numbersOfLi oldNumOfLines++ } } + + // "git diff" outputs "@@ -1 +1,3 @@" for "OLD" => "A\nB\nC" + // FIXME: GIT-DIFF-CUT-BUG But there is a bug in CutDiffAroundLine, then the "Patch" stored in the comment model becomes "@@ -1,1 +0,4 @@" + // It may generate incorrect results for difference cases, for example: delete 2 line add 1 line, delete 2 line add 2 line etc, need to double check. + // For example: "L1\nL2" => "A\nB", then the patch shows "L2" as line 1 on the left (deleted part) + // construct the new hunk header newHunk[headerLines] = fmt.Sprintf("@@ -%d,%d +%d,%d @@", oldBegin, oldNumOfLines, newBegin, newNumOfLines) diff --git a/modules/git/diff_test.go b/modules/git/diff_test.go index 0f865c52a8..7671fffcc1 100644 --- a/modules/git/diff_test.go +++ b/modules/git/diff_test.go @@ -154,7 +154,7 @@ func TestCutDiffAroundLine(t *testing.T) { } func BenchmarkCutDiffAroundLine(b *testing.B) { - for n := 0; n < b.N; n++ { + for b.Loop() { CutDiffAroundLine(strings.NewReader(exampleDiff), 3, true, 3) } } @@ -177,8 +177,8 @@ func ExampleCutDiffAroundLine() { func TestParseDiffHunkString(t *testing.T) { leftLine, leftHunk, rightLine, rightHunk := ParseDiffHunkString("@@ -19,3 +19,5 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER") - assert.EqualValues(t, 19, leftLine) - assert.EqualValues(t, 3, leftHunk) - assert.EqualValues(t, 19, rightLine) - assert.EqualValues(t, 5, rightHunk) + assert.Equal(t, 19, leftLine) + assert.Equal(t, 3, leftHunk) + assert.Equal(t, 19, rightLine) + assert.Equal(t, 5, rightHunk) } diff --git a/modules/git/error.go b/modules/git/error.go index 10fb37be07..7d131345d0 100644 --- a/modules/git/error.go +++ b/modules/git/error.go @@ -32,22 +32,6 @@ func (err ErrNotExist) Unwrap() error { return util.ErrNotExist } -// ErrBadLink entry.FollowLink error -type ErrBadLink struct { - Name string - Message string -} - -func (err ErrBadLink) Error() string { - return fmt.Sprintf("%s: %s", err.Name, err.Message) -} - -// IsErrBadLink if some error is ErrBadLink -func IsErrBadLink(err error) bool { - _, ok := err.(ErrBadLink) - return ok -} - // ErrBranchNotExist represents a "BranchNotExist" kind of error. type ErrBranchNotExist struct { Name string diff --git a/modules/git/foreachref/format.go b/modules/git/foreachref/format.go index 97e8ee4724..d9573a55d6 100644 --- a/modules/git/foreachref/format.go +++ b/modules/git/foreachref/format.go @@ -76,7 +76,7 @@ func (f Format) Parser(r io.Reader) *Parser { // would turn into "%0a%00". func (f Format) hexEscaped(delim []byte) string { escaped := "" - for i := 0; i < len(delim); i++ { + for i := range delim { escaped += "%" + hex.EncodeToString([]byte{delim[i]}) } return escaped diff --git a/modules/git/git.go b/modules/git/git.go index 2b593910a2..a2ffd6d289 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -30,6 +30,7 @@ type Features struct { SupportProcReceive bool // >= 2.29 SupportHashSha256 bool // >= 2.42, SHA-256 repositories no longer an ‘experimental curiosity’ SupportedObjectFormats []ObjectFormat // sha1, sha256 + SupportCheckAttrOnBare bool // >= 2.40 } var ( @@ -77,6 +78,7 @@ func loadGitVersionFeatures() (*Features, error) { if features.SupportHashSha256 { features.SupportedObjectFormats = append(features.SupportedObjectFormats, Sha256ObjectFormat) } + features.SupportCheckAttrOnBare = features.CheckVersionAtLeast("2.40") return features, nil } diff --git a/modules/git/git_test.go b/modules/git/git_test.go index 5472842b76..58ba01cabc 100644 --- a/modules/git/git_test.go +++ b/modules/git/git_test.go @@ -10,18 +10,19 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/tempdir" "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" ) func testRun(m *testing.M) error { - gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + gitHomePath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("git-home") if err != nil { return fmt.Errorf("unable to create temp dir: %w", err) } - defer util.RemoveAll(gitHomePath) + defer cleanup() + setting.Git.HomePath = gitHomePath if err = InitFull(context.Background()); err != nil { diff --git a/modules/git/grep.go b/modules/git/grep.go index 44ec6ca2be..66711650c9 100644 --- a/modules/git/grep.go +++ b/modules/git/grep.go @@ -61,14 +61,15 @@ func GrepSearch(ctx context.Context, repo *Repository, search string, opts GrepO */ var results []*GrepResult cmd := NewCommand("grep", "--null", "--break", "--heading", "--line-number", "--full-name") - cmd.AddOptionValues("--context", fmt.Sprint(opts.ContextLineNumber)) - if opts.GrepMode == GrepModeExact { + cmd.AddOptionValues("--context", strconv.Itoa(opts.ContextLineNumber)) + switch opts.GrepMode { + case GrepModeExact: cmd.AddArguments("--fixed-strings") cmd.AddOptionValues("-e", strings.TrimLeft(search, "-")) - } else if opts.GrepMode == GrepModeRegexp { + case GrepModeRegexp: cmd.AddArguments("--perl-regexp") cmd.AddOptionValues("-e", strings.TrimLeft(search, "-")) - } else /* words */ { + default: /* words */ words := strings.Fields(search) cmd.AddArguments("--fixed-strings", "--ignore-case") for i, word := range words { diff --git a/modules/git/hook.go b/modules/git/hook.go index 46f93ce13e..548a59971d 100644 --- a/modules/git/hook.go +++ b/modules/git/hook.go @@ -7,11 +7,10 @@ package git import ( "errors" "os" - "path" "path/filepath" + "slices" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -27,12 +26,7 @@ var ErrNotValidHook = errors.New("not a valid Git hook") // IsValidHookName returns true if given name is a valid Git hook. func IsValidHookName(name string) bool { - for _, hn := range hookNames { - if hn == name { - return true - } - } - return false + return slices.Contains(hookNames, name) } // Hook represents a Git hook. @@ -51,17 +45,28 @@ func GetHook(repoPath, name string) (*Hook, error) { } h := &Hook{ name: name, - path: path.Join(repoPath, "hooks", name+".d", name), + path: filepath.Join(repoPath, "hooks", name+".d", name), } - samplePath := filepath.Join(repoPath, "hooks", name+".sample") - if isFile(h.path) { + isFile, err := util.IsFile(h.path) + if err != nil { + return nil, err + } + if isFile { data, err := os.ReadFile(h.path) if err != nil { return nil, err } h.IsActive = true h.Content = string(data) - } else if isFile(samplePath) { + return h, nil + } + + samplePath := filepath.Join(repoPath, "hooks", name+".sample") + isFile, err = util.IsFile(samplePath) + if err != nil { + return nil, err + } + if isFile { data, err := os.ReadFile(samplePath) if err != nil { return nil, err @@ -79,7 +84,11 @@ func (h *Hook) Name() string { // Update updates hook settings. func (h *Hook) Update() error { if len(strings.TrimSpace(h.Content)) == 0 { - if isExist(h.path) { + exist, err := util.IsExist(h.path) + if err != nil { + return err + } + if exist { err := util.Remove(h.path) if err != nil { return err @@ -103,7 +112,10 @@ func (h *Hook) Update() error { // ListHooks returns a list of Git hooks of given repository. func ListHooks(repoPath string) (_ []*Hook, err error) { - if !isDir(path.Join(repoPath, "hooks")) { + exist, err := util.IsDir(filepath.Join(repoPath, "hooks")) + if err != nil { + return nil, err + } else if !exist { return nil, errors.New("hooks path does not exist") } @@ -116,28 +128,3 @@ func ListHooks(repoPath string) (_ []*Hook, err error) { } return hooks, nil } - -const ( - // HookPathUpdate hook update path - HookPathUpdate = "hooks/update" -) - -// SetUpdateHook writes given content to update hook of the repository. -func SetUpdateHook(repoPath, content string) (err error) { - log.Debug("Setting update hook: %s", repoPath) - hookPath := path.Join(repoPath, HookPathUpdate) - isExist, err := util.IsExist(hookPath) - if err != nil { - log.Debug("Unable to check if %s exists. Error: %v", hookPath, err) - return err - } - if isExist { - err = util.Remove(hookPath) - } else { - err = os.MkdirAll(path.Dir(hookPath), os.ModePerm) - } - if err != nil { - return err - } - return os.WriteFile(hookPath, []byte(content), 0o777) -} diff --git a/modules/git/key.go b/modules/git/key.go new file mode 100644 index 0000000000..2513c048b7 --- /dev/null +++ b/modules/git/key.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +// Based on https://git-scm.com/docs/git-config#Documentation/git-config.txt-gpgformat +const ( + SigningKeyFormatOpenPGP = "openpgp" // for GPG keys, the expected default of git cli + SigningKeyFormatSSH = "ssh" +) + +type SigningKey struct { + KeyID string + Format string +} diff --git a/modules/git/repo_language_stats.go b/modules/git/languagestats/language_stats.go index 8551ea9d24..a71284c3e4 100644 --- a/modules/git/repo_language_stats.go +++ b/modules/git/languagestats/language_stats.go @@ -1,13 +1,15 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package git +package languagestats import ( + "context" "strings" "unicode" - "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" ) const ( @@ -49,19 +51,15 @@ func mergeLanguageStats(stats map[string]int64) map[string]int64 { return res } -func TryReadLanguageAttribute(attrs map[string]string) optional.Option[string] { - language := AttributeToString(attrs, AttributeLinguistLanguage) - if language.Value() == "" { - language = AttributeToString(attrs, AttributeGitlabLanguage) - if language.Has() { - raw := language.Value() - // gitlab-language may have additional parameters after the language - // ignore them and just use the main language - // https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type - if idx := strings.IndexByte(raw, '?'); idx >= 0 { - language = optional.Some(raw[:idx]) - } - } +// GetFileLanguage tries to get the (linguist) language of the file content +func GetFileLanguage(ctx context.Context, gitRepo *git.Repository, treeish, treePath string) (string, error) { + attributesMap, err := attribute.CheckAttributes(ctx, gitRepo, treeish, attribute.CheckAttributeOpts{ + Attributes: []string{attribute.LinguistLanguage, attribute.GitlabLanguage}, + Filenames: []string{treePath}, + }) + if err != nil { + return "", err } - return language + + return attributesMap[treePath].GetLanguage().Value(), nil } diff --git a/modules/git/repo_language_stats_gogit.go b/modules/git/languagestats/language_stats_gogit.go index a34c03c781..418c05b157 100644 --- a/modules/git/repo_language_stats_gogit.go +++ b/modules/git/languagestats/language_stats_gogit.go @@ -3,13 +3,15 @@ //go:build gogit -package git +package languagestats import ( "bytes" "io" "code.gitea.io/gitea/modules/analyze" + git_module "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/optional" "github.com/go-enry/go-enry/v2" @@ -19,7 +21,7 @@ import ( ) // GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { +func GetLanguageStats(repo *git_module.Repository, commitID string) (map[string]int64, error) { r, err := git.PlainOpen(repo.Path) if err != nil { return nil, err @@ -40,8 +42,11 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } - checker, deferable := repo.CheckAttributeReader(commitID) - defer deferable() + checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes) + if err != nil { + return nil, err + } + defer checker.Close() // sizes contains the current calculated size of all files by language sizes := make(map[string]int64) @@ -62,43 +67,41 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err isDocumentation := optional.None[bool]() isDetectable := optional.None[bool]() - if checker != nil { - attrs, err := checker.CheckPath(f.Name) - if err == nil { - isVendored = AttributeToBool(attrs, AttributeLinguistVendored) - if isVendored.ValueOrDefault(false) { - return nil - } - - isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) - if isGenerated.ValueOrDefault(false) { - return nil - } + attrs, err := checker.CheckPath(f.Name) + if err == nil { + isVendored = attrs.GetVendored() + if isVendored.ValueOrDefault(false) { + return nil + } - isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) - if isDocumentation.ValueOrDefault(false) { - return nil - } + isGenerated = attrs.GetGenerated() + if isGenerated.ValueOrDefault(false) { + return nil + } - isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) - if !isDetectable.ValueOrDefault(true) { - return nil - } + isDocumentation = attrs.GetDocumentation() + if isDocumentation.ValueOrDefault(false) { + return nil + } - hasLanguage := TryReadLanguageAttribute(attrs) - if hasLanguage.Value() != "" { - language := hasLanguage.Value() + isDetectable = attrs.GetDetectable() + if !isDetectable.ValueOrDefault(true) { + return nil + } - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } + hasLanguage := attrs.GetLanguage() + if hasLanguage.Value() != "" { + language := hasLanguage.Value() - // this language will always be added to the size - sizes[language] += f.Size - return nil + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if len(group) != 0 { + language = group } + + // this language will always be added to the size + sizes[language] += f.Size + return nil } } diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/languagestats/language_stats_nogogit.go index de7707bd6c..94cf9fff8c 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/languagestats/language_stats_nogogit.go @@ -3,13 +3,15 @@ //go:build !gogit -package git +package languagestats import ( "bytes" "io" "code.gitea.io/gitea/modules/analyze" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" @@ -17,7 +19,7 @@ import ( ) // GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { +func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64, error) { // We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. // so let's create a batch stdin and stdout batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx) @@ -34,19 +36,19 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if err := writeID(commitID); err != nil { return nil, err } - shaBytes, typ, size, err := ReadBatchLine(batchReader) + shaBytes, typ, size, err := git.ReadBatchLine(batchReader) if typ != "commit" { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} + return nil, git.ErrNotExist{ID: commitID} } - sha, err := NewIDFromString(string(shaBytes)) + sha, err := git.NewIDFromString(string(shaBytes)) if err != nil { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} + return nil, git.ErrNotExist{ID: commitID} } - commit, err := CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) + commit, err := git.CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) if err != nil { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) return nil, err @@ -62,8 +64,11 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } - checker, deferable := repo.CheckAttributeReader(commitID) - defer deferable() + checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes) + if err != nil { + return nil, err + } + defer checker.Close() contentBuf := bytes.Buffer{} var content []byte @@ -92,47 +97,40 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err } isVendored := optional.None[bool]() - isGenerated := optional.None[bool]() isDocumentation := optional.None[bool]() isDetectable := optional.None[bool]() - if checker != nil { - attrs, err := checker.CheckPath(f.Name()) - if err == nil { - isVendored = AttributeToBool(attrs, AttributeLinguistVendored) - if isVendored.ValueOrDefault(false) { - continue - } - - isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) - if isGenerated.ValueOrDefault(false) { - continue - } + attrs, err := checker.CheckPath(f.Name()) + attrLinguistGenerated := optional.None[bool]() + if err == nil { + if isVendored = attrs.GetVendored(); isVendored.ValueOrDefault(false) { + continue + } - isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) - if isDocumentation.ValueOrDefault(false) { - continue - } + if attrLinguistGenerated = attrs.GetGenerated(); attrLinguistGenerated.ValueOrDefault(false) { + continue + } - isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) - if !isDetectable.ValueOrDefault(true) { - continue - } + if isDocumentation = attrs.GetDocumentation(); isDocumentation.ValueOrDefault(false) { + continue + } - hasLanguage := TryReadLanguageAttribute(attrs) - if hasLanguage.Value() != "" { - language := hasLanguage.Value() + if isDetectable = attrs.GetDetectable(); !isDetectable.ValueOrDefault(true) { + continue + } - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } + if hasLanguage := attrs.GetLanguage(); hasLanguage.Value() != "" { + language := hasLanguage.Value() - // this language will always be added to the size - sizes[language] += f.Size() - continue + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if len(group) != 0 { + language = group } + + // this language will always be added to the size + sizes[language] += f.Size() + continue } } @@ -149,7 +147,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if err := writeID(f.ID.String()); err != nil { return nil, err } - _, _, size, err := ReadBatchLine(batchReader) + _, _, size, err := git.ReadBatchLine(batchReader) if err != nil { log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err) return nil, err @@ -167,11 +165,19 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } content = contentBuf.Bytes() - if err := DiscardFull(batchReader, discard); err != nil { + if err := git.DiscardFull(batchReader, discard); err != nil { return nil, err } } - if !isGenerated.Has() && enry.IsGenerated(f.Name(), content) { + + // if "generated" attribute is set, use it, otherwise use enry.IsGenerated to guess + var isGenerated bool + if attrLinguistGenerated.Has() { + isGenerated = attrLinguistGenerated.Value() + } else { + isGenerated = enry.IsGenerated(f.Name(), content) + } + if isGenerated { continue } diff --git a/modules/git/repo_language_stats_test.go b/modules/git/languagestats/language_stats_test.go index 1ee5f4c3af..b908ae6413 100644 --- a/modules/git/repo_language_stats_test.go +++ b/modules/git/languagestats/language_stats_test.go @@ -3,34 +3,36 @@ //go:build !gogit -package git +package languagestats import ( - "path/filepath" "testing" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRepository_GetLanguageStats(t *testing.T) { - repoPath := filepath.Join(testReposDir, "language_stats_repo") - gitRepo, err := openRepositoryWithDefaultContext(repoPath) + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) require.NoError(t, err) - defer gitRepo.Close() - stats, err := gitRepo.GetLanguageStats("8fee858da5796dfb37704761701bb8e800ad9ef3") + stats, err := GetLanguageStats(gitRepo, "8fee858da5796dfb37704761701bb8e800ad9ef3") require.NoError(t, err) - assert.EqualValues(t, map[string]int64{ + assert.Equal(t, map[string]int64{ "Python": 134, "Java": 112, }, stats) } func TestMergeLanguageStats(t *testing.T) { - assert.EqualValues(t, map[string]int64{ + assert.Equal(t, map[string]int64{ "PHP": 1, "python": 10, "JAVA": 700, diff --git a/modules/git/languagestats/main_test.go b/modules/git/languagestats/main_test.go new file mode 100644 index 0000000000..707d268c81 --- /dev/null +++ b/modules/git/languagestats/main_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package languagestats + +import ( + "context" + "fmt" + "os" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func testRun(m *testing.M) error { + gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + if err != nil { + return fmt.Errorf("unable to create temp dir: %w", err) + } + defer util.RemoveAll(gitHomePath) + setting.Git.HomePath = gitHomePath + + if err = git.InitFull(context.Background()); err != nil { + return fmt.Errorf("failed to call Init: %w", err) + } + + exitCode := m.Run() + if exitCode != 0 { + return fmt.Errorf("run test failed, ExitCode=%d", exitCode) + } + return nil +} + +func TestMain(m *testing.M) { + if err := testRun(m); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Test failed: %v", err) + os.Exit(1) + } +} diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index cf9c10d7b4..cff2556083 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -13,7 +13,7 @@ import ( ) func getCacheKey(repoPath, commitID, entryPath string) string { - hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, commitID, entryPath))) + hashBytes := sha256.Sum256(fmt.Appendf(nil, "%s:%s:%s", repoPath, commitID, entryPath)) return fmt.Sprintf("last_commit:%x", hashBytes) } diff --git a/modules/git/log_name_status.go b/modules/git/log_name_status.go index 0e9e22f1dc..dfdef38ef9 100644 --- a/modules/git/log_name_status.go +++ b/modules/git/log_name_status.go @@ -118,11 +118,12 @@ func (g *LogNameStatusRepoParser) Next(treepath string, paths2ids map[string]int g.buffull = false g.next, err = g.rd.ReadSlice('\x00') if err != nil { - if err == bufio.ErrBufferFull { + switch err { + case bufio.ErrBufferFull: g.buffull = true - } else if err == io.EOF { + case io.EOF: return nil, nil - } else { + default: return nil, err } } @@ -132,11 +133,12 @@ func (g *LogNameStatusRepoParser) Next(treepath string, paths2ids map[string]int if bytes.Equal(g.next, []byte("commit\000")) { g.next, err = g.rd.ReadSlice('\x00') if err != nil { - if err == bufio.ErrBufferFull { + switch err { + case bufio.ErrBufferFull: g.buffull = true - } else if err == io.EOF { + case io.EOF: return nil, nil - } else { + default: return nil, err } } @@ -214,11 +216,12 @@ diffloop: } g.next, err = g.rd.ReadSlice('\x00') if err != nil { - if err == bufio.ErrBufferFull { + switch err { + case bufio.ErrBufferFull: g.buffull = true - } else if err == io.EOF { + case io.EOF: return &ret, nil - } else { + default: return nil, err } } @@ -343,10 +346,7 @@ func WalkGitLog(ctx context.Context, repo *Repository, head *Commit, treepath st results := make([]string, len(paths)) remaining := len(paths) - nextRestart := (len(paths) * 3) / 4 - if nextRestart > 70 { - nextRestart = 70 - } + nextRestart := min((len(paths)*3)/4, 70) lastEmptyParent := head.ID.String() commitSinceLastEmptyParent := uint64(0) commitSinceNextRestart := uint64(0) diff --git a/modules/git/object_id.go b/modules/git/object_id.go index 82d30184df..25dfef3ec5 100644 --- a/modules/git/object_id.go +++ b/modules/git/object_id.go @@ -99,5 +99,5 @@ type ErrInvalidSHA struct { } func (err ErrInvalidSHA) Error() string { - return fmt.Sprintf("invalid sha: %s", err.SHA) + return "invalid sha: " + err.SHA } diff --git a/modules/git/parse_nogogit_test.go b/modules/git/parse_nogogit_test.go index a4436ce499..6594c84269 100644 --- a/modules/git/parse_nogogit_test.go +++ b/modules/git/parse_nogogit_test.go @@ -58,7 +58,7 @@ func TestParseTreeEntriesLong(t *testing.T) { assert.NoError(t, err) assert.Len(t, entries, len(testCase.Expected)) for i, entry := range entries { - assert.EqualValues(t, testCase.Expected[i], entry) + assert.Equal(t, testCase.Expected[i], entry) } } } @@ -91,7 +91,7 @@ func TestParseTreeEntriesShort(t *testing.T) { assert.NoError(t, err) assert.Len(t, entries, len(testCase.Expected)) for i, entry := range entries { - assert.EqualValues(t, testCase.Expected[i], entry) + assert.Equal(t, testCase.Expected[i], entry) } } } diff --git a/modules/git/ref.go b/modules/git/ref.go index f20a175e42..56b2db858a 100644 --- a/modules/git/ref.go +++ b/modules/git/ref.go @@ -109,8 +109,8 @@ func (ref RefName) IsFor() bool { } func (ref RefName) nameWithoutPrefix(prefix string) string { - if strings.HasPrefix(string(ref), prefix) { - return strings.TrimPrefix(string(ref), prefix) + if after, ok := strings.CutPrefix(string(ref), prefix); ok { + return after } return "" } diff --git a/modules/git/repo.go b/modules/git/repo.go index 6459adf851..f1f6902773 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -18,6 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" ) // GPGSettings represents the default GPG settings for this repository @@ -27,6 +28,7 @@ type GPGSettings struct { Email string Name string PublicKeyContent string + Format string } const prettyLogFormat = `--pretty=format:%H` @@ -42,9 +44,9 @@ func (repo *Repository) parsePrettyFormatLogToList(logs []byte) ([]*Commit, erro return commits, nil } - parts := bytes.Split(logs, []byte{'\n'}) + parts := bytes.SplitSeq(logs, []byte{'\n'}) - for _, commitID := range parts { + for commitID := range parts { commit, err := repo.GetCommit(string(commitID)) if err != nil { return nil, err @@ -266,11 +268,11 @@ func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch // CreateBundle create bundle content to the target path func (repo *Repository) CreateBundle(ctx context.Context, commit string, out io.Writer) error { - tmp, err := os.MkdirTemp(os.TempDir(), "gitea-bundle") + tmp, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-bundle") if err != nil { return err } - defer os.RemoveAll(tmp) + defer cleanup() env := append(os.Environ(), "GIT_OBJECT_DIRECTORY="+filepath.Join(repo.Path, "objects")) _, _, err = NewCommand("init", "--bare").RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) diff --git a/modules/git/repo_attribute.go b/modules/git/repo_attribute.go deleted file mode 100644 index 89101e5af3..0000000000 --- a/modules/git/repo_attribute.go +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - "path/filepath" - "time" - - "code.gitea.io/gitea/modules/log" -) - -// CheckAttributeOpts represents the possible options to CheckAttribute -type CheckAttributeOpts struct { - CachedOnly bool - AllAttributes bool - Attributes []string - Filenames []string - IndexFile string - WorkTree string -} - -// CheckAttribute return the Blame object of file -func (repo *Repository) CheckAttribute(opts CheckAttributeOpts) (map[string]map[string]string, error) { - env := []string{} - - if len(opts.IndexFile) > 0 { - env = append(env, "GIT_INDEX_FILE="+opts.IndexFile) - } - if len(opts.WorkTree) > 0 { - env = append(env, "GIT_WORK_TREE="+opts.WorkTree) - } - - if len(env) > 0 { - env = append(os.Environ(), env...) - } - - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - - cmd := NewCommand("check-attr", "-z") - - if opts.AllAttributes { - cmd.AddArguments("-a") - } else { - for _, attribute := range opts.Attributes { - if attribute != "" { - cmd.AddDynamicArguments(attribute) - } - } - } - - if opts.CachedOnly { - cmd.AddArguments("--cached") - } - - cmd.AddDashesAndList(opts.Filenames...) - - if err := cmd.Run(repo.Ctx, &RunOpts{ - Env: env, - Dir: repo.Path, - Stdout: stdOut, - Stderr: stdErr, - }); err != nil { - return nil, fmt.Errorf("failed to run check-attr: %w\n%s\n%s", err, stdOut.String(), stdErr.String()) - } - - // FIXME: This is incorrect on versions < 1.8.5 - fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) - - if len(fields)%3 != 1 { - return nil, fmt.Errorf("wrong number of fields in return from check-attr") - } - - name2attribute2info := make(map[string]map[string]string) - - for i := 0; i < (len(fields) / 3); i++ { - filename := string(fields[3*i]) - attribute := string(fields[3*i+1]) - info := string(fields[3*i+2]) - attribute2info := name2attribute2info[filename] - if attribute2info == nil { - attribute2info = make(map[string]string) - } - attribute2info[attribute] = info - name2attribute2info[filename] = attribute2info - } - - return name2attribute2info, nil -} - -// CheckAttributeReader provides a reader for check-attribute content that can be long running -type CheckAttributeReader struct { - // params - Attributes []string - Repo *Repository - IndexFile string - WorkTree string - - stdinReader io.ReadCloser - stdinWriter *os.File - stdOut *nulSeparatedAttributeWriter - cmd *Command - env []string - ctx context.Context - cancel context.CancelFunc -} - -// Init initializes the CheckAttributeReader -func (c *CheckAttributeReader) Init(ctx context.Context) error { - if len(c.Attributes) == 0 { - lw := new(nulSeparatedAttributeWriter) - lw.attributes = make(chan attributeTriple) - lw.closed = make(chan struct{}) - - c.stdOut = lw - c.stdOut.Close() - return fmt.Errorf("no provided Attributes to check") - } - - c.ctx, c.cancel = context.WithCancel(ctx) - c.cmd = NewCommand("check-attr", "--stdin", "-z") - - if len(c.IndexFile) > 0 { - c.cmd.AddArguments("--cached") - c.env = append(c.env, "GIT_INDEX_FILE="+c.IndexFile) - } - - if len(c.WorkTree) > 0 { - c.env = append(c.env, "GIT_WORK_TREE="+c.WorkTree) - } - - c.env = append(c.env, "GIT_FLUSH=1") - - c.cmd.AddDynamicArguments(c.Attributes...) - - var err error - - c.stdinReader, c.stdinWriter, err = os.Pipe() - if err != nil { - c.cancel() - return err - } - - lw := new(nulSeparatedAttributeWriter) - lw.attributes = make(chan attributeTriple, 5) - lw.closed = make(chan struct{}) - c.stdOut = lw - return nil -} - -func (c *CheckAttributeReader) Run() error { - defer func() { - _ = c.stdinReader.Close() - _ = c.stdOut.Close() - }() - stdErr := new(bytes.Buffer) - err := c.cmd.Run(c.ctx, &RunOpts{ - Env: c.env, - Dir: c.Repo.Path, - Stdin: c.stdinReader, - Stdout: c.stdOut, - Stderr: stdErr, - }) - if err != nil && !IsErrCanceledOrKilled(err) { - return fmt.Errorf("failed to run attr-check. Error: %w\nStderr: %s", err, stdErr.String()) - } - return nil -} - -// CheckPath check attr for given path -func (c *CheckAttributeReader) CheckPath(path string) (rs map[string]string, err error) { - defer func() { - if err != nil && err != c.ctx.Err() { - log.Error("Unexpected error when checking path %s in %s, error: %v", path, filepath.Base(c.Repo.Path), err) - } - }() - - select { - case <-c.ctx.Done(): - return nil, c.ctx.Err() - default: - } - - if _, err = c.stdinWriter.Write([]byte(path + "\x00")); err != nil { - defer c.Close() - return nil, err - } - - reportTimeout := func() error { - stdOutClosed := false - select { - case <-c.stdOut.closed: - stdOutClosed = true - default: - } - debugMsg := fmt.Sprintf("check path %q in repo %q", path, filepath.Base(c.Repo.Path)) - debugMsg += fmt.Sprintf(", stdOut: tmp=%q, pos=%d, closed=%v", string(c.stdOut.tmp), c.stdOut.pos, stdOutClosed) - if c.cmd.cmd != nil { - debugMsg += fmt.Sprintf(", process state: %q", c.cmd.cmd.ProcessState.String()) - } - _ = c.Close() - return fmt.Errorf("CheckPath timeout: %s", debugMsg) - } - - rs = make(map[string]string) - for range c.Attributes { - select { - case <-time.After(5 * time.Second): - // There is a strange "hang" problem in gitdiff.GetDiff -> CheckPath - // So add a timeout here to mitigate the problem, and output more logs for debug purpose - // In real world, if CheckPath runs long than seconds, it blocks the end user's operation, - // and at the moment the CheckPath result is not so important, so we can just ignore it. - return nil, reportTimeout() - case attr, ok := <-c.stdOut.ReadAttribute(): - if !ok { - return nil, c.ctx.Err() - } - rs[attr.Attribute] = attr.Value - case <-c.ctx.Done(): - return nil, c.ctx.Err() - } - } - return rs, nil -} - -func (c *CheckAttributeReader) Close() error { - c.cancel() - err := c.stdinWriter.Close() - return err -} - -type attributeTriple struct { - Filename string - Attribute string - Value string -} - -type nulSeparatedAttributeWriter struct { - tmp []byte - attributes chan attributeTriple - closed chan struct{} - working attributeTriple - pos int -} - -func (wr *nulSeparatedAttributeWriter) Write(p []byte) (n int, err error) { - l, read := len(p), 0 - - nulIdx := bytes.IndexByte(p, '\x00') - for nulIdx >= 0 { - wr.tmp = append(wr.tmp, p[:nulIdx]...) - switch wr.pos { - case 0: - wr.working = attributeTriple{ - Filename: string(wr.tmp), - } - case 1: - wr.working.Attribute = string(wr.tmp) - case 2: - wr.working.Value = string(wr.tmp) - } - wr.tmp = wr.tmp[:0] - wr.pos++ - if wr.pos > 2 { - wr.attributes <- wr.working - wr.pos = 0 - } - read += nulIdx + 1 - if l > read { - p = p[nulIdx+1:] - nulIdx = bytes.IndexByte(p, '\x00') - } else { - return l, nil - } - } - wr.tmp = append(wr.tmp, p...) - return len(p), nil -} - -func (wr *nulSeparatedAttributeWriter) ReadAttribute() <-chan attributeTriple { - return wr.attributes -} - -func (wr *nulSeparatedAttributeWriter) Close() error { - select { - case <-wr.closed: - return nil - default: - } - close(wr.attributes) - close(wr.closed) - return nil -} - -// CheckAttributeReader creates a check attribute reader for the current repository and provided commit ID -func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeReader, context.CancelFunc) { - indexFilename, worktree, deleteTemporaryFile, err := repo.ReadTreeToTemporaryIndex(commitID) - if err != nil { - return nil, func() {} - } - - checker := &CheckAttributeReader{ - Attributes: []string{ - AttributeLinguistVendored, - AttributeLinguistGenerated, - AttributeLinguistDocumentation, - AttributeLinguistDetectable, - AttributeLinguistLanguage, - AttributeGitlabLanguage, - }, - Repo: repo, - IndexFile: indexFilename, - WorkTree: worktree, - } - ctx, cancel := context.WithCancel(repo.Ctx) - if err := checker.Init(ctx); err != nil { - log.Error("Unable to open attribute checker for commit %s, error: %v", commitID, err) - } else { - go func() { - err := checker.Run() - if err != nil && !IsErrCanceledOrKilled(err) { - log.Error("Attribute checker for commit %s exits with error: %v", commitID, err) - } - cancel() - }() - } - deferrable := func() { - _ = checker.Close() - cancel() - deleteTemporaryFile() - } - - return checker, deferrable -} diff --git a/modules/git/repo_attribute_test.go b/modules/git/repo_attribute_test.go deleted file mode 100644 index e0bde146f6..0000000000 --- a/modules/git/repo_attribute_test.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "context" - mathRand "math/rand/v2" - "path/filepath" - "slices" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { - wr := &nulSeparatedAttributeWriter{ - attributes: make(chan attributeTriple, 5), - } - - testStr := ".gitignore\"\n\x00linguist-vendored\x00unspecified\x00" - - n, err := wr.Write([]byte(testStr)) - - assert.Len(t, testStr, n) - assert.NoError(t, err) - select { - case attr := <-wr.ReadAttribute(): - assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, AttributeLinguistVendored, attr.Attribute) - assert.Equal(t, "unspecified", attr.Value) - case <-time.After(100 * time.Millisecond): - assert.FailNow(t, "took too long to read an attribute from the list") - } - // Write a second attribute again - n, err = wr.Write([]byte(testStr)) - - assert.Len(t, testStr, n) - assert.NoError(t, err) - - select { - case attr := <-wr.ReadAttribute(): - assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, AttributeLinguistVendored, attr.Attribute) - assert.Equal(t, "unspecified", attr.Value) - case <-time.After(100 * time.Millisecond): - assert.FailNow(t, "took too long to read an attribute from the list") - } - - // Write a partial attribute - _, err = wr.Write([]byte("incomplete-file")) - assert.NoError(t, err) - _, err = wr.Write([]byte("name\x00")) - assert.NoError(t, err) - - select { - case <-wr.ReadAttribute(): - assert.FailNow(t, "There should not be an attribute ready to read") - case <-time.After(100 * time.Millisecond): - } - _, err = wr.Write([]byte("attribute\x00")) - assert.NoError(t, err) - select { - case <-wr.ReadAttribute(): - assert.FailNow(t, "There should not be an attribute ready to read") - case <-time.After(100 * time.Millisecond): - } - - _, err = wr.Write([]byte("value\x00")) - assert.NoError(t, err) - - attr := <-wr.ReadAttribute() - assert.Equal(t, "incomplete-filename", attr.Filename) - assert.Equal(t, "attribute", attr.Attribute) - assert.Equal(t, "value", attr.Value) - - _, err = wr.Write([]byte("shouldbe.vendor\x00linguist-vendored\x00set\x00shouldbe.vendor\x00linguist-generated\x00unspecified\x00shouldbe.vendor\x00linguist-language\x00unspecified\x00")) - assert.NoError(t, err) - attr = <-wr.ReadAttribute() - assert.NoError(t, err) - assert.EqualValues(t, attributeTriple{ - Filename: "shouldbe.vendor", - Attribute: AttributeLinguistVendored, - Value: "set", - }, attr) - attr = <-wr.ReadAttribute() - assert.NoError(t, err) - assert.EqualValues(t, attributeTriple{ - Filename: "shouldbe.vendor", - Attribute: AttributeLinguistGenerated, - Value: "unspecified", - }, attr) - attr = <-wr.ReadAttribute() - assert.NoError(t, err) - assert.EqualValues(t, attributeTriple{ - Filename: "shouldbe.vendor", - Attribute: AttributeLinguistLanguage, - Value: "unspecified", - }, attr) -} - -func TestAttributeReader(t *testing.T) { - t.Skip() // for debug purpose only, do not run in CI - - ctx := t.Context() - - timeout := 1 * time.Second - repoPath := filepath.Join(testReposDir, "language_stats_repo") - commitRef := "HEAD" - - oneRound := func(t *testing.T, roundIdx int) { - ctx, cancel := context.WithTimeout(ctx, timeout) - _ = cancel - gitRepo, err := OpenRepository(ctx, repoPath) - require.NoError(t, err) - defer gitRepo.Close() - - commit, err := gitRepo.GetCommit(commitRef) - require.NoError(t, err) - - files, err := gitRepo.LsFiles() - require.NoError(t, err) - - randomFiles := slices.Clone(files) - randomFiles = append(randomFiles, "any-file-1", "any-file-2") - - t.Logf("Round %v with %d files", roundIdx, len(randomFiles)) - - attrReader, deferrable := gitRepo.CheckAttributeReader(commit.ID.String()) - defer deferrable() - - wg := sync.WaitGroup{} - wg.Add(1) - - go func() { - for { - file := randomFiles[mathRand.IntN(len(randomFiles))] - _, err := attrReader.CheckPath(file) - if err != nil { - for i := 0; i < 10; i++ { - _, _ = attrReader.CheckPath(file) - } - break - } - } - wg.Done() - }() - wg.Wait() - } - - for i := 0; i < 100; i++ { - oneRound(t, i) - } -} diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go index 0ca1ea79c2..293aca159c 100644 --- a/modules/git/repo_base_gogit.go +++ b/modules/git/repo_base_gogit.go @@ -49,7 +49,12 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { repoPath, err := filepath.Abs(repoPath) if err != nil { return nil, err - } else if !isDir(repoPath) { + } + exist, err := util.IsDir(repoPath) + if err != nil { + return nil, err + } + if !exist { return nil, util.NewNotExistErrorf("no such file or directory") } diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go index 477e3b8742..6f9bfd4b43 100644 --- a/modules/git/repo_base_nogogit.go +++ b/modules/git/repo_base_nogogit.go @@ -47,7 +47,12 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { repoPath, err := filepath.Abs(repoPath) if err != nil { return nil, err - } else if !isDir(repoPath) { + } + exist, err := util.IsDir(repoPath) + if err != nil { + return nil, err + } + if !exist { return nil, util.NewNotExistErrorf("no such file or directory") } @@ -62,7 +67,7 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { if repo.batch == nil { var err error - repo.batch, err = repo.NewBatch(ctx) + repo.batch, err = NewBatch(ctx, repo.Path) if err != nil { return nil, nil, nil, err } @@ -76,7 +81,7 @@ func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bu } log.Debug("Opening temporary cat file batch for: %s", repo.Path) - tempBatch, err := repo.NewBatch(ctx) + tempBatch, err := NewBatch(ctx, repo.Path) if err != nil { return nil, nil, nil, err } @@ -87,7 +92,7 @@ func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bu func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { if repo.check == nil { var err error - repo.check, err = repo.NewBatchCheck(ctx) + repo.check, err = NewBatchCheck(ctx, repo.Path) if err != nil { return nil, nil, nil, err } @@ -101,7 +106,7 @@ func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError } log.Debug("Opening temporary cat file batch-check for: %s", repo.Path) - tempBatchCheck, err := repo.NewBatchCheck(ctx) + tempBatchCheck, err := NewBatchCheck(ctx, repo.Path) if err != nil { return nil, nil, nil, err } diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go index 916391f167..e7ecf53f51 100644 --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -7,7 +7,6 @@ package git import ( "context" "errors" - "fmt" "strings" ) @@ -25,36 +24,6 @@ func IsBranchExist(ctx context.Context, repoPath, name string) bool { return IsReferenceExist(ctx, repoPath, BranchPrefix+name) } -// Branch represents a Git branch. -type Branch struct { - Name string - Path string - - gitRepo *Repository -} - -// GetHEADBranch returns corresponding branch of HEAD. -func (repo *Repository) GetHEADBranch() (*Branch, error) { - if repo == nil { - return nil, fmt.Errorf("nil repo") - } - stdout, _, err := NewCommand("symbolic-ref", "HEAD").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) - if err != nil { - return nil, err - } - stdout = strings.TrimSpace(stdout) - - if !strings.HasPrefix(stdout, BranchPrefix) { - return nil, fmt.Errorf("invalid HEAD branch: %v", stdout) - } - - return &Branch{ - Name: stdout[len(BranchPrefix):], - Path: stdout, - gitRepo: repo, - }, nil -} - func GetDefaultBranch(ctx context.Context, repoPath string) (string, error) { stdout, _, err := NewCommand("symbolic-ref", "HEAD").RunStdString(ctx, &RunOpts{Dir: repoPath}) if err != nil { @@ -67,37 +36,6 @@ func GetDefaultBranch(ctx context.Context, repoPath string) (string, error) { return strings.TrimPrefix(stdout, BranchPrefix), nil } -// GetBranch returns a branch by it's name -func (repo *Repository) GetBranch(branch string) (*Branch, error) { - if !repo.IsBranchExist(branch) { - return nil, ErrBranchNotExist{branch} - } - return &Branch{ - Path: repo.Path, - Name: branch, - gitRepo: repo, - }, nil -} - -// GetBranches returns a slice of *git.Branch -func (repo *Repository) GetBranches(skip, limit int) ([]*Branch, int, error) { - brs, countAll, err := repo.GetBranchNames(skip, limit) - if err != nil { - return nil, 0, err - } - - branches := make([]*Branch, len(brs)) - for i := range brs { - branches[i] = &Branch{ - Path: repo.Path, - Name: brs[i], - gitRepo: repo, - } - } - - return branches, countAll, nil -} - // DeleteBranchOptions Option(s) for delete branch type DeleteBranchOptions struct { Force bool @@ -147,11 +85,6 @@ func (repo *Repository) RemoveRemote(name string) error { return err } -// GetCommit returns the head commit of a branch -func (branch *Branch) GetCommit() (*Commit, error) { - return branch.gitRepo.GetBranchCommit(branch.Name) -} - // RenameBranch rename a branch func (repo *Repository) RenameBranch(from, to string) error { _, _, err := NewCommand("branch", "-m").AddDynamicArguments(from, to).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) diff --git a/modules/git/repo_branch_test.go b/modules/git/repo_branch_test.go index cda170d976..8e8ea16fcd 100644 --- a/modules/git/repo_branch_test.go +++ b/modules/git/repo_branch_test.go @@ -21,21 +21,21 @@ func TestRepository_GetBranches(t *testing.T) { assert.NoError(t, err) assert.Len(t, branches, 2) - assert.EqualValues(t, 3, countAll) + assert.Equal(t, 3, countAll) assert.ElementsMatch(t, []string{"master", "branch2"}, branches) branches, countAll, err = bareRepo1.GetBranchNames(0, 0) assert.NoError(t, err) assert.Len(t, branches, 3) - assert.EqualValues(t, 3, countAll) + assert.Equal(t, 3, countAll) assert.ElementsMatch(t, []string{"master", "branch2", "branch1"}, branches) branches, countAll, err = bareRepo1.GetBranchNames(5, 1) assert.NoError(t, err) assert.Empty(t, branches) - assert.EqualValues(t, 3, countAll) + assert.Equal(t, 3, countAll) assert.ElementsMatch(t, []string{}, branches) } @@ -71,15 +71,15 @@ func TestGetRefsBySha(t *testing.T) { // refs/pull/1/head branches, err = bareRepo5.GetRefsBySha("c83380d7056593c51a699d12b9c00627bd5743e9", PullPrefix) assert.NoError(t, err) - assert.EqualValues(t, []string{"refs/pull/1/head"}, branches) + assert.Equal(t, []string{"refs/pull/1/head"}, branches) branches, err = bareRepo5.GetRefsBySha("d8e0bbb45f200e67d9a784ce55bd90821af45ebd", BranchPrefix) assert.NoError(t, err) - assert.EqualValues(t, []string{"refs/heads/master", "refs/heads/master-clone"}, branches) + assert.Equal(t, []string{"refs/heads/master", "refs/heads/master-clone"}, branches) branches, err = bareRepo5.GetRefsBySha("58a4bcc53ac13e7ff76127e0fb518b5262bf09af", BranchPrefix) assert.NoError(t, err) - assert.EqualValues(t, []string{"refs/heads/test-patch-1"}, branches) + assert.Equal(t, []string{"refs/heads/test-patch-1"}, branches) } func BenchmarkGetRefsBySha(b *testing.B) { diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index 72f35711f0..4066a1ca7b 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -89,7 +89,8 @@ func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) { return commits[0], nil } -func (repo *Repository) commitsByRange(id ObjectID, page, pageSize int, not string) ([]*Commit, error) { +// commitsByRangeWithTime returns the specific page commits before current revision, with not, since, until support +func (repo *Repository) commitsByRangeWithTime(id ObjectID, page, pageSize int, not, since, until string) ([]*Commit, error) { cmd := NewCommand("log"). AddOptionFormat("--skip=%d", (page-1)*pageSize). AddOptionFormat("--max-count=%d", pageSize). @@ -99,6 +100,12 @@ func (repo *Repository) commitsByRange(id ObjectID, page, pageSize int, not stri if not != "" { cmd.AddOptionValues("--not", not) } + if since != "" { + cmd.AddOptionFormat("--since=%s", since) + } + if until != "" { + cmd.AddOptionFormat("--until=%s", until) + } stdout, _, err := cmd.RunStdBytes(repo.Ctx, &RunOpts{Dir: repo.Path}) if err != nil { @@ -212,6 +219,8 @@ type CommitsByFileAndRangeOptions struct { File string Not string Page int + Since string + Until string } // CommitsByFileAndRange return the commits according revision file and the page @@ -231,6 +240,12 @@ func (repo *Repository) CommitsByFileAndRange(opts CommitsByFileAndRangeOptions) if opts.Not != "" { gitCmd.AddOptionValues("--not", opts.Not) } + if opts.Since != "" { + gitCmd.AddOptionFormat("--since=%s", opts.Since) + } + if opts.Until != "" { + gitCmd.AddOptionFormat("--until=%s", opts.Until) + } gitCmd.AddDashesAndList(opts.File) err := gitCmd.Run(repo.Ctx, &RunOpts{ @@ -532,11 +547,11 @@ func (repo *Repository) GetCommitBranchStart(env []string, branch, endCommitID s return "", runErr } - parts := bytes.Split(bytes.TrimSpace(stdout), []byte{'\n'}) + parts := bytes.SplitSeq(bytes.TrimSpace(stdout), []byte{'\n'}) // check the commits one by one until we find a commit contained by another branch // and we think this commit is the divergence point - for _, commitID := range parts { + for commitID := range parts { branches, err := repo.getBranches(env, string(commitID), 2) if err != nil { return "", err diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index 5aa0e9ec04..3ead3e2216 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -81,10 +81,10 @@ func (repo *Repository) getCommit(id ObjectID) (*Commit, error) { _, _ = wr.Write([]byte(id.String() + "\n")) - return repo.getCommitFromBatchReader(rd, id) + return repo.getCommitFromBatchReader(wr, rd, id) } -func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id ObjectID) (*Commit, error) { +func (repo *Repository) getCommitFromBatchReader(wr WriteCloserError, rd *bufio.Reader, id ObjectID) (*Commit, error) { _, typ, size, err := ReadBatchLine(rd) if err != nil { if errors.Is(err, io.EOF) || IsErrNotExist(err) { @@ -112,7 +112,11 @@ func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id ObjectID) return nil, err } - commit, err := tag.Commit(repo) + if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { + return nil, err + } + + commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) if err != nil { return nil, err } diff --git a/modules/git/repo_commitgraph_gogit.go b/modules/git/repo_commitgraph_gogit.go index d3182f15c6..c0082b62c8 100644 --- a/modules/git/repo_commitgraph_gogit.go +++ b/modules/git/repo_commitgraph_gogit.go @@ -8,7 +8,7 @@ package git import ( "os" - "path" + "path/filepath" gitealog "code.gitea.io/gitea/modules/log" @@ -18,7 +18,7 @@ import ( // CommitNodeIndex returns the index for walking commit graph func (r *Repository) CommitNodeIndex() (cgobject.CommitNodeIndex, *os.File) { - indexPath := path.Join(r.Path, "objects", "info", "commit-graph") + indexPath := filepath.Join(r.Path, "objects", "info", "commit-graph") file, err := os.Open(indexPath) if err == nil { diff --git a/modules/git/repo_gpg.go b/modules/git/repo_gpg.go index 8f91b4dce5..0021a7bda7 100644 --- a/modules/git/repo_gpg.go +++ b/modules/git/repo_gpg.go @@ -6,6 +6,7 @@ package git import ( "fmt" + "os" "strings" "code.gitea.io/gitea/modules/process" @@ -13,6 +14,14 @@ import ( // LoadPublicKeyContent will load the key from gpg func (gpgSettings *GPGSettings) LoadPublicKeyContent() error { + if gpgSettings.Format == SigningKeyFormatSSH { + content, err := os.ReadFile(gpgSettings.KeyID) + if err != nil { + return fmt.Errorf("unable to read SSH public key file: %s, %w", gpgSettings.KeyID, err) + } + gpgSettings.PublicKeyContent = string(content) + return nil + } content, stderr, err := process.GetManager().Exec( "gpg -a --export", "gpg", "-a", "--export", gpgSettings.KeyID) @@ -44,6 +53,9 @@ func (repo *Repository) GetDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, signingKey, _, _ := NewCommand("config", "--get", "user.signingkey").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) gpgSettings.KeyID = strings.TrimSpace(signingKey) + format, _, _ := NewCommand("config", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) + gpgSettings.Format = strings.TrimSpace(format) + defaultEmail, _, _ := NewCommand("config", "--get", "user.email").RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) gpgSettings.Email = strings.TrimSpace(defaultEmail) diff --git a/modules/git/repo_index.go b/modules/git/repo_index.go index 1c7fcc063e..4879121a41 100644 --- a/modules/git/repo_index.go +++ b/modules/git/repo_index.go @@ -10,8 +10,7 @@ import ( "path/filepath" "strings" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/setting" ) // ReadTreeToIndex reads a treeish to the index @@ -59,26 +58,18 @@ func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (tmpIndexFilena } }() - removeDirFn := func(dir string) func() { // it can't use the return value "tmpDir" directly because it is empty when error occurs - return func() { - if err := util.RemoveAll(dir); err != nil { - log.Error("failed to remove tmp index dir: %v", err) - } - } - } - - tmpDir, err = os.MkdirTemp("", "index") + tmpDir, cancel, err = setting.AppDataTempDir("git-repo-content").MkdirTempRandom("index") if err != nil { return "", "", nil, err } tmpIndexFilename = filepath.Join(tmpDir, ".tmp-index") - cancel = removeDirFn(tmpDir) + err = repo.ReadTreeToIndex(treeish, tmpIndexFilename) if err != nil { return "", "", cancel, err } - return tmpIndexFilename, tmpDir, cancel, err + return tmpIndexFilename, tmpDir, cancel, nil } // EmptyIndex empties the index @@ -95,7 +86,7 @@ func (repo *Repository) LsFiles(filenames ...string) ([]string, error) { return nil, err } filelist := make([]string, 0, len(filenames)) - for _, line := range bytes.Split(res, []byte{'\000'}) { + for line := range bytes.SplitSeq(res, []byte{'\000'}) { filelist = append(filelist, string(line)) } diff --git a/modules/git/repo_object.go b/modules/git/repo_object.go index 1d34305270..08e0413311 100644 --- a/modules/git/repo_object.go +++ b/modules/git/repo_object.go @@ -85,17 +85,3 @@ func (repo *Repository) hashObject(reader io.Reader, save bool) (string, error) } return strings.TrimSpace(stdout.String()), nil } - -// GetRefType gets the type of the ref based on the string -func (repo *Repository) GetRefType(ref string) ObjectType { - if repo.IsTagExist(ref) { - return ObjectTag - } else if repo.IsBranchExist(ref) { - return ObjectBranch - } else if repo.IsCommitExist(ref) { - return ObjectCommit - } else if _, err := repo.GetBlob(ref); err == nil { - return ObjectBlob - } - return ObjectType("invalid") -} diff --git a/modules/git/repo_ref.go b/modules/git/repo_ref.go index 739cfb972c..554f9f73e1 100644 --- a/modules/git/repo_ref.go +++ b/modules/git/repo_ref.go @@ -19,11 +19,12 @@ func (repo *Repository) GetRefs() ([]*Reference, error) { // refType should only be a literal "branch" or "tag" and nothing else func (repo *Repository) ListOccurrences(ctx context.Context, refType, commitSHA string) ([]string, error) { cmd := NewCommand() - if refType == "branch" { + switch refType { + case "branch": cmd.AddArguments("branch") - } else if refType == "tag" { + case "tag": cmd.AddArguments("tag") - } else { + default: return nil, util.NewInvalidArgumentErrorf(`can only use "branch" or "tag" for refType, but got %q`, refType) } stdout, _, err := cmd.AddArguments("--no-color", "--sort=-creatordate", "--contains").AddDynamicArguments(commitSHA).RunStdString(ctx, &RunOpts{Dir: repo.Path}) diff --git a/modules/git/repo_stats.go b/modules/git/repo_stats.go index 76fe92bb34..8c6f31c38c 100644 --- a/modules/git/repo_stats.go +++ b/modules/git/repo_stats.go @@ -40,7 +40,9 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) since := fromTime.Format(time.RFC3339) - stdout, _, runErr := NewCommand("rev-list", "--count", "--no-merges", "--branches=*", "--date=iso").AddOptionFormat("--since='%s'", since).RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand("rev-list", "--count", "--no-merges", "--branches=*", "--date=iso"). + AddOptionFormat("--since=%s", since). + RunStdString(repo.Ctx, &RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -60,7 +62,8 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) _ = stdoutWriter.Close() }() - gitCmd := NewCommand("log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso").AddOptionFormat("--since='%s'", since) + gitCmd := NewCommand("log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso"). + AddOptionFormat("--since=%s", since) if len(branch) == 0 { gitCmd.AddArguments("--branches=*") } else { diff --git a/modules/git/repo_stats_test.go b/modules/git/repo_stats_test.go index 3d032385ee..85d8807a6e 100644 --- a/modules/git/repo_stats_test.go +++ b/modules/git/repo_stats_test.go @@ -30,7 +30,7 @@ func TestRepository_GetCodeActivityStats(t *testing.T) { assert.EqualValues(t, 10, code.Additions) assert.EqualValues(t, 1, code.Deletions) assert.Len(t, code.Authors, 3) - assert.EqualValues(t, "tris.git@shoddynet.org", code.Authors[1].Email) + assert.Equal(t, "tris.git@shoddynet.org", code.Authors[1].Email) assert.EqualValues(t, 3, code.Authors[1].Commits) assert.EqualValues(t, 5, code.Authors[0].Commits) } diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index c74618471a..c8d72eee02 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -39,8 +39,8 @@ func (repo *Repository) GetTagNameBySHA(sha string) (string, error) { return "", err } - tagRefs := strings.Split(stdout, "\n") - for _, tagRef := range tagRefs { + tagRefs := strings.SplitSeq(stdout, "\n") + for tagRef := range tagRefs { if len(strings.TrimSpace(tagRef)) > 0 { fields := strings.Fields(tagRef) if strings.HasPrefix(fields[0], sha) && strings.HasPrefix(fields[1], TagPrefix) { @@ -62,7 +62,7 @@ func (repo *Repository) GetTagID(name string) (string, error) { return "", err } // Make sure exact match is used: "v1" != "release/v1" - for _, line := range strings.Split(stdout, "\n") { + for line := range strings.SplitSeq(stdout, "\n") { fields := strings.Fields(line) if len(fields) == 2 && fields[1] == "refs/tags/"+name { return fields[0], nil diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index e0a3104249..3d2b4f52bd 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -41,8 +41,11 @@ func (repo *Repository) GetTagType(id ObjectID) (string, error) { return "", err } _, typ, _, err := ReadBatchLine(rd) - if IsErrNotExist(err) { - return "", ErrNotExist{ID: id.String()} + if err != nil { + if IsErrNotExist(err) { + return "", ErrNotExist{ID: id.String()} + } + return "", err } return typ, nil } diff --git a/modules/git/repo_tag_test.go b/modules/git/repo_tag_test.go index f1f081680a..f1f5ff6664 100644 --- a/modules/git/repo_tag_test.go +++ b/modules/git/repo_tag_test.go @@ -27,12 +27,12 @@ func TestRepository_GetTags(t *testing.T) { } assert.Len(t, tags, 2) assert.Len(t, tags, total) - assert.EqualValues(t, "signed-tag", tags[0].Name) - assert.EqualValues(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", tags[0].ID.String()) - assert.EqualValues(t, "tag", tags[0].Type) - assert.EqualValues(t, "test", tags[1].Name) - assert.EqualValues(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[1].ID.String()) - assert.EqualValues(t, "tag", tags[1].Type) + assert.Equal(t, "signed-tag", tags[0].Name) + assert.Equal(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", tags[0].ID.String()) + assert.Equal(t, "tag", tags[0].Type) + assert.Equal(t, "test", tags[1].Name) + assert.Equal(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[1].ID.String()) + assert.Equal(t, "tag", tags[1].Type) } func TestRepository_GetTag(t *testing.T) { @@ -64,18 +64,13 @@ func TestRepository_GetTag(t *testing.T) { // and try to get the Tag for lightweight tag lTag, err := bareRepo1.GetTag(lTagName) - if err != nil { - assert.NoError(t, err) - return - } - if lTag == nil { - assert.NotNil(t, lTag) - assert.FailNow(t, "nil lTag: %s", lTagName) - } - assert.EqualValues(t, lTagName, lTag.Name) - assert.EqualValues(t, lTagCommitID, lTag.ID.String()) - assert.EqualValues(t, lTagCommitID, lTag.Object.String()) - assert.EqualValues(t, "commit", lTag.Type) + require.NoError(t, err) + require.NotNil(t, lTag, "nil lTag: %s", lTagName) + + assert.Equal(t, lTagName, lTag.Name) + assert.Equal(t, lTagCommitID, lTag.ID.String()) + assert.Equal(t, lTagCommitID, lTag.Object.String()) + assert.Equal(t, "commit", lTag.Type) // ANNOTATED TAGS aTagCommitID := "8006ff9adbf0cb94da7dad9e537e53817f9fa5c0" @@ -97,19 +92,14 @@ func TestRepository_GetTag(t *testing.T) { } aTag, err := bareRepo1.GetTag(aTagName) - if err != nil { - assert.NoError(t, err) - return - } - if aTag == nil { - assert.NotNil(t, aTag) - assert.FailNow(t, "nil aTag: %s", aTagName) - } - assert.EqualValues(t, aTagName, aTag.Name) - assert.EqualValues(t, aTagID, aTag.ID.String()) + require.NoError(t, err) + require.NotNil(t, aTag, "nil aTag: %s", aTagName) + + assert.Equal(t, aTagName, aTag.Name) + assert.Equal(t, aTagID, aTag.ID.String()) assert.NotEqual(t, aTagID, aTag.Object.String()) - assert.EqualValues(t, aTagCommitID, aTag.Object.String()) - assert.EqualValues(t, "tag", aTag.Type) + assert.Equal(t, aTagCommitID, aTag.Object.String()) + assert.Equal(t, "tag", aTag.Type) // RELEASE TAGS @@ -127,14 +117,14 @@ func TestRepository_GetTag(t *testing.T) { assert.NoError(t, err) return } - assert.EqualValues(t, rTagCommitID, rTagID) + assert.Equal(t, rTagCommitID, rTagID) oTagID, err := bareRepo1.GetTagID(lTagName) if err != nil { assert.NoError(t, err) return } - assert.EqualValues(t, lTagCommitID, oTagID) + assert.Equal(t, lTagCommitID, oTagID) } func TestRepository_GetAnnotatedTag(t *testing.T) { @@ -170,9 +160,9 @@ func TestRepository_GetAnnotatedTag(t *testing.T) { return } assert.NotNil(t, tag) - assert.EqualValues(t, aTagName, tag.Name) - assert.EqualValues(t, aTagID, tag.ID.String()) - assert.EqualValues(t, "tag", tag.Type) + assert.Equal(t, aTagName, tag.Name) + assert.Equal(t, aTagID, tag.ID.String()) + assert.Equal(t, "tag", tag.Type) // Annotated tag's Commit ID should fail tag2, err := bareRepo1.GetAnnotatedTag(aTagCommitID) diff --git a/modules/git/repo_tree.go b/modules/git/repo_tree.go index 70e5aee023..309a73d759 100644 --- a/modules/git/repo_tree.go +++ b/modules/git/repo_tree.go @@ -15,7 +15,7 @@ import ( type CommitTreeOpts struct { Parents []string Message string - KeyID string + Key *SigningKey NoGPGSign bool AlwaysSign bool } @@ -43,8 +43,13 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt _, _ = messageBytes.WriteString(opts.Message) _, _ = messageBytes.WriteString("\n") - if opts.KeyID != "" || opts.AlwaysSign { - cmd.AddOptionFormat("-S%s", opts.KeyID) + if opts.Key != nil { + if opts.Key.Format != "" { + cmd.AddConfig("gpg.format", opts.Key.Format) + } + cmd.AddOptionFormat("-S%s", opts.Key.KeyID) + } else if opts.AlwaysSign { + cmd.AddOptionFormat("-S") } if opts.NoGPGSign { diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go index d74769ccb2..1954f85162 100644 --- a/modules/git/repo_tree_nogogit.go +++ b/modules/git/repo_tree_nogogit.go @@ -35,7 +35,11 @@ func (repo *Repository) getTree(id ObjectID) (*Tree, error) { if err != nil { return nil, err } - commit, err := tag.Commit(repo) + + if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { + return nil, err + } + commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) if err != nil { return nil, err } diff --git a/modules/git/signature_test.go b/modules/git/signature_test.go index 92681feea9..b9b5aff61b 100644 --- a/modules/git/signature_test.go +++ b/modules/git/signature_test.go @@ -42,6 +42,6 @@ func TestParseSignatureFromCommitLine(t *testing.T) { } for _, test := range tests { got := parseSignatureFromCommitLine(test.line) - assert.EqualValues(t, test.want, got) + assert.Equal(t, test.want, got) } } diff --git a/modules/git/submodule_test.go b/modules/git/submodule_test.go index fbf8c32e1e..7893b95e3a 100644 --- a/modules/git/submodule_test.go +++ b/modules/git/submodule_test.go @@ -19,11 +19,11 @@ func TestGetTemplateSubmoduleCommits(t *testing.T) { assert.Len(t, submodules, 2) - assert.EqualValues(t, "<°)))><", submodules[0].Path) - assert.EqualValues(t, "d2932de67963f23d43e1c7ecf20173e92ee6c43c", submodules[0].Commit) + assert.Equal(t, "<°)))><", submodules[0].Path) + assert.Equal(t, "d2932de67963f23d43e1c7ecf20173e92ee6c43c", submodules[0].Commit) - assert.EqualValues(t, "libtest", submodules[1].Path) - assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[1].Commit) + assert.Equal(t, "libtest", submodules[1].Path) + assert.Equal(t, "1234567890123456789012345678901234567890", submodules[1].Commit) } func TestAddTemplateSubmoduleIndexes(t *testing.T) { @@ -42,6 +42,6 @@ func TestAddTemplateSubmoduleIndexes(t *testing.T) { submodules, err := GetTemplateSubmoduleCommits(DefaultContext, tmpDir) require.NoError(t, err) assert.Len(t, submodules, 1) - assert.EqualValues(t, "new-dir", submodules[0].Path) - assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[0].Commit) + assert.Equal(t, "new-dir", submodules[0].Path) + assert.Equal(t, "1234567890123456789012345678901234567890", submodules[0].Commit) } diff --git a/modules/git/tag.go b/modules/git/tag.go index f7666aa89b..8bf3658d62 100644 --- a/modules/git/tag.go +++ b/modules/git/tag.go @@ -21,11 +21,6 @@ type Tag struct { Signature *CommitSignature } -// Commit return the commit of the tag reference -func (tag *Tag) Commit(gitRepo *Repository) (*Commit, error) { - return gitRepo.getCommit(tag.Object) -} - func parsePayloadSignature(data []byte, messageStart int) (payload, msg, sign string) { pos := messageStart signStart, signEnd := -1, -1 diff --git a/modules/git/tests/repos/language_stats_repo/config b/modules/git/tests/repos/language_stats_repo/config index 515f483629..a4ef456cbc 100644 --- a/modules/git/tests/repos/language_stats_repo/config +++ b/modules/git/tests/repos/language_stats_repo/config @@ -1,5 +1,5 @@ [core] repositoryformatversion = 0 filemode = true - bare = false + bare = true logallrefupdates = true diff --git a/modules/git/tests/repos/repo3_notes/config b/modules/git/tests/repos/repo3_notes/config index d545cdabdb..5ed22e23d1 100644 --- a/modules/git/tests/repos/repo3_notes/config +++ b/modules/git/tests/repos/repo3_notes/config @@ -1,7 +1,7 @@ [core] repositoryformatversion = 0 filemode = false - bare = false + bare = true logallrefupdates = true symlinks = false ignorecase = true diff --git a/modules/git/tests/repos/repo4_commitsbetween/config b/modules/git/tests/repos/repo4_commitsbetween/config index d545cdabdb..5ed22e23d1 100644 --- a/modules/git/tests/repos/repo4_commitsbetween/config +++ b/modules/git/tests/repos/repo4_commitsbetween/config @@ -1,7 +1,7 @@ [core] repositoryformatversion = 0 filemode = false - bare = false + bare = true logallrefupdates = true symlinks = false ignorecase = true diff --git a/modules/git/tree.go b/modules/git/tree.go index f6fdff97d0..38fb45f3b1 100644 --- a/modules/git/tree.go +++ b/modules/git/tree.go @@ -56,7 +56,7 @@ func (repo *Repository) LsTree(ref string, filenames ...string) ([]string, error return nil, err } filelist := make([]string, 0, len(filenames)) - for _, line := range bytes.Split(res, []byte{'\000'}) { + for line := range bytes.SplitSeq(res, []byte{'\000'}) { filelist = append(filelist, string(line)) } diff --git a/modules/git/tree_blob_nogogit.go b/modules/git/tree_blob_nogogit.go index b7bcf40edd..b18d0fa05e 100644 --- a/modules/git/tree_blob_nogogit.go +++ b/modules/git/tree_blob_nogogit.go @@ -11,7 +11,7 @@ import ( ) // GetTreeEntryByPath get the tree entries according the sub dir -func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { +func (t *Tree) GetTreeEntryByPath(relpath string) (_ *TreeEntry, err error) { if len(relpath) == 0 { return &TreeEntry{ ptree: t, @@ -21,27 +21,25 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { }, nil } - // FIXME: This should probably use git cat-file --batch to be a bit more efficient relpath = path.Clean(relpath) parts := strings.Split(relpath, "/") - var err error + tree := t - for i, name := range parts { - if i == len(parts)-1 { - entries, err := tree.ListEntries() - if err != nil { - return nil, err - } - for _, v := range entries { - if v.Name() == name { - return v, nil - } - } - } else { - tree, err = tree.SubTree(name) - if err != nil { - return nil, err - } + for _, name := range parts[:len(parts)-1] { + tree, err = tree.SubTree(name) + if err != nil { + return nil, err + } + } + + name := parts[len(parts)-1] + entries, err := tree.ListEntries() + if err != nil { + return nil, err + } + for _, v := range entries { + if v.Name() == name { + return v, nil } } return nil, ErrNotExist{"", relpath} diff --git a/modules/git/tree_entry.go b/modules/git/tree_entry.go index 9513121487..5099d8ee79 100644 --- a/modules/git/tree_entry.go +++ b/modules/git/tree_entry.go @@ -5,9 +5,11 @@ package git import ( - "io" + "path" "sort" "strings" + + "code.gitea.io/gitea/modules/util" ) // Type returns the type of the entry (commit, tree, blob) @@ -22,83 +24,57 @@ func (te *TreeEntry) Type() string { } } -// FollowLink returns the entry pointed to by a symlink -func (te *TreeEntry) FollowLink() (*TreeEntry, error) { +type EntryFollowResult struct { + SymlinkContent string + TargetFullPath string + TargetEntry *TreeEntry +} + +func EntryFollowLink(commit *Commit, fullPath string, te *TreeEntry) (*EntryFollowResult, error) { if !te.IsLink() { - return nil, ErrBadLink{te.Name(), "not a symlink"} + return nil, util.ErrorWrap(util.ErrUnprocessableContent, "%q is not a symlink", fullPath) } - // read the link - r, err := te.Blob().DataAsync() - if err != nil { - return nil, err + // git's filename max length is 4096, hopefully a link won't be longer than multiple of that + const maxSymlinkSize = 20 * 4096 + if te.Blob().Size() > maxSymlinkSize { + return nil, util.ErrorWrap(util.ErrUnprocessableContent, "%q content exceeds symlink limit", fullPath) } - closed := false - defer func() { - if !closed { - _ = r.Close() - } - }() - buf := make([]byte, te.Size()) - _, err = io.ReadFull(r, buf) + + link, err := te.Blob().GetBlobContent(maxSymlinkSize) if err != nil { return nil, err } - _ = r.Close() - closed = true - - lnk := string(buf) - t := te.ptree - - // traverse up directories - for ; t != nil && strings.HasPrefix(lnk, "../"); lnk = lnk[3:] { - t = t.ptree + if strings.HasPrefix(link, "/") { + // It's said that absolute path will be stored as is in Git + return &EntryFollowResult{SymlinkContent: link}, util.ErrorWrap(util.ErrUnprocessableContent, "%q is an absolute symlink", fullPath) } - if t == nil { - return nil, ErrBadLink{te.Name(), "points outside of repo"} - } - - target, err := t.GetTreeEntryByPath(lnk) + targetFullPath := path.Join(path.Dir(fullPath), link) + targetEntry, err := commit.GetTreeEntryByPath(targetFullPath) if err != nil { - if IsErrNotExist(err) { - return nil, ErrBadLink{te.Name(), "broken link"} - } - return nil, err + return &EntryFollowResult{SymlinkContent: link}, err } - return target, nil + return &EntryFollowResult{SymlinkContent: link, TargetFullPath: targetFullPath, TargetEntry: targetEntry}, nil } -// FollowLinks returns the entry ultimately pointed to by a symlink -func (te *TreeEntry) FollowLinks() (*TreeEntry, error) { - if !te.IsLink() { - return nil, ErrBadLink{te.Name(), "not a symlink"} - } - entry := te - for i := 0; i < 999; i++ { - if entry.IsLink() { - next, err := entry.FollowLink() - if err != nil { - return nil, err - } - if next.ID == entry.ID { - return nil, ErrBadLink{ - entry.Name(), - "recursive link", - } - } - entry = next - } else { +func EntryFollowLinks(commit *Commit, firstFullPath string, firstTreeEntry *TreeEntry, optLimit ...int) (res *EntryFollowResult, err error) { + limit := util.OptionalArg(optLimit, 10) + treeEntry, fullPath := firstTreeEntry, firstFullPath + for range limit { + res, err = EntryFollowLink(commit, fullPath, treeEntry) + if err != nil { + return res, err + } + treeEntry, fullPath = res.TargetEntry, res.TargetFullPath + if !treeEntry.IsLink() { break } } - if entry.IsLink() { - return nil, ErrBadLink{ - te.Name(), - "too many levels of symbolic links", - } + if treeEntry.IsLink() { + return res, util.ErrorWrap(util.ErrUnprocessableContent, "%q has too many links", firstFullPath) } - return entry, nil + return res, nil } // returns the Tree pointed to by this TreeEntry, or nil if this is not a tree diff --git a/modules/git/tree_entry_common_test.go b/modules/git/tree_entry_common_test.go new file mode 100644 index 0000000000..8b63bbb993 --- /dev/null +++ b/modules/git/tree_entry_common_test.go @@ -0,0 +1,76 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "testing" + + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFollowLink(t *testing.T) { + r, err := openRepositoryWithDefaultContext("tests/repos/repo1_bare") + require.NoError(t, err) + defer r.Close() + + commit, err := r.GetCommit("37991dec2c8e592043f47155ce4808d4580f9123") + require.NoError(t, err) + + // get the symlink + { + lnkFullPath := "foo/bar/link_to_hello" + lnk, err := commit.Tree.GetTreeEntryByPath("foo/bar/link_to_hello") + require.NoError(t, err) + assert.True(t, lnk.IsLink()) + + // should be able to dereference to target + res, err := EntryFollowLink(commit, lnkFullPath, lnk) + require.NoError(t, err) + assert.Equal(t, "hello", res.TargetEntry.Name()) + assert.Equal(t, "foo/nar/hello", res.TargetFullPath) + assert.False(t, res.TargetEntry.IsLink()) + assert.Equal(t, "b14df6442ea5a1b382985a6549b85d435376c351", res.TargetEntry.ID.String()) + } + + { + // should error when called on a normal file + entry, err := commit.Tree.GetTreeEntryByPath("file1.txt") + require.NoError(t, err) + res, err := EntryFollowLink(commit, "file1.txt", entry) + assert.ErrorIs(t, err, util.ErrUnprocessableContent) + assert.Nil(t, res) + } + + { + // should error for broken links + entry, err := commit.Tree.GetTreeEntryByPath("foo/broken_link") + require.NoError(t, err) + assert.True(t, entry.IsLink()) + res, err := EntryFollowLink(commit, "foo/broken_link", entry) + assert.ErrorIs(t, err, util.ErrNotExist) + assert.Equal(t, "nar/broken_link", res.SymlinkContent) + } + + { + // should error for external links + entry, err := commit.Tree.GetTreeEntryByPath("foo/outside_repo") + require.NoError(t, err) + assert.True(t, entry.IsLink()) + res, err := EntryFollowLink(commit, "foo/outside_repo", entry) + assert.ErrorIs(t, err, util.ErrNotExist) + assert.Equal(t, "../../outside_repo", res.SymlinkContent) + } + + { + // testing fix for short link bug + entry, err := commit.Tree.GetTreeEntryByPath("foo/link_short") + require.NoError(t, err) + res, err := EntryFollowLink(commit, "foo/link_short", entry) + assert.ErrorIs(t, err, util.ErrNotExist) + assert.Equal(t, "a", res.SymlinkContent) + } +} diff --git a/modules/git/tree_entry_gogit.go b/modules/git/tree_entry_gogit.go index eb9b012681..e6845f1c77 100644 --- a/modules/git/tree_entry_gogit.go +++ b/modules/git/tree_entry_gogit.go @@ -19,16 +19,12 @@ type TreeEntry struct { gogitTreeEntry *object.TreeEntry ptree *Tree - size int64 - sized bool - fullName string + size int64 + sized bool } // Name returns the name of the entry func (te *TreeEntry) Name() string { - if te.fullName != "" { - return te.fullName - } return te.gogitTreeEntry.Name } @@ -55,7 +51,7 @@ func (te *TreeEntry) Size() int64 { return te.size } -// IsSubModule if the entry is a sub module +// IsSubModule if the entry is a submodule func (te *TreeEntry) IsSubModule() bool { return te.gogitTreeEntry.Mode == filemode.Submodule } diff --git a/modules/git/tree_entry_mode.go b/modules/git/tree_entry_mode.go index ec4487549d..f36c07bc2a 100644 --- a/modules/git/tree_entry_mode.go +++ b/modules/git/tree_entry_mode.go @@ -15,18 +15,14 @@ type EntryMode int // one of these. const ( // EntryModeNoEntry is possible if the file was added or removed in a commit. In the case of - // added the base commit will not have the file in its tree so a mode of 0o000000 is used. + // when adding the base commit doesn't have the file in its tree, a mode of 0o000000 is used. EntryModeNoEntry EntryMode = 0o000000 - // EntryModeBlob - EntryModeBlob EntryMode = 0o100644 - // EntryModeExec - EntryModeExec EntryMode = 0o100755 - // EntryModeSymlink + + EntryModeBlob EntryMode = 0o100644 + EntryModeExec EntryMode = 0o100755 EntryModeSymlink EntryMode = 0o120000 - // EntryModeCommit - EntryModeCommit EntryMode = 0o160000 - // EntryModeTree - EntryModeTree EntryMode = 0o040000 + EntryModeCommit EntryMode = 0o160000 + EntryModeTree EntryMode = 0o040000 ) // String converts an EntryMode to a string @@ -34,10 +30,29 @@ func (e EntryMode) String() string { return strconv.FormatInt(int64(e), 8) } -// ToEntryMode converts a string to an EntryMode -func ToEntryMode(value string) EntryMode { - v, _ := strconv.ParseInt(value, 8, 32) - return EntryMode(v) +// IsSubModule if the entry is a submodule +func (e EntryMode) IsSubModule() bool { + return e == EntryModeCommit +} + +// IsDir if the entry is a sub dir +func (e EntryMode) IsDir() bool { + return e == EntryModeTree +} + +// IsLink if the entry is a symlink +func (e EntryMode) IsLink() bool { + return e == EntryModeSymlink +} + +// IsRegular if the entry is a regular file +func (e EntryMode) IsRegular() bool { + return e == EntryModeBlob +} + +// IsExecutable if the entry is an executable file (not necessarily binary) +func (e EntryMode) IsExecutable() bool { + return e == EntryModeExec } func ParseEntryMode(mode string) (EntryMode, error) { diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go index 81fb638d56..8fad96cdf8 100644 --- a/modules/git/tree_entry_nogogit.go +++ b/modules/git/tree_entry_nogogit.go @@ -18,7 +18,7 @@ type TreeEntry struct { sized bool } -// Name returns the name of the entry +// Name returns the name of the entry (base name) func (te *TreeEntry) Name() string { return te.name } @@ -57,29 +57,29 @@ func (te *TreeEntry) Size() int64 { return te.size } -// IsSubModule if the entry is a sub module +// IsSubModule if the entry is a submodule func (te *TreeEntry) IsSubModule() bool { - return te.entryMode == EntryModeCommit + return te.entryMode.IsSubModule() } // IsDir if the entry is a sub dir func (te *TreeEntry) IsDir() bool { - return te.entryMode == EntryModeTree + return te.entryMode.IsDir() } // IsLink if the entry is a symlink func (te *TreeEntry) IsLink() bool { - return te.entryMode == EntryModeSymlink + return te.entryMode.IsLink() } // IsRegular if the entry is a regular file func (te *TreeEntry) IsRegular() bool { - return te.entryMode == EntryModeBlob + return te.entryMode.IsRegular() } // IsExecutable if the entry is an executable file (not necessarily binary) func (te *TreeEntry) IsExecutable() bool { - return te.entryMode == EntryModeExec + return te.entryMode.IsExecutable() } // Blob returns the blob object the entry diff --git a/modules/git/tree_entry_test.go b/modules/git/tree_entry_test.go index 30eee13669..9ca82675e0 100644 --- a/modules/git/tree_entry_test.go +++ b/modules/git/tree_entry_test.go @@ -53,50 +53,3 @@ func TestEntriesCustomSort(t *testing.T) { assert.Equal(t, "bcd", entries[6].Name()) assert.Equal(t, "abc", entries[7].Name()) } - -func TestFollowLink(t *testing.T) { - r, err := openRepositoryWithDefaultContext("tests/repos/repo1_bare") - assert.NoError(t, err) - defer r.Close() - - commit, err := r.GetCommit("37991dec2c8e592043f47155ce4808d4580f9123") - assert.NoError(t, err) - - // get the symlink - lnk, err := commit.Tree.GetTreeEntryByPath("foo/bar/link_to_hello") - assert.NoError(t, err) - assert.True(t, lnk.IsLink()) - - // should be able to dereference to target - target, err := lnk.FollowLink() - assert.NoError(t, err) - assert.Equal(t, "hello", target.Name()) - assert.False(t, target.IsLink()) - assert.Equal(t, "b14df6442ea5a1b382985a6549b85d435376c351", target.ID.String()) - - // should error when called on normal file - target, err = commit.Tree.GetTreeEntryByPath("file1.txt") - assert.NoError(t, err) - _, err = target.FollowLink() - assert.EqualError(t, err, "file1.txt: not a symlink") - - // should error for broken links - target, err = commit.Tree.GetTreeEntryByPath("foo/broken_link") - assert.NoError(t, err) - assert.True(t, target.IsLink()) - _, err = target.FollowLink() - assert.EqualError(t, err, "broken_link: broken link") - - // should error for external links - target, err = commit.Tree.GetTreeEntryByPath("foo/outside_repo") - assert.NoError(t, err) - assert.True(t, target.IsLink()) - _, err = target.FollowLink() - assert.EqualError(t, err, "outside_repo: points outside of repo") - - // testing fix for short link bug - target, err = commit.Tree.GetTreeEntryByPath("foo/link_short") - assert.NoError(t, err) - _, err = target.FollowLink() - assert.EqualError(t, err, "link_short: broken link") -} diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go index 421b0ecb0f..272b018ffd 100644 --- a/modules/git/tree_gogit.go +++ b/modules/git/tree_gogit.go @@ -69,7 +69,7 @@ func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { seen := map[plumbing.Hash]bool{} walker := object.NewTreeWalker(t.gogitTree, true, seen) for { - fullName, entry, err := walker.Next() + _, entry, err := walker.Next() if err == io.EOF { break } @@ -84,7 +84,6 @@ func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { ID: ParseGogitHash(entry.Hash), gogitTreeEntry: &entry, ptree: t, - fullName: fullName, } entries = append(entries, convertedEntry) } diff --git a/modules/git/tree_test.go b/modules/git/tree_test.go index 5fee64b038..cae11c4b1b 100644 --- a/modules/git/tree_test.go +++ b/modules/git/tree_test.go @@ -19,7 +19,7 @@ func TestSubTree_Issue29101(t *testing.T) { assert.NoError(t, err) // old code could produce a different error if called multiple times - for i := 0; i < 10; i++ { + for range 10 { _, err = commit.SubTree("file1.txt") assert.Error(t, err) assert.True(t, IsErrNotExist(err)) @@ -33,10 +33,10 @@ func Test_GetTreePathLatestCommit(t *testing.T) { commitID, err := repo.GetBranchCommitID("master") assert.NoError(t, err) - assert.EqualValues(t, "544d8f7a3b15927cddf2299b4b562d6ebd71b6a7", commitID) + assert.Equal(t, "544d8f7a3b15927cddf2299b4b562d6ebd71b6a7", commitID) commit, err := repo.GetTreePathLatestCommit("master", "blame.txt") assert.NoError(t, err) assert.NotNil(t, commit) - assert.EqualValues(t, "45fb6cbc12f970b04eacd5cd4165edd11c8d7376", commit.ID.String()) + assert.Equal(t, "45fb6cbc12f970b04eacd5cd4165edd11c8d7376", commit.ID.String()) } diff --git a/modules/git/url/url.go b/modules/git/url/url.go index 1c5e8377a6..aa6fa31c5e 100644 --- a/modules/git/url/url.go +++ b/modules/git/url/url.go @@ -133,12 +133,13 @@ func ParseRepositoryURL(ctx context.Context, repoURL string) (*RepositoryURL, er } } - if parsed.URL.Scheme == "http" || parsed.URL.Scheme == "https" { + switch parsed.URL.Scheme { + case "http", "https": if !httplib.IsCurrentGiteaSiteURL(ctx, repoURL) { return ret, nil } fillPathParts(strings.TrimPrefix(parsed.URL.Path, setting.AppSubURL)) - } else if parsed.URL.Scheme == "ssh" || parsed.URL.Scheme == "git+ssh" { + case "ssh", "git+ssh": domainSSH := setting.SSH.Domain domainCur := httplib.GuessCurrentHostDomain(ctx) urlDomain, _, _ := net.SplitHostPort(parsed.URL.Host) @@ -166,9 +167,10 @@ func MakeRepositoryWebLink(repoURL *RepositoryURL) string { // now, let's guess, for example: // * git@github.com:owner/submodule.git // * https://github.com/example/submodule1.git - if repoURL.GitURL.Scheme == "http" || repoURL.GitURL.Scheme == "https" { + switch repoURL.GitURL.Scheme { + case "http", "https": return strings.TrimSuffix(repoURL.GitURL.String(), ".git") - } else if repoURL.GitURL.Scheme == "ssh" || repoURL.GitURL.Scheme == "git+ssh" { + case "ssh", "git+ssh": hostname, _, _ := net.SplitHostPort(repoURL.GitURL.Host) hostname = util.IfZero(hostname, repoURL.GitURL.Host) urlPath := strings.TrimSuffix(repoURL.GitURL.Path, ".git") diff --git a/modules/git/url/url_test.go b/modules/git/url/url_test.go index 681da564f9..6655c20be3 100644 --- a/modules/git/url/url_test.go +++ b/modules/git/url/url_test.go @@ -165,8 +165,8 @@ func TestParseGitURLs(t *testing.T) { t.Run(kase.kase, func(t *testing.T) { u, err := ParseGitURL(kase.kase) assert.NoError(t, err) - assert.EqualValues(t, kase.expected.extraMark, u.extraMark) - assert.EqualValues(t, *kase.expected, *u) + assert.Equal(t, kase.expected.extraMark, u.extraMark) + assert.Equal(t, *kase.expected, *u) }) } } diff --git a/modules/git/utils.go b/modules/git/utils.go index 56cba9087a..897306efd0 100644 --- a/modules/git/utils.go +++ b/modules/git/utils.go @@ -8,7 +8,6 @@ import ( "encoding/hex" "fmt" "io" - "os" "strconv" "strings" "sync" @@ -41,33 +40,6 @@ func (oc *ObjectCache[T]) Get(id string) (T, bool) { return obj, has } -// isDir returns true if given path is a directory, -// or returns false when it's a file or does not exist. -func isDir(dir string) bool { - f, e := os.Stat(dir) - if e != nil { - return false - } - return f.IsDir() -} - -// isFile returns true if given path is a file, -// or returns false when it's a directory or does not exist. -func isFile(filePath string) bool { - f, e := os.Stat(filePath) - if e != nil { - return false - } - return !f.IsDir() -} - -// isExist checks whether a file or directory exists. -// It returns false when the file or directory does not exist. -func isExist(path string) bool { - _, err := os.Stat(path) - return err == nil || os.IsExist(err) -} - // ConcatenateError concatenats an error with stderr string func ConcatenateError(err error, stderr string) error { if len(stderr) == 0 { diff --git a/modules/gitrepo/branch.go b/modules/gitrepo/branch.go index 9c4bdc5bdf..d7857819e4 100644 --- a/modules/gitrepo/branch.go +++ b/modules/gitrepo/branch.go @@ -11,14 +11,14 @@ import ( // GetBranchesByPath returns a branch by its path // if limit = 0 it will not limit -func GetBranchesByPath(ctx context.Context, repo Repository, skip, limit int) ([]*git.Branch, int, error) { +func GetBranchesByPath(ctx context.Context, repo Repository, skip, limit int) ([]string, int, error) { gitRepo, err := OpenRepository(ctx, repo) if err != nil { return nil, 0, err } defer gitRepo.Close() - return gitRepo.GetBranches(skip, limit) + return gitRepo.GetBranchNames(skip, limit) } func GetBranchCommitID(ctx context.Context, repo Repository, branch string) (string, error) { @@ -44,24 +44,12 @@ func GetDefaultBranch(ctx context.Context, repo Repository) (string, error) { return git.GetDefaultBranch(ctx, repoPath(repo)) } -func GetWikiDefaultBranch(ctx context.Context, repo Repository) (string, error) { - return git.GetDefaultBranch(ctx, wikiPath(repo)) -} - // IsReferenceExist returns true if given reference exists in the repository. func IsReferenceExist(ctx context.Context, repo Repository, name string) bool { return git.IsReferenceExist(ctx, repoPath(repo), name) } -func IsWikiReferenceExist(ctx context.Context, repo Repository, name string) bool { - return git.IsReferenceExist(ctx, wikiPath(repo), name) -} - // IsBranchExist returns true if given branch exists in the repository. func IsBranchExist(ctx context.Context, repo Repository, name string) bool { return IsReferenceExist(ctx, repo, git.BranchPrefix+name) } - -func IsWikiBranchExist(ctx context.Context, repo Repository, name string) bool { - return IsWikiReferenceExist(ctx, repo, git.BranchPrefix+name) -} diff --git a/modules/gitrepo/gitrepo.go b/modules/gitrepo/gitrepo.go index 5e2ec9ed1e..5da65e2452 100644 --- a/modules/gitrepo/gitrepo.go +++ b/modules/gitrepo/gitrepo.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "path/filepath" - "strings" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/reqctx" @@ -16,21 +15,15 @@ import ( "code.gitea.io/gitea/modules/util" ) +// Repository represents a git repository which stored in a disk type Repository interface { - GetName() string - GetOwnerName() string -} - -func absPath(owner, name string) string { - return filepath.Join(setting.RepoRootPath, strings.ToLower(owner), strings.ToLower(name)+".git") + RelativePath() string // We don't assume how the directory structure of the repository is, so we only need the relative path } +// RelativePath should be an unix style path like username/reponame.git +// This method should change it according to the current OS. func repoPath(repo Repository) string { - return absPath(repo.GetOwnerName(), repo.GetName()) -} - -func wikiPath(repo Repository) string { - return filepath.Join(setting.RepoRootPath, strings.ToLower(repo.GetOwnerName()), strings.ToLower(repo.GetName())+".wiki.git") + return filepath.Join(setting.RepoRootPath, filepath.FromSlash(repo.RelativePath())) } // OpenRepository opens the repository at the given relative path with the provided context. @@ -38,10 +31,6 @@ func OpenRepository(ctx context.Context, repo Repository) (*git.Repository, erro return git.OpenRepository(ctx, repoPath(repo)) } -func OpenWikiRepository(ctx context.Context, repo Repository) (*git.Repository, error) { - return git.OpenRepository(ctx, wikiPath(repo)) -} - // contextKey is a value for use with context.WithValue. type contextKey struct { repoPath string @@ -86,9 +75,8 @@ func DeleteRepository(ctx context.Context, repo Repository) error { } // RenameRepository renames a repository's name on disk -func RenameRepository(ctx context.Context, repo Repository, newName string) error { - newRepoPath := absPath(repo.GetOwnerName(), newName) - if err := util.Rename(repoPath(repo), newRepoPath); err != nil { +func RenameRepository(ctx context.Context, repo, newRepo Repository) error { + if err := util.Rename(repoPath(repo), repoPath(newRepo)); err != nil { return fmt.Errorf("rename repository directory: %w", err) } return nil diff --git a/modules/gitrepo/hooks.go b/modules/gitrepo/hooks.go index cf44f792c6..d9d4a88ff1 100644 --- a/modules/gitrepo/hooks.go +++ b/modules/gitrepo/hooks.go @@ -106,16 +106,11 @@ done return hookNames, hookTpls, giteaHookTpls } -// CreateDelegateHooksForRepo creates all the hooks scripts for the repo -func CreateDelegateHooksForRepo(_ context.Context, repo Repository) (err error) { +// CreateDelegateHooks creates all the hooks scripts for the repo +func CreateDelegateHooks(_ context.Context, repo Repository) (err error) { return createDelegateHooks(filepath.Join(repoPath(repo), "hooks")) } -// CreateDelegateHooksForWiki creates all the hooks scripts for the wiki repo -func CreateDelegateHooksForWiki(_ context.Context, repo Repository) (err error) { - return createDelegateHooks(filepath.Join(wikiPath(repo), "hooks")) -} - func createDelegateHooks(hookDir string) (err error) { hookNames, hookTpls, giteaHookTpls := getHookTemplates() @@ -178,16 +173,11 @@ func ensureExecutable(filename string) error { return os.Chmod(filename, mode) } -// CheckDelegateHooksForRepo checks the hooks scripts for the repo -func CheckDelegateHooksForRepo(_ context.Context, repo Repository) ([]string, error) { +// CheckDelegateHooks checks the hooks scripts for the repo +func CheckDelegateHooks(_ context.Context, repo Repository) ([]string, error) { return checkDelegateHooks(filepath.Join(repoPath(repo), "hooks")) } -// CheckDelegateHooksForWiki checks the hooks scripts for the repo -func CheckDelegateHooksForWiki(_ context.Context, repo Repository) ([]string, error) { - return checkDelegateHooks(filepath.Join(wikiPath(repo), "hooks")) -} - func checkDelegateHooks(hookDir string) ([]string, error) { hookNames, hookTpls, giteaHookTpls := getHookTemplates() diff --git a/modules/globallock/globallock_test.go b/modules/globallock/globallock_test.go index 0143fc6833..8d55d9f699 100644 --- a/modules/globallock/globallock_test.go +++ b/modules/globallock/globallock_test.go @@ -70,7 +70,7 @@ func testLockAndDo(t *testing.T) { count := 0 wg := sync.WaitGroup{} wg.Add(concurrency) - for i := 0; i < concurrency; i++ { + for range concurrency { go func() { defer wg.Done() err := LockAndDo(ctx, "test", func(ctx context.Context) error { diff --git a/modules/globallock/redis_locker.go b/modules/globallock/redis_locker.go index 34ed9e389b..45dc769fd4 100644 --- a/modules/globallock/redis_locker.go +++ b/modules/globallock/redis_locker.go @@ -6,7 +6,6 @@ package globallock import ( "context" "errors" - "fmt" "sync" "sync/atomic" "time" @@ -78,7 +77,7 @@ func (l *redisLocker) Close() error { func (l *redisLocker) lock(ctx context.Context, key string, tries int) (ReleaseFunc, error) { if l.closed.Load() { - return func() {}, fmt.Errorf("locker is closed") + return func() {}, errors.New("locker is closed") } options := []redsync.Option{ diff --git a/modules/graceful/manager_windows.go b/modules/graceful/manager_windows.go index d776e0e9f9..457768d6ca 100644 --- a/modules/graceful/manager_windows.go +++ b/modules/graceful/manager_windows.go @@ -41,8 +41,7 @@ func (g *Manager) start() { // Make SVC process run := svc.Run - //lint:ignore SA1019 We use IsAnInteractiveSession because IsWindowsService has a different permissions profile - isAnInteractiveSession, err := svc.IsAnInteractiveSession() //nolint:staticcheck + isAnInteractiveSession, err := svc.IsAnInteractiveSession() //nolint:staticcheck // must use IsAnInteractiveSession because IsWindowsService has a different permissions profile if err != nil { log.Error("Unable to ascertain if running as an Windows Service: %v", err) return diff --git a/modules/graceful/releasereopen/releasereopen_test.go b/modules/graceful/releasereopen/releasereopen_test.go index 0e8b48257d..46e67c2046 100644 --- a/modules/graceful/releasereopen/releasereopen_test.go +++ b/modules/graceful/releasereopen/releasereopen_test.go @@ -30,14 +30,14 @@ func TestManager(t *testing.T) { _ = m.Register(t3) assert.NoError(t, m.ReleaseReopen()) - assert.EqualValues(t, 1, t1.count) - assert.EqualValues(t, 1, t2.count) - assert.EqualValues(t, 1, t3.count) + assert.Equal(t, 1, t1.count) + assert.Equal(t, 1, t2.count) + assert.Equal(t, 1, t3.count) c2() assert.NoError(t, m.ReleaseReopen()) - assert.EqualValues(t, 2, t1.count) - assert.EqualValues(t, 1, t2.count) - assert.EqualValues(t, 2, t3.count) + assert.Equal(t, 2, t1.count) + assert.Equal(t, 1, t2.count) + assert.Equal(t, 2, t3.count) } diff --git a/modules/gtprof/trace_builtin.go b/modules/gtprof/trace_builtin.go index 41743a25e4..2590ed3a13 100644 --- a/modules/gtprof/trace_builtin.go +++ b/modules/gtprof/trace_builtin.go @@ -40,7 +40,7 @@ func (t *traceBuiltinSpan) toString(out *strings.Builder, indent int) { if t.ts.endTime.IsZero() { out.WriteString(" duration: (not ended)") } else { - out.WriteString(fmt.Sprintf(" duration=%.4fs", t.ts.endTime.Sub(t.ts.startTime).Seconds())) + fmt.Fprintf(out, " duration=%.4fs", t.ts.endTime.Sub(t.ts.startTime).Seconds()) } for _, a := range t.ts.attributes { out.WriteString(" ") diff --git a/modules/highlight/highlight_test.go b/modules/highlight/highlight_test.go index 659688bd0f..b36de98c5c 100644 --- a/modules/highlight/highlight_test.go +++ b/modules/highlight/highlight_test.go @@ -114,7 +114,7 @@ c=2 t.Run(tt.name, func(t *testing.T) { out, lexerName, err := File(tt.name, "", []byte(tt.code)) assert.NoError(t, err) - assert.EqualValues(t, tt.want, out) + assert.Equal(t, tt.want, out) assert.Equal(t, tt.lexerName, lexerName) }) } @@ -177,7 +177,7 @@ c=2`), for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { out := PlainText([]byte(tt.code)) - assert.EqualValues(t, tt.want, out) + assert.Equal(t, tt.want, out) }) } } diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go index 1069310316..15c6371422 100644 --- a/modules/hostmatcher/hostmatcher.go +++ b/modules/hostmatcher/hostmatcher.go @@ -6,6 +6,7 @@ package hostmatcher import ( "net" "path/filepath" + "slices" "strings" ) @@ -38,7 +39,7 @@ func isBuiltin(s string) bool { // ParseHostMatchList parses the host list HostMatchList func ParseHostMatchList(settingKeyHint, hostList string) *HostMatchList { hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList} - for _, s := range strings.Split(hostList, ",") { + for s := range strings.SplitSeq(hostList, ",") { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { continue @@ -61,7 +62,7 @@ func ParseSimpleMatchList(settingKeyHint, matchList string) *HostMatchList { SettingKeyHint: settingKeyHint, SettingValue: matchList, } - for _, s := range strings.Split(matchList, ",") { + for s := range strings.SplitSeq(matchList, ",") { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { continue @@ -98,10 +99,8 @@ func (hl *HostMatchList) checkPattern(host string) bool { } func (hl *HostMatchList) checkIP(ip net.IP) bool { - for _, pattern := range hl.patterns { - if pattern == "*" { - return true - } + if slices.Contains(hl.patterns, "*") { + return true } for _, builtin := range hl.builtins { switch builtin { diff --git a/modules/htmlutil/html.go b/modules/htmlutil/html.go index 0ab0e71689..efbc174b2e 100644 --- a/modules/htmlutil/html.go +++ b/modules/htmlutil/html.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "slices" + "strings" ) // ParseSizeAndClass get size and class from string with default values @@ -31,6 +32,9 @@ func ParseSizeAndClass(defaultSize int, defaultClass string, others ...any) (int } func HTMLFormat(s template.HTML, rawArgs ...any) template.HTML { + if !strings.Contains(string(s), "%") || len(rawArgs) == 0 { + panic("HTMLFormat requires one or more arguments") + } args := slices.Clone(rawArgs) for i, v := range args { switch v := v.(type) { @@ -38,6 +42,8 @@ func HTMLFormat(s template.HTML, rawArgs ...any) template.HTML { // for most basic types (including template.HTML which is safe), just do nothing and use it case string: args[i] = template.HTMLEscapeString(v) + case template.URL: + args[i] = template.HTMLEscapeString(string(v)) case fmt.Stringer: args[i] = template.HTMLEscapeString(v.String()) default: diff --git a/modules/htmlutil/html_test.go b/modules/htmlutil/html_test.go index 5ff05d75b3..22258ce59d 100644 --- a/modules/htmlutil/html_test.go +++ b/modules/htmlutil/html_test.go @@ -10,6 +10,15 @@ import ( "github.com/stretchr/testify/assert" ) +type testStringer struct{} + +func (t testStringer) String() string { + return "&StringMethod" +} + func TestHTMLFormat(t *testing.T) { assert.Equal(t, template.HTML("<a>< < 1</a>"), HTMLFormat("<a>%s %s %d</a>", "<", template.HTML("<"), 1)) + assert.Equal(t, template.HTML("%!s(<nil>)"), HTMLFormat("%s", nil)) + assert.Equal(t, template.HTML("<>"), HTMLFormat("%s", template.URL("<>"))) + assert.Equal(t, template.HTML("&StringMethod &StringMethod"), HTMLFormat("%s %s", testStringer{}, &testStringer{})) } diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go index 045b00d944..dd3efab7a5 100644 --- a/modules/httpcache/httpcache.go +++ b/modules/httpcache/httpcache.go @@ -79,7 +79,7 @@ func HandleGenericETagCache(req *http.Request, w http.ResponseWriter, etag strin func checkIfNoneMatchIsValid(req *http.Request, etag string) bool { ifNoneMatch := req.Header.Get("If-None-Match") if len(ifNoneMatch) > 0 { - for _, item := range strings.Split(ifNoneMatch, ",") { + for item := range strings.SplitSeq(ifNoneMatch, ",") { item = strings.TrimPrefix(strings.TrimSpace(item), "W/") // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag#directives if item == etag { return true diff --git a/modules/httplib/request.go b/modules/httplib/request.go index 5e40922896..49ea6f4b73 100644 --- a/modules/httplib/request.go +++ b/modules/httplib/request.go @@ -108,7 +108,7 @@ func (r *Request) Body(data any) *Request { switch t := data.(type) { case nil: // do nothing case string: - bf := bytes.NewBufferString(t) + bf := strings.NewReader(t) r.req.Body = io.NopCloser(bf) r.req.ContentLength = int64(len(t)) case []byte: @@ -143,13 +143,13 @@ func (r *Request) getResponse() (*http.Response, error) { paramBody = paramBody[0 : len(paramBody)-1] } - if r.req.Method == "GET" && len(paramBody) > 0 { + if r.req.Method == http.MethodGet && len(paramBody) > 0 { if strings.Contains(r.url, "?") { r.url += "&" + paramBody } else { r.url = r.url + "?" + paramBody } - } else if r.req.Method == "POST" && r.req.Body == nil && len(paramBody) > 0 { + } else if r.req.Method == http.MethodPost && r.req.Body == nil && len(paramBody) > 0 { r.Header("Content-Type", "application/x-www-form-urlencoded") r.Body(paramBody) // string } diff --git a/modules/httplib/serve_test.go b/modules/httplib/serve_test.go index 06c95bc594..78b88c9b5f 100644 --- a/modules/httplib/serve_test.go +++ b/modules/httplib/serve_test.go @@ -4,11 +4,11 @@ package httplib import ( - "fmt" "net/http" "net/http/httptest" "net/url" "os" + "strconv" "strings" "testing" @@ -23,14 +23,14 @@ func TestServeContentByReader(t *testing.T) { _, rangeStr, _ := strings.Cut(t.Name(), "_range_") r := &http.Request{Header: http.Header{}, Form: url.Values{}} if rangeStr != "" { - r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + r.Header.Set("Range", "bytes="+rangeStr) } reader := strings.NewReader(data) w := httptest.NewRecorder() ServeContentByReader(r, w, int64(len(data)), reader, &ServeHeaderOptions{}) assert.Equal(t, expectedStatusCode, w.Code) if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { - assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, strconv.Itoa(len(expectedContent)), w.Header().Get("Content-Length")) assert.Equal(t, expectedContent, w.Body.String()) } } @@ -68,7 +68,7 @@ func TestServeContentByReadSeeker(t *testing.T) { _, rangeStr, _ := strings.Cut(t.Name(), "_range_") r := &http.Request{Header: http.Header{}, Form: url.Values{}} if rangeStr != "" { - r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + r.Header.Set("Range", "bytes="+rangeStr) } seekReader, err := os.OpenFile(tmpFile, os.O_RDONLY, 0o644) @@ -79,7 +79,7 @@ func TestServeContentByReadSeeker(t *testing.T) { ServeContentByReadSeeker(r, w, nil, seekReader, &ServeHeaderOptions{}) assert.Equal(t, expectedStatusCode, w.Code) if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { - assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, strconv.Itoa(len(expectedContent)), w.Header().Get("Content-Length")) assert.Equal(t, expectedContent, w.Body.String()) } } diff --git a/modules/httplib/url.go b/modules/httplib/url.go index 5d5b64dc0c..f51506ac3b 100644 --- a/modules/httplib/url.go +++ b/modules/httplib/url.go @@ -53,28 +53,34 @@ func getRequestScheme(req *http.Request) string { return "" } -// GuessCurrentAppURL tries to guess the current full app URL (with sub-path) by http headers. It always has a '/' suffix, exactly the same as setting.AppURL +// GuessCurrentAppURL tries to guess the current full public URL (with sub-path) by http headers. It always has a '/' suffix, exactly the same as setting.AppURL +// TODO: should rename it to GuessCurrentPublicURL in the future func GuessCurrentAppURL(ctx context.Context) string { return GuessCurrentHostURL(ctx) + setting.AppSubURL + "/" } // GuessCurrentHostURL tries to guess the current full host URL (no sub-path) by http headers, there is no trailing slash. func GuessCurrentHostURL(ctx context.Context) string { - req, ok := ctx.Value(RequestContextKey).(*http.Request) - if !ok { - return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") - } - // If no scheme provided by reverse proxy, then do not guess the AppURL, use the configured one. + // Try the best guess to get the current host URL (will be used for public URL) by http headers. // At the moment, if site admin doesn't configure the proxy headers correctly, then Gitea would guess wrong. // There are some cases: // 1. The reverse proxy is configured correctly, it passes "X-Forwarded-Proto/Host" headers. Perfect, Gitea can handle it correctly. // 2. The reverse proxy is not configured correctly, doesn't pass "X-Forwarded-Proto/Host" headers, eg: only one "proxy_pass http://gitea:3000" in Nginx. // 3. There is no reverse proxy. - // Without an extra config option, Gitea is impossible to distinguish between case 2 and case 3, - // then case 2 would result in wrong guess like guessed AppURL becomes "http://gitea:3000/", which is not accessible by end users. - // So in the future maybe it should introduce a new config option, to let site admin decide how to guess the AppURL. + // Without more information, Gitea is impossible to distinguish between case 2 and case 3, then case 2 would result in + // wrong guess like guessed public URL becomes "http://gitea:3000/" behind a "https" reverse proxy, which is not accessible by end users. + // So we introduced "PUBLIC_URL_DETECTION" option, to control the guessing behavior to satisfy different use cases. + req, ok := ctx.Value(RequestContextKey).(*http.Request) + if !ok { + return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") + } reqScheme := getRequestScheme(req) if reqScheme == "" { + // if no reverse proxy header, try to use "Host" header for absolute URL + if setting.PublicURLDetection == setting.PublicURLAuto && req.Host != "" { + return util.Iif(req.TLS == nil, "http://", "https://") + req.Host + } + // fall back to default AppURL return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") } // X-Forwarded-Host has many problems: non-standard, not well-defined (X-Forwarded-Port or not), conflicts with Host header. @@ -88,8 +94,8 @@ func GuessCurrentHostDomain(ctx context.Context) string { return util.IfZero(domain, host) } -// MakeAbsoluteURL tries to make a link to an absolute URL: -// * If link is empty, it returns the current app URL. +// MakeAbsoluteURL tries to make a link to an absolute public URL: +// * If link is empty, it returns the current public URL. // * If link is absolute, it returns the link. // * Otherwise, it returns the current host URL + link, the link itself should have correct sub-path (AppSubURL) if needed. func MakeAbsoluteURL(ctx context.Context, link string) string { diff --git a/modules/httplib/url_test.go b/modules/httplib/url_test.go index d57653646b..0ffb0cac05 100644 --- a/modules/httplib/url_test.go +++ b/modules/httplib/url_test.go @@ -5,6 +5,7 @@ package httplib import ( "context" + "crypto/tls" "net/http" "testing" @@ -39,6 +40,42 @@ func TestIsRelativeURL(t *testing.T) { } } +func TestGuessCurrentHostURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + headersWithProto := http.Header{"X-Forwarded-Proto": {"https"}} + + t.Run("Legacy", func(t *testing.T) { + defer test.MockVariableValue(&setting.PublicURLDetection, setting.PublicURLLegacy)() + + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(t.Context())) + + // legacy: "Host" is not used when there is no "X-Forwarded-Proto" header + ctx := context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000"}) + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(ctx)) + + // if "X-Forwarded-Proto" exists, then use it and "Host" header + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000", Header: headersWithProto}) + assert.Equal(t, "https://req-host:3000", GuessCurrentHostURL(ctx)) + }) + + t.Run("Auto", func(t *testing.T) { + defer test.MockVariableValue(&setting.PublicURLDetection, setting.PublicURLAuto)() + + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(t.Context())) + + // auto: always use "Host" header, the scheme is determined by "X-Forwarded-Proto" header, or TLS config if no "X-Forwarded-Proto" header + ctx := context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000"}) + assert.Equal(t, "http://req-host:3000", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host", TLS: &tls.ConnectionState{}}) + assert.Equal(t, "https://req-host", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000", Header: headersWithProto}) + assert.Equal(t, "https://req-host:3000", GuessCurrentHostURL(ctx)) + }) +} + func TestMakeAbsoluteURL(t *testing.T) { defer test.MockVariableValue(&setting.Protocol, "http")() defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index e1a381f992..70f0995a01 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -16,7 +16,6 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/indexer" path_filter "code.gitea.io/gitea/modules/indexer/code/bleve/token/path" "code.gitea.io/gitea/modules/indexer/code/internal" @@ -30,7 +29,6 @@ import ( "github.com/blevesearch/bleve/v2" analyzer_custom "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - "github.com/blevesearch/bleve/v2/analysis/token/camelcase" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter" @@ -72,7 +70,7 @@ const ( filenameIndexerAnalyzer = "filenameIndexerAnalyzer" filenameIndexerTokenizer = "filenameIndexerTokenizer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 8 + repoIndexerLatestVersion = 9 ) // generateBleveIndexMapping generates a bleve index mapping for the repo indexer @@ -109,7 +107,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { "type": analyzer_custom.Name, "char_filters": []string{}, "tokenizer": letter.Name, - "token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name}, + "token_filters": []string{unicodeNormalizeName, lowercase.Name}, }); err != nil { return nil, err } @@ -191,7 +189,8 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro return err } else if !typesniffer.DetectContentType(fileContents).IsText() { // FIXME: UTF-16 files will probably fail here - return nil + // Even if the file is not recognized as a "text file", we could still put its name into the indexers to make the filename become searchable, while leave the content to empty. + fileContents = nil } if _, err = batchReader.Discard(1); err != nil { @@ -217,12 +216,7 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize) if len(changes.Updates) > 0 { - r, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - return err - } - defer r.Close() - gitBatch, err := r.NewBatch(ctx) + gitBatch, err := git.NewBatch(ctx, repo.RepoPath()) if err != nil { return err } diff --git a/modules/indexer/code/bleve/token/path/path.go b/modules/indexer/code/bleve/token/path/path.go index 107e0da109..6dfc12f146 100644 --- a/modules/indexer/code/bleve/token/path/path.go +++ b/modules/indexer/code/bleve/token/path/path.go @@ -51,13 +51,13 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke slices.Reverse(input) } - for i := 0; i < len(input); i++ { + for i := range input { var sb strings.Builder - sb.WriteString(string(input[0].Term)) + sb.Write(input[0].Term) for j := 1; j < i; j++ { sb.WriteString("/") - sb.WriteString(string(input[j].Term)) + sb.Write(input[j].Term) } term := sb.String() @@ -97,5 +97,9 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke } func init() { - registry.RegisterTokenFilter(Name, TokenFilterConstructor) + // FIXME: move it to the bleve's init function, but do not call it in global init + err := registry.RegisterTokenFilter(Name, TokenFilterConstructor) + if err != nil { + panic(err) + } } diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index be8efad5fd..f925ce396a 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -15,7 +15,6 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/code/internal" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" @@ -209,12 +208,7 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elasti func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { reqs := make([]elastic.BulkableRequest, 0) if len(changes.Updates) > 0 { - r, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - return err - } - defer r.Close() - batch, err := r.NewBatch(ctx) + batch, err := git.NewBatch(ctx, repo.RepoPath()) if err != nil { return err } diff --git a/modules/indexer/code/elasticsearch/elasticsearch_test.go b/modules/indexer/code/elasticsearch/elasticsearch_test.go index a6d2af92b2..e8f1f202ce 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/code/elasticsearch/elasticsearch_test.go @@ -11,6 +11,6 @@ import ( func TestIndexPos(t *testing.T) { startIdx, endIdx := contentMatchIndexPos("test index start and end", "start", "end") - assert.EqualValues(t, 11, startIdx) - assert.EqualValues(t, 15, endIdx) + assert.Equal(t, 11, startIdx) + assert.Equal(t, 15, endIdx) } diff --git a/modules/indexer/code/git.go b/modules/indexer/code/git.go index 0089dd259f..41bc74e6ec 100644 --- a/modules/indexer/code/git.go +++ b/modules/indexer/code/git.go @@ -129,8 +129,8 @@ func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revisio changes.Updates = append(changes.Updates, updates...) return nil } - lines := strings.Split(stdout, "\n") - for _, line := range lines { + lines := strings.SplitSeq(stdout, "\n") + for line := range lines { line = strings.TrimSpace(line) if len(line) == 0 { continue diff --git a/modules/indexer/code/gitgrep/gitgrep.go b/modules/indexer/code/gitgrep/gitgrep.go index 093c189ba3..6f6e0b47b9 100644 --- a/modules/indexer/code/gitgrep/gitgrep.go +++ b/modules/indexer/code/gitgrep/gitgrep.go @@ -26,9 +26,10 @@ func indexSettingToGitGrepPathspecList() (list []string) { func PerformSearch(ctx context.Context, page int, repoID int64, gitRepo *git.Repository, ref git.RefName, keyword string, searchMode indexer.SearchModeType) (searchResults []*code_indexer.Result, total int, err error) { grepMode := git.GrepModeWords - if searchMode == indexer.SearchModeExact { + switch searchMode { + case indexer.SearchModeExact: grepMode = git.GrepModeExact - } else if searchMode == indexer.SearchModeRegexp { + case indexer.SearchModeRegexp: grepMode = git.GrepModeRegexp } res, err := git.GrepSearch(ctx, gitRepo, keyword, git.GrepOptions{ diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index 1d9bf99d40..78fea22f10 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -310,7 +310,7 @@ func TestESIndexAndSearch(t *testing.T) { if indexer != nil { indexer.Close() } - assert.FailNow(t, "Unable to init ES indexer Error: %v", err) + require.NoError(t, err, "Unable to init ES indexer") } defer indexer.Close() diff --git a/modules/indexer/code/internal/indexer.go b/modules/indexer/code/internal/indexer.go index 6c9a8af635..d58b028124 100644 --- a/modules/indexer/code/internal/indexer.go +++ b/modules/indexer/code/internal/indexer.go @@ -5,7 +5,7 @@ package internal import ( "context" - "fmt" + "errors" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" @@ -48,13 +48,13 @@ func (d *dummyIndexer) SupportedSearchModes() []indexer.SearchMode { } func (d *dummyIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Delete(ctx context.Context, repoID int64) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Search(ctx context.Context, opts *SearchOptions) (int64, []*SearchResult, []*SearchResultLanguages, error) { - return 0, nil, nil, fmt.Errorf("indexer is not ready") + return 0, nil, nil, errors.New("indexer is not ready") } diff --git a/modules/indexer/code/search.go b/modules/indexer/code/search.go index e37aff8e59..a7a5d7d2e3 100644 --- a/modules/indexer/code/search.go +++ b/modules/indexer/code/search.go @@ -77,7 +77,7 @@ func HighlightSearchResultCode(filename, language string, lineNums []int, code s // The lineNums outputted by highlight.Code might not match the original lineNums, because "highlight" removes the last `\n` lines := make([]*ResultLine, min(len(highlightedLines), len(lineNums))) - for i := 0; i < len(lines); i++ { + for i := range lines { lines[i] = &ResultLine{ Num: lineNums[i], FormattedContent: template.HTML(highlightedLines[i]), diff --git a/modules/indexer/internal/bleve/indexer.go b/modules/indexer/internal/bleve/indexer.go index 01e53ca636..9d1e24a874 100644 --- a/modules/indexer/internal/bleve/indexer.go +++ b/modules/indexer/internal/bleve/indexer.go @@ -5,7 +5,7 @@ package bleve import ( "context" - "fmt" + "errors" "code.gitea.io/gitea/modules/indexer/internal" "code.gitea.io/gitea/modules/log" @@ -39,11 +39,11 @@ func NewIndexer(indexDir string, version int, mappingGetter func() (mapping.Inde // Init initializes the indexer func (i *Indexer) Init(_ context.Context) (bool, error) { if i == nil { - return false, fmt.Errorf("cannot init nil indexer") + return false, errors.New("cannot init nil indexer") } if i.Indexer != nil { - return false, fmt.Errorf("indexer is already initialized") + return false, errors.New("indexer is already initialized") } indexer, version, err := openIndexer(i.indexDir, i.version) @@ -83,10 +83,10 @@ func (i *Indexer) Init(_ context.Context) (bool, error) { // Ping checks if the indexer is available func (i *Indexer) Ping(_ context.Context) error { if i == nil { - return fmt.Errorf("cannot ping nil indexer") + return errors.New("cannot ping nil indexer") } if i.Indexer == nil { - return fmt.Errorf("indexer is not initialized") + return errors.New("indexer is not initialized") } return nil } diff --git a/modules/indexer/internal/elasticsearch/indexer.go b/modules/indexer/internal/elasticsearch/indexer.go index 395eea3bce..265ce26585 100644 --- a/modules/indexer/internal/elasticsearch/indexer.go +++ b/modules/indexer/internal/elasticsearch/indexer.go @@ -5,6 +5,7 @@ package elasticsearch import ( "context" + "errors" "fmt" "code.gitea.io/gitea/modules/indexer/internal" @@ -36,10 +37,10 @@ func NewIndexer(url, indexName string, version int, mapping string) *Indexer { // Init initializes the indexer func (i *Indexer) Init(ctx context.Context) (bool, error) { if i == nil { - return false, fmt.Errorf("cannot init nil indexer") + return false, errors.New("cannot init nil indexer") } if i.Client != nil { - return false, fmt.Errorf("indexer is already initialized") + return false, errors.New("indexer is already initialized") } client, err := i.initClient() @@ -66,10 +67,10 @@ func (i *Indexer) Init(ctx context.Context) (bool, error) { // Ping checks if the indexer is available func (i *Indexer) Ping(ctx context.Context) error { if i == nil { - return fmt.Errorf("cannot ping nil indexer") + return errors.New("cannot ping nil indexer") } if i.Client == nil { - return fmt.Errorf("indexer is not initialized") + return errors.New("indexer is not initialized") } resp, err := i.Client.ClusterHealth().Do(ctx) diff --git a/modules/indexer/internal/indexer.go b/modules/indexer/internal/indexer.go index c7f356da1e..3442bbaff2 100644 --- a/modules/indexer/internal/indexer.go +++ b/modules/indexer/internal/indexer.go @@ -5,7 +5,7 @@ package internal import ( "context" - "fmt" + "errors" ) // Indexer defines an basic indexer interface @@ -27,11 +27,11 @@ func NewDummyIndexer() Indexer { type dummyIndexer struct{} func (d *dummyIndexer) Init(ctx context.Context) (bool, error) { - return false, fmt.Errorf("indexer is not ready") + return false, errors.New("indexer is not ready") } func (d *dummyIndexer) Ping(ctx context.Context) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Close() {} diff --git a/modules/indexer/internal/meilisearch/indexer.go b/modules/indexer/internal/meilisearch/indexer.go index 01bb49bbfc..65db75bb55 100644 --- a/modules/indexer/internal/meilisearch/indexer.go +++ b/modules/indexer/internal/meilisearch/indexer.go @@ -5,6 +5,7 @@ package meilisearch import ( "context" + "errors" "fmt" "github.com/meilisearch/meilisearch-go" @@ -33,11 +34,11 @@ func NewIndexer(url, apiKey, indexName string, version int, settings *meilisearc // Init initializes the indexer func (i *Indexer) Init(_ context.Context) (bool, error) { if i == nil { - return false, fmt.Errorf("cannot init nil indexer") + return false, errors.New("cannot init nil indexer") } if i.Client != nil { - return false, fmt.Errorf("indexer is already initialized") + return false, errors.New("indexer is already initialized") } i.Client = meilisearch.New(i.url, meilisearch.WithAPIKey(i.apiKey)) @@ -62,10 +63,10 @@ func (i *Indexer) Init(_ context.Context) (bool, error) { // Ping checks if the indexer is available func (i *Indexer) Ping(ctx context.Context) error { if i == nil { - return fmt.Errorf("cannot ping nil indexer") + return errors.New("cannot ping nil indexer") } if i.Client == nil { - return fmt.Errorf("indexer is not initialized") + return errors.New("indexer is not initialized") } resp, err := i.Client.Health() if err != nil { diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 9534b0b750..39d96cab98 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -5,11 +5,13 @@ package bleve import ( "context" + "strconv" "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/util" "github.com/blevesearch/bleve/v2" @@ -246,12 +248,20 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectColumnID.Value(), "project_board_id")) } - if options.PosterID.Has() { - queries = append(queries, inner_bleve.NumericEqualityQuery(options.PosterID.Value(), "poster_id")) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + queries = append(queries, inner_bleve.NumericEqualityQuery(posterIDInt64, "poster_id")) } - if options.AssigneeID.Has() { - queries = append(queries, inner_bleve.NumericEqualityQuery(options.AssigneeID.Value(), "assignee_id")) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + queries = append(queries, inner_bleve.NumericRangeInclusiveQuery(optional.Some[int64](1), optional.None[int64](), "assignee_id")) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + queries = append(queries, inner_bleve.NumericEqualityQuery(assigneeIDInt64, "assignee_id")) + } } if options.MentionID.Has() { diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 6124ad2515..50951f9c88 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -6,6 +6,7 @@ package db import ( "context" "strings" + "sync" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" @@ -18,7 +19,7 @@ import ( "xorm.io/builder" ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = (*Indexer)(nil) // Indexer implements Indexer interface to use database's like search type Indexer struct { @@ -29,11 +30,9 @@ func (i *Indexer) SupportedSearchModes() []indexer.SearchMode { return indexer.SearchModesExactWords() } -func NewIndexer() *Indexer { - return &Indexer{ - Indexer: &inner_db.Indexer{}, - } -} +var GetIndexer = sync.OnceValue(func() *Indexer { + return &Indexer{Indexer: &inner_db.Indexer{}} +}) // Index dummy function func (i *Indexer) Index(_ context.Context, _ ...*internal.IndexerData) error { @@ -122,7 +121,11 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( }, nil } - ids, total, err := issue_model.IssueIDs(ctx, opt, cond) + return i.FindWithIssueOptions(ctx, opt, cond) +} + +func (i *Indexer) FindWithIssueOptions(ctx context.Context, opt *issue_model.IssuesOptions, otherConds ...builder.Cond) (*internal.SearchResult, error) { + ids, total, err := issue_model.IssueIDs(ctx, opt, otherConds...) if err != nil { return nil, err } diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index 87ce398a20..380a25dc23 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -6,6 +6,7 @@ package db import ( "context" "fmt" + "strings" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" @@ -34,7 +35,11 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m case internal.SortByDeadlineAsc: sortType = "nearduedate" default: - sortType = "newest" + if strings.HasPrefix(string(options.SortBy), issue_model.ScopeSortPrefix) { + sortType = string(options.SortBy) + } else { + sortType = "newest" + } } // See the comment of issues_model.SearchOptions for the reason why we need to convert @@ -54,7 +59,7 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m RepoIDs: options.RepoIDs, AllPublic: options.AllPublic, RepoCond: nil, - AssigneeID: optional.Some(convertID(options.AssigneeID)), + AssigneeID: options.AssigneeID, PosterID: options.PosterID, MentionedID: convertID(options.MentionID), ReviewRequestedID: convertID(options.ReviewRequestedID), @@ -68,7 +73,6 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m ExcludedLabelNames: nil, IncludeMilestones: nil, SortType: sortType, - IssueIDs: nil, UpdatedAfterUnix: options.UpdatedAfterUnix.Value(), UpdatedBeforeUnix: options.UpdatedBeforeUnix.Value(), PriorityRepoID: 0, diff --git a/modules/indexer/issues/dboptions.go b/modules/indexer/issues/dboptions.go index 4f6ad96d22..f17724664d 100644 --- a/modules/indexer/issues/dboptions.go +++ b/modules/indexer/issues/dboptions.go @@ -4,12 +4,19 @@ package issues import ( + "strings" + "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/indexer/issues/internal" "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/setting" ) func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOptions { + if opts.IssueIDs != nil { + setting.PanicInDevOrTesting("Indexer SearchOptions doesn't support IssueIDs") + } searchOpt := &SearchOptions{ Keyword: keyword, RepoIDs: opts.RepoIDs, @@ -45,11 +52,7 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp searchOpt.ProjectID = optional.Some[int64](0) // Those issues with no project(projectid==0) } - if opts.AssigneeID.Value() == db.NoConditionID { - searchOpt.AssigneeID = optional.Some[int64](0) // FIXME: this is inconsistent from other places, 0 means "no assignee" - } else if opts.AssigneeID.Value() != 0 { - searchOpt.AssigneeID = opts.AssigneeID - } + searchOpt.AssigneeID = opts.AssigneeID // See the comment of issues_model.SearchOptions for the reason why we need to convert convertID := func(id int64) optional.Option[int64] { @@ -99,7 +102,11 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp // Unsupported sort type for search fallthrough default: - searchOpt.SortBy = SortByUpdatedDesc + if strings.HasPrefix(opts.SortType, issues_model.ScopeSortPrefix) { + searchOpt.SortBy = internal.SortBy(opts.SortType) + } else { + searchOpt.SortBy = SortByUpdatedDesc + } } return searchOpt diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 464c0028f2..9d627466ef 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -5,7 +5,6 @@ package elasticsearch import ( "context" - "fmt" "strconv" "strings" @@ -96,7 +95,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er issue := issues[0] _, err := b.inner.Client.Index(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", issue.ID)). + Id(strconv.FormatInt(issue.ID, 10)). BodyJson(issue). Do(ctx) return err @@ -107,7 +106,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er reqs = append(reqs, elastic.NewBulkIndexRequest(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", issue.ID)). + Id(strconv.FormatInt(issue.ID, 10)). Doc(issue), ) } @@ -126,7 +125,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { } else if len(ids) == 1 { _, err := b.inner.Client.Delete(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", ids[0])). + Id(strconv.FormatInt(ids[0], 10)). Do(ctx) return err } @@ -136,7 +135,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { reqs = append(reqs, elastic.NewBulkDeleteRequest(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", id)), + Id(strconv.FormatInt(id, 10)), ) } @@ -212,12 +211,22 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query.Must(elastic.NewTermQuery("project_board_id", options.ProjectColumnID.Value())) } - if options.PosterID.Has() { - query.Must(elastic.NewTermQuery("poster_id", options.PosterID.Value())) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + query.Must(elastic.NewTermQuery("poster_id", posterIDInt64)) } - if options.AssigneeID.Has() { - query.Must(elastic.NewTermQuery("assignee_id", options.AssigneeID.Value())) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + q := elastic.NewRangeQuery("assignee_id") + q.Gte(1) + query.Must(q) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + query.Must(elastic.NewTermQuery("assignee_id", assigneeIDInt64)) + } } if options.MentionID.Has() { diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 4741235d47..8f25c84b76 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -103,7 +103,7 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err) } case "db": - issueIndexer = db.NewIndexer() + issueIndexer = db.GetIndexer() case "meilisearch": issueIndexer = meilisearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName) existed, err = issueIndexer.Init(ctx) @@ -217,7 +217,7 @@ func PopulateIssueIndexer(ctx context.Context) error { return fmt.Errorf("shutdown before completion: %w", ctx.Err()) default: } - repos, _, err := repo_model.SearchRepositoryByName(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{ ListOptions: db_model.ListOptions{Page: page, PageSize: repo_model.RepositoryListDefaultPageSize}, OrderBy: db_model.SearchOrderByID, Private: true, @@ -291,19 +291,22 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, err // So if the user creates an issue and list issues immediately, the issue may not be listed because the indexer needs time to index the issue. // Even worse, the external indexer like elastic search may not be available for a while, // and the user may not be able to list issues completely until it is available again. - ix = db.NewIndexer() + ix = db.GetIndexer() } + result, err := ix.Search(ctx, opts) if err != nil { return nil, 0, err } + return SearchResultToIDSlice(result), result.Total, nil +} +func SearchResultToIDSlice(result *internal.SearchResult) []int64 { ret := make([]int64, 0, len(result.Hits)) for _, hit := range result.Hits { ret = append(ret, hit.ID) } - - return ret, result.Total, nil + return ret } // CountIssues counts issues by options. It is a shortcut of SearchIssues(ctx, opts) but only returns the total count. diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index 14dd6ba101..3e38ac49b7 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -44,6 +44,7 @@ func TestDBSearchIssues(t *testing.T) { t.Run("search issues with order", searchIssueWithOrder) t.Run("search issues in project", searchIssueInProject) t.Run("search issues with paginator", searchIssueWithPaginator) + t.Run("search issues with any assignee", searchIssueWithAnyAssignee) } func searchIssueWithKeyword(t *testing.T) { @@ -176,19 +177,19 @@ func searchIssueByID(t *testing.T) { }{ { opts: SearchOptions{ - PosterID: optional.Some(int64(1)), + PosterID: "1", }, expectedIDs: []int64{11, 6, 3, 2, 1}, }, { opts: SearchOptions{ - AssigneeID: optional.Some(int64(1)), + AssigneeID: "1", }, expectedIDs: []int64{6, 1}, }, { - // NOTE: This tests no assignees filtering and also ToSearchOptions() to ensure it will set AssigneeID to 0 when it is passed as -1. - opts: *ToSearchOptions("", &issues.IssuesOptions{AssigneeID: optional.Some(db.NoConditionID)}), + // NOTE: This tests no assignees filtering and also ToSearchOptions() to ensure it handles the filter correctly + opts: *ToSearchOptions("", &issues.IssuesOptions{AssigneeID: "(none)"}), expectedIDs: []int64{22, 21, 16, 15, 14, 13, 12, 11, 20, 5, 19, 18, 10, 7, 4, 9, 8, 3, 2}, }, { @@ -462,3 +463,25 @@ func searchIssueWithPaginator(t *testing.T) { assert.Equal(t, test.expectedTotal, total) } } + +func searchIssueWithAnyAssignee(t *testing.T) { + tests := []struct { + opts SearchOptions + expectedIDs []int64 + expectedTotal int64 + }{ + { + SearchOptions{ + AssigneeID: "(any)", + }, + []int64{17, 6, 1}, + 3, + }, + } + for _, test := range tests { + issueIDs, total, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) + assert.Equal(t, test.expectedIDs, issueIDs) + assert.Equal(t, test.expectedTotal, total) + } +} diff --git a/modules/indexer/issues/internal/indexer.go b/modules/indexer/issues/internal/indexer.go index 415f442d0c..59c6f48485 100644 --- a/modules/indexer/issues/internal/indexer.go +++ b/modules/indexer/issues/internal/indexer.go @@ -5,7 +5,7 @@ package internal import ( "context" - "fmt" + "errors" "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/internal" @@ -36,13 +36,13 @@ func (d *dummyIndexer) SupportedSearchModes() []indexer.SearchMode { } func (d *dummyIndexer) Index(_ context.Context, _ ...*IndexerData) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Delete(_ context.Context, _ ...int64) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Search(_ context.Context, _ *SearchOptions) (*SearchResult, error) { - return nil, fmt.Errorf("indexer is not ready") + return nil, errors.New("indexer is not ready") } diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 976e2d696b..0d4f0f727d 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -97,9 +97,8 @@ type SearchOptions struct { ProjectID optional.Option[int64] // project the issues belong to ProjectColumnID optional.Option[int64] // project column the issues belong to - PosterID optional.Option[int64] // poster of the issues - - AssigneeID optional.Option[int64] // assignee of the issues, zero means no assignee + PosterID string // poster of the issues, "(none)" or "(any)" or a user ID + AssigneeID string // assignee of the issues, "(none)" or "(any)" or a user ID MentionID optional.Option[int64] // mentioned user of the issues diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 0483853dfd..7aebbbcd58 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -8,7 +8,6 @@ package tests import ( - "context" "fmt" "slices" "testing" @@ -40,7 +39,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { data[v.ID] = v } require.NoError(t, indexer.Index(t.Context(), d...)) - require.NoError(t, waitData(indexer, int64(len(data)))) + waitData(t, indexer, int64(len(data))) } defer func() { @@ -54,13 +53,13 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { for _, v := range c.ExtraData { data[v.ID] = v } - require.NoError(t, waitData(indexer, int64(len(data)))) + waitData(t, indexer, int64(len(data))) defer func() { for _, v := range c.ExtraData { require.NoError(t, indexer.Delete(t.Context(), v.ID)) delete(data, v.ID) } - require.NoError(t, waitData(indexer, int64(len(data)))) + waitData(t, indexer, int64(len(data))) }() } @@ -93,7 +92,7 @@ var cases = []*testIndexerCase{ Name: "default", SearchOptions: &internal.SearchOptions{}, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) }, }, @@ -379,7 +378,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - PosterID: optional.Some(int64(1)), + PosterID: "1", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -397,7 +396,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - AssigneeID: optional.Some(int64(1)), + AssigneeID: "1", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -415,7 +414,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - AssigneeID: optional.Some(int64(0)), + AssigneeID: "(none)", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -526,7 +525,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCreatedDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -542,7 +541,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByUpdatedDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -558,7 +557,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCommentsDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -574,7 +573,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByDeadlineDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -590,7 +589,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCreatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -606,7 +605,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByUpdatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -622,7 +621,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCommentsAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -638,7 +637,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByDeadlineAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -647,6 +646,21 @@ var cases = []*testIndexerCase{ } }, }, + { + Name: "SearchAnyAssignee", + SearchOptions: &internal.SearchOptions{ + AssigneeID: "(any)", + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Len(t, result.Hits, 180) + for _, v := range result.Hits { + assert.GreaterOrEqual(t, data[v.ID].AssigneeID, int64(1)) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.AssigneeID >= 1 + }), result.Total) + }, + }, } type testIndexerCase struct { @@ -736,22 +750,10 @@ func countIndexerData(data map[int64]*internal.IndexerData, f func(v *internal.I // waitData waits for the indexer to index all data. // Some engines like Elasticsearch index data asynchronously, so we need to wait for a while. -func waitData(indexer internal.Indexer, total int64) error { - var actual int64 - for i := 0; i < 100; i++ { - result, err := indexer.Search(context.Background(), &internal.SearchOptions{ - Paginator: &db.ListOptions{ - PageSize: 0, - }, - }) - if err != nil { - return err - } - actual = result.Total - if actual == total { - return nil - } - time.Sleep(100 * time.Millisecond) - } - return fmt.Errorf("waitData: expected %d, actual %d", total, actual) +func waitData(t *testing.T, indexer internal.Indexer, total int64) { + assert.Eventually(t, func() bool { + result, err := indexer.Search(t.Context(), &internal.SearchOptions{Paginator: &db.ListOptions{}}) + require.NoError(t, err) + return result.Total == total + }, 10*time.Second, 100*time.Millisecond, "expected total=%d", total) } diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index a1746f5954..759a98473f 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -187,12 +187,20 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query.And(inner_meilisearch.NewFilterEq("project_board_id", options.ProjectColumnID.Value())) } - if options.PosterID.Has() { - query.And(inner_meilisearch.NewFilterEq("poster_id", options.PosterID.Value())) - } - - if options.AssigneeID.Has() { - query.And(inner_meilisearch.NewFilterEq("assignee_id", options.AssigneeID.Value())) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + query.And(inner_meilisearch.NewFilterEq("poster_id", posterIDInt64)) + } + + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + query.And(inner_meilisearch.NewFilterGte("assignee_id", 1)) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + query.And(inner_meilisearch.NewFilterEq("assignee_id", assigneeIDInt64)) + } } if options.MentionID.Has() { diff --git a/modules/indexer/issues/meilisearch/meilisearch_test.go b/modules/indexer/issues/meilisearch/meilisearch_test.go index a3a332554a..2fea4004cb 100644 --- a/modules/indexer/issues/meilisearch/meilisearch_test.go +++ b/modules/indexer/issues/meilisearch/meilisearch_test.go @@ -74,13 +74,13 @@ func TestConvertHits(t *testing.T) { } hits, err := convertHits(validResponse) assert.NoError(t, err) - assert.EqualValues(t, []internal.Match{{ID: 11}, {ID: 22}, {ID: 33}}, hits) + assert.Equal(t, []internal.Match{{ID: 11}, {ID: 22}, {ID: 33}}, hits) } func TestDoubleQuoteKeyword(t *testing.T) { - assert.EqualValues(t, "", doubleQuoteKeyword("")) - assert.EqualValues(t, `"a" "b" "c"`, doubleQuoteKeyword("a b c")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword(`a "" "d" """g`)) + assert.Empty(t, doubleQuoteKeyword("")) + assert.Equal(t, `"a" "b" "c"`, doubleQuoteKeyword("a b c")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword(`a "" "d" """g`)) } diff --git a/modules/indexer/stats/db.go b/modules/indexer/stats/db.go index 067a6f609b..199d493e97 100644 --- a/modules/indexer/stats/db.go +++ b/modules/indexer/stats/db.go @@ -8,6 +8,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/languagestats" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" @@ -62,7 +63,7 @@ func (db *DBIndexer) Index(id int64) error { } // Calculate and save language statistics to database - stats, err := gitRepo.GetLanguageStats(commitID) + stats, err := languagestats.GetLanguageStats(gitRepo, commitID) if err != nil { if !setting.IsInTesting { log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err) diff --git a/modules/indexer/stats/queue.go b/modules/indexer/stats/queue.go index d002bd57cf..69cde321d8 100644 --- a/modules/indexer/stats/queue.go +++ b/modules/indexer/stats/queue.go @@ -4,7 +4,7 @@ package stats import ( - "fmt" + "errors" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" @@ -31,7 +31,7 @@ func handler(items ...int64) []int64 { func initStatsQueue() error { statsQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_stats_update", handler) if statsQueue == nil { - return fmt.Errorf("unable to create repo_stats_update queue") + return errors.New("unable to create repo_stats_update queue") } go graceful.GetManager().RunWithCancel(statsQueue) return nil diff --git a/modules/issue/template/template.go b/modules/issue/template/template.go index a2b9d6b33e..192aaf8e01 100644 --- a/modules/issue/template/template.go +++ b/modules/issue/template/template.go @@ -4,9 +4,11 @@ package template import ( + "errors" "fmt" "net/url" "regexp" + "slices" "strconv" "strings" @@ -31,17 +33,17 @@ func Validate(template *api.IssueTemplate) error { func validateMetadata(template *api.IssueTemplate) error { if strings.TrimSpace(template.Name) == "" { - return fmt.Errorf("'name' is required") + return errors.New("'name' is required") } if strings.TrimSpace(template.About) == "" { - return fmt.Errorf("'about' is required") + return errors.New("'about' is required") } return nil } func validateYaml(template *api.IssueTemplate) error { if len(template.Fields) == 0 { - return fmt.Errorf("'body' is required") + return errors.New("'body' is required") } ids := make(container.Set[string]) for idx, field := range template.Fields { @@ -401,7 +403,7 @@ func (f *valuedField) Render() string { } func (f *valuedField) Value() string { - return strings.TrimSpace(f.Get(fmt.Sprintf("form-field-%s", f.ID))) + return strings.TrimSpace(f.Get("form-field-" + f.ID)) } func (f *valuedField) Options() []*valuedOption { @@ -444,14 +446,9 @@ func (o *valuedOption) Label() string { func (o *valuedOption) IsChecked() bool { switch o.field.Type { case api.IssueFormFieldTypeDropdown: - checks := strings.Split(o.field.Get(fmt.Sprintf("form-field-%s", o.field.ID)), ",") + checks := strings.Split(o.field.Get("form-field-"+o.field.ID), ",") idx := strconv.Itoa(o.index) - for _, v := range checks { - if v == idx { - return true - } - } - return false + return slices.Contains(checks, idx) case api.IssueFormFieldTypeCheckboxes: return o.field.Get(fmt.Sprintf("form-field-%s-%d", o.field.ID, o.index)) == "on" } diff --git a/modules/issue/template/template_test.go b/modules/issue/template/template_test.go index 575e23def9..7fec9431b6 100644 --- a/modules/issue/template/template_test.go +++ b/modules/issue/template/template_test.go @@ -904,7 +904,7 @@ Option 1 of dropdown, Option 2 of dropdown t.Fatal(err) } if got := RenderToMarkdown(template, tt.args.values); got != tt.want { - assert.EqualValues(t, tt.want, got) + assert.Equal(t, tt.want, got) } }) } diff --git a/modules/json/json.go b/modules/json/json.go index 34568c75c6..444dc8526a 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -3,11 +3,10 @@ package json -// Allow "encoding/json" import. import ( "bytes" "encoding/binary" - "encoding/json" //nolint:depguard + "encoding/json" //nolint:depguard // this package wraps it "io" jsoniter "github.com/json-iterator/go" @@ -145,6 +144,12 @@ func Valid(data []byte) bool { // UnmarshalHandleDoubleEncode - due to a bug in xorm (see https://gitea.com/xorm/xorm/pulls/1957) - it's // possible that a Blob may be double encoded or gain an unwanted prefix of 0xff 0xfe. func UnmarshalHandleDoubleEncode(bs []byte, v any) error { + if len(bs) == 0 { + // json.Unmarshal will report errors if input is empty (nil or zero-length) + // It seems that XORM ignores the nil but still passes zero-length string into this function + // To be consistent, we should treat all empty inputs as success + return nil + } err := json.Unmarshal(bs, v) if err != nil { ok := true diff --git a/modules/json/json_test.go b/modules/json/json_test.go new file mode 100644 index 0000000000..ace7167913 --- /dev/null +++ b/modules/json/json_test.go @@ -0,0 +1,18 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package json + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGiteaDBJSONUnmarshal(t *testing.T) { + var m map[any]any + err := UnmarshalHandleDoubleEncode(nil, &m) + assert.NoError(t, err) + err = UnmarshalHandleDoubleEncode([]byte(""), &m) + assert.NoError(t, err) +} diff --git a/modules/label/label.go b/modules/label/label.go index d3ef0e1dc9..3e68c4d26e 100644 --- a/modules/label/label.go +++ b/modules/label/label.go @@ -7,19 +7,24 @@ import ( "fmt" "regexp" "strings" -) + "sync" -// colorPattern is a regexp which can validate label color -var colorPattern = regexp.MustCompile("^#?(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{3})$") + "code.gitea.io/gitea/modules/util" +) // Label represents label information loaded from template type Label struct { - Name string `yaml:"name"` - Color string `yaml:"color"` - Description string `yaml:"description,omitempty"` - Exclusive bool `yaml:"exclusive,omitempty"` + Name string `yaml:"name"` + Color string `yaml:"color"` + Description string `yaml:"description,omitempty"` + Exclusive bool `yaml:"exclusive,omitempty"` + ExclusiveOrder int `yaml:"exclusive_order,omitempty"` } +var colorPattern = sync.OnceValue(func() *regexp.Regexp { + return regexp.MustCompile(`^#([\da-fA-F]{3}|[\da-fA-F]{6})$`) +}) + // NormalizeColor normalizes a color string to a 6-character hex code func NormalizeColor(color string) (string, error) { // normalize case @@ -30,8 +35,8 @@ func NormalizeColor(color string) (string, error) { color = "#" + color } - if !colorPattern.MatchString(color) { - return "", fmt.Errorf("bad color code: %s", color) + if !colorPattern().MatchString(color) { + return "", util.NewInvalidArgumentErrorf("invalid color: %s", color) } // convert 3-character shorthand into 6-character version diff --git a/modules/label/parser.go b/modules/label/parser.go index 511bac823f..2a10152062 100644 --- a/modules/label/parser.go +++ b/modules/label/parser.go @@ -72,7 +72,7 @@ func parseYamlFormat(fileName string, data []byte) ([]*Label, error) { func parseLegacyFormat(fileName string, data []byte) ([]*Label, error) { lines := strings.Split(string(data), "\n") list := make([]*Label, 0, len(lines)) - for i := 0; i < len(lines); i++ { + for i := range lines { line := strings.TrimSpace(lines[i]) if len(line) == 0 { continue @@ -108,7 +108,7 @@ func LoadTemplateDescription(fileName string) (string, error) { return "", err } - for i := 0; i < len(list); i++ { + for i := range list { if i > 0 { buf.WriteString(", ") } diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 0a27fb0c86..4b51193846 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -70,7 +70,7 @@ func (c *HTTPClient) transferNames() []string { func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { log.Trace("BATCH operation with objects: %v", objects) - url := fmt.Sprintf("%s/objects/batch", c.endpoint) + url := c.endpoint + "/objects/batch" // Original: In some lfs server implementations, they require the ref attribute. #32838 // `ref` is an "optional object describing the server ref that the objects belong to" diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index 7869c0a21a..179bcdb29a 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -31,7 +31,7 @@ func (a *DummyTransferAdapter) Name() string { } func (a *DummyTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) { - return io.NopCloser(bytes.NewBufferString("dummy")), nil + return io.NopCloser(strings.NewReader("dummy")), nil } func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error { @@ -49,7 +49,7 @@ func lfsTestRoundtripHandler(req *http.Request) *http.Response { if strings.Contains(url, "status-not-ok") { return &http.Response{StatusCode: http.StatusBadRequest} } else if strings.Contains(url, "invalid-json-response") { - return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBufferString("invalid json"))} + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("invalid json"))} } else if strings.Contains(url, "valid-batch-request-download") { batchResponse = &BatchResponse{ Transfer: "dummy", diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index ebde20f826..9c95613057 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -15,15 +15,13 @@ import ( "strings" ) +// spec: https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md const ( - blobSizeCutoff = 1024 + MetaFileMaxSize = 1024 // spec says the maximum size of a pointer file must be smaller than 1024 - // MetaFileIdentifier is the string appearing at the first line of LFS pointer files. - // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" + MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" // the first line of a pointer file - // MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. - MetaFileOidPrefix = "oid sha256:" + MetaFileOidPrefix = "oid sha256:" // spec says the only supported hash is sha256 at the moment ) var ( @@ -39,7 +37,7 @@ var ( // ReadPointer tries to read LFS pointer data from the reader func ReadPointer(reader io.Reader) (Pointer, error) { - buf := make([]byte, blobSizeCutoff) + buf := make([]byte, MetaFileMaxSize) n, err := io.ReadFull(reader, buf) if err != nil && err != io.ErrUnexpectedEOF { return Pointer{}, err @@ -65,6 +63,7 @@ func ReadPointerFromBuffer(buf []byte) (Pointer, error) { return p, ErrInvalidStructure } + // spec says "key/value pairs MUST be sorted alphabetically in ascending order (version is exception and must be the first)" oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix) if len(oid) != 64 || !oidPattern.MatchString(oid) { return p, ErrInvalidOIDFormat diff --git a/modules/lfs/pointer_scanner_gogit.go b/modules/lfs/pointer_scanner_gogit.go index f4302c23bc..e153b8e24e 100644 --- a/modules/lfs/pointer_scanner_gogit.go +++ b/modules/lfs/pointer_scanner_gogit.go @@ -31,7 +31,7 @@ func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan c default: } - if blob.Size > blobSizeCutoff { + if blob.Size > MetaFileMaxSize { return nil } diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index 8bbd45771a..ef72d76db4 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -32,7 +32,7 @@ func TestBasicTransferAdapter(t *testing.T) { if strings.Contains(url, "download-request") { assert.Equal(t, "GET", req.Method) - return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBufferString("dummy"))} + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("dummy"))} } else if strings.Contains(url, "upload-request") { assert.Equal(t, "PUT", req.Method) assert.Equal(t, "application/octet-stream", req.Header.Get("Content-Type")) @@ -126,7 +126,7 @@ func TestBasicTransferAdapter(t *testing.T) { } for n, c := range cases { - err := a.Upload(t.Context(), c.link, p, bytes.NewBufferString("dummy")) + err := a.Upload(t.Context(), c.link, p, strings.NewReader("dummy")) if len(c.expectederror) > 0 { assert.Contains(t, err.Error(), c.expectederror, "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { diff --git a/modules/lfstransfer/backend/backend.go b/modules/lfstransfer/backend/backend.go index 1328d93a48..dd4108ea56 100644 --- a/modules/lfstransfer/backend/backend.go +++ b/modules/lfstransfer/backend/backend.go @@ -47,7 +47,7 @@ func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (t return nil, err } server = server.JoinPath("api/internal/repo", repo, "info/lfs") - return &GiteaBackend{ctx: ctx, server: server, op: op, authToken: token, internalAuth: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil + return &GiteaBackend{ctx: ctx, server: server, op: op, authToken: token, internalAuth: "Bearer " + setting.InternalToken, logger: logger}, nil } // Batch implements transfer.Backend diff --git a/modules/lfstransfer/backend/lock.go b/modules/lfstransfer/backend/lock.go index 639f8b184e..2c3c16a9bb 100644 --- a/modules/lfstransfer/backend/lock.go +++ b/modules/lfstransfer/backend/lock.go @@ -5,6 +5,7 @@ package backend import ( "context" + "errors" "fmt" "io" "net/http" @@ -74,7 +75,7 @@ func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) { if respBody.Lock == nil { g.logger.Log("api returned nil lock") - return nil, fmt.Errorf("api returned nil lock") + return nil, errors.New("api returned nil lock") } respLock := respBody.Lock owner := userUnknown @@ -263,7 +264,7 @@ func (g *giteaLock) CurrentUser() (string, error) { // AsLockSpec implements transfer.Lock func (g *giteaLock) AsLockSpec(ownerID bool) ([]string, error) { msgs := []string{ - fmt.Sprintf("lock %s", g.ID()), + "lock " + g.ID(), fmt.Sprintf("path %s %s", g.ID(), g.Path()), fmt.Sprintf("locked-at %s %s", g.ID(), g.FormattedTimestamp()), fmt.Sprintf("ownername %s %s", g.ID(), g.OwnerName()), @@ -285,9 +286,9 @@ func (g *giteaLock) AsLockSpec(ownerID bool) ([]string, error) { // AsArguments implements transfer.Lock func (g *giteaLock) AsArguments() []string { return []string{ - fmt.Sprintf("id=%s", g.ID()), - fmt.Sprintf("path=%s", g.Path()), - fmt.Sprintf("locked-at=%s", g.FormattedTimestamp()), - fmt.Sprintf("ownername=%s", g.OwnerName()), + "id=" + g.ID(), + "path=" + g.Path(), + "locked-at=" + g.FormattedTimestamp(), + "ownername=" + g.OwnerName(), } } diff --git a/modules/lfstransfer/backend/util.go b/modules/lfstransfer/backend/util.go index 98ce0b1e62..afe02f799c 100644 --- a/modules/lfstransfer/backend/util.go +++ b/modules/lfstransfer/backend/util.go @@ -132,6 +132,7 @@ func newInternalRequestLFS(ctx context.Context, internalURL, method string, head return nil } req := private.NewInternalRequest(ctx, internalURL, method) + req.SetReadWriteTimeout(0) for k, v := range headers { req.Header(k, v) } diff --git a/modules/log/event_format.go b/modules/log/event_format.go index c23b3b411b..4cf471d223 100644 --- a/modules/log/event_format.go +++ b/modules/log/event_format.go @@ -212,7 +212,7 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms } } if hasColorValue { - msg = []byte(fmt.Sprintf(msgFormat, msgArgs...)) + msg = fmt.Appendf(nil, msgFormat, msgArgs...) } } // try to re-use the pre-formatted simple text message @@ -243,8 +243,8 @@ func EventFormatTextMessage(mode *WriterMode, event *Event, msgFormat string, ms buf = append(buf, msg...) if event.Stacktrace != "" && mode.StacktraceLevel <= event.Level { - lines := bytes.Split([]byte(event.Stacktrace), []byte("\n")) - for _, line := range lines { + lines := bytes.SplitSeq([]byte(event.Stacktrace), []byte("\n")) + for line := range lines { buf = append(buf, "\n\t"...) buf = append(buf, line...) } diff --git a/modules/log/event_writer_base.go b/modules/log/event_writer_base.go index c327c48ca2..9189ca4e90 100644 --- a/modules/log/event_writer_base.go +++ b/modules/log/event_writer_base.go @@ -105,7 +105,7 @@ func (b *EventWriterBaseImpl) Run(ctx context.Context) { case io.WriterTo: _, err = msg.WriteTo(b.OutputWriteCloser) default: - _, err = b.OutputWriteCloser.Write([]byte(fmt.Sprint(msg))) + _, err = fmt.Fprint(b.OutputWriteCloser, msg) } if err != nil { FallbackErrorf("unable to write log message of %q (%v): %v", b.Name, err, event.Msg) diff --git a/modules/log/flags.go b/modules/log/flags.go index 8064c91745..f409261150 100644 --- a/modules/log/flags.go +++ b/modules/log/flags.go @@ -123,7 +123,7 @@ func FlagsFromString(from string, def ...uint32) Flags { return Flags{defined: true, flags: def[0]} } flags := uint32(0) - for _, flag := range strings.Split(strings.ToLower(from), ",") { + for flag := range strings.SplitSeq(strings.ToLower(from), ",") { flags |= flagFromString[strings.TrimSpace(flag)] } return Flags{defined: true, flags: flags} diff --git a/modules/log/flags_test.go b/modules/log/flags_test.go index 03972a9fb0..6eb65d8114 100644 --- a/modules/log/flags_test.go +++ b/modules/log/flags_test.go @@ -12,19 +12,19 @@ import ( ) func TestFlags(t *testing.T) { - assert.EqualValues(t, Ldefault, Flags{}.Bits()) + assert.Equal(t, Ldefault, Flags{}.Bits()) assert.EqualValues(t, 0, FlagsFromString("").Bits()) - assert.EqualValues(t, Lgopid, FlagsFromString("", Lgopid).Bits()) + assert.Equal(t, Lgopid, FlagsFromString("", Lgopid).Bits()) assert.EqualValues(t, 0, FlagsFromString("none", Lgopid).Bits()) - assert.EqualValues(t, Ldate|Ltime, FlagsFromString("date,time", Lgopid).Bits()) + assert.Equal(t, Ldate|Ltime, FlagsFromString("date,time", Lgopid).Bits()) - assert.EqualValues(t, "stdflags", FlagsFromString("stdflags").String()) - assert.EqualValues(t, "medfile", FlagsFromString("medfile").String()) + assert.Equal(t, "stdflags", FlagsFromString("stdflags").String()) + assert.Equal(t, "medfile", FlagsFromString("medfile").String()) bs, err := json.Marshal(FlagsFromString("utc,level")) assert.NoError(t, err) - assert.EqualValues(t, `"level,utc"`, string(bs)) + assert.Equal(t, `"level,utc"`, string(bs)) var flags Flags assert.NoError(t, json.Unmarshal(bs, &flags)) - assert.EqualValues(t, LUTC|Llevel, flags.Bits()) + assert.Equal(t, LUTC|Llevel, flags.Bits()) } diff --git a/modules/log/level_test.go b/modules/log/level_test.go index cd18a807d8..0e59af6cb7 100644 --- a/modules/log/level_test.go +++ b/modules/log/level_test.go @@ -32,11 +32,11 @@ func TestLevelMarshalUnmarshalJSON(t *testing.T) { assert.NoError(t, err) assert.Equal(t, INFO, testLevel.Level) - err = json.Unmarshal([]byte(fmt.Sprintf(`{"level":%d}`, 2)), &testLevel) + err = json.Unmarshal(fmt.Appendf(nil, `{"level":%d}`, 2), &testLevel) assert.NoError(t, err) assert.Equal(t, INFO, testLevel.Level) - err = json.Unmarshal([]byte(fmt.Sprintf(`{"level":%d}`, 10012)), &testLevel) + err = json.Unmarshal(fmt.Appendf(nil, `{"level":%d}`, 10012), &testLevel) assert.NoError(t, err) assert.Equal(t, INFO, testLevel.Level) @@ -51,5 +51,5 @@ func TestLevelMarshalUnmarshalJSON(t *testing.T) { } func makeTestLevelBytes(level string) []byte { - return []byte(fmt.Sprintf(`{"level":"%s"}`, level)) + return fmt.Appendf(nil, `{"level":"%s"}`, level) } diff --git a/modules/log/logger.go b/modules/log/logger.go index 3fc524d55e..8b89e0eb5a 100644 --- a/modules/log/logger.go +++ b/modules/log/logger.go @@ -45,6 +45,6 @@ type Logger interface { LevelLogger } -type LogStringer interface { //nolint:revive +type LogStringer interface { //nolint:revive // export stutter LogString() string } diff --git a/modules/log/logger_test.go b/modules/log/logger_test.go index 9f604815f6..a74139dc51 100644 --- a/modules/log/logger_test.go +++ b/modules/log/logger_test.go @@ -57,16 +57,16 @@ func TestLogger(t *testing.T) { dump := logger.DumpWriters() assert.Empty(t, dump) - assert.EqualValues(t, NONE, logger.GetLevel()) + assert.Equal(t, NONE, logger.GetLevel()) assert.False(t, logger.IsEnabled()) w1 := newDummyWriter("dummy-1", DEBUG, 0) logger.AddWriters(w1) - assert.EqualValues(t, DEBUG, logger.GetLevel()) + assert.Equal(t, DEBUG, logger.GetLevel()) w2 := newDummyWriter("dummy-2", WARN, 200*time.Millisecond) logger.AddWriters(w2) - assert.EqualValues(t, DEBUG, logger.GetLevel()) + assert.Equal(t, DEBUG, logger.GetLevel()) dump = logger.DumpWriters() assert.Len(t, dump, 2) diff --git a/modules/markup/common/footnote.go b/modules/markup/common/footnote.go index 4406803694..1ece436c66 100644 --- a/modules/markup/common/footnote.go +++ b/modules/markup/common/footnote.go @@ -53,7 +53,7 @@ type FootnoteLink struct { // Dump implements Node.Dump. func (n *FootnoteLink) Dump(source []byte, level int) { m := map[string]string{} - m["Index"] = fmt.Sprintf("%v", n.Index) + m["Index"] = strconv.Itoa(n.Index) m["Name"] = fmt.Sprintf("%v", n.Name) ast.DumpHelper(n, source, level, m, nil) } @@ -85,7 +85,7 @@ type FootnoteBackLink struct { // Dump implements Node.Dump. func (n *FootnoteBackLink) Dump(source []byte, level int) { m := map[string]string{} - m["Index"] = fmt.Sprintf("%v", n.Index) + m["Index"] = strconv.Itoa(n.Index) m["Name"] = fmt.Sprintf("%v", n.Name) ast.DumpHelper(n, source, level, m, nil) } @@ -151,7 +151,7 @@ type FootnoteList struct { // Dump implements Node.Dump. func (n *FootnoteList) Dump(source []byte, level int) { m := map[string]string{} - m["Count"] = fmt.Sprintf("%v", n.Count) + m["Count"] = strconv.Itoa(n.Count) ast.DumpHelper(n, source, level, m, nil) } @@ -197,7 +197,7 @@ func (b *footnoteBlockParser) Open(parent ast.Node, reader text.Reader, pc parse return nil, parser.NoChildren } open := pos + 1 - closure := util.FindClosure(line[pos+1:], '[', ']', false, false) //nolint + closure := util.FindClosure(line[pos+1:], '[', ']', false, false) //nolint:staticcheck // deprecated function closes := pos + 1 + closure next := closes + 1 if closure > -1 { @@ -287,7 +287,7 @@ func (s *footnoteParser) Parse(parent ast.Node, block text.Reader, pc parser.Con return nil } open := pos - closure := util.FindClosure(line[pos:], '[', ']', false, false) //nolint + closure := util.FindClosure(line[pos:], '[', ']', false, false) //nolint:staticcheck // deprecated function if closure < 0 { return nil } @@ -409,9 +409,9 @@ func (r *FootnoteHTMLRenderer) renderFootnoteLink(w util.BufWriter, source []byt _, _ = w.Write(n.Name) _, _ = w.WriteString(`"><a href="#fn:`) _, _ = w.Write(n.Name) - _, _ = w.WriteString(`" class="footnote-ref" role="doc-noteref">`) + _, _ = w.WriteString(`" class="footnote-ref" role="doc-noteref">`) // FIXME: here and below, need to keep the classes _, _ = w.WriteString(is) - _, _ = w.WriteString(`</a></sup>`) + _, _ = w.WriteString(` </a></sup>`) // the style doesn't work at the moment, so add a space to separate the names } return ast.WalkContinue, nil } diff --git a/modules/markup/common/linkify.go b/modules/markup/common/linkify.go index 52888958fa..3eecb97eac 100644 --- a/modules/markup/common/linkify.go +++ b/modules/markup/common/linkify.go @@ -85,9 +85,10 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont } else if lastChar == ')' { closing := 0 for i := m[1] - 1; i >= m[0]; i-- { - if line[i] == ')' { + switch line[i] { + case ')': closing++ - } else if line[i] == '(' { + case '(': closing-- } } diff --git a/modules/markup/console/console.go b/modules/markup/console/console.go index 06f3acfa68..492579b0a5 100644 --- a/modules/markup/console/console.go +++ b/modules/markup/console/console.go @@ -6,13 +6,14 @@ package console import ( "bytes" "io" - "path" + "unicode/utf8" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" trend "github.com/buildkite/terminal-to-html/v3" - "github.com/go-enry/go-enry/v2" ) func init() { @@ -22,6 +23,8 @@ func init() { // Renderer implements markup.Renderer type Renderer struct{} +var _ markup.RendererContentDetector = (*Renderer)(nil) + // Name implements markup.Renderer func (Renderer) Name() string { return "console" @@ -40,15 +43,36 @@ func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { } // CanRender implements markup.RendererContentDetector -func (Renderer) CanRender(filename string, input io.Reader) bool { - buf, err := io.ReadAll(input) - if err != nil { +func (Renderer) CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool { + if !sniffedType.IsTextPlain() { return false } - if enry.GetLanguage(path.Base(filename), buf) != enry.OtherLanguage { + + s := util.UnsafeBytesToString(prefetchBuf) + rs := []rune(s) + cnt := 0 + firstErrPos := -1 + isCtrlSep := func(p int) bool { + return p < len(rs) && (rs[p] == ';' || rs[p] == 'm') + } + for i, c := range rs { + if c == 0 { + return false + } + if c == '\x1b' { + match := i+1 < len(rs) && rs[i+1] == '[' + if match && (isCtrlSep(i+2) || isCtrlSep(i+3) || isCtrlSep(i+4) || isCtrlSep(i+5)) { + cnt++ + } + } + if c == utf8.RuneError && firstErrPos == -1 { + firstErrPos = i + } + } + if firstErrPos != -1 && firstErrPos != len(rs)-1 { return false } - return bytes.ContainsRune(buf, '\x1b') + return cnt >= 2 // only render it as console output if there are at least two escape sequences } // Render renders terminal colors to HTML with all specific handling stuff. diff --git a/modules/markup/console/console_test.go b/modules/markup/console/console_test.go index 040ec151f4..d1192bebc2 100644 --- a/modules/markup/console/console_test.go +++ b/modules/markup/console/console_test.go @@ -8,23 +8,39 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/typesniffer" "github.com/stretchr/testify/assert" ) func TestRenderConsole(t *testing.T) { - var render Renderer - kases := map[string]string{ - "\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok": "<span class=\"term-fg37 term-bg40\">npm</span> <span class=\"term-fg32\">info</span> <span class=\"term-fg35\">it worked if it ends with</span> ok", + cases := []struct { + input string + expected string + }{ + {"\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok", `<span class="term-fg37 term-bg40">npm</span> <span class="term-fg32">info</span> <span class="term-fg35">it worked if it ends with</span> ok`}, + {"\x1b[1;2m \x1b[123m 啊", `<span class="term-fg2"> 啊</span>`}, + {"\x1b[1;2m \x1b[123m \xef", `<span class="term-fg2"> �</span>`}, + {"\x1b[1;2m \x1b[123m \xef \xef", ``}, + {"\x1b[12", ``}, + {"\x1b[1", ``}, + {"\x1b[FOO\x1b[", ``}, + {"\x1b[mFOO\x1b[m", `FOO`}, } - for k, v := range kases { + var render Renderer + for i, c := range cases { var buf strings.Builder - canRender := render.CanRender("test", strings.NewReader(k)) - assert.True(t, canRender) + st := typesniffer.DetectContentType([]byte(c.input)) + canRender := render.CanRender("test", st, []byte(c.input)) + if c.expected == "" { + assert.False(t, canRender, "case %d: expected not to render", i) + continue + } - err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(k), &buf) + assert.True(t, canRender) + err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(c.input), &buf) assert.NoError(t, err) - assert.EqualValues(t, v, buf.String()) + assert.Equal(t, c.expected, buf.String()) } } diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index b0b18ab467..fff7f0baca 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -25,6 +25,6 @@ func TestRenderCSV(t *testing.T) { var buf strings.Builder err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(k), &buf) assert.NoError(t, err) - assert.EqualValues(t, v, buf.String()) + assert.Equal(t, v, buf.String()) } } diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 03242e569e..39861ade12 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -12,11 +12,9 @@ import ( "runtime" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) // RegisterRenderers registers all supported third part renderers according settings @@ -77,27 +75,22 @@ func envMark(envName string) string { // Render renders the data of the document to HTML via the external tool. func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { - var ( - command = strings.NewReplacer( - envMark("GITEA_PREFIX_SRC"), ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault), - envMark("GITEA_PREFIX_RAW"), ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw), - ).Replace(p.Command) - commands = strings.Fields(command) - args = commands[1:] - ) + baseLinkSrc := ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault) + baseLinkRaw := ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw) + command := strings.NewReplacer( + envMark("GITEA_PREFIX_SRC"), baseLinkSrc, + envMark("GITEA_PREFIX_RAW"), baseLinkRaw, + ).Replace(p.Command) + commands := strings.Fields(command) + args := commands[1:] if p.IsInputFile { // write to temp file - f, err := os.CreateTemp("", "gitea_input") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("gitea_input") if err != nil { return fmt.Errorf("%s create temp file when rendering %s failed: %w", p.Name(), p.Command, err) } - tmpPath := f.Name() - defer func() { - if err := util.Remove(tmpPath); err != nil { - log.Warn("Unable to remove temporary file: %s: Error: %v", tmpPath, err) - } - }() + defer cleanup() _, err = io.Copy(f, input) if err != nil { @@ -112,14 +105,14 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. args = append(args, f.Name()) } - processCtx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Render [%s] for %s", commands[0], ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault))) + processCtx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Render [%s] for %s", commands[0], baseLinkSrc)) defer finished() cmd := exec.CommandContext(processCtx, commands[0], args...) cmd.Env = append( os.Environ(), - "GITEA_PREFIX_SRC="+ctx.RenderHelper.ResolveLink("", markup.LinkTypeDefault), - "GITEA_PREFIX_RAW="+ctx.RenderHelper.ResolveLink("", markup.LinkTypeRaw), + "GITEA_PREFIX_SRC="+baseLinkSrc, + "GITEA_PREFIX_RAW="+baseLinkRaw, ) if !p.IsInputFile { cmd.Stdin = input diff --git a/modules/markup/html.go b/modules/markup/html.go index 3aaf669c63..51afd4be00 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "regexp" + "slices" "strings" "sync" @@ -32,7 +33,6 @@ type globalVarsType struct { comparePattern *regexp.Regexp fullURLPattern *regexp.Regexp emailRegex *regexp.Regexp - blackfridayExtRegex *regexp.Regexp emojiShortCodeRegex *regexp.Regexp issueFullPattern *regexp.Regexp filesChangedFullPattern *regexp.Regexp @@ -72,10 +72,8 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // it is still accepted by the CommonMark specification, as well as the HTML5 spec: // http://spec.commonmark.org/0.28/#email-address // https://html.spec.whatwg.org/multipage/input.html#e-mail-state-(type%3Demail) - v.emailRegex = regexp.MustCompile("(?:\\s|^|\\(|\\[)([a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)(?:\\s|$|\\)|\\]|;|,|\\?|!|\\.(\\s|$))") - - // blackfridayExtRegex is for blackfriday extensions create IDs like fn:user-content-footnote - v.blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`) + // At the moment, we use stricter rule for rendering purpose: only allow the "name" part starting after the word boundary + v.emailRegex = regexp.MustCompile(`\b([-\w.!#$%&'*+/=?^{|}~]*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)\b`) // emojiShortCodeRegex find emoji by alias like :smile: v.emojiShortCodeRegex = regexp.MustCompile(`:[-+\w]+:`) @@ -89,22 +87,18 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // codePreviewPattern matches "http://domain/.../{owner}/{repo}/src/commit/{commit}/{filepath}#L10-L20" v.codePreviewPattern = regexp.MustCompile(`https?://\S+/([^\s/]+)/([^\s/]+)/src/commit/([0-9a-f]{7,64})(/\S+)#(L\d+(-L\d+)?)`) - v.tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`) + // cleans: "<foo/bar", "<any words/", ("<html", "<head", "<script", "<style", "<?", "<%") + v.tagCleaner = regexp.MustCompile(`(?i)<(/?\w+/\w+|/[\w ]+/|/?(html|head|script|style|%|\?)\b)`) v.nulCleaner = strings.NewReplacer("\000", "") return v }) -// IsFullURLBytes reports whether link fits valid format. -func IsFullURLBytes(link []byte) bool { - return globalVars().fullURLPattern.Match(link) -} - func IsFullURLString(link string) bool { return globalVars().fullURLPattern.MatchString(link) } func IsNonEmptyRelativePath(link string) bool { - return link != "" && !IsFullURLString(link) && link[0] != '/' && link[0] != '?' && link[0] != '#' + return link != "" && !IsFullURLString(link) && link[0] != '?' && link[0] != '#' } // CustomLinkURLSchemes allows for additional schemes to be detected when parsing links within text @@ -116,13 +110,7 @@ func CustomLinkURLSchemes(schemes []string) { if !validScheme.MatchString(s) { continue } - without := false - for _, sna := range xurls.SchemesNoAuthority { - if s == sna { - without = true - break - } - } + without := slices.Contains(xurls.SchemesNoAuthority, s) if without { s += ":" } else { @@ -260,7 +248,7 @@ func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output node, err := html.Parse(io.MultiReader( // prepend "<html><body>" strings.NewReader("<html><body>"), - // Strip out nuls - they're always invalid + // strip out NULLs (they're always invalid), and escape known tags bytes.NewReader(globalVars().tagCleaner.ReplaceAll([]byte(globalVars().nulCleaner.Replace(string(rawHTML))), []byte("<$1"))), // close the tags strings.NewReader("</body></html>"), @@ -316,44 +304,39 @@ func isEmojiNode(node *html.Node) bool { } func visitNode(ctx *RenderContext, procs []processor, node *html.Node) *html.Node { - // Add user-content- to IDs and "#" links if they don't already have them - for idx, attr := range node.Attr { - val := strings.TrimPrefix(attr.Val, "#") - notHasPrefix := !(strings.HasPrefix(val, "user-content-") || globalVars().blackfridayExtRegex.MatchString(val)) - - if attr.Key == "id" && notHasPrefix { - node.Attr[idx].Val = "user-content-" + attr.Val - } - - if attr.Key == "href" && strings.HasPrefix(attr.Val, "#") && notHasPrefix { - node.Attr[idx].Val = "#user-content-" + val - } - } - - switch node.Type { - case html.TextNode: + if node.Type == html.TextNode { for _, proc := range procs { proc(ctx, node) // it might add siblings } + return node.NextSibling + } + if node.Type != html.ElementNode { + return node.NextSibling + } - case html.ElementNode: - if isEmojiNode(node) { - // TextNode emoji will be converted to `<span class="emoji">`, then the next iteration will visit the "span" - // if we don't stop it, it will go into the TextNode again and create an infinite recursion - return node.NextSibling - } else if node.Data == "code" || node.Data == "pre" { - return node.NextSibling // ignore code and pre nodes - } else if node.Data == "img" { - return visitNodeImg(ctx, node) - } else if node.Data == "video" { - return visitNodeVideo(ctx, node) - } else if node.Data == "a" { - procs = emojiProcessors // Restrict text in links to emojis - } - for n := node.FirstChild; n != nil; { - n = visitNode(ctx, procs, n) - } - default: + processNodeAttrID(node) + processFootnoteNode(ctx, node) // FIXME: the footnote processing should be done in the "footnote.go" renderer directly + + if isEmojiNode(node) { + // TextNode emoji will be converted to `<span class="emoji">`, then the next iteration will visit the "span" + // if we don't stop it, it will go into the TextNode again and create an infinite recursion + return node.NextSibling + } else if node.Data == "code" || node.Data == "pre" { + return node.NextSibling // ignore code and pre nodes + } else if node.Data == "img" { + return visitNodeImg(ctx, node) + } else if node.Data == "video" { + return visitNodeVideo(ctx, node) + } + + if node.Data == "a" { + processNodeA(ctx, node) + // only use emoji processors for the content in the "A" tag, + // because the content there is not processable, for example: the content is a commit id or a full URL. + procs = emojiProcessors + } + for n := node.FirstChild; n != nil; { + n = visitNode(ctx, procs, n) } return node.NextSibling } diff --git a/modules/markup/html_commit.go b/modules/markup/html_commit.go index aa1b7d034a..fe7a034967 100644 --- a/modules/markup/html_commit.go +++ b/modules/markup/html_commit.go @@ -43,7 +43,6 @@ func createCodeLink(href, content, class string) *html.Node { code := &html.Node{ Type: html.ElementNode, Data: atom.Code.String(), - Attr: []html.Attribute{{Key: "class", Val: "nohighlight"}}, } code.AppendChild(text) @@ -63,7 +62,7 @@ func anyHashPatternExtract(s string) (ret anyHashPatternResult, ok bool) { // if url ends in '.', it's very likely that it is not part of the actual url but used to finish a sentence. ret.PosEnd-- ret.FullURL = ret.FullURL[:len(ret.FullURL)-1] - for i := 0; i < len(m); i++ { + for i := range m { m[i] = min(m[i], ret.PosEnd) } } @@ -189,7 +188,7 @@ func hashCurrentPatternProcessor(ctx *RenderContext, node *html.Node) { continue } - link := ctx.RenderHelper.ResolveLink(util.URLJoin(ctx.RenderOptions.Metas["user"], ctx.RenderOptions.Metas["repo"], "commit", hash), LinkTypeApp) + link := "/:root/" + util.URLJoin(ctx.RenderOptions.Metas["user"], ctx.RenderOptions.Metas["repo"], "commit", hash) replaceContent(node, m[2], m[3], createCodeLink(link, base.ShortSha(hash), "commit")) start = 0 node = node.NextSibling.NextSibling @@ -205,9 +204,9 @@ func commitCrossReferencePatternProcessor(ctx *RenderContext, node *html.Node) { return } - reftext := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) - linkHref := ctx.RenderHelper.ResolveLink(util.URLJoin(ref.Owner, ref.Name, "commit", ref.CommitSha), LinkTypeApp) - link := createLink(ctx, linkHref, reftext, "commit") + refText := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) + linkHref := "/:root/" + util.URLJoin(ref.Owner, ref.Name, "commit", ref.CommitSha) + link := createLink(ctx, linkHref, refText, "commit") replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link) node = node.NextSibling.NextSibling diff --git a/modules/markup/html_email.go b/modules/markup/html_email.go index cbfae8b829..cf18e99d98 100644 --- a/modules/markup/html_email.go +++ b/modules/markup/html_email.go @@ -3,7 +3,11 @@ package markup -import "golang.org/x/net/html" +import ( + "strings" + + "golang.org/x/net/html" +) // emailAddressProcessor replaces raw email addresses with a mailto: link. func emailAddressProcessor(ctx *RenderContext, node *html.Node) { @@ -14,6 +18,14 @@ func emailAddressProcessor(ctx *RenderContext, node *html.Node) { return } + var nextByte byte + if len(node.Data) > m[3] { + nextByte = node.Data[m[3]] + } + if strings.IndexByte(":/", nextByte) != -1 { + // for cases: "git@gitea.com:owner/repo.git", "https://git@gitea.com/owner/repo.git" + return + } mail := node.Data[m[2]:m[3]] replaceContent(node, m[2], m[3], createLink(ctx, "mailto:"+mail, mail, "" /*mailto*/)) node = node.NextSibling.NextSibling diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index 159d712955..467cc509d0 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -107,7 +107,7 @@ func TestRender_IssueIndexPattern2(t *testing.T) { isExternal := false if marker == "!" { path = "pulls" - prefix = "http://localhost:3000/someUser/someRepo/pulls/" + prefix = "/someUser/someRepo/pulls/" } else { path = "issues" prefix = "https://someurl.com/someUser/someRepo/" @@ -116,7 +116,7 @@ func TestRender_IssueIndexPattern2(t *testing.T) { links := make([]any, len(indices)) for i, index := range indices { - links[i] = numericIssueLink(util.URLJoin(TestRepoURL, path), "ref-issue", index, marker) + links[i] = numericIssueLink(util.URLJoin("/test-owner/test-repo", path), "ref-issue", index, marker) } expectedNil := fmt.Sprintf(expectedFmt, links...) testRenderIssueIndexPattern(t, s, expectedNil, NewTestRenderContext(TestAppURL, localMetas)) @@ -293,13 +293,13 @@ func TestRender_AutoLink(t *testing.T) { // render valid commit URLs tmp := util.URLJoin(TestRepoURL, "commit", "d8a994ef243349f321568f9e36d5c3f444b99cae") - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24</code></a>") tmp += "#diff-2" - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24 (diff-2)</code></a>") // render other commit URLs tmp = "https://external-link.gitea.io/go-gitea/gitea/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2" - test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code>d8a994ef24 (diff-2)</code></a>") } func TestRender_FullIssueURLs(t *testing.T) { @@ -405,10 +405,10 @@ func TestRegExp_anySHA1Pattern(t *testing.T) { if v.CommitID == "" { assert.False(t, ok) } else { - assert.EqualValues(t, strings.TrimSuffix(k, "."), ret.FullURL) - assert.EqualValues(t, v.CommitID, ret.CommitID) - assert.EqualValues(t, v.SubPath, ret.SubPath) - assert.EqualValues(t, v.QueryHash, ret.QueryHash) + assert.Equal(t, strings.TrimSuffix(k, "."), ret.FullURL) + assert.Equal(t, v.CommitID, ret.CommitID) + assert.Equal(t, v.SubPath, ret.SubPath) + assert.Equal(t, v.QueryHash, ret.QueryHash) } } } diff --git a/modules/markup/html_issue.go b/modules/markup/html_issue.go index 7a6f33011a..85bec5db20 100644 --- a/modules/markup/html_issue.go +++ b/modules/markup/html_issue.go @@ -82,7 +82,7 @@ func createIssueLinkContentWithSummary(ctx *RenderContext, linkHref string, ref h, err := DefaultRenderHelperFuncs.RenderRepoIssueIconTitle(ctx, RenderIssueIconTitleOptions{ OwnerName: ref.Owner, RepoName: ref.Name, - LinkHref: linkHref, + LinkHref: ctx.RenderHelper.ResolveLink(linkHref, LinkTypeDefault), IssueIndex: issueIndex, }) if err != nil { @@ -162,7 +162,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { issueOwner := util.Iif(ref.Owner == "", ctx.RenderOptions.Metas["user"], ref.Owner) issueRepo := util.Iif(ref.Owner == "", ctx.RenderOptions.Metas["repo"], ref.Name) issuePath := util.Iif(ref.IsPull, "pulls", "issues") - linkHref := ctx.RenderHelper.ResolveLink(util.URLJoin(issueOwner, issueRepo, issuePath, ref.Issue), LinkTypeApp) + linkHref := "/:root/" + util.URLJoin(issueOwner, issueRepo, issuePath, ref.Issue) // at the moment, only render the issue index in a full line (or simple line) as icon+title // otherwise it would be too noisy for "take #1 as an example" in a sentence diff --git a/modules/markup/html_issue_test.go b/modules/markup/html_issue_test.go index 8d189fbdf6..39cd9dcf6a 100644 --- a/modules/markup/html_issue_test.go +++ b/modules/markup/html_issue_test.go @@ -30,6 +30,7 @@ func TestRender_IssueList(t *testing.T) { rctx := markup.NewTestRenderContext(markup.TestAppURL, map[string]string{ "user": "test-user", "repo": "test-repo", "markupAllowShortIssuePattern": "true", + "footnoteContextId": "12345", }) out, err := markdown.RenderString(rctx, input) require.NoError(t, err) @@ -39,7 +40,7 @@ func TestRender_IssueList(t *testing.T) { t.Run("NormalIssueRef", func(t *testing.T) { test( "#12345", - `<p><a href="http://localhost:3000/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a></p>`, + `<p><a href="/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a></p>`, ) }) @@ -56,7 +57,7 @@ func TestRender_IssueList(t *testing.T) { test( "* foo #12345 bar", `<ul> -<li>foo <a href="http://localhost:3000/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a> bar</li> +<li>foo <a href="/test-user/test-repo/issues/12345" class="ref-issue" rel="nofollow">#12345</a> bar</li> </ul>`, ) }) @@ -69,4 +70,22 @@ func TestRender_IssueList(t *testing.T) { </ul>`, ) }) + + t.Run("IssueFootnote", func(t *testing.T) { + test( + "foo[^1][^2]\n\n[^1]: bar\n[^2]: baz", + `<p>foo<sup id="fnref:user-content-1-12345"><a href="#fn:user-content-1-12345" rel="nofollow">1 </a></sup><sup id="fnref:user-content-2-12345"><a href="#fn:user-content-2-12345" rel="nofollow">2 </a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-1-12345"> +<p>bar <a href="#fnref:user-content-1-12345" rel="nofollow">↩︎</a></p> +</li> +<li id="fn:user-content-2-12345"> +<p>baz <a href="#fnref:user-content-2-12345" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div>`, + ) + }) } diff --git a/modules/markup/html_link.go b/modules/markup/html_link.go index 0e7a988d36..43faef1681 100644 --- a/modules/markup/html_link.go +++ b/modules/markup/html_link.go @@ -31,8 +31,8 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { // It makes page handling terrible, but we prefer GitHub syntax // And fall back to MediaWiki only when it is obvious from the look // Of text and link contents - sl := strings.Split(content, "|") - for _, v := range sl { + sl := strings.SplitSeq(content, "|") + for v := range sl { if equalPos := strings.IndexByte(v, '='); equalPos == -1 { // There is no equal in this argument; this is a mandatory arg if props["name"] == "" { @@ -125,7 +125,6 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { } } if image { - link = ctx.RenderHelper.ResolveLink(link, LinkTypeMedia) title := props["title"] if title == "" { title = props["alt"] @@ -151,7 +150,6 @@ func shortLinkProcessor(ctx *RenderContext, node *html.Node) { childNode.Attr = childNode.Attr[:2] } } else { - link = ctx.RenderHelper.ResolveLink(link, LinkTypeDefault) childNode.Type = html.TextNode childNode.Data = name } diff --git a/modules/markup/html_mention.go b/modules/markup/html_mention.go index fffa12e7b7..f97c034cf3 100644 --- a/modules/markup/html_mention.go +++ b/modules/markup/html_mention.go @@ -33,7 +33,7 @@ func mentionProcessor(ctx *RenderContext, node *html.Node) { if ok && strings.Contains(mention, "/") { mentionOrgAndTeam := strings.Split(mention, "/") if mentionOrgAndTeam[0][1:] == ctx.RenderOptions.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") { - link := ctx.RenderHelper.ResolveLink(util.URLJoin("org", ctx.RenderOptions.Metas["org"], "teams", mentionOrgAndTeam[1]), LinkTypeApp) + link := "/:root/" + util.URLJoin("org", ctx.RenderOptions.Metas["org"], "teams", mentionOrgAndTeam[1]) replaceContent(node, loc.Start, loc.End, createLink(ctx, link, mention, "" /*mention*/)) node = node.NextSibling.NextSibling start = 0 @@ -45,7 +45,7 @@ func mentionProcessor(ctx *RenderContext, node *html.Node) { mentionedUsername := mention[1:] if DefaultRenderHelperFuncs != nil && DefaultRenderHelperFuncs.IsUsernameMentionable(ctx, mentionedUsername) { - link := ctx.RenderHelper.ResolveLink(mentionedUsername, LinkTypeApp) + link := "/:root/" + mentionedUsername replaceContent(node, loc.Start, loc.End, createLink(ctx, link, mention, "" /*mention*/)) node = node.NextSibling.NextSibling start = 0 diff --git a/modules/markup/html_node.go b/modules/markup/html_node.go index 6e8ca67900..4eb78fdd2b 100644 --- a/modules/markup/html_node.go +++ b/modules/markup/html_node.go @@ -4,42 +4,105 @@ package markup import ( + "strings" + "golang.org/x/net/html" ) +func isAnchorIDUserContent(s string) bool { + // blackfridayExtRegex is for blackfriday extensions create IDs like fn:user-content-footnote + // old logic: blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`) + return strings.HasPrefix(s, "user-content-") || strings.Contains(s, ":user-content-") +} + +func isAnchorIDFootnote(s string) bool { + return strings.HasPrefix(s, "fnref:user-content-") || strings.HasPrefix(s, "fn:user-content-") +} + +func isAnchorHrefFootnote(s string) bool { + return strings.HasPrefix(s, "#fnref:user-content-") || strings.HasPrefix(s, "#fn:user-content-") +} + +func processNodeAttrID(node *html.Node) { + // Add user-content- to IDs and "#" links if they don't already have them, + // and convert the link href to a relative link to the host root + for idx, attr := range node.Attr { + if attr.Key == "id" { + if !isAnchorIDUserContent(attr.Val) { + node.Attr[idx].Val = "user-content-" + attr.Val + } + } + } +} + +func processFootnoteNode(ctx *RenderContext, node *html.Node) { + for idx, attr := range node.Attr { + if (attr.Key == "id" && isAnchorIDFootnote(attr.Val)) || + (attr.Key == "href" && isAnchorHrefFootnote(attr.Val)) { + if footnoteContextID := ctx.RenderOptions.Metas["footnoteContextId"]; footnoteContextID != "" { + node.Attr[idx].Val = attr.Val + "-" + footnoteContextID + } + continue + } + } +} + +func processNodeA(ctx *RenderContext, node *html.Node) { + for idx, attr := range node.Attr { + if attr.Key == "href" { + if anchorID, ok := strings.CutPrefix(attr.Val, "#"); ok { + if !isAnchorIDUserContent(attr.Val) { + node.Attr[idx].Val = "#user-content-" + anchorID + } + } else { + node.Attr[idx].Val = ctx.RenderHelper.ResolveLink(attr.Val, LinkTypeDefault) + } + } + } +} + func visitNodeImg(ctx *RenderContext, img *html.Node) (next *html.Node) { next = img.NextSibling - for i, attr := range img.Attr { - if attr.Key != "src" { + attrSrc, hasLazy := "", false + for i, imgAttr := range img.Attr { + hasLazy = hasLazy || imgAttr.Key == "loading" && imgAttr.Val == "lazy" + if imgAttr.Key != "src" { + attrSrc = imgAttr.Val continue } - if IsNonEmptyRelativePath(attr.Val) { - attr.Val = ctx.RenderHelper.ResolveLink(attr.Val, LinkTypeMedia) + imgSrcOrigin := imgAttr.Val + isLinkable := imgSrcOrigin != "" && !strings.HasPrefix(imgSrcOrigin, "data:") - // By default, the "<img>" tag should also be clickable, - // because frontend use `<img>` to paste the re-scaled image into the markdown, - // so it must match the default markdown image behavior. - hasParentAnchor := false - for p := img.Parent; p != nil; p = p.Parent { - if hasParentAnchor = p.Type == html.ElementNode && p.Data == "a"; hasParentAnchor { - break - } - } - if !hasParentAnchor { - imgA := &html.Node{Type: html.ElementNode, Data: "a", Attr: []html.Attribute{ - {Key: "href", Val: attr.Val}, - {Key: "target", Val: "_blank"}, - }} - parent := img.Parent - imgNext := img.NextSibling - parent.RemoveChild(img) - parent.InsertBefore(imgA, imgNext) - imgA.AppendChild(img) + // By default, the "<img>" tag should also be clickable, + // because frontend uses `<img>` to paste the re-scaled image into the Markdown, + // so it must match the default Markdown image behavior. + cnt := 0 + for p := img.Parent; isLinkable && p != nil && cnt < 2; p = p.Parent { + if hasParentAnchor := p.Type == html.ElementNode && p.Data == "a"; hasParentAnchor { + isLinkable = false + break } + cnt++ } - attr.Val = camoHandleLink(attr.Val) - img.Attr[i] = attr + if isLinkable { + wrapper := &html.Node{Type: html.ElementNode, Data: "a", Attr: []html.Attribute{ + {Key: "href", Val: ctx.RenderHelper.ResolveLink(imgSrcOrigin, LinkTypeDefault)}, + {Key: "target", Val: "_blank"}, + }} + parent := img.Parent + imgNext := img.NextSibling + parent.RemoveChild(img) + parent.InsertBefore(wrapper, imgNext) + wrapper.AppendChild(img) + } + + imgAttr.Val = ctx.RenderHelper.ResolveLink(imgSrcOrigin, LinkTypeMedia) + imgAttr.Val = camoHandleLink(imgAttr.Val) + img.Attr[i] = imgAttr + } + if !RenderBehaviorForTesting.DisableAdditionalAttributes && !hasLazy && !strings.HasPrefix(attrSrc, "data:") { + img.Attr = append(img.Attr, html.Attribute{Key: "loading", Val: "lazy"}) } return next } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index f0f062fa64..5fdbf43f7c 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -35,6 +35,7 @@ func TestRender_Commits(t *testing.T) { sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" repo := markup.TestAppURL + testRepoOwnerName + "/" + testRepoName + "/" commit := util.URLJoin(repo, "commit", sha) + commitPath := "/user13/repo11/commit/" + sha tree := util.URLJoin(repo, "tree", sha, "src") file := util.URLJoin(repo, "commit", sha, "example.txt") @@ -44,9 +45,9 @@ func TestRender_Commits(t *testing.T) { commitCompare := util.URLJoin(repo, "compare", sha+"..."+sha) commitCompareWithHash := commitCompare + "#L2" - test(sha, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) - test(sha[:7], `<p><a href="`+commit[:len(commit)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`) - test(sha[:39], `<p><a href="`+commit[:len(commit)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(sha, `<p><a href="`+commitPath+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(sha[:7], `<p><a href="`+commitPath[:len(commitPath)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`) + test(sha[:39], `<p><a href="`+commitPath[:len(commitPath)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test(commit, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test(tree, `<p><a href="`+tree+`" rel="nofollow"><code>65f1bf27bc/src</code></a></p>`) @@ -57,13 +58,13 @@ func TestRender_Commits(t *testing.T) { test(commitCompare, `<p><a href="`+commitCompare+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc</code></a></p>`) test(commitCompareWithHash, `<p><a href="`+commitCompareWithHash+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc (L2)</code></a></p>`) - test("commit "+sha, `<p>commit <a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test("commit "+sha, `<p>commit <a href="`+commitPath+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) test("/home/gitea/"+sha, "<p>/home/gitea/"+sha+"</p>") test("deadbeef", `<p>deadbeef</p>`) test("d27ace93", `<p>d27ace93</p>`) test(sha[:14]+".x", `<p>`+sha[:14]+`.x</p>`) - expected14 := `<a href="` + commit[:len(commit)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>` + expected14 := `<a href="` + commitPath[:len(commitPath)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>` test(sha[:14]+".", `<p>`+expected14+`.</p>`) test(sha[:14]+",", `<p>`+expected14+`,</p>`) test("["+sha[:14]+"]", `<p>[`+expected14+`]</p>`) @@ -80,10 +81,10 @@ func TestRender_CrossReferences(t *testing.T) { test( "test-owner/test-repo#12345", - `<p><a href="`+util.URLJoin(markup.TestAppURL, "test-owner", "test-repo", "issues", "12345")+`" class="ref-issue" rel="nofollow">test-owner/test-repo#12345</a></p>`) + `<p><a href="/test-owner/test-repo/issues/12345" class="ref-issue" rel="nofollow">test-owner/test-repo#12345</a></p>`) test( "go-gitea/gitea#12345", - `<p><a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) + `<p><a href="/go-gitea/gitea/issues/12345" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) test( "/home/gitea/go-gitea/gitea#12345", `<p>/home/gitea/go-gitea/gitea#12345</p>`) @@ -224,10 +225,10 @@ func TestRender_email(t *testing.T) { test := func(input, expected string) { res, err := markup.RenderString(markup.NewTestRenderContext().WithRelativePath("a.md"), input) assert.NoError(t, err) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res)) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res), "input: %s", input) } - // Text that should be turned into email link + // Text that should be turned into email link test( "info@gitea.com", `<p><a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a></p>`) @@ -259,28 +260,48 @@ func TestRender_email(t *testing.T) { <a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>? <a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>!</p>`) + // match GitHub behavior + test("email@domain@domain.com", `<p>email@<a href="mailto:domain@domain.com" rel="nofollow">domain@domain.com</a></p>`) + + // match GitHub behavior + test(`"info@gitea.com"`, `<p>"<a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a>"</p>`) + // Test that should *not* be turned into email links test( - "\"info@gitea.com\"", - `<p>"info@gitea.com"</p>`) - test( "/home/gitea/mailstore/info@gitea/com", `<p>/home/gitea/mailstore/info@gitea/com</p>`) test( "git@try.gitea.io:go-gitea/gitea.git", `<p>git@try.gitea.io:go-gitea/gitea.git</p>`) test( + "https://foo:bar@gitea.io", + `<p><a href="https://foo:bar@gitea.io" rel="nofollow">https://foo:bar@gitea.io</a></p>`) + test( "gitea@3", `<p>gitea@3</p>`) test( "gitea@gmail.c", `<p>gitea@gmail.c</p>`) test( - "email@domain@domain.com", - `<p>email@domain@domain.com</p>`) - test( "email@domain..com", `<p>email@domain..com</p>`) + + cases := []struct { + input, expected string + }{ + // match GitHub behavior + {"?a@d.zz", `<p>?<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + {"*a@d.zz", `<p>*<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + {"~a@d.zz", `<p>~<a href="mailto:a@d.zz" rel="nofollow">a@d.zz</a></p>`}, + + // the following cases don't match GitHub behavior, but they are valid email addresses ... + // maybe we should reduce the candidate characters for the "name" part in the future + {"a*a@d.zz", `<p><a href="mailto:a*a@d.zz" rel="nofollow">a*a@d.zz</a></p>`}, + {"a~a@d.zz", `<p><a href="mailto:a~a@d.zz" rel="nofollow">a~a@d.zz</a></p>`}, + } + for _, c := range cases { + test(c.input, c.expected) + } } func TestRender_emoji(t *testing.T) { @@ -468,7 +489,7 @@ func Test_ParseClusterFuzz(t *testing.T) { assert.NotContains(t, res.String(), "<html") } -func TestPostProcess_RenderDocument(t *testing.T) { +func TestPostProcess(t *testing.T) { setting.StaticURLPrefix = markup.TestAppURL // can't run standalone defer testModule.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() @@ -479,7 +500,7 @@ func TestPostProcess_RenderDocument(t *testing.T) { assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res.String())) } - // Issue index shouldn't be post processing in a document. + // Issue index shouldn't be post-processing in a document. test( "#1", "#1") @@ -487,9 +508,9 @@ func TestPostProcess_RenderDocument(t *testing.T) { // But cross-referenced issue index should work. test( "go-gitea/gitea#12345", - `<a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue">go-gitea/gitea#12345</a>`) + `<a href="/go-gitea/gitea/issues/12345" class="ref-issue">go-gitea/gitea#12345</a>`) - // Test that other post processing still works. + // Test that other post-processing still works. test( ":gitea:", `<span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span>`) @@ -498,6 +519,16 @@ func TestPostProcess_RenderDocument(t *testing.T) { `Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle`) test("http://localhost:3000/person/repo/issues/4#issuecomment-1234", `<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234" class="ref-issue">person/repo#4 (comment)</a>`) + + // special tags, GitHub's behavior, and for unclosed tags, output as text content as much as possible + test("<script>a", `<script>a`) + test("<script>a</script>", `<script>a</script>`) + test("<STYLE>a", `<STYLE>a`) + test("<style>a</STYLE>", `<style>a</STYLE>`) + + // other special tags, our special behavior + test("<?php\nfoo", "<?php\nfoo") + test("<%asp\nfoo", "<%asp\nfoo") } func TestIssue16020(t *testing.T) { @@ -543,7 +574,7 @@ func TestIssue18471(t *testing.T) { err := markup.PostProcessDefault(markup.NewTestRenderContext(localMetas), strings.NewReader(data), &res) assert.NoError(t, err) - assert.Equal(t, `<a href="http://domain/org/repo/compare/783b039...da951ce" class="compare"><code class="nohighlight">783b039...da951ce</code></a>`, res.String()) + assert.Equal(t, `<a href="http://domain/org/repo/compare/783b039...da951ce" class="compare"><code>783b039...da951ce</code></a>`, res.String()) } func TestIsFullURL(t *testing.T) { diff --git a/modules/markup/internal/internal_test.go b/modules/markup/internal/internal_test.go index 98ff3bc079..590bcbb67f 100644 --- a/modules/markup/internal/internal_test.go +++ b/modules/markup/internal/internal_test.go @@ -35,7 +35,7 @@ func TestRenderInternal(t *testing.T) { assert.EqualValues(t, c.protected, protected) _, _ = io.WriteString(in, string(protected)) _ = in.Close() - assert.EqualValues(t, c.recovered, out.String()) + assert.Equal(t, c.recovered, out.String()) } var r1, r2 RenderInternal @@ -44,11 +44,11 @@ func TestRenderInternal(t *testing.T) { _ = r1.init("sec", nil) protected = r1.ProtectSafeAttrs(`<div class="test"></div>`) assert.EqualValues(t, `<div data-attr-class="sec:test"></div>`, protected) - assert.EqualValues(t, "data-attr-class", r1.SafeAttr("class")) - assert.EqualValues(t, "sec:val", r1.SafeValue("val")) + assert.Equal(t, "data-attr-class", r1.SafeAttr("class")) + assert.Equal(t, "sec:val", r1.SafeValue("val")) recovered, ok := r1.RecoverProtectedValue("sec:val") assert.True(t, ok) - assert.EqualValues(t, "val", recovered) + assert.Equal(t, "val", recovered) recovered, ok = r1.RecoverProtectedValue("other:val") assert.False(t, ok) assert.Empty(t, recovered) @@ -57,5 +57,5 @@ func TestRenderInternal(t *testing.T) { in2 := r2.init("sec-other", out2) _, _ = io.WriteString(in2, string(protected)) _ = in2.Close() - assert.EqualValues(t, `<div data-attr-class="sec:test"></div>`, out2.String(), "different secureID should not recover the value") + assert.Equal(t, `<div data-attr-class="sec:test"></div>`, out2.String(), "different secureID should not recover the value") } diff --git a/modules/markup/markdown/ast.go b/modules/markup/markdown/ast.go index ca165b1ba0..f29f883734 100644 --- a/modules/markup/markdown/ast.go +++ b/modules/markup/markdown/ast.go @@ -4,6 +4,7 @@ package markdown import ( + "html/template" "strconv" "github.com/yuin/goldmark/ast" @@ -29,9 +30,7 @@ func (n *Details) Kind() ast.NodeKind { // NewDetails returns a new Paragraph node. func NewDetails() *Details { - return &Details{ - BaseBlock: ast.BaseBlock{}, - } + return &Details{} } // Summary is a block that contains the summary of details block @@ -54,9 +53,7 @@ func (n *Summary) Kind() ast.NodeKind { // NewSummary returns a new Summary node. func NewSummary() *Summary { - return &Summary{ - BaseBlock: ast.BaseBlock{}, - } + return &Summary{} } // TaskCheckBoxListItem is a block that represents a list item of a markdown block with a checkbox @@ -95,29 +92,6 @@ type Icon struct { Name []byte } -// Dump implements Node.Dump . -func (n *Icon) Dump(source []byte, level int) { - m := map[string]string{} - m["Name"] = string(n.Name) - ast.DumpHelper(n, source, level, m, nil) -} - -// KindIcon is the NodeKind for Icon -var KindIcon = ast.NewNodeKind("Icon") - -// Kind implements Node.Kind. -func (n *Icon) Kind() ast.NodeKind { - return KindIcon -} - -// NewIcon returns a new Paragraph node. -func NewIcon(name string) *Icon { - return &Icon{ - BaseInline: ast.BaseInline{}, - Name: []byte(name), - } -} - // ColorPreview is an inline for a color preview type ColorPreview struct { ast.BaseInline @@ -175,3 +149,24 @@ func NewAttention(attentionType string) *Attention { AttentionType: attentionType, } } + +var KindRawHTML = ast.NewNodeKind("RawHTML") + +type RawHTML struct { + ast.BaseBlock + rawHTML template.HTML +} + +func (n *RawHTML) Dump(source []byte, level int) { + m := map[string]string{} + m["RawHTML"] = string(n.rawHTML) + ast.DumpHelper(n, source, level, m, nil) +} + +func (n *RawHTML) Kind() ast.NodeKind { + return KindRawHTML +} + +func NewRawHTML(rawHTML template.HTML) *RawHTML { + return &RawHTML{rawHTML: rawHTML} +} diff --git a/modules/markup/markdown/convertyaml.go b/modules/markup/markdown/convertyaml.go index 1675b68be2..04664a9c1d 100644 --- a/modules/markup/markdown/convertyaml.go +++ b/modules/markup/markdown/convertyaml.go @@ -4,23 +4,22 @@ package markdown import ( + "strings" + + "code.gitea.io/gitea/modules/htmlutil" + "code.gitea.io/gitea/modules/svg" + "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" "gopkg.in/yaml.v3" ) func nodeToTable(meta *yaml.Node) ast.Node { - for { - if meta == nil { - return nil - } - switch meta.Kind { - case yaml.DocumentNode: - meta = meta.Content[0] - continue - default: - } - break + for meta != nil && meta.Kind == yaml.DocumentNode { + meta = meta.Content[0] + } + if meta == nil { + return nil } switch meta.Kind { case yaml.MappingNode: @@ -72,12 +71,28 @@ func sequenceNodeToTable(meta *yaml.Node) ast.Node { return table } -func nodeToDetails(meta *yaml.Node, icon string) ast.Node { +func nodeToDetails(g *ASTTransformer, meta *yaml.Node) ast.Node { + for meta != nil && meta.Kind == yaml.DocumentNode { + meta = meta.Content[0] + } + if meta == nil { + return nil + } + if meta.Kind != yaml.MappingNode { + return nil + } + var keys []string + for i := 0; i < len(meta.Content); i += 2 { + if meta.Content[i].Kind == yaml.ScalarNode { + keys = append(keys, meta.Content[i].Value) + } + } details := NewDetails() + details.SetAttributeString(g.renderInternal.SafeAttr("class"), g.renderInternal.SafeValue("frontmatter-content")) summary := NewSummary() - summary.AppendChild(summary, NewIcon(icon)) + summaryInnerHTML := htmlutil.HTMLFormat("%s %s", svg.RenderHTML("octicon-table", 12), strings.Join(keys, ", ")) + summary.AppendChild(summary, NewRawHTML(summaryInnerHTML)) details.AppendChild(details, summary) details.AppendChild(details, nodeToTable(meta)) - return details } diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 69c2a96ff1..b28fa9824e 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -5,14 +5,10 @@ package markdown import ( "fmt" - "regexp" - "strings" - "sync" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/internal" - "code.gitea.io/gitea/modules/setting" "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" @@ -51,7 +47,7 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa tocList := make([]Header, 0, 20) if rc.yamlNode != nil { - metaNode := rc.toMetaNode() + metaNode := rc.toMetaNode(g) if metaNode != nil { node.InsertBefore(node, firstChild, metaNode) } @@ -68,23 +64,12 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa g.transformHeading(ctx, v, reader, &tocList) case *ast.Paragraph: g.applyElementDir(v) - case *ast.Image: - g.transformImage(ctx, v) - case *ast.Link: - g.transformLink(ctx, v) case *ast.List: g.transformList(ctx, v, rc) case *ast.Text: if v.SoftLineBreak() && !v.HardLineBreak() { - // TODO: this was a quite unclear part, old code: `if metas["mode"] != "document" { use comment link break setting }` - // many places render non-comment contents with no mode=document, then these contents also use comment's hard line break setting - // especially in many tests. - markdownLineBreakStyle := ctx.RenderOptions.Metas["markdownLineBreakStyle"] - if markdownLineBreakStyle == "comment" { - v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInComments) - } else if markdownLineBreakStyle == "document" { - v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInDocuments) - } + newLineHardBreak := ctx.RenderOptions.Metas["markdownNewLineHardBreak"] == "true" + v.SetHardLineBreak(newLineHardBreak) } case *ast.CodeSpan: g.transformCodeSpan(ctx, v, reader) @@ -111,11 +96,6 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa } } -// it is copied from old code, which is quite doubtful whether it is correct -var reValidIconName = sync.OnceValue(func() *regexp.Regexp { - return regexp.MustCompile(`^[-\w]+$`) // old: regexp.MustCompile("^[a-z ]+$") -}) - // NewHTMLRenderer creates a HTMLRenderer to render in the gitea form. func NewHTMLRenderer(renderInternal *internal.RenderInternal, opts ...html.Option) renderer.NodeRenderer { r := &HTMLRenderer{ @@ -140,11 +120,11 @@ func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { reg.Register(ast.KindDocument, r.renderDocument) reg.Register(KindDetails, r.renderDetails) reg.Register(KindSummary, r.renderSummary) - reg.Register(KindIcon, r.renderIcon) reg.Register(ast.KindCodeSpan, r.renderCodeSpan) reg.Register(KindAttention, r.renderAttention) reg.Register(KindTaskCheckBoxListItem, r.renderTaskCheckBoxListItem) reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox) + reg.Register(KindRawHTML, r.renderRawHTML) } func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { @@ -155,7 +135,7 @@ func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast. if entering { _, err = w.WriteString("<div") if err == nil { - _, err = w.WriteString(fmt.Sprintf(` lang=%q`, val)) + _, err = fmt.Fprintf(w, ` lang=%q`, val) } if err == nil { _, err = w.WriteRune('>') @@ -206,30 +186,14 @@ func (r *HTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.N return ast.WalkContinue, nil } -func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *HTMLRenderer) renderRawHTML(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } - - n := node.(*Icon) - - name := strings.TrimSpace(strings.ToLower(string(n.Name))) - - if len(name) == 0 { - // skip this - return ast.WalkContinue, nil - } - - if !reValidIconName().MatchString(name) { - // skip this - return ast.WalkContinue, nil - } - - // FIXME: the "icon xxx" is from Fomantic UI, it's really questionable whether it still works correctly - err := r.renderInternal.FormatWithSafeAttrs(w, `<i class="icon %s"></i>`, name) + n := node.(*RawHTML) + _, err := w.WriteString(string(r.renderInternal.ProtectSafeAttrs(n.rawHTML))) if err != nil { return ast.WalkStop, err } - return ast.WalkContinue, nil } diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index ace31eb540..3b788432ba 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -5,7 +5,7 @@ package markdown import ( - "fmt" + "errors" "html/template" "io" "strings" @@ -48,7 +48,7 @@ func (l *limitWriter) Write(data []byte) (int, error) { if err != nil { return n, err } - return n, fmt.Errorf("rendered content too large - truncating render") + return n, errors.New("rendered content too large - truncating render") } n, err := l.w.Write(data) l.sum += int64(n) @@ -86,20 +86,15 @@ func (r *GlodmarkRender) highlightingRenderer(w util.BufWriter, c highlighting.C preClasses += " is-loading" } - err := r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<pre class="%s">`, preClasses) - if err != nil { - return - } - // include language-x class as part of commonmark spec, "chroma" class is used to highlight the code // the "display" class is used by "js/markup/math.ts" to render the code element as a block // the "math.ts" strictly depends on the structure: <pre class="code-block is-loading"><code class="language-math display">...</code></pre> - err = r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<code class="chroma language-%s display">`, languageStr) + err := r.ctx.RenderInternal.FormatWithSafeAttrs(w, `<div class="code-block-container code-overflow-scroll"><pre class="%s"><code class="chroma language-%s display">`, preClasses, languageStr) if err != nil { return } } else { - _, err := w.WriteString("</code></pre>") + _, err := w.WriteString("</code></pre></div>") if err != nil { return } @@ -126,11 +121,11 @@ func SpecializedMarkdown(ctx *markup.RenderContext) *GlodmarkRender { highlighting.WithWrapperRenderer(r.highlightingRenderer), ), math.NewExtension(&ctx.RenderInternal, math.Options{ - Enabled: setting.Markdown.EnableMath, - ParseDollarInline: true, - ParseDollarBlock: true, - ParseSquareBlock: true, // TODO: this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping, it should be deprecated in the future (by some config options) - // ParseBracketInline: true, // TODO: this is also a bad syntax "\( ... \)", it also conflicts, it should be deprecated in the future + Enabled: setting.Markdown.EnableMath, + ParseInlineDollar: setting.Markdown.MathCodeBlockOptions.ParseInlineDollar, + ParseInlineParentheses: setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses, // this is a bad syntax "\( ... \)", it conflicts with normal markdown escaping + ParseBlockDollar: setting.Markdown.MathCodeBlockOptions.ParseBlockDollar, + ParseBlockSquareBrackets: setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets, // this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping }), meta.Meta, ), @@ -184,17 +179,10 @@ func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error // Preserve original length. bufWithMetadataLength := len(buf) - rc := &RenderConfig{ - Meta: markup.RenderMetaAsDetails, - Icon: "table", - Lang: "", - } + rc := &RenderConfig{Meta: markup.RenderMetaAsDetails} buf, _ = ExtractMetadataBytes(buf, rc) - metaLength := bufWithMetadataLength - len(buf) - if metaLength < 0 { - metaLength = 0 - } + metaLength := max(bufWithMetadataLength-len(buf), 0) rc.metaLength = metaLength pc.Set(renderConfigKey, rc) diff --git a/modules/markup/markdown/markdown_math_test.go b/modules/markup/markdown/markdown_math_test.go index 813f050965..a75f18d36a 100644 --- a/modules/markup/markdown/markdown_math_test.go +++ b/modules/markup/markdown/markdown_math_test.go @@ -8,6 +8,8 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" ) @@ -15,6 +17,7 @@ import ( const nl = "\n" func TestMathRender(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseInlineParentheses: true} testcases := []struct { testcase string expected string @@ -69,7 +72,7 @@ func TestMathRender(t *testing.T) { }, { "$$a$$", - `<code class="language-math display">a</code>` + nl, + `<p><code class="language-math">a</code></p>` + nl, }, { "$$a$$ test", @@ -111,6 +114,7 @@ func TestMathRender(t *testing.T) { } func TestMathRenderBlockIndent(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseBlockDollar: true, ParseBlockSquareBrackets: true} testcases := []struct { name string testcase string @@ -243,3 +247,64 @@ x }) } } + +func TestMathRenderOptions(t *testing.T) { + setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{} + defer test.MockVariableValue(&setting.Markdown.MathCodeBlockOptions) + test := func(t *testing.T, expected, input string) { + res, err := RenderString(markup.NewTestRenderContext(), input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(res)), "input: %s", input) + } + + // default (non-conflict) inline syntax + test(t, `<p><code class="language-math">a</code></p>`, "$`a`$") + + // ParseInlineDollar + test(t, `<p>$a$</p>`, `$a$`) + setting.Markdown.MathCodeBlockOptions.ParseInlineDollar = true + test(t, `<p><code class="language-math">a</code></p>`, `$a$`) + + // ParseInlineParentheses + test(t, `<p>(a)</p>`, `\(a\)`) + setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses = true + test(t, `<p><code class="language-math">a</code></p>`, `\(a\)`) + + // ParseBlockDollar + test(t, `<p>$$ +a +$$</p> +`, ` +$$ +a +$$ +`) + setting.Markdown.MathCodeBlockOptions.ParseBlockDollar = true + test(t, `<pre class="code-block is-loading"><code class="language-math display"> +a +</code></pre> +`, ` +$$ +a +$$ +`) + + // ParseBlockSquareBrackets + test(t, `<p>[ +a +]</p> +`, ` +\[ +a +\] +`) + setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets = true + test(t, `<pre class="code-block is-loading"><code class="language-math display"> +a +</code></pre> +`, ` +\[ +a +\] +`) +} diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index 7a09be8665..4eb01bcc2d 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -47,7 +47,7 @@ func TestRender_StandardLinks(t *testing.T) { func TestRender_Images(t *testing.T) { setting.AppURL = AppURL - test := func(input, expected string) { + render := func(input, expected string) { buffer, err := markdown.RenderString(markup.NewTestRenderContext(FullURL), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) @@ -59,27 +59,32 @@ func TestRender_Images(t *testing.T) { result := util.URLJoin(FullURL, url) // hint: With Markdown v2.5.2, there is a new syntax: [link](URL){:target="_blank"} , but we do not support it now - test( + render( "!["+title+"]("+url+")", `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "[["+title+"|"+url+"]]", `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) - test( + render( "[!["+title+"]("+url+")]("+href+")", `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "!["+title+"]("+url+")", `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) - test( + render( "[["+title+"|"+url+"]]", `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) - test( + render( "[!["+title+"]("+url+")]("+href+")", `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) + + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, false)() + render( + "<a><img src='a.jpg'></a>", // by the way, empty "a" tag will be removed + `<p dir="auto"><img src="http://localhost:3000/user13/repo11/a.jpg" loading="lazy"/></p>`) } func TestTotal_RenderString(t *testing.T) { @@ -223,7 +228,7 @@ This PR has been generated by [Renovate Bot](https://github.com/renovatebot/reno <dd>This is another definition of the second term.</dd> </dl> <h3 id="user-content-footnotes">Footnotes</h3> -<p>Here is a simple footnote,<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1</a></sup> and here is a longer one.<sup id="fnref:user-content-bignote"><a href="#fn:user-content-bignote" rel="nofollow">2</a></sup></p> +<p>Here is a simple footnote,<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1 </a></sup> and here is a longer one.<sup id="fnref:user-content-bignote"><a href="#fn:user-content-bignote" rel="nofollow">2 </a></sup></p> <div> <hr/> <ol> @@ -252,7 +257,7 @@ This PR has been generated by [Renovate Bot](https://github.com/renovatebot/reno return username == "r-lyeh" }, }) - for i := 0; i < len(sameCases); i++ { + for i := range sameCases { line, err := markdown.RenderString(markup.NewTestRenderContext(localMetas), sameCases[i]) assert.NoError(t, err) assert.Equal(t, testAnswers[i], string(line)) @@ -308,12 +313,12 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) { testcase := `  ` - expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"></a> -<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"></a></p> + expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"/></a> +<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"/></a></p> ` - res, err := markdown.RenderRawString(markup.NewTestRenderContext(), testcase) + res, err := markdown.RenderString(markup.NewTestRenderContext(), testcase) assert.NoError(t, err) - assert.Equal(t, expected, res) + assert.Equal(t, expected, string(res)) } func TestRenderEmojiInLinks_Issue12331(t *testing.T) { @@ -383,18 +388,74 @@ func TestColorPreview(t *testing.T) { } } -func TestTaskList(t *testing.T) { +func TestMarkdownFrontmatter(t *testing.T) { testcases := []struct { - testcase string + name string + input string expected string }{ { + "MapInFrontmatter", + `--- +key1: val1 +key2: val2 +--- +test +`, + `<details class="frontmatter-content"><summary><span>octicon-table(12/)</span> key1, key2</summary><table> +<thead> +<tr> +<th>key1</th> +<th>key2</th> +</tr> +</thead> +<tbody> +<tr> +<td>val1</td> +<td>val2</td> +</tr> +</tbody> +</table> +</details><p>test</p> +`, + }, + + { + "ListInFrontmatter", + `--- +- item1 +- item2 +--- +test +`, + `- item1 +- item2 + +<p>test</p> +`, + }, + + { + "StringInFrontmatter", + `--- +anything +--- +test +`, + `anything + +<p>test</p> +`, + }, + + { // data-source-position should take into account YAML frontmatter. + "ListAfterFrontmatter", `--- foo: bar --- - [ ] task 1`, - `<details><summary><i class="icon table"></i></summary><table> + `<details class="frontmatter-content"><summary><span>octicon-table(12/)</span> foo</summary><table> <thead> <tr> <th>foo</th> @@ -414,9 +475,9 @@ foo: bar } for _, test := range testcases { - res, err := markdown.RenderString(markup.NewTestRenderContext(), test.testcase) - assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) - assert.Equal(t, template.HTML(test.expected), res, "Unexpected result in testcase %q", test.testcase) + res, err := markdown.RenderString(markup.NewTestRenderContext(), test.input) + assert.NoError(t, err, "Unexpected error in testcase: %q", test.name) + assert.Equal(t, test.expected, string(res), "Unexpected result in testcase %q", test.name) } } @@ -473,3 +534,16 @@ space</p> assert.NoError(t, err) assert.Equal(t, expected, string(result)) } + +func TestMarkdownLink(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + input := `<a href=foo>link1</a> +<a href='/foo'>link2</a> +<a href="#foo">link3</a>` + result, err := markdown.RenderString(markup.NewTestRenderContext("/base", localMetas), input) + assert.NoError(t, err) + assert.Equal(t, `<p><a href="/base/foo" rel="nofollow">link1</a> +<a href="/base/foo" rel="nofollow">link2</a> +<a href="#user-content-foo" rel="nofollow">link3</a></p> +`, string(result)) +} diff --git a/modules/markup/markdown/math/block_renderer.go b/modules/markup/markdown/math/block_renderer.go index 412e4d0dee..95a336a02c 100644 --- a/modules/markup/markdown/math/block_renderer.go +++ b/modules/markup/markdown/math/block_renderer.go @@ -42,7 +42,7 @@ func (r *BlockRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { func (r *BlockRenderer) writeLines(w util.BufWriter, source []byte, n gast.Node) { l := n.Lines().Len() - for i := 0; i < l; i++ { + for i := range l { line := n.Lines().At(i) _, _ = w.Write(util.EscapeHTML(line.Value(source))) } @@ -51,8 +51,8 @@ func (r *BlockRenderer) writeLines(w util.BufWriter, source []byte, n gast.Node) func (r *BlockRenderer) renderBlock(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { n := node.(*Block) if entering { - code := giteaUtil.Iif(n.Inline, "", `<pre class="code-block is-loading">`) + `<code class="language-math display">` - _ = r.renderInternal.FormatWithSafeAttrs(w, template.HTML(code)) + codeHTML := giteaUtil.Iif[template.HTML](n.Inline, "", `<pre class="code-block is-loading">`) + `<code class="language-math display">` + _, _ = w.WriteString(string(r.renderInternal.ProtectSafeAttrs(codeHTML))) r.writeLines(w, source, n) } else { _, _ = w.WriteString(`</code>` + giteaUtil.Iif(n.Inline, "", `</pre>`) + "\n") diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go index a57abe9f9b..a711d1e1cd 100644 --- a/modules/markup/markdown/math/inline_parser.go +++ b/modules/markup/markdown/math/inline_parser.go @@ -15,26 +15,26 @@ type inlineParser struct { trigger []byte endBytesSingleDollar []byte endBytesDoubleDollar []byte - endBytesBracket []byte + endBytesParentheses []byte + enableInlineDollar bool } -var defaultInlineDollarParser = &inlineParser{ - trigger: []byte{'$'}, - endBytesSingleDollar: []byte{'$'}, - endBytesDoubleDollar: []byte{'$', '$'}, -} - -func NewInlineDollarParser() parser.InlineParser { - return defaultInlineDollarParser +func NewInlineDollarParser(enableInlineDollar bool) parser.InlineParser { + return &inlineParser{ + trigger: []byte{'$'}, + endBytesSingleDollar: []byte{'$'}, + endBytesDoubleDollar: []byte{'$', '$'}, + enableInlineDollar: enableInlineDollar, + } } -var defaultInlineBracketParser = &inlineParser{ - trigger: []byte{'\\', '('}, - endBytesBracket: []byte{'\\', ')'}, +var defaultInlineParenthesesParser = &inlineParser{ + trigger: []byte{'\\', '('}, + endBytesParentheses: []byte{'\\', ')'}, } -func NewInlineBracketParser() parser.InlineParser { - return defaultInlineBracketParser +func NewInlineParenthesesParser() parser.InlineParser { + return defaultInlineParenthesesParser } // Trigger triggers this parser on $ or \ @@ -46,7 +46,7 @@ func isPunctuation(b byte) bool { return b == '.' || b == '!' || b == '?' || b == ',' || b == ';' || b == ':' } -func isBracket(b byte) bool { +func isParenthesesClose(b byte) bool { return b == ')' } @@ -70,10 +70,11 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. startMarkLen = 1 stopMark = parser.endBytesSingleDollar if len(line) > 1 { - if line[1] == '$' { + switch line[1] { + case '$': startMarkLen = 2 stopMark = parser.endBytesDoubleDollar - } else if line[1] == '`' { + case '`': pos := 1 for ; pos < len(line) && line[pos] == '`'; pos++ { } @@ -85,7 +86,11 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. } } else { startMarkLen = 2 - stopMark = parser.endBytesBracket + stopMark = parser.endBytesParentheses + } + + if line[0] == '$' && !parser.enableInlineDollar && (len(line) == 1 || line[1] != '`') { + return nil } if checkSurrounding { @@ -109,7 +114,7 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. succeedingCharacter = line[i+len(stopMark)] } // check valid ending character - isValidEndingChar := isPunctuation(succeedingCharacter) || isBracket(succeedingCharacter) || + isValidEndingChar := isPunctuation(succeedingCharacter) || isParenthesesClose(succeedingCharacter) || succeedingCharacter == ' ' || succeedingCharacter == '\n' || succeedingCharacter == 0 if checkSurrounding && !isValidEndingChar { break @@ -121,9 +126,10 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. i++ continue } - if line[i] == '{' { + switch line[i] { + case '{': depth++ - } else if line[i] == '}' { + case '}': depth-- } } diff --git a/modules/markup/markdown/math/inline_renderer.go b/modules/markup/markdown/math/inline_renderer.go index d000a7b317..eeeb60cc7e 100644 --- a/modules/markup/markdown/math/inline_renderer.go +++ b/modules/markup/markdown/math/inline_renderer.go @@ -28,7 +28,7 @@ func NewInlineRenderer(renderInternal *internal.RenderInternal) renderer.NodeRen func (r *InlineRenderer) renderInline(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { if entering { - _ = r.renderInternal.FormatWithSafeAttrs(w, `<code class="language-math">`) + _, _ = w.WriteString(string(r.renderInternal.ProtectSafeAttrs(`<code class="language-math">`))) for c := n.FirstChild(); c != nil; c = c.NextSibling() { segment := c.(*ast.Text).Segment value := util.EscapeHTML(segment.Value(source)) diff --git a/modules/markup/markdown/math/math.go b/modules/markup/markdown/math/math.go index a6ff593d62..4b74db2d76 100644 --- a/modules/markup/markdown/math/math.go +++ b/modules/markup/markdown/math/math.go @@ -14,10 +14,11 @@ import ( ) type Options struct { - Enabled bool - ParseDollarInline bool - ParseDollarBlock bool - ParseSquareBlock bool + Enabled bool + ParseInlineDollar bool // inline $$ xxx $$ text + ParseInlineParentheses bool // inline \( xxx \) text + ParseBlockDollar bool // block $$ multiple-line $$ text + ParseBlockSquareBrackets bool // block \[ multiple-line \] text } // Extension is a math extension @@ -42,16 +43,16 @@ func (e *Extension) Extend(m goldmark.Markdown) { return } - inlines := []util.PrioritizedValue{util.Prioritized(NewInlineBracketParser(), 501)} - if e.options.ParseDollarInline { - inlines = append(inlines, util.Prioritized(NewInlineDollarParser(), 502)) + var inlines []util.PrioritizedValue + if e.options.ParseInlineParentheses { + inlines = append(inlines, util.Prioritized(NewInlineParenthesesParser(), 501)) } - m.Parser().AddOptions(parser.WithInlineParsers(inlines...)) + inlines = append(inlines, util.Prioritized(NewInlineDollarParser(e.options.ParseInlineDollar), 502)) + m.Parser().AddOptions(parser.WithInlineParsers(inlines...)) m.Parser().AddOptions(parser.WithBlockParsers( - util.Prioritized(NewBlockParser(e.options.ParseDollarBlock, e.options.ParseSquareBlock), 701), + util.Prioritized(NewBlockParser(e.options.ParseBlockDollar, e.options.ParseBlockSquareBrackets), 701), )) - m.Renderer().AddOptions(renderer.WithNodeRenderers( util.Prioritized(NewBlockRenderer(e.renderInternal), 501), util.Prioritized(NewInlineRenderer(e.renderInternal), 502), diff --git a/modules/markup/markdown/meta_test.go b/modules/markup/markdown/meta_test.go index 278c33f1d2..283d289d48 100644 --- a/modules/markup/markdown/meta_test.go +++ b/modules/markup/markdown/meta_test.go @@ -51,7 +51,7 @@ func TestExtractMetadata(t *testing.T) { var meta IssueTemplate body, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) assert.NoError(t, err) - assert.Equal(t, "", body) + assert.Empty(t, body) assert.Equal(t, metaTest, meta) assert.True(t, meta.Valid()) }) @@ -60,7 +60,7 @@ func TestExtractMetadata(t *testing.T) { func TestExtractMetadataBytes(t *testing.T) { t.Run("ValidFrontAndBody", func(t *testing.T) { var meta IssueTemplate - body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta) + body, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest), &meta) assert.NoError(t, err) assert.Equal(t, bodyTest, string(body)) assert.Equal(t, metaTest, meta) @@ -69,21 +69,21 @@ func TestExtractMetadataBytes(t *testing.T) { t.Run("NoFirstSeparator", func(t *testing.T) { var meta IssueTemplate - _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", frontTest, sepTest, bodyTest)), &meta) + _, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", frontTest, sepTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoLastSeparator", func(t *testing.T) { var meta IssueTemplate - _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, bodyTest)), &meta) + _, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", sepTest, frontTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoBody", func(t *testing.T) { var meta IssueTemplate - body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta) + body, err := ExtractMetadataBytes(fmt.Appendf(nil, "%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) assert.NoError(t, err) - assert.Equal(t, "", string(body)) + assert.Empty(t, string(body)) assert.Equal(t, metaTest, meta) assert.True(t, meta.Valid()) }) diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index f4c48d1b3d..d8b1b10ce6 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -16,7 +16,6 @@ import ( // RenderConfig represents rendering configuration for this file type RenderConfig struct { Meta markup.RenderMetaMode - Icon string TOC string // "false": hide, "side"/empty: in sidebar, "main"/"true": in main view Lang string yamlNode *yaml.Node @@ -74,7 +73,7 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { type yamlRenderConfig struct { Meta *string `yaml:"meta"` - Icon *string `yaml:"details_icon"` + Icon *string `yaml:"details_icon"` // deprecated, because there is no font icon, so no custom icon TOC *string `yaml:"include_toc"` Lang *string `yaml:"lang"` } @@ -96,10 +95,6 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { rc.Meta = renderMetaModeFromString(*cfg.Gitea.Meta) } - if cfg.Gitea.Icon != nil { - rc.Icon = strings.TrimSpace(strings.ToLower(*cfg.Gitea.Icon)) - } - if cfg.Gitea.Lang != nil && *cfg.Gitea.Lang != "" { rc.Lang = *cfg.Gitea.Lang } @@ -111,7 +106,7 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { return nil } -func (rc *RenderConfig) toMetaNode() ast.Node { +func (rc *RenderConfig) toMetaNode(g *ASTTransformer) ast.Node { if rc.yamlNode == nil { return nil } @@ -119,7 +114,7 @@ func (rc *RenderConfig) toMetaNode() ast.Node { case markup.RenderMetaAsTable: return nodeToTable(rc.yamlNode) case markup.RenderMetaAsDetails: - return nodeToDetails(rc.yamlNode, rc.Icon) + return nodeToDetails(g, rc.yamlNode) default: return nil } diff --git a/modules/markup/markdown/renderconfig_test.go b/modules/markup/markdown/renderconfig_test.go index 13346570fa..53c52177a7 100644 --- a/modules/markup/markdown/renderconfig_test.go +++ b/modules/markup/markdown/renderconfig_test.go @@ -21,42 +21,36 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "empty", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", }, "", }, { "lang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "test", }, "lang: test", }, { "metatable", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", }, "gitea: table", }, { "metanone", &RenderConfig{ Meta: "none", - Icon: "table", Lang: "", }, "gitea: none", }, { "metadetails", &RenderConfig{ Meta: "details", - Icon: "table", Lang: "", }, "gitea: details", }, { "metawrong", &RenderConfig{ Meta: "details", - Icon: "table", Lang: "", }, "gitea: wrong", }, @@ -64,7 +58,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { "toc", &RenderConfig{ TOC: "true", Meta: "table", - Icon: "table", Lang: "", }, "include_toc: true", }, @@ -72,14 +65,12 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { "tocfalse", &RenderConfig{ TOC: "false", Meta: "table", - Icon: "table", Lang: "", }, "include_toc: false", }, { "toclang", &RenderConfig{ Meta: "table", - Icon: "table", TOC: "true", Lang: "testlang", }, ` @@ -90,7 +81,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` gitea: @@ -100,7 +90,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang2", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` lang: notright @@ -111,7 +100,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { { "complexlang", &RenderConfig{ Meta: "table", - Icon: "table", Lang: "testlang", }, ` gitea: @@ -123,7 +111,6 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Lang: "two", Meta: "table", TOC: "true", - Icon: "smiley", }, ` lang: one include_toc: true @@ -139,14 +126,12 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got := &RenderConfig{ Meta: "table", - Icon: "table", Lang: "", } err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got) require.NoError(t, err) assert.Equal(t, tt.expected.Meta, got.Meta) - assert.Equal(t, tt.expected.Icon, got.Icon) assert.Equal(t, tt.expected.Lang, got.Lang) assert.Equal(t, tt.expected.TOC, got.TOC) }) diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go index ea1af83a3e..a11b9d0390 100644 --- a/modules/markup/markdown/toc.go +++ b/modules/markup/markdown/toc.go @@ -4,7 +4,6 @@ package markdown import ( - "fmt" "net/url" "code.gitea.io/gitea/modules/translation" @@ -50,7 +49,7 @@ func createTOCNode(toc []Header, lang string, detailsAttrs map[string]string) as } li := ast.NewListItem(currentLevel * 2) a := ast.NewLink() - a.Destination = []byte(fmt.Sprintf("#%s", url.QueryEscape(header.ID))) + a.Destination = []byte("#" + url.QueryEscape(header.ID)) a.AppendChild(a, ast.NewString([]byte(header.Text))) li.AppendChild(li, a) ul.AppendChild(ul, li) diff --git a/modules/markup/markdown/transform_blockquote.go b/modules/markup/markdown/transform_blockquote.go index 3a8c6fa018..bf17f01681 100644 --- a/modules/markup/markdown/transform_blockquote.go +++ b/modules/markup/markdown/transform_blockquote.go @@ -46,7 +46,7 @@ func (g *ASTTransformer) extractBlockquoteAttentionEmphasis(firstParagraph ast.N if !ok { return "", nil } - val1 := string(node1.Text(reader.Source())) //nolint:staticcheck + val1 := string(node1.Text(reader.Source())) //nolint:staticcheck // Text is deprecated attentionType := strings.ToLower(val1) if g.attentionTypes.Contains(attentionType) { return attentionType, []ast.Node{node1} diff --git a/modules/markup/markdown/transform_codespan.go b/modules/markup/markdown/transform_codespan.go index bccc43aad2..c2e4295bc2 100644 --- a/modules/markup/markdown/transform_codespan.go +++ b/modules/markup/markdown/transform_codespan.go @@ -68,7 +68,7 @@ func cssColorHandler(value string) bool { } func (g *ASTTransformer) transformCodeSpan(_ *markup.RenderContext, v *ast.CodeSpan, reader text.Reader) { - colorContent := v.Text(reader.Source()) //nolint:staticcheck + colorContent := v.Text(reader.Source()) //nolint:staticcheck // Text is deprecated if cssColorHandler(string(colorContent)) { v.AppendChild(v, NewColorPreview(colorContent)) } diff --git a/modules/markup/markdown/transform_heading.go b/modules/markup/markdown/transform_heading.go index 5f8a12794d..a229a7b1a4 100644 --- a/modules/markup/markdown/transform_heading.go +++ b/modules/markup/markdown/transform_heading.go @@ -16,10 +16,10 @@ import ( func (g *ASTTransformer) transformHeading(_ *markup.RenderContext, v *ast.Heading, reader text.Reader, tocList *[]Header) { for _, attr := range v.Attributes() { if _, ok := attr.Value.([]byte); !ok { - v.SetAttribute(attr.Name, []byte(fmt.Sprintf("%v", attr.Value))) + v.SetAttribute(attr.Name, fmt.Appendf(nil, "%v", attr.Value)) } } - txt := v.Text(reader.Source()) //nolint:staticcheck + txt := v.Text(reader.Source()) //nolint:staticcheck // Text is deprecated header := Header{ Text: util.UnsafeBytesToString(txt), Level: v.Level, diff --git a/modules/markup/markdown/transform_image.go b/modules/markup/markdown/transform_image.go deleted file mode 100644 index 36512e59a8..0000000000 --- a/modules/markup/markdown/transform_image.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markdown - -import ( - "code.gitea.io/gitea/modules/markup" - - "github.com/yuin/goldmark/ast" -) - -func (g *ASTTransformer) transformImage(ctx *markup.RenderContext, v *ast.Image) { - // Images need two things: - // - // 1. Their src needs to munged to be a real value - // 2. If they're not wrapped with a link they need a link wrapper - - // Check if the destination is a real link - if len(v.Destination) > 0 && !markup.IsFullURLBytes(v.Destination) { - v.Destination = []byte(ctx.RenderHelper.ResolveLink(string(v.Destination), markup.LinkTypeMedia)) - } - - parent := v.Parent() - // Create a link around image only if parent is not already a link - if _, ok := parent.(*ast.Link); !ok && parent != nil { - next := v.NextSibling() - - // Create a link wrapper - wrap := ast.NewLink() - wrap.Destination = v.Destination - wrap.Title = v.Title - wrap.SetAttributeString("target", []byte("_blank")) - - // Duplicate the current image node - image := ast.NewImage(ast.NewLink()) - image.Destination = v.Destination - image.Title = v.Title - for _, attr := range v.Attributes() { - image.SetAttribute(attr.Name, attr.Value) - } - for child := v.FirstChild(); child != nil; { - next := child.NextSibling() - image.AppendChild(image, child) - child = next - } - - // Append our duplicate image to the wrapper link - wrap.AppendChild(wrap, image) - - // Wire in the next sibling - wrap.SetNextSibling(next) - - // Replace the current node with the wrapper link - parent.ReplaceChild(parent, v, wrap) - - // But most importantly ensure the next sibling is still on the old image too - v.SetNextSibling(next) - } -} diff --git a/modules/markup/markdown/transform_link.go b/modules/markup/markdown/transform_link.go deleted file mode 100644 index 51c2c915d8..0000000000 --- a/modules/markup/markdown/transform_link.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package markdown - -import ( - "code.gitea.io/gitea/modules/markup" - - "github.com/yuin/goldmark/ast" -) - -func resolveLink(ctx *markup.RenderContext, link, userContentAnchorPrefix string) (result string, resolved bool) { - isAnchorFragment := link != "" && link[0] == '#' - if !isAnchorFragment && !markup.IsFullURLString(link) { - link, resolved = ctx.RenderHelper.ResolveLink(link, markup.LinkTypeDefault), true - } - if isAnchorFragment && userContentAnchorPrefix != "" { - link, resolved = userContentAnchorPrefix+link[1:], true - } - return link, resolved -} - -func (g *ASTTransformer) transformLink(ctx *markup.RenderContext, v *ast.Link) { - if link, resolved := resolveLink(ctx, string(v.Destination), "#user-content-"); resolved { - v.Destination = []byte(link) - } -} diff --git a/modules/markup/mdstripper/mdstripper.go b/modules/markup/mdstripper/mdstripper.go index fe0eabb473..5a6504416a 100644 --- a/modules/markup/mdstripper/mdstripper.go +++ b/modules/markup/mdstripper/mdstripper.go @@ -46,7 +46,7 @@ func (r *stripRenderer) Render(w io.Writer, source []byte, doc ast.Node) error { coalesce := prevSibIsText r.processString( w, - v.Text(source), //nolint:staticcheck + v.Text(source), //nolint:staticcheck // Text is deprecated coalesce) if v.SoftLineBreak() { r.doubleSpace(w) @@ -91,8 +91,7 @@ func (r *stripRenderer) processAutoLink(w io.Writer, link []byte) { } // Note: we're not attempting to match the URL scheme (http/https) - host := strings.ToLower(u.Host) - if host != "" && host != strings.ToLower(r.localhost.Host) { + if u.Host != "" && !strings.EqualFold(u.Host, r.localhost.Host) { // Process out of band r.links = append(r.links, linkStr) return @@ -107,11 +106,12 @@ func (r *stripRenderer) processAutoLink(w io.Writer, link []byte) { } var sep string - if parts[3] == "issues" { + switch parts[3] { + case "issues": sep = "#" - } else if parts[3] == "pulls" { + case "pulls": sep = "!" - } else { + default: // Process out of band r.links = append(r.links, linkStr) return diff --git a/modules/markup/mdstripper/mdstripper_test.go b/modules/markup/mdstripper/mdstripper_test.go index ea34df0a3b..7fb49c1e01 100644 --- a/modules/markup/mdstripper/mdstripper_test.go +++ b/modules/markup/mdstripper/mdstripper_test.go @@ -79,7 +79,7 @@ A HIDDEN ` + "`" + `GHOST` + "`" + ` IN THIS LINE. lines = append(lines, line) } } - assert.EqualValues(t, test.expectedText, lines) - assert.EqualValues(t, test.expectedLinks, links) + assert.Equal(t, test.expectedText, lines) + assert.Equal(t, test.expectedLinks, links) } } diff --git a/modules/markup/orgmode/orgmode.go b/modules/markup/orgmode/orgmode.go index 70d02c1321..93c335d244 100644 --- a/modules/markup/orgmode/orgmode.go +++ b/modules/markup/orgmode/orgmode.go @@ -1,7 +1,7 @@ // Copyright 2017 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package markup +package orgmode import ( "fmt" @@ -125,27 +125,13 @@ type orgWriter struct { var _ org.Writer = (*orgWriter)(nil) -func (r *orgWriter) resolveLink(kind, link string) string { - link = strings.TrimPrefix(link, "file:") - if !strings.HasPrefix(link, "#") && // not a URL fragment - !markup.IsFullURLString(link) { - if kind == "regular" { - // orgmode reports the link kind as "regular" for "[[ImageLink.svg][The Image Desc]]" - // so we need to try to guess the link kind again here - kind = org.RegularLink{URL: link}.Kind() - } - if kind == "image" || kind == "video" { - link = r.rctx.RenderHelper.ResolveLink(link, markup.LinkTypeMedia) - } else { - link = r.rctx.RenderHelper.ResolveLink(link, markup.LinkTypeDefault) - } - } - return link +func (r *orgWriter) resolveLink(link string) string { + return strings.TrimPrefix(link, "file:") } // WriteRegularLink renders images, links or videos func (r *orgWriter) WriteRegularLink(l org.RegularLink) { - link := r.resolveLink(l.Kind(), l.URL) + link := r.resolveLink(l.URL) printHTML := func(html template.HTML, a ...any) { _, _ = fmt.Fprint(r, htmlutil.HTMLFormat(html, a...)) @@ -156,14 +142,14 @@ func (r *orgWriter) WriteRegularLink(l org.RegularLink) { if l.Description == nil { printHTML(`<img src="%s" alt="%s">`, link, link) } else { - imageSrc := r.resolveLink(l.Kind(), org.String(l.Description...)) + imageSrc := r.resolveLink(org.String(l.Description...)) printHTML(`<a href="%s"><img src="%s" alt="%s"></a>`, link, imageSrc, imageSrc) } case "video": if l.Description == nil { printHTML(`<video src="%s">%s</video>`, link, link) } else { - videoSrc := r.resolveLink(l.Kind(), org.String(l.Description...)) + videoSrc := r.resolveLink(org.String(l.Description...)) printHTML(`<a href="%s"><video src="%s">%s</video></a>`, link, videoSrc, videoSrc) } default: diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go index de39bafebe..df4bb38ad1 100644 --- a/modules/markup/orgmode/orgmode_test.go +++ b/modules/markup/orgmode/orgmode_test.go @@ -1,7 +1,7 @@ // Copyright 2017 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package markup +package orgmode_test import ( "os" @@ -9,6 +9,7 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/orgmode" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" @@ -22,7 +23,7 @@ func TestMain(m *testing.M) { func TestRender_StandardLinks(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("/relative-path/media/branch/main/"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -30,37 +31,37 @@ func TestRender_StandardLinks(t *testing.T) { test("[[https://google.com/]]", `<p><a href="https://google.com/">https://google.com/</a></p>`) test("[[ImageLink.svg][The Image Desc]]", - `<p><a href="/relative-path/media/branch/main/ImageLink.svg">The Image Desc</a></p>`) + `<p><a href="ImageLink.svg">The Image Desc</a></p>`) } func TestRender_InternalLinks(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("/relative-path/src/branch/main"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } test("[[file:test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="test.org">Test</a></p>`) test("[[./test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="./test.org">Test</a></p>`) test("[[test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/test.org">Test</a></p>`) + `<p><a href="test.org">Test</a></p>`) test("[[path/to/test.org][Test]]", - `<p><a href="/relative-path/src/branch/main/path/to/test.org">Test</a></p>`) + `<p><a href="path/to/test.org">Test</a></p>`) } func TestRender_Media(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext("./relative-path"), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } test("[[file:../../.images/src/02/train.jpg]]", - `<p><img src=".images/src/02/train.jpg" alt=".images/src/02/train.jpg"></p>`) + `<p><img src="../../.images/src/02/train.jpg" alt="../../.images/src/02/train.jpg"></p>`) test("[[file:train.jpg]]", - `<p><img src="relative-path/train.jpg" alt="relative-path/train.jpg"></p>`) + `<p><img src="train.jpg" alt="train.jpg"></p>`) // With description. test("[[https://example.com][https://example.com/example.svg]]", @@ -91,7 +92,7 @@ func TestRender_Media(t *testing.T) { func TestRender_Source(t *testing.T) { test := func(input, expected string) { - buffer, err := RenderString(markup.NewTestRenderContext(), input) + buffer, err := orgmode.RenderString(markup.NewTestRenderContext(), input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } diff --git a/modules/markup/render.go b/modules/markup/render.go index 37a2a86687..79f1f473c2 100644 --- a/modules/markup/render.go +++ b/modules/markup/render.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/url" + "strconv" "strings" "time" @@ -46,7 +47,7 @@ type RenderOptions struct { // user&repo, format&style®exp (for external issue pattern), teams&org (for mention) // RefTypeNameSubURL (for iframe&asciicast) // markupAllowShortIssuePattern - // markdownLineBreakStyle (comment, document) + // markdownNewLineHardBreak Metas map[string]string // used by external render. the router "/org/repo/render/..." will output the rendered content in a standalone page @@ -247,7 +248,8 @@ func Init(renderHelpFuncs *RenderHelperFuncs) { } func ComposeSimpleDocumentMetas() map[string]string { - return map[string]string{"markdownLineBreakStyle": "document"} + // TODO: there is no separate config option for "simple document" rendering, so temporarily use the same config as "repo file" + return map[string]string{"markdownNewLineHardBreak": strconv.FormatBool(setting.Markdown.RenderOptionsRepoFile.NewLineHardBreak)} } type TestRenderHelper struct { @@ -261,8 +263,14 @@ func (r *TestRenderHelper) IsCommitIDExisting(commitID string) bool { return strings.HasPrefix(commitID, "65f1bf2") //|| strings.HasPrefix(commitID, "88fc37a") } -func (r *TestRenderHelper) ResolveLink(link string, likeType LinkType) string { - return r.ctx.ResolveLinkRelative(r.BaseLink, "", link) +func (r *TestRenderHelper) ResolveLink(link, preferLinkType string) string { + linkType, link := ParseRenderedLink(link, preferLinkType) + switch linkType { + case LinkTypeRoot: + return r.ctx.ResolveLinkRoot(link) + default: + return r.ctx.ResolveLinkRelative(r.BaseLink, "", link) + } } var _ RenderHelper = (*TestRenderHelper)(nil) diff --git a/modules/markup/render_helper.go b/modules/markup/render_helper.go index 8ff0e7d6fb..b16f1189c5 100644 --- a/modules/markup/render_helper.go +++ b/modules/markup/render_helper.go @@ -10,13 +10,11 @@ import ( "code.gitea.io/gitea/modules/setting" ) -type LinkType string - const ( - LinkTypeApp LinkType = "app" // the link is relative to the AppSubURL - LinkTypeDefault LinkType = "default" // the link is relative to the default base (eg: repo link, or current ref tree path) - LinkTypeMedia LinkType = "media" // the link should be used to access media files (images, videos) - LinkTypeRaw LinkType = "raw" // not really useful, mainly for environment GITEA_PREFIX_RAW for external renders + LinkTypeDefault = "" + LinkTypeRoot = "/:root" // the link is relative to the AppSubURL(ROOT_URL) + LinkTypeMedia = "/:media" // the link should be used to access media files (images, videos) + LinkTypeRaw = "/:raw" // not really useful, mainly for environment GITEA_PREFIX_RAW for external renders ) type RenderHelper interface { @@ -27,7 +25,7 @@ type RenderHelper interface { // but not make processors to guess "is it rendering a comment or a wiki?" or "does it need to check commit ID?" IsCommitIDExisting(commitID string) bool - ResolveLink(link string, likeType LinkType) string + ResolveLink(link, preferLinkType string) string } // RenderHelperFuncs is used to decouple cycle-import @@ -51,7 +49,8 @@ func (r *SimpleRenderHelper) IsCommitIDExisting(commitID string) bool { return false } -func (r *SimpleRenderHelper) ResolveLink(link string, likeType LinkType) string { +func (r *SimpleRenderHelper) ResolveLink(link, preferLinkType string) string { + _, link = ParseRenderedLink(link, preferLinkType) return resolveLinkRelative(context.Background(), setting.AppSubURL+"/", "", link, false) } diff --git a/modules/markup/render_link.go b/modules/markup/render_link.go index b2e0699681..046544ce81 100644 --- a/modules/markup/render_link.go +++ b/modules/markup/render_link.go @@ -33,10 +33,24 @@ func resolveLinkRelative(ctx context.Context, base, cur, link string, absolute b return finalLink } -func (ctx *RenderContext) ResolveLinkRelative(base, cur, link string) (finalLink string) { +func (ctx *RenderContext) ResolveLinkRelative(base, cur, link string) string { + if strings.HasPrefix(link, "/:") { + setting.PanicInDevOrTesting("invalid link %q, forgot to cut?", link) + } return resolveLinkRelative(ctx, base, cur, link, ctx.RenderOptions.UseAbsoluteLink) } -func (ctx *RenderContext) ResolveLinkApp(link string) string { +func (ctx *RenderContext) ResolveLinkRoot(link string) string { return ctx.ResolveLinkRelative(setting.AppSubURL+"/", "", link) } + +func ParseRenderedLink(s, preferLinkType string) (linkType, link string) { + if strings.HasPrefix(s, "/:") { + p := strings.IndexByte(s[1:], '/') + if p == -1 { + return s, "" + } + return s[:p+1], s[p+2:] + } + return preferLinkType, s +} diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 35f90eb46c..b6e9c348b7 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -4,12 +4,12 @@ package markup import ( - "bytes" "io" "path" "strings" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" ) // Renderer defines an interface for rendering markup file to HTML @@ -37,7 +37,7 @@ type ExternalRenderer interface { // RendererContentDetector detects if the content can be rendered // by specified renderer type RendererContentDetector interface { - CanRender(filename string, input io.Reader) bool + CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool } var ( @@ -60,13 +60,9 @@ func GetRendererByFileName(filename string) Renderer { } // DetectRendererType detects the markup type of the content -func DetectRendererType(filename string, input io.Reader) string { - buf, err := io.ReadAll(input) - if err != nil { - return "" - } +func DetectRendererType(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) string { for _, renderer := range renderers { - if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, bytes.NewReader(buf)) { + if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, sniffedType, prefetchBuf) { return renderer.Name() } } diff --git a/modules/markup/sanitizer_default.go b/modules/markup/sanitizer_default.go index 14161eb533..0fbf0f0b24 100644 --- a/modules/markup/sanitizer_default.go +++ b/modules/markup/sanitizer_default.go @@ -4,6 +4,7 @@ package markup import ( + "html/template" "io" "net/url" "regexp" @@ -52,6 +53,8 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy { policy.AllowAttrs("src", "autoplay", "controls").OnElements("video") + policy.AllowAttrs("loading").OnElements("img") + // Allow generally safe attributes (reference: https://github.com/jch/html-pipeline) generalSafeAttrs := []string{ "abbr", "accept", "accept-charset", @@ -90,9 +93,9 @@ func (st *Sanitizer) createDefaultPolicy() *bluemonday.Policy { return policy } -// Sanitize takes a string that contains a HTML fragment or document and applies policy whitelist. -func Sanitize(s string) string { - return GetDefaultSanitizer().defaultPolicy.Sanitize(s) +// Sanitize use default sanitizer policy to sanitize a string +func Sanitize(s string) template.HTML { + return template.HTML(GetDefaultSanitizer().defaultPolicy.Sanitize(s)) } // SanitizeReader sanitizes a Reader diff --git a/modules/markup/sanitizer_default_test.go b/modules/markup/sanitizer_default_test.go index 5282916944..e5ba018e1b 100644 --- a/modules/markup/sanitizer_default_test.go +++ b/modules/markup/sanitizer_default_test.go @@ -69,6 +69,6 @@ func TestSanitizer(t *testing.T) { } for i := 0; i < len(testCases); i += 2 { - assert.Equal(t, testCases[i+1], Sanitize(testCases[i])) + assert.Equal(t, testCases[i+1], string(Sanitize(testCases[i]))) } } diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 230260ff94..4d2ec287a9 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -184,7 +184,7 @@ func NewCollector() Collector { Users: prometheus.NewDesc( namespace+"users", "Number of Users", - nil, nil, + []string{"state"}, nil, ), Watches: prometheus.NewDesc( namespace+"watches", @@ -373,7 +373,14 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( c.Users, prometheus.GaugeValue, - float64(stats.Counter.User), + float64(stats.Counter.UsersActive), + "active", // state label + ) + ch <- prometheus.MustNewConstMetric( + c.Users, + prometheus.GaugeValue, + float64(stats.Counter.UsersNotActive), + "inactive", // state label ) ch <- prometheus.MustNewConstMetric( c.Watches, diff --git a/modules/migration/schemas_bindata.go b/modules/migration/schemas_bindata.go index c5db3b3461..695c2c1135 100644 --- a/modules/migration/schemas_bindata.go +++ b/modules/migration/schemas_bindata.go @@ -3,6 +3,28 @@ //go:build bindata +//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas bindata.dat + package migration -//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas migration bindata.go +import ( + "io" + "io/fs" + "path" + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() fs.FS { + return assetfs.NewEmbeddedFS(bindata) +}) + +func openSchema(filename string) (io.ReadCloser, error) { + return BuiltinAssets().Open(path.Base(filename)) +} diff --git a/modules/migration/schemas_static.go b/modules/migration/schemas_static.go deleted file mode 100644 index 8a0c340a65..0000000000 --- a/modules/migration/schemas_static.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package migration - -import ( - "io" - "path" -) - -func openSchema(filename string) (io.ReadCloser, error) { - return Assets.Open(path.Base(filename)) -} diff --git a/modules/optional/option.go b/modules/optional/option.go index af9e5ac852..cbecf86987 100644 --- a/modules/optional/option.go +++ b/modules/optional/option.go @@ -3,6 +3,14 @@ package optional +import "strconv" + +// Option is a generic type that can hold a value of type T or be empty (None). +// +// It must use the slice type to work with "chi" form values binding: +// * non-existing value are represented as an empty slice (None) +// * existing value is represented as a slice with one element (Some) +// * multiple values are represented as a slice with multiple elements (Some), the Value is the first element (not well-defined in this case) type Option[T any] []T func None[T any]() Option[T] { @@ -20,6 +28,13 @@ func FromPtr[T any](v *T) Option[T] { return Some(*v) } +func FromMapLookup[K comparable, V any](m map[K]V, k K) Option[V] { + if v, ok := m[k]; ok { + return Some(v) + } + return None[V]() +} + func FromNonDefault[T comparable](v T) Option[T] { var zero T if v == zero { @@ -43,3 +58,12 @@ func (o Option[T]) ValueOrDefault(v T) T { } return v } + +// ParseBool get the corresponding optional.Option[bool] of a string using strconv.ParseBool +func ParseBool(s string) Option[bool] { + v, e := strconv.ParseBool(s) + if e != nil { + return None[bool]() + } + return Some(v) +} diff --git a/modules/optional/option_test.go b/modules/optional/option_test.go index 203e9221e3..ea80a2e3cb 100644 --- a/modules/optional/option_test.go +++ b/modules/optional/option_test.go @@ -56,4 +56,23 @@ func TestOption(t *testing.T) { opt3 := optional.FromNonDefault(1) assert.True(t, opt3.Has()) assert.Equal(t, int(1), opt3.Value()) + + opt4 := optional.FromMapLookup(map[string]int{"a": 1}, "a") + assert.True(t, opt4.Has()) + assert.Equal(t, 1, opt4.Value()) + opt4 = optional.FromMapLookup(map[string]int{"a": 1}, "b") + assert.False(t, opt4.Has()) +} + +func Test_ParseBool(t *testing.T) { + assert.Equal(t, optional.None[bool](), optional.ParseBool("")) + assert.Equal(t, optional.None[bool](), optional.ParseBool("x")) + + assert.Equal(t, optional.Some(false), optional.ParseBool("0")) + assert.Equal(t, optional.Some(false), optional.ParseBool("f")) + assert.Equal(t, optional.Some(false), optional.ParseBool("False")) + + assert.Equal(t, optional.Some(true), optional.ParseBool("1")) + assert.Equal(t, optional.Some(true), optional.ParseBool("t")) + assert.Equal(t, optional.Some(true), optional.ParseBool("True")) } diff --git a/modules/optional/serialization_test.go b/modules/optional/serialization_test.go index 09a4bddea0..cf81a94cfc 100644 --- a/modules/optional/serialization_test.go +++ b/modules/optional/serialization_test.go @@ -4,7 +4,7 @@ package optional_test import ( - std_json "encoding/json" //nolint:depguard + std_json "encoding/json" //nolint:depguard // for testing purpose "testing" "code.gitea.io/gitea/modules/json" @@ -51,11 +51,11 @@ func TestOptionalToJson(t *testing.T) { t.Run(tc.name, func(t *testing.T) { b, err := json.Marshal(tc.obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, string(b), "gitea json module returned unexpected") + assert.Equal(t, tc.want, string(b), "gitea json module returned unexpected") b, err = std_json.Marshal(tc.obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, string(b), "std json module returned unexpected") + assert.Equal(t, tc.want, string(b), "std json module returned unexpected") }) } } @@ -89,12 +89,12 @@ func TestOptionalFromJson(t *testing.T) { var obj1 testSerializationStruct err := json.Unmarshal([]byte(tc.data), &obj1) assert.NoError(t, err) - assert.EqualValues(t, tc.want, obj1, "gitea json module returned unexpected") + assert.Equal(t, tc.want, obj1, "gitea json module returned unexpected") var obj2 testSerializationStruct err = std_json.Unmarshal([]byte(tc.data), &obj2) assert.NoError(t, err) - assert.EqualValues(t, tc.want, obj2, "std json module returned unexpected") + assert.Equal(t, tc.want, obj2, "std json module returned unexpected") }) } } @@ -135,7 +135,7 @@ optional_two_string: null t.Run(tc.name, func(t *testing.T) { b, err := yaml.Marshal(tc.obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, string(b), "yaml module returned unexpected") + assert.Equal(t, tc.want, string(b), "yaml module returned unexpected") }) } } @@ -184,7 +184,7 @@ optional_twostring: null var obj testSerializationStruct err := yaml.Unmarshal([]byte(tc.data), &obj) assert.NoError(t, err) - assert.EqualValues(t, tc.want, obj, "yaml module returned unexpected") + assert.Equal(t, tc.want, obj, "yaml module returned unexpected") }) } } diff --git a/modules/options/options_bindata.go b/modules/options/options_bindata.go index 29151cb3cb..b2321d7eb5 100644 --- a/modules/options/options_bindata.go +++ b/modules/options/options_bindata.go @@ -3,6 +3,21 @@ //go:build bindata +//go:generate go run ../../build/generate-bindata.go ../../options bindata.dat + package options -//go:generate go run ../../build/generate-bindata.go ../../options options bindata.go +import ( + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", assetfs.NewEmbeddedFS(bindata)) +}) diff --git a/modules/options/dynamic.go b/modules/options/options_dynamic.go index 085492d11c..085492d11c 100644 --- a/modules/options/dynamic.go +++ b/modules/options/options_dynamic.go diff --git a/modules/options/static.go b/modules/options/static.go deleted file mode 100644 index 72b28e990e..0000000000 --- a/modules/options/static.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package options - -import ( - "code.gitea.io/gitea/modules/assetfs" -) - -func BuiltinAssets() *assetfs.Layer { - return assetfs.Bindata("builtin(bindata)", Assets) -} diff --git a/modules/packages/container/const.go b/modules/packages/container/const.go new file mode 100644 index 0000000000..6c7c9b46d1 --- /dev/null +++ b/modules/packages/container/const.go @@ -0,0 +1,11 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package container + +const ( + ContentTypeDockerDistributionManifestV2 = "application/vnd.docker.distribution.manifest.v2+json" + + ManifestFilename = "manifest.json" + UploadVersion = "_upload" +) diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go index 2a41fb9105..3ef0684d13 100644 --- a/modules/packages/container/metadata.go +++ b/modules/packages/container/metadata.go @@ -71,14 +71,34 @@ type Manifest struct { Size int64 `json:"size"` } +func IsMediaTypeValid(mt string) bool { + return strings.HasPrefix(mt, "application/vnd.docker.") || strings.HasPrefix(mt, "application/vnd.oci.") +} + +func IsMediaTypeImageManifest(mt string) bool { + return strings.EqualFold(mt, oci.MediaTypeImageManifest) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.v2+json") +} + +func IsMediaTypeImageIndex(mt string) bool { + return strings.EqualFold(mt, oci.MediaTypeImageIndex) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.list.v2+json") +} + // ParseImageConfig parses the metadata of an image config -func ParseImageConfig(mt string, r io.Reader) (*Metadata, error) { - if strings.EqualFold(mt, helm.ConfigMediaType) { +func ParseImageConfig(mediaType string, r io.Reader) (*Metadata, error) { + if strings.EqualFold(mediaType, helm.ConfigMediaType) { return parseHelmConfig(r) } // fallback to OCI Image Config - return parseOCIImageConfig(r) + // FIXME: this fallback is not right, we should strictly check the media type in the future + metadata, err := parseOCIImageConfig(r) + if err != nil { + if !IsMediaTypeImageManifest(mediaType) { + return &Metadata{Platform: "unknown/unknown"}, nil + } + return nil, err + } + return metadata, nil } func parseOCIImageConfig(r io.Reader) (*Metadata, error) { diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go index 665499b2e6..0f2d702925 100644 --- a/modules/packages/container/metadata_test.go +++ b/modules/packages/container/metadata_test.go @@ -11,6 +11,7 @@ import ( oci "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestParseImageConfig(t *testing.T) { @@ -58,4 +59,8 @@ func TestParseImageConfig(t *testing.T) { assert.ElementsMatch(t, []string{author}, metadata.Authors) assert.Equal(t, projectURL, metadata.ProjectURL) assert.Equal(t, repositoryURL, metadata.RepositoryURL) + + metadata, err = ParseImageConfig("anything-unknown", strings.NewReader("")) + require.NoError(t, err) + assert.Equal(t, &Metadata{Platform: "unknown/unknown"}, metadata) } diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index 37612556d7..dadb7eaefc 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -28,8 +28,7 @@ func NewContentStore() *ContentStore { return contentStore } -// Get gets a package blob -func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { +func (s *ContentStore) OpenBlob(key BlobHash256Key) (storage.Object, error) { return s.store.Open(KeyToRelativePath(key)) } diff --git a/modules/packages/goproxy/metadata.go b/modules/packages/goproxy/metadata.go index 40f7d20508..a67b149f4d 100644 --- a/modules/packages/goproxy/metadata.go +++ b/modules/packages/goproxy/metadata.go @@ -5,7 +5,6 @@ package goproxy import ( "archive/zip" - "fmt" "io" "path" "strings" @@ -88,7 +87,7 @@ func ParsePackage(r io.ReaderAt, size int64) (*Package, error) { return nil, ErrInvalidStructure } - p.GoMod = fmt.Sprintf("module %s", p.Name) + p.GoMod = "module " + p.Name return p, nil } diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go index 4ab45edcec..0cd657cd44 100644 --- a/modules/packages/hashed_buffer.go +++ b/modules/packages/hashed_buffer.go @@ -6,6 +6,7 @@ package packages import ( "io" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util/filebuffer" ) @@ -34,11 +35,11 @@ func NewHashedBuffer() (*HashedBuffer, error) { // NewHashedBufferWithSize creates a hashed buffer with a specific memory size func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { - b, err := filebuffer.New(maxMemorySize) + tempDir, err := setting.AppDataTempDir("package-hashed-buffer").MkdirAllSub("") if err != nil { return nil, err } - + b := filebuffer.New(maxMemorySize, tempDir) hash := NewMultiHasher() combinedWriter := io.MultiWriter(b, hash) diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go index 564e782f18..5104c1fb25 100644 --- a/modules/packages/hashed_buffer_test.go +++ b/modules/packages/hashed_buffer_test.go @@ -9,10 +9,13 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestHashedBuffer(t *testing.T) { + setting.AppDataPath = t.TempDir() cases := []struct { MaxMemorySize int Data string diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 8ba4dbfba7..11b5123c27 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -58,7 +58,7 @@ type PackageMetadata struct { Time map[string]time.Time `json:"time,omitempty"` Homepage string `json:"homepage,omitempty"` Keywords []string `json:"keywords,omitempty"` - Repository Repository `json:"repository,omitempty"` + Repository Repository `json:"repository"` Author User `json:"author"` ReadmeFilename string `json:"readmeFilename,omitempty"` Users map[string]bool `json:"users,omitempty"` @@ -75,7 +75,7 @@ type PackageMetadataVersion struct { Author User `json:"author"` Homepage string `json:"homepage,omitempty"` License string `json:"license,omitempty"` - Repository Repository `json:"repository,omitempty"` + Repository Repository `json:"repository"` Keywords []string `json:"keywords,omitempty"` Dependencies map[string]string `json:"dependencies,omitempty"` BundleDependencies []string `json:"bundleDependencies,omitempty"` diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index d1d0263387..362d0470d5 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -23,5 +23,5 @@ type Metadata struct { OptionalDependencies map[string]string `json:"optional_dependencies,omitempty"` Bin map[string]string `json:"bin,omitempty"` Readme string `json:"readme,omitempty"` - Repository Repository `json:"repository,omitempty"` + Repository Repository `json:"repository"` } diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 1e98ddffde..513b4dd2b9 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -57,14 +57,25 @@ type Package struct { // Metadata represents the metadata of a Nuget package type Metadata struct { - Description string `json:"description,omitempty"` - ReleaseNotes string `json:"release_notes,omitempty"` - Readme string `json:"readme,omitempty"` - Authors string `json:"authors,omitempty"` - ProjectURL string `json:"project_url,omitempty"` - RepositoryURL string `json:"repository_url,omitempty"` - RequireLicenseAcceptance bool `json:"require_license_acceptance"` - Dependencies map[string][]Dependency `json:"dependencies,omitempty"` + Authors string `json:"authors,omitempty"` + Copyright string `json:"copyright,omitempty"` + Description string `json:"description,omitempty"` + DevelopmentDependency bool `json:"development_dependency,omitempty"` + IconURL string `json:"icon_url,omitempty"` + Language string `json:"language,omitempty"` + LicenseURL string `json:"license_url,omitempty"` + MinClientVersion string `json:"min_client_version,omitempty"` + Owners string `json:"owners,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Readme string `json:"readme,omitempty"` + ReleaseNotes string `json:"release_notes,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + RequireLicenseAcceptance bool `json:"require_license_acceptance"` + Summary string `json:"summary,omitempty"` + Tags string `json:"tags,omitempty"` + Title string `json:"title,omitempty"` + + Dependencies map[string][]Dependency `json:"dependencies,omitempty"` } // Dependency represents a dependency of a Nuget package @@ -74,24 +85,31 @@ type Dependency struct { } // https://learn.microsoft.com/en-us/nuget/reference/nuspec +// https://github.com/NuGet/NuGet.Client/blob/dev/src/NuGet.Core/NuGet.Packaging/compiler/resources/nuspec.xsd type nuspecPackage struct { Metadata struct { - ID string `xml:"id"` - Version string `xml:"version"` - Authors string `xml:"authors"` - RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + // required fields + Authors string `xml:"authors"` + Description string `xml:"description"` + ID string `xml:"id"` + Version string `xml:"version"` + + // optional fields + Copyright string `xml:"copyright"` + DevelopmentDependency bool `xml:"developmentDependency"` + IconURL string `xml:"iconUrl"` + Language string `xml:"language"` + LicenseURL string `xml:"licenseUrl"` + MinClientVersion string `xml:"minClientVersion,attr"` + Owners string `xml:"owners"` ProjectURL string `xml:"projectUrl"` - Description string `xml:"description"` - ReleaseNotes string `xml:"releaseNotes"` Readme string `xml:"readme"` - PackageTypes struct { - PackageType []struct { - Name string `xml:"name,attr"` - } `xml:"packageType"` - } `xml:"packageTypes"` - Repository struct { - URL string `xml:"url,attr"` - } `xml:"repository"` + ReleaseNotes string `xml:"releaseNotes"` + RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + Summary string `xml:"summary"` + Tags string `xml:"tags"` + Title string `xml:"title"` + Dependencies struct { Dependency []struct { ID string `xml:"id,attr"` @@ -107,6 +125,14 @@ type nuspecPackage struct { } `xml:"dependency"` } `xml:"group"` } `xml:"dependencies"` + PackageTypes struct { + PackageType []struct { + Name string `xml:"name,attr"` + } `xml:"packageType"` + } `xml:"packageTypes"` + Repository struct { + URL string `xml:"url,attr"` + } `xml:"repository"` } `xml:"metadata"` } @@ -167,13 +193,24 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { } m := &Metadata{ - Description: p.Metadata.Description, - ReleaseNotes: p.Metadata.ReleaseNotes, Authors: p.Metadata.Authors, + Copyright: p.Metadata.Copyright, + Description: p.Metadata.Description, + DevelopmentDependency: p.Metadata.DevelopmentDependency, + IconURL: p.Metadata.IconURL, + Language: p.Metadata.Language, + LicenseURL: p.Metadata.LicenseURL, + MinClientVersion: p.Metadata.MinClientVersion, + Owners: p.Metadata.Owners, ProjectURL: p.Metadata.ProjectURL, + ReleaseNotes: p.Metadata.ReleaseNotes, RepositoryURL: p.Metadata.Repository.URL, RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance, - Dependencies: make(map[string][]Dependency), + Summary: p.Metadata.Summary, + Tags: p.Metadata.Tags, + Title: p.Metadata.Title, + + Dependencies: make(map[string][]Dependency), } if p.Metadata.Readme != "" { @@ -227,13 +264,13 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { func toNormalizedVersion(v *version.Version) string { var buf bytes.Buffer segments := v.Segments64() - fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) + _, _ = fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) if len(segments) > 3 && segments[3] > 0 { - fmt.Fprintf(&buf, ".%d", segments[3]) + _, _ = fmt.Fprintf(&buf, ".%d", segments[3]) } pre := v.Prerelease() if pre != "" { - fmt.Fprint(&buf, "-", pre) + _, _ = fmt.Fprint(&buf, "-", pre) } return buf.String() } diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go index f466492f8a..90c3e8dfeb 100644 --- a/modules/packages/nuget/metadata_test.go +++ b/modules/packages/nuget/metadata_test.go @@ -12,44 +12,62 @@ import ( ) const ( - id = "System.Gitea" - semver = "1.0.1" - authors = "Gitea Authors" - projectURL = "https://gitea.io" - description = "Package Description" - releaseNotes = "Package Release Notes" - readme = "Readme" - repositoryURL = "https://gitea.io/gitea/gitea" - targetFramework = ".NETStandard2.1" - dependencyID = "System.Text.Json" - dependencyVersion = "5.0.0" + authors = "Gitea Authors" + copyright = "Package Copyright" + dependencyID = "System.Text.Json" + dependencyVersion = "5.0.0" + developmentDependency = true + description = "Package Description" + iconURL = "https://gitea.io/favicon.png" + id = "System.Gitea" + language = "Package Language" + licenseURL = "https://gitea.io/license" + minClientVersion = "1.0.0.0" + owners = "Package Owners" + projectURL = "https://gitea.io" + readme = "Readme" + releaseNotes = "Package Release Notes" + repositoryURL = "https://gitea.io/gitea/gitea" + requireLicenseAcceptance = true + tags = "tag_1 tag_2 tag_3" + targetFramework = ".NETStandard2.1" + title = "Package Title" + versionStr = "1.0.1" ) const nuspecContent = `<?xml version="1.0" encoding="utf-8"?> <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> - <metadata> - <id>` + id + `</id> - <version>` + semver + `</version> - <authors>` + authors + `</authors> - <requireLicenseAcceptance>true</requireLicenseAcceptance> - <projectUrl>` + projectURL + `</projectUrl> - <description>` + description + `</description> - <releaseNotes>` + releaseNotes + `</releaseNotes> - <repository url="` + repositoryURL + `" /> - <readme>README.md</readme> - <dependencies> - <group targetFramework="` + targetFramework + `"> - <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" /> - </group> - </dependencies> - </metadata> + <metadata minClientVersion="` + minClientVersion + `"> + <authors>` + authors + `</authors> + <copyright>` + copyright + `</copyright> + <description>` + description + `</description> + <developmentDependency>true</developmentDependency> + <iconUrl>` + iconURL + `</iconUrl> + <id>` + id + `</id> + <language>` + language + `</language> + <licenseUrl>` + licenseURL + `</licenseUrl> + <owners>` + owners + `</owners> + <projectUrl>` + projectURL + `</projectUrl> + <readme>README.md</readme> + <releaseNotes>` + releaseNotes + `</releaseNotes> + <repository url="` + repositoryURL + `" /> + <requireLicenseAcceptance>true</requireLicenseAcceptance> + <tags>` + tags + `</tags> + <title>` + title + `</title> + <version>` + versionStr + `</version> + <dependencies> + <group targetFramework="` + targetFramework + `"> + <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" /> + </group> + </dependencies> + </metadata> </package>` const symbolsNuspecContent = `<?xml version="1.0" encoding="utf-8"?> <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> <metadata> <id>` + id + `</id> - <version>` + semver + `</version> + <version>` + versionStr + `</version> <description>` + description + `</description> <packageTypes> <packageType name="SymbolsPackage" /> @@ -140,14 +158,26 @@ func TestParsePackageMetaData(t *testing.T) { assert.NotNil(t, np) assert.Equal(t, DependencyPackage, np.PackageType) - assert.Equal(t, id, np.ID) - assert.Equal(t, semver, np.Version) assert.Equal(t, authors, np.Metadata.Authors) - assert.Equal(t, projectURL, np.Metadata.ProjectURL) assert.Equal(t, description, np.Metadata.Description) - assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) + assert.Equal(t, id, np.ID) + assert.Equal(t, versionStr, np.Version) + + assert.Equal(t, copyright, np.Metadata.Copyright) + assert.Equal(t, developmentDependency, np.Metadata.DevelopmentDependency) + assert.Equal(t, iconURL, np.Metadata.IconURL) + assert.Equal(t, language, np.Metadata.Language) + assert.Equal(t, licenseURL, np.Metadata.LicenseURL) + assert.Equal(t, minClientVersion, np.Metadata.MinClientVersion) + assert.Equal(t, owners, np.Metadata.Owners) + assert.Equal(t, projectURL, np.Metadata.ProjectURL) assert.Equal(t, readme, np.Metadata.Readme) + assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL) + assert.Equal(t, requireLicenseAcceptance, np.Metadata.RequireLicenseAcceptance) + assert.Equal(t, tags, np.Metadata.Tags) + assert.Equal(t, title, np.Metadata.Title) + assert.Len(t, np.Metadata.Dependencies, 1) assert.Contains(t, np.Metadata.Dependencies, targetFramework) deps := np.Metadata.Dependencies[targetFramework] @@ -180,7 +210,7 @@ func TestParsePackageMetaData(t *testing.T) { assert.Equal(t, SymbolsPackage, np.PackageType) assert.Equal(t, id, np.ID) - assert.Equal(t, semver, np.Version) + assert.Equal(t, versionStr, np.Version) assert.Equal(t, description, np.Metadata.Description) assert.Empty(t, np.Metadata.Dependencies) }) diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go index 81bf0371a0..9c952e1f10 100644 --- a/modules/packages/nuget/symbol_extractor.go +++ b/modules/packages/nuget/symbol_extractor.go @@ -34,7 +34,7 @@ type PortablePdbList []*PortablePdb func (l PortablePdbList) Close() { for _, pdb := range l { - pdb.Content.Close() + _ = pdb.Content.Close() } } @@ -65,7 +65,7 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { buf, err := packages.CreateHashedBufferFromReader(f) - f.Close() + _ = f.Close() if err != nil { return err @@ -73,12 +73,12 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { id, err := ParseDebugHeaderID(buf) if err != nil { - buf.Close() + _ = buf.Close() return fmt.Errorf("Invalid PDB file: %w", err) } if _, err := buf.Seek(0, io.SeekStart); err != nil { - buf.Close() + _ = buf.Close() return err } diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go index fa1b80ee82..e841e377d9 100644 --- a/modules/packages/nuget/symbol_extractor_test.go +++ b/modules/packages/nuget/symbol_extractor_test.go @@ -9,6 +9,8 @@ import ( "encoding/base64" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) @@ -17,18 +19,19 @@ fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==` func TestExtractPortablePdb(t *testing.T) { + setting.AppDataPath = t.TempDir() createArchive := func(name string, content []byte) []byte { var buf bytes.Buffer archive := zip.NewWriter(&buf) w, _ := archive.Create(name) - w.Write(content) - archive.Close() + _, _ = w.Write(content) + _ = archive.Close() return buf.Bytes() } t.Run("MissingPdbFiles", func(t *testing.T) { var buf bytes.Buffer - zip.NewWriter(&buf).Close() + _ = zip.NewWriter(&buf).Close() pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len())) assert.ErrorIs(t, err, ErrMissingPdbFiles) diff --git a/modules/packages/pub/metadata.go b/modules/packages/pub/metadata.go index afb464e462..9b00472eb2 100644 --- a/modules/packages/pub/metadata.go +++ b/modules/packages/pub/metadata.go @@ -88,7 +88,7 @@ func ParsePackage(r io.Reader) (*Package, error) { if err != nil { return nil, err } - } else if strings.ToLower(hd.Name) == "readme.md" { + } else if strings.EqualFold(hd.Name, "readme.md") { data, err := io.ReadAll(tr) if err != nil { return nil, err diff --git a/modules/packages/rubygems/marshal.go b/modules/packages/rubygems/marshal.go index 4e6a5fc5f8..1505221acc 100644 --- a/modules/packages/rubygems/marshal.go +++ b/modules/packages/rubygems/marshal.go @@ -250,7 +250,7 @@ func (e *MarshalEncoder) marshalArray(arr reflect.Value) error { return err } - for i := 0; i < length; i++ { + for i := range length { if err := e.marshal(arr.Index(i).Interface()); err != nil { return err } diff --git a/modules/packages/swift/metadata.go b/modules/packages/swift/metadata.go index 24c4262ab7..85beb57607 100644 --- a/modules/packages/swift/metadata.go +++ b/modules/packages/swift/metadata.go @@ -47,7 +47,7 @@ type Metadata struct { Keywords []string `json:"keywords,omitempty"` RepositoryURL string `json:"repository_url,omitempty"` License string `json:"license,omitempty"` - Author Person `json:"author,omitempty"` + Author Person `json:"author"` Manifests map[string]*Manifest `json:"manifests,omitempty"` } diff --git a/modules/paginator/paginator.go b/modules/paginator/paginator.go index 8258d194c2..0f64e89d9a 100644 --- a/modules/paginator/paginator.go +++ b/modules/paginator/paginator.go @@ -4,6 +4,8 @@ package paginator +import "code.gitea.io/gitea/modules/util" + /* In template: @@ -32,25 +34,43 @@ Output: // Paginator represents a set of results of pagination calculations. type Paginator struct { - total int // total rows count + total int // total rows count, -1 means unknown + totalPages int // total pages count, -1 means unknown + current int // current page number + curRows int // current page rows count + pagingNum int // how many rows in one page - current int // current page number numPages int // how many pages to show on the UI } // New initialize a new pagination calculation and returns a Paginator as result. func New(total, pagingNum, current, numPages int) *Paginator { - if pagingNum <= 0 { - pagingNum = 1 + pagingNum = max(pagingNum, 1) + totalPages := util.Iif(total == -1, -1, (total+pagingNum-1)/pagingNum) + if total >= 0 { + current = min(current, totalPages) } - if current <= 0 { - current = 1 + current = max(current, 1) + return &Paginator{ + total: total, + totalPages: totalPages, + current: current, + pagingNum: pagingNum, + numPages: numPages, } - p := &Paginator{total, pagingNum, current, numPages} - if p.current > p.TotalPages() { - p.current = p.TotalPages() +} + +func (p *Paginator) SetCurRows(rows int) { + // For "unlimited paging", we need to know the rows of current page to determine if there is a next page. + // There is still an edge case: when curRows==pagingNum, then the "next page" will be an empty page. + // Ideally we should query one more row to determine if there is really a next page, but it's impossible in current framework. + p.curRows = rows + if p.total == -1 && p.current == 1 && !p.HasNext() { + // if there is only one page for the "unlimited paging", set total rows/pages count + // then the tmpl could decide to hide the nav bar. + p.total = rows + p.totalPages = util.Iif(p.total == 0, 0, 1) } - return p } // IsFirst returns true if current page is the first page. @@ -72,7 +92,10 @@ func (p *Paginator) Previous() int { // HasNext returns true if there is a next page relative to current page. func (p *Paginator) HasNext() bool { - return p.total > p.current*p.pagingNum + if p.total == -1 { + return p.curRows >= p.pagingNum + } + return p.current*p.pagingNum < p.total } func (p *Paginator) Next() int { @@ -84,10 +107,7 @@ func (p *Paginator) Next() int { // IsLast returns true if current page is the last page. func (p *Paginator) IsLast() bool { - if p.total == 0 { - return true - } - return p.total > (p.current-1)*p.pagingNum && !p.HasNext() + return !p.HasNext() } // Total returns number of total rows. @@ -97,10 +117,7 @@ func (p *Paginator) Total() int { // TotalPages returns number of total pages. func (p *Paginator) TotalPages() int { - if p.total == 0 { - return 1 - } - return (p.total + p.pagingNum - 1) / p.pagingNum + return p.totalPages } // Current returns current page number. @@ -135,10 +152,10 @@ func getMiddleIdx(numPages int) int { // If value is -1 means "..." that more pages are not showing. func (p *Paginator) Pages() []*Page { if p.numPages == 0 { - return []*Page{} - } else if p.numPages == 1 && p.TotalPages() == 1 { + return nil + } else if p.total == -1 || (p.numPages == 1 && p.TotalPages() == 1) { // Only show current page. - return []*Page{{1, true}} + return []*Page{{p.current, true}} } // Total page number is less or equal. diff --git a/modules/paginator/paginator_test.go b/modules/paginator/paginator_test.go index 8a56ee5121..ed46ecea94 100644 --- a/modules/paginator/paginator_test.go +++ b/modules/paginator/paginator_test.go @@ -76,9 +76,7 @@ func TestPaginator(t *testing.T) { t.Run("Only current page", func(t *testing.T) { p := New(0, 10, 1, 1) pages := p.Pages() - assert.Len(t, pages, 1) - assert.Equal(t, 1, pages[0].Num()) - assert.True(t, pages[0].IsCurrent()) + assert.Empty(t, pages) // no "total", so no pages p = New(1, 10, 1, 1) pages = p.Pages() diff --git a/modules/private/hook.go b/modules/private/hook.go index 87d6549f9c..215996b9b9 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -7,9 +7,9 @@ import ( "context" "fmt" "net/url" - "time" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" ) @@ -82,29 +82,32 @@ type HookProcReceiveRefResult struct { HeadBranch string } +func newInternalRequestAPIForHooks(ctx context.Context, hookName, ownerName, repoName string, opts HookOptions) *httplib.Request { + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/%s/%s/%s", hookName, url.PathEscape(ownerName), url.PathEscape(repoName)) + req := newInternalRequestAPI(ctx, reqURL, "POST", opts) + // This "timeout" applies to http.Client's timeout: A Timeout of zero means no timeout. + // This "timeout" was previously set to `time.Duration(60+len(opts.OldCommitIDs))` seconds, but it caused unnecessary timeout failures. + // It should be good enough to remove the client side timeout, only respect the "ctx" and server side timeout. + req.SetReadWriteTimeout(0) + return req +} + // HookPreReceive check whether the provided commits are allowed func HookPreReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) ResponseExtra { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - req := newInternalRequestAPI(ctx, reqURL, "POST", opts) - req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second) + req := newInternalRequestAPIForHooks(ctx, "pre-receive", ownerName, repoName, opts) _, extra := requestJSONResp(req, &ResponseText{}) return extra } // HookPostReceive updates services and users func HookPostReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) (*HookPostReceiveResult, ResponseExtra) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/post-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - req := newInternalRequestAPI(ctx, reqURL, "POST", opts) - req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second) + req := newInternalRequestAPIForHooks(ctx, "post-receive", ownerName, repoName, opts) return requestJSONResp(req, &HookPostReceiveResult{}) } // HookProcReceive proc-receive hook func HookProcReceive(ctx context.Context, ownerName, repoName string, opts HookOptions) (*HookProcReceiveResult, ResponseExtra) { - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/proc-receive/%s/%s", url.PathEscape(ownerName), url.PathEscape(repoName)) - - req := newInternalRequestAPI(ctx, reqURL, "POST", opts) - req.SetReadWriteTimeout(time.Duration(60+len(opts.OldCommitIDs)) * time.Second) + req := newInternalRequestAPIForHooks(ctx, "proc-receive", ownerName, repoName, opts) return requestJSONResp(req, &HookProcReceiveResult{}) } diff --git a/modules/private/internal.go b/modules/private/internal.go index 35eed1d608..e599c6eb8e 100644 --- a/modules/private/internal.go +++ b/modules/private/internal.go @@ -6,7 +6,6 @@ package private import ( "context" "crypto/tls" - "fmt" "net" "net/http" "os" @@ -47,7 +46,7 @@ Ensure you are running in the correct environment or set the correct configurati req := httplib.NewRequest(url, method). SetContext(ctx). Header("X-Real-IP", getClientIP()). - Header("X-Gitea-Internal-Auth", fmt.Sprintf("Bearer %s", setting.InternalToken)). + Header("X-Gitea-Internal-Auth", "Bearer "+setting.InternalToken). SetTLSClientConfig(&tls.Config{ InsecureSkipVerify: true, ServerName: setting.Domain, diff --git a/modules/private/serv.go b/modules/private/serv.go index 2ccc6c1129..b1dafbd81b 100644 --- a/modules/private/serv.go +++ b/modules/private/serv.go @@ -46,18 +46,16 @@ type ServCommandResults struct { } // ServCommand preps for a serv call -func ServCommand(ctx context.Context, keyID int64, ownerName, repoName string, mode perm.AccessMode, verbs ...string) (*ServCommandResults, ResponseExtra) { +func ServCommand(ctx context.Context, keyID int64, ownerName, repoName string, mode perm.AccessMode, verb, lfsVerb string) (*ServCommandResults, ResponseExtra) { reqURL := setting.LocalURL + fmt.Sprintf("api/internal/serv/command/%d/%s/%s?mode=%d", keyID, url.PathEscape(ownerName), url.PathEscape(repoName), mode, ) - for _, verb := range verbs { - if verb != "" { - reqURL += fmt.Sprintf("&verb=%s", url.QueryEscape(verb)) - } - } + reqURL += "&verb=" + url.QueryEscape(verb) + // reqURL += "&lfs_verb=" + url.QueryEscape(lfsVerb) // TODO: actually there is no use of this parameter. In the future, the URL construction should be more flexible + _ = lfsVerb req := newInternalRequestAPI(ctx, reqURL, "GET") return requestJSONResp(req, &ServCommandResults{}) } diff --git a/modules/proxyprotocol/errors.go b/modules/proxyprotocol/errors.go index 5439a86bd8..f76c82b7f6 100644 --- a/modules/proxyprotocol/errors.go +++ b/modules/proxyprotocol/errors.go @@ -20,7 +20,7 @@ type ErrBadAddressType struct { } func (e *ErrBadAddressType) Error() string { - return fmt.Sprintf("Unexpected proxy header address type: %s", e.Address) + return "Unexpected proxy header address type: " + e.Address } // ErrBadRemote is an error demonstrating a bad proxy header with bad Remote diff --git a/modules/public/public.go b/modules/public/public.go index 7f8ce29056..a7eace1538 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -44,7 +44,7 @@ func FileHandlerFunc() http.HandlerFunc { func parseAcceptEncoding(val string) container.Set[string] { parts := strings.Split(val, ";") types := make(container.Set[string]) - for _, v := range strings.Split(parts[0], ",") { + for v := range strings.SplitSeq(parts[0], ",") { types.Add(strings.TrimSpace(v)) } return types @@ -89,19 +89,16 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, servePublicAsset(w, req, fi, fi.ModTime(), f) } -type GzipBytesProvider interface { - GzipBytes() []byte -} - // servePublicAsset serve http content func servePublicAsset(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { setWellKnownContentType(w, fi.Name()) httpcache.SetCacheControlInHeader(w.Header(), httpcache.CacheControlForPublicStatic()) encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) - if encodings.Contains("gzip") { - // try to provide gzip content directly from bindata (provided by vfsgen۰CompressedFileInfo) - if compressed, ok := fi.(GzipBytesProvider); ok { - rdGzip := bytes.NewReader(compressed.GzipBytes()) + fiEmbedded, _ := fi.(assetfs.EmbeddedFileInfo) + if encodings.Contains("gzip") && fiEmbedded != nil { + // try to provide gzip content directly from bindata + if gzipBytes, ok := fiEmbedded.GetGzipContent(); ok { + rdGzip := bytes.NewReader(gzipBytes) // all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name // then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data if w.Header().Get("Content-Type") == "" { @@ -113,5 +110,4 @@ func servePublicAsset(w http.ResponseWriter, req *http.Request, fi os.FileInfo, } } http.ServeContent(w, req, fi.Name(), modtime, content) - return } diff --git a/modules/public/public_bindata.go b/modules/public/public_bindata.go index 4878f88ad1..2dcf3e72e4 100644 --- a/modules/public/public_bindata.go +++ b/modules/public/public_bindata.go @@ -5,4 +5,19 @@ package public -//go:generate go run ../../build/generate-bindata.go ../../public public bindata.go true +//go:generate go run ../../build/generate-bindata.go ../../public bindata.dat + +import ( + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", assetfs.NewEmbeddedFS(bindata)) +}) diff --git a/modules/public/serve_dynamic.go b/modules/public/public_dynamic.go index a668b17c34..a668b17c34 100644 --- a/modules/public/serve_dynamic.go +++ b/modules/public/public_dynamic.go diff --git a/modules/public/serve_static.go b/modules/public/serve_static.go deleted file mode 100644 index e79085021e..0000000000 --- a/modules/public/serve_static.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package public - -import ( - "time" - - "code.gitea.io/gitea/modules/assetfs" - "code.gitea.io/gitea/modules/timeutil" -) - -var _ GzipBytesProvider = (*vfsgen۰CompressedFileInfo)(nil) - -// GlobalModTime provide a global mod time for embedded asset files -func GlobalModTime(filename string) time.Time { - return timeutil.GetExecutableModTime() -} - -func BuiltinAssets() *assetfs.Layer { - return assetfs.Bindata("builtin(bindata)", Assets) -} diff --git a/modules/queue/base_levelqueue_common.go b/modules/queue/base_levelqueue_common.go index 78d3b85a8a..d37093b84d 100644 --- a/modules/queue/base_levelqueue_common.go +++ b/modules/queue/base_levelqueue_common.go @@ -83,7 +83,7 @@ func prepareLevelDB(cfg *BaseConfig) (conn string, db *leveldb.DB, err error) { } conn = cfg.ConnStr } - for i := 0; i < 10; i++ { + for range 10 { if db, err = nosql.GetManager().GetLevelDB(conn); err == nil { break } diff --git a/modules/queue/base_redis.go b/modules/queue/base_redis.go index a1e234943d..bea0fd7a98 100644 --- a/modules/queue/base_redis.go +++ b/modules/queue/base_redis.go @@ -29,7 +29,7 @@ func newBaseRedisGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { client := nosql.GetManager().GetRedisClient(cfg.ConnStr) var err error - for i := 0; i < 10; i++ { + for range 10 { err = client.Ping(graceful.GetManager().ShutdownContext()).Err() if err == nil { break diff --git a/modules/queue/base_test.go b/modules/queue/base_test.go index 73abf49091..8e7c18d740 100644 --- a/modules/queue/base_test.go +++ b/modules/queue/base_test.go @@ -21,7 +21,7 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) _ = q.RemoveAll(ctx) cnt, err := q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 0, cnt) + assert.Equal(t, 0, cnt) // push the first item err = q.PushItem(ctx, []byte("foo")) @@ -29,7 +29,7 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) cnt, err = q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 1, cnt) + assert.Equal(t, 1, cnt) // push a duplicate item err = q.PushItem(ctx, []byte("foo")) @@ -45,10 +45,10 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) has, err := q.HasItem(ctx, []byte("foo")) assert.NoError(t, err) if !isUnique { - assert.EqualValues(t, 2, cnt) + assert.Equal(t, 2, cnt) assert.False(t, has) // non-unique queues don't check for duplicates } else { - assert.EqualValues(t, 1, cnt) + assert.Equal(t, 1, cnt) assert.True(t, has) } @@ -59,18 +59,18 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) // pop the first item (and the duplicate if non-unique) it, err := q.PopItem(ctx) assert.NoError(t, err) - assert.EqualValues(t, "foo", string(it)) + assert.Equal(t, "foo", string(it)) if !isUnique { it, err = q.PopItem(ctx) assert.NoError(t, err) - assert.EqualValues(t, "foo", string(it)) + assert.Equal(t, "foo", string(it)) } // pop another item it, err = q.PopItem(ctx) assert.NoError(t, err) - assert.EqualValues(t, "bar", string(it)) + assert.Equal(t, "bar", string(it)) // pop an empty queue (timeout, cancel) ctxTimed, cancel := context.WithTimeout(ctx, 10*time.Millisecond) @@ -87,7 +87,7 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) // test blocking push if queue is full for i := 0; i < cfg.Length; i++ { - err = q.PushItem(ctx, []byte(fmt.Sprintf("item-%d", i))) + err = q.PushItem(ctx, fmt.Appendf(nil, "item-%d", i)) assert.NoError(t, err) } ctxTimed, cancel = context.WithTimeout(ctx, 10*time.Millisecond) @@ -107,13 +107,13 @@ func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error) // remove all cnt, err = q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, cfg.Length, cnt) + assert.Equal(t, cfg.Length, cnt) _ = q.RemoveAll(ctx) cnt, err = q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 0, cnt) + assert.Equal(t, 0, cnt) }) } @@ -126,7 +126,7 @@ func TestBaseDummy(t *testing.T) { cnt, err := q.Len(ctx) assert.NoError(t, err) - assert.EqualValues(t, 0, cnt) + assert.Equal(t, 0, cnt) has, err := q.HasItem(ctx, []byte("foo")) assert.NoError(t, err) diff --git a/modules/queue/manager.go b/modules/queue/manager.go index 079e2bee7a..ae6c51872d 100644 --- a/modules/queue/manager.go +++ b/modules/queue/manager.go @@ -6,6 +6,7 @@ package queue import ( "context" "errors" + "maps" "sync" "time" @@ -70,9 +71,7 @@ func (m *Manager) ManagedQueues() map[int64]ManagedWorkerPoolQueue { defer m.mu.Unlock() queues := make(map[int64]ManagedWorkerPoolQueue, len(m.Queues)) - for k, v := range m.Queues { - queues[k] = v - } + maps.Copy(queues, m.Queues) return queues } diff --git a/modules/queue/manager_test.go b/modules/queue/manager_test.go index f55ee85a22..fda498cc84 100644 --- a/modules/queue/manager_test.go +++ b/modules/queue/manager_test.go @@ -47,7 +47,7 @@ CONN_STR = redis:// assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/common"), q.baseConfig.DataFullDir) assert.Equal(t, 100000, q.baseConfig.Length) assert.Equal(t, 20, q.batchLength) - assert.Equal(t, "", q.baseConfig.ConnStr) + assert.Empty(t, q.baseConfig.ConnStr) assert.Equal(t, "default_queue", q.baseConfig.QueueFullName) assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName) assert.NotZero(t, q.GetWorkerMaxNumber()) @@ -101,7 +101,7 @@ MAX_WORKERS = 123 assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir2"), q2.baseConfig.DataFullDir) assert.Equal(t, 102, q2.baseConfig.Length) assert.Equal(t, 22, q2.batchLength) - assert.Equal(t, "", q2.baseConfig.ConnStr) + assert.Empty(t, q2.baseConfig.ConnStr) assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName) assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName) assert.Equal(t, 123, q2.GetWorkerMaxNumber()) diff --git a/modules/queue/workerqueue_test.go b/modules/queue/workerqueue_test.go index 0ca2be1d21..a6c369d5f9 100644 --- a/modules/queue/workerqueue_test.go +++ b/modules/queue/workerqueue_test.go @@ -63,9 +63,9 @@ func TestWorkerPoolQueueUnhandled(t *testing.T) { ok := true for i := 0; i < queueSetting.Length; i++ { if i%2 == 0 { - ok = ok && assert.EqualValues(t, 2, m[i], "test %s: item %d", t.Name(), i) + ok = ok && assert.Equal(t, 2, m[i], "test %s: item %d", t.Name(), i) } else { - ok = ok && assert.EqualValues(t, 1, m[i], "test %s: item %d", t.Name(), i) + ok = ok && assert.Equal(t, 1, m[i], "test %s: item %d", t.Name(), i) } } if !ok { @@ -77,17 +77,17 @@ func TestWorkerPoolQueueUnhandled(t *testing.T) { runCount := 2 // we can run these tests even hundreds times to see its stability t.Run("1/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { test(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1}) } }) t.Run("3/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { test(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1}) } }) t.Run("4/5", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { test(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5}) } }) @@ -96,17 +96,17 @@ func TestWorkerPoolQueueUnhandled(t *testing.T) { func TestWorkerPoolQueuePersistence(t *testing.T) { runCount := 2 // we can run these tests even hundreds times to see its stability t.Run("1/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1, Length: 100}) } }) t.Run("3/1", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1, Length: 100}) } }) t.Run("4/5", func(t *testing.T) { - for i := 0; i < runCount; i++ { + for range runCount { testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5, Length: 100}) } }) @@ -141,7 +141,7 @@ func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSett q, _ := newWorkerPoolQueueForTest("pr_patch_checker_test", queueSetting, testHandler, true) stop := runWorkerPoolQueue(q) - for i := 0; i < testCount; i++ { + for i := range testCount { _ = q.Push("task-" + strconv.Itoa(i)) } close(startWhenAllReady) @@ -173,7 +173,7 @@ func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSett assert.NotEmpty(t, tasksQ1) assert.NotEmpty(t, tasksQ2) - assert.EqualValues(t, testCount, len(tasksQ1)+len(tasksQ2)) + assert.Equal(t, testCount, len(tasksQ1)+len(tasksQ2)) } func TestWorkerPoolQueueActiveWorkers(t *testing.T) { @@ -186,34 +186,34 @@ func TestWorkerPoolQueueActiveWorkers(t *testing.T) { q, _ := newWorkerPoolQueueForTest("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 1, Length: 100}, handler, false) stop := runWorkerPoolQueue(q) - for i := 0; i < 5; i++ { + for i := range 5 { assert.NoError(t, q.Push(i)) } time.Sleep(50 * time.Millisecond) - assert.EqualValues(t, 1, q.GetWorkerNumber()) - assert.EqualValues(t, 1, q.GetWorkerActiveNumber()) + assert.Equal(t, 1, q.GetWorkerNumber()) + assert.Equal(t, 1, q.GetWorkerActiveNumber()) time.Sleep(500 * time.Millisecond) - assert.EqualValues(t, 1, q.GetWorkerNumber()) - assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 1, q.GetWorkerNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) time.Sleep(workerIdleDuration) - assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + assert.Equal(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working stop() q, _ = newWorkerPoolQueueForTest("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 3, Length: 100}, handler, false) stop = runWorkerPoolQueue(q) - for i := 0; i < 15; i++ { + for i := range 15 { assert.NoError(t, q.Push(i)) } time.Sleep(50 * time.Millisecond) - assert.EqualValues(t, 3, q.GetWorkerNumber()) - assert.EqualValues(t, 3, q.GetWorkerActiveNumber()) + assert.Equal(t, 3, q.GetWorkerNumber()) + assert.Equal(t, 3, q.GetWorkerActiveNumber()) time.Sleep(500 * time.Millisecond) - assert.EqualValues(t, 3, q.GetWorkerNumber()) - assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 3, q.GetWorkerNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) time.Sleep(workerIdleDuration) - assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + assert.Equal(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working stop() } @@ -240,13 +240,13 @@ func TestWorkerPoolQueueShutdown(t *testing.T) { } <-handlerCalled time.Sleep(200 * time.Millisecond) // wait for a while to make sure all workers are active - assert.EqualValues(t, 4, q.GetWorkerActiveNumber()) + assert.Equal(t, 4, q.GetWorkerActiveNumber()) stop() // stop triggers shutdown - assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) // no item was ever handled, so we still get all of them again q, _ = newWorkerPoolQueueForTest("test-workpoolqueue", qs, handler, false) - assert.EqualValues(t, 20, q.GetQueueItemNumber()) + assert.Equal(t, 20, q.GetQueueItemNumber()) } func TestWorkerPoolQueueWorkerIdleReset(t *testing.T) { @@ -274,7 +274,7 @@ func TestWorkerPoolQueueWorkerIdleReset(t *testing.T) { } q, _ = newWorkerPoolQueueForTest("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 2, Length: 100}, handler, false) stop := runWorkerPoolQueue(q) - for i := 0; i < 100; i++ { + for i := range 100 { assert.NoError(t, q.Push(i)) } time.Sleep(500 * time.Millisecond) diff --git a/modules/references/references.go b/modules/references/references.go index a5b102b7f2..592bd4cbe4 100644 --- a/modules/references/references.go +++ b/modules/references/references.go @@ -462,11 +462,12 @@ func findAllIssueReferencesBytes(content []byte, links []string) []*rawReference continue } var sep string - if parts[3] == "issues" { + switch parts[3] { + case "issues": sep = "#" - } else if parts[3] == "pulls" { + case "pulls": sep = "!" - } else { + default: continue } // Note: closing/reopening keywords not supported with URLs diff --git a/modules/references/references_test.go b/modules/references/references_test.go index 1b6a968d6a..a15ae99f79 100644 --- a/modules/references/references_test.go +++ b/modules/references/references_test.go @@ -46,7 +46,7 @@ owner/repo!123456789 contentBytes := []byte(test) convertFullHTMLReferencesToShortRefs(re, &contentBytes) result := string(contentBytes) - assert.EqualValues(t, expect, result) + assert.Equal(t, expect, result) } func TestFindAllIssueReferences(t *testing.T) { @@ -283,9 +283,9 @@ func testFixtures(t *testing.T, fixtures []testFixture, context string) { } expref := rawToIssueReferenceList(expraw) refs := FindAllIssueReferencesMarkdown(fixture.input) - assert.EqualValues(t, expref, refs, "[%s] Failed to parse: {%s}", context, fixture.input) + assert.Equal(t, expref, refs, "[%s] Failed to parse: {%s}", context, fixture.input) rawrefs := findAllIssueReferencesMarkdown(fixture.input) - assert.EqualValues(t, expraw, rawrefs, "[%s] Failed to parse: {%s}", context, fixture.input) + assert.Equal(t, expraw, rawrefs, "[%s] Failed to parse: {%s}", context, fixture.input) } // Restore for other tests that may rely on the original value @@ -294,7 +294,7 @@ func testFixtures(t *testing.T, fixtures []testFixture, context string) { func TestFindAllMentions(t *testing.T) { res := FindAllMentionsBytes([]byte("@tasha, @mike; @lucy: @john")) - assert.EqualValues(t, []RefSpan{ + assert.Equal(t, []RefSpan{ {Start: 0, End: 6}, {Start: 8, End: 13}, {Start: 15, End: 20}, @@ -554,7 +554,7 @@ func TestParseCloseKeywords(t *testing.T) { res := pat.FindAllStringSubmatch(test.match, -1) assert.Len(t, res, 1) assert.Len(t, res[0], 2) - assert.EqualValues(t, test.expected, res[0][1]) + assert.Equal(t, test.expected, res[0][1]) } } } diff --git a/modules/regexplru/regexplru_test.go b/modules/regexplru/regexplru_test.go index 9c24b23fa9..4b539c31e9 100644 --- a/modules/regexplru/regexplru_test.go +++ b/modules/regexplru/regexplru_test.go @@ -18,9 +18,9 @@ func TestRegexpLru(t *testing.T) { assert.NoError(t, err) assert.True(t, r.MatchString("a")) - assert.EqualValues(t, 1, lruCache.Len()) + assert.Equal(t, 1, lruCache.Len()) _, err = GetCompiled("(") assert.Error(t, err) - assert.EqualValues(t, 2, lruCache.Len()) + assert.Equal(t, 2, lruCache.Len()) } diff --git a/modules/repository/branch.go b/modules/repository/branch.go index 2bf9930f19..30aa0a6e85 100644 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -41,11 +41,12 @@ func SyncRepoBranchesWithRepo(ctx context.Context, repo *repo_model.Repository, if err != nil { return 0, fmt.Errorf("GetObjectFormat: %w", err) } - _, err = db.GetEngine(ctx).ID(repo.ID).Update(&repo_model.Repository{ObjectFormatName: objFmt.Name()}) - if err != nil { - return 0, fmt.Errorf("UpdateRepository: %w", err) + if objFmt.Name() != repo.ObjectFormatName { + repo.ObjectFormatName = objFmt.Name() + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "object_format_name"); err != nil { + return 0, fmt.Errorf("UpdateRepositoryColsWithAutoTime: %w", err) + } } - repo.ObjectFormatName = objFmt.Name() // keep consistent with db allBranches := container.Set[string]{} { diff --git a/modules/repository/branch_test.go b/modules/repository/branch_test.go index acf75a1ac0..ead28aa141 100644 --- a/modules/repository/branch_test.go +++ b/modules/repository/branch_test.go @@ -27,5 +27,5 @@ func TestSyncRepoBranches(t *testing.T) { assert.Equal(t, "sha1", repo.ObjectFormatName) branch, err := git_model.GetBranch(db.DefaultContext, 1, "master") assert.NoError(t, err) - assert.EqualValues(t, "master", branch.Name) + assert.Equal(t, "master", branch.Name) } diff --git a/modules/repository/commits.go b/modules/repository/commits.go index 16520fb28a..878fdc1603 100644 --- a/modules/repository/commits.go +++ b/modules/repository/commits.go @@ -13,6 +13,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -131,7 +132,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repo *repo_model func (pc *PushCommits) AvatarLink(ctx context.Context, email string) string { size := avatars.DefaultAvatarPixelSize * setting.Avatar.RenderedSizeFactor - v, _ := cache.GetWithContextCache(ctx, "push_commits", email, func() (string, error) { + v, _ := cache.GetWithContextCache(ctx, cachegroup.EmailAvatarLink, email, func(ctx context.Context, email string) (string, error) { u, err := user_model.GetUserByEmail(ctx, email) if err != nil { if !user_model.IsErrUserNotExist(err) { diff --git a/modules/repository/commits_test.go b/modules/repository/commits_test.go index e8dbf0b4e9..030cd7714d 100644 --- a/modules/repository/commits_test.go +++ b/modules/repository/commits_test.go @@ -62,9 +62,9 @@ func TestPushCommits_ToAPIPayloadCommits(t *testing.T) { assert.Equal(t, "user2", payloadCommits[0].Committer.UserName) assert.Equal(t, "User2", payloadCommits[0].Author.Name) assert.Equal(t, "user2", payloadCommits[0].Author.UserName) - assert.EqualValues(t, []string{}, payloadCommits[0].Added) - assert.EqualValues(t, []string{}, payloadCommits[0].Removed) - assert.EqualValues(t, []string{"readme.md"}, payloadCommits[0].Modified) + assert.Equal(t, []string{}, payloadCommits[0].Added) + assert.Equal(t, []string{}, payloadCommits[0].Removed) + assert.Equal(t, []string{"readme.md"}, payloadCommits[0].Modified) assert.Equal(t, "27566bd", payloadCommits[1].ID) assert.Equal(t, "good signed commit (with not yet validated email)", payloadCommits[1].Message) @@ -73,9 +73,9 @@ func TestPushCommits_ToAPIPayloadCommits(t *testing.T) { assert.Equal(t, "user2", payloadCommits[1].Committer.UserName) assert.Equal(t, "User2", payloadCommits[1].Author.Name) assert.Equal(t, "user2", payloadCommits[1].Author.UserName) - assert.EqualValues(t, []string{}, payloadCommits[1].Added) - assert.EqualValues(t, []string{}, payloadCommits[1].Removed) - assert.EqualValues(t, []string{"readme.md"}, payloadCommits[1].Modified) + assert.Equal(t, []string{}, payloadCommits[1].Added) + assert.Equal(t, []string{}, payloadCommits[1].Removed) + assert.Equal(t, []string{"readme.md"}, payloadCommits[1].Modified) assert.Equal(t, "5099b81", payloadCommits[2].ID) assert.Equal(t, "good signed commit", payloadCommits[2].Message) @@ -84,9 +84,9 @@ func TestPushCommits_ToAPIPayloadCommits(t *testing.T) { assert.Equal(t, "user2", payloadCommits[2].Committer.UserName) assert.Equal(t, "User2", payloadCommits[2].Author.Name) assert.Equal(t, "user2", payloadCommits[2].Author.UserName) - assert.EqualValues(t, []string{"readme.md"}, payloadCommits[2].Added) - assert.EqualValues(t, []string{}, payloadCommits[2].Removed) - assert.EqualValues(t, []string{}, payloadCommits[2].Modified) + assert.Equal(t, []string{"readme.md"}, payloadCommits[2].Added) + assert.Equal(t, []string{}, payloadCommits[2].Removed) + assert.Equal(t, []string{}, payloadCommits[2].Modified) assert.Equal(t, "69554a6", headCommit.ID) assert.Equal(t, "not signed commit", headCommit.Message) @@ -95,9 +95,9 @@ func TestPushCommits_ToAPIPayloadCommits(t *testing.T) { assert.Equal(t, "user2", headCommit.Committer.UserName) assert.Equal(t, "User2", headCommit.Author.Name) assert.Equal(t, "user2", headCommit.Author.UserName) - assert.EqualValues(t, []string{}, headCommit.Added) - assert.EqualValues(t, []string{}, headCommit.Removed) - assert.EqualValues(t, []string{"readme.md"}, headCommit.Modified) + assert.Equal(t, []string{}, headCommit.Added) + assert.Equal(t, []string{}, headCommit.Removed) + assert.Equal(t, []string{"readme.md"}, headCommit.Modified) } func TestPushCommits_AvatarLink(t *testing.T) { @@ -200,5 +200,3 @@ func TestListToPushCommits(t *testing.T) { assert.Equal(t, now, pushCommits.Commits[1].Timestamp) } } - -// TODO TestPushUpdate diff --git a/modules/repository/create.go b/modules/repository/create.go index b4f7033bd7..a75598a84b 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -7,19 +7,10 @@ import ( "context" "fmt" "os" - "path" "path/filepath" - "strings" - activities_model "code.gitea.io/gitea/models/activities" - "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" - access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" - issue_indexer "code.gitea.io/gitea/modules/indexer/issues" - "code.gitea.io/gitea/modules/log" - api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/util" ) const notRegularFileMode = os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular @@ -64,97 +55,3 @@ func UpdateRepoSize(ctx context.Context, repo *repo_model.Repository) error { return repo_model.UpdateRepoSize(ctx, repo.ID, size, lfsSize) } - -// CheckDaemonExportOK creates/removes git-daemon-export-ok for git-daemon... -func CheckDaemonExportOK(ctx context.Context, repo *repo_model.Repository) error { - if err := repo.LoadOwner(ctx); err != nil { - return err - } - - // Create/Remove git-daemon-export-ok for git-daemon... - daemonExportFile := path.Join(repo.RepoPath(), `git-daemon-export-ok`) - - isExist, err := util.IsExist(daemonExportFile) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err) - return err - } - - isPublic := !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePublic - if !isPublic && isExist { - if err = util.Remove(daemonExportFile); err != nil { - log.Error("Failed to remove %s: %v", daemonExportFile, err) - } - } else if isPublic && !isExist { - if f, err := os.Create(daemonExportFile); err != nil { - log.Error("Failed to create %s: %v", daemonExportFile, err) - } else { - f.Close() - } - } - - return nil -} - -// UpdateRepository updates a repository with db context -func UpdateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { - repo.LowerName = strings.ToLower(repo.Name) - - e := db.GetEngine(ctx) - - if _, err = e.ID(repo.ID).AllCols().Update(repo); err != nil { - return fmt.Errorf("update: %w", err) - } - - if err = UpdateRepoSize(ctx, repo); err != nil { - log.Error("Failed to update size for repository: %v", err) - } - - if visibilityChanged { - if err = repo.LoadOwner(ctx); err != nil { - return fmt.Errorf("LoadOwner: %w", err) - } - if repo.Owner.IsOrganization() { - // Organization repository need to recalculate access table when visibility is changed. - if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { - return fmt.Errorf("recalculateTeamAccesses: %w", err) - } - } - - // If repo has become private, we need to set its actions to private. - if repo.IsPrivate { - _, err = e.Where("repo_id = ?", repo.ID).Cols("is_private").Update(&activities_model.Action{ - IsPrivate: true, - }) - if err != nil { - return err - } - - if err = repo_model.ClearRepoStars(ctx, repo.ID); err != nil { - return err - } - } - - // Create/Remove git-daemon-export-ok for git-daemon... - if err := CheckDaemonExportOK(ctx, repo); err != nil { - return err - } - - forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) - if err != nil { - return fmt.Errorf("getRepositoriesByForkID: %w", err) - } - for i := range forkRepos { - forkRepos[i].IsPrivate = repo.IsPrivate || repo.Owner.Visibility == api.VisibleTypePrivate - if err = UpdateRepository(ctx, forkRepos[i], true); err != nil { - return fmt.Errorf("updateRepository[%d]: %w", forkRepos[i].ID, err) - } - } - - // If visibility is changed, we need to update the issue indexer. - // Since the data in the issue indexer have field to indicate if the repo is public or not. - issue_indexer.UpdateRepoIndexer(ctx, repo.ID) - } - - return nil -} diff --git a/modules/repository/create_test.go b/modules/repository/create_test.go index a9151482b4..b85a10adad 100644 --- a/modules/repository/create_test.go +++ b/modules/repository/create_test.go @@ -6,7 +6,6 @@ package repository import ( "testing" - activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" @@ -14,26 +13,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestUpdateRepositoryVisibilityChanged(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - - // Get sample repo and change visibility - repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 9) - assert.NoError(t, err) - repo.IsPrivate = true - - // Update it - err = UpdateRepository(db.DefaultContext, repo, true) - assert.NoError(t, err) - - // Check visibility of action has become private - act := activities_model.Action{} - _, err = db.GetEngine(db.DefaultContext).ID(3).Get(&act) - - assert.NoError(t, err) - assert.True(t, act.IsPrivate) -} - func TestGetDirectorySize(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 1) @@ -41,5 +20,5 @@ func TestGetDirectorySize(t *testing.T) { size, err := getDirectorySize(repo.RepoPath()) assert.NoError(t, err) repo.Size = 8165 // real size on the disk - assert.EqualValues(t, repo.Size, size) + assert.Equal(t, repo.Size, size) } diff --git a/modules/repository/env.go b/modules/repository/env.go index e4f32092fc..78e06f86fb 100644 --- a/modules/repository/env.go +++ b/modules/repository/env.go @@ -4,8 +4,8 @@ package repository import ( - "fmt" "os" + "strconv" "strings" repo_model "code.gitea.io/gitea/models/repo" @@ -72,9 +72,9 @@ func FullPushingEnvironment(author, committer *user_model.User, repo *repo_model EnvRepoUsername+"="+repo.OwnerName, EnvRepoIsWiki+"="+isWiki, EnvPusherName+"="+committer.Name, - EnvPusherID+"="+fmt.Sprintf("%d", committer.ID), - EnvRepoID+"="+fmt.Sprintf("%d", repo.ID), - EnvPRID+"="+fmt.Sprintf("%d", prID), + EnvPusherID+"="+strconv.FormatInt(committer.ID, 10), + EnvRepoID+"="+strconv.FormatInt(repo.ID, 10), + EnvPRID+"="+strconv.FormatInt(prID, 10), EnvAppURL+"="+setting.AppURL, "SSH_ORIGINAL_COMMAND=gitea-internal", ) diff --git a/modules/repository/init.go b/modules/repository/init.go index 3fc1261baa..12e9606c74 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -11,11 +11,7 @@ import ( "strings" issues_model "code.gitea.io/gitea/models/issues" - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/label" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -121,29 +117,6 @@ func LoadRepoConfig() error { return nil } -func CheckInitRepository(ctx context.Context, repo *repo_model.Repository) (err error) { - // Somehow the directory could exist. - isExist, err := gitrepo.IsRepositoryExist(ctx, repo) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) - return err - } - if isExist { - return repo_model.ErrRepoFilesAlreadyExist{ - Uname: repo.OwnerName, - Name: repo.Name, - } - } - - // Init git bare new repository. - if err = git.InitRepository(ctx, repo.RepoPath(), true, repo.ObjectFormatName); err != nil { - return fmt.Errorf("git.InitRepository: %w", err) - } else if err = gitrepo.CreateDelegateHooksForRepo(ctx, repo); err != nil { - return fmt.Errorf("createDelegateHooks: %w", err) - } - return nil -} - // InitializeLabels adds a label set to a repository using a template func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg bool) error { list, err := LoadTemplateLabelsByDisplayName(labelTemplate) @@ -152,12 +125,13 @@ func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg } labels := make([]*issues_model.Label, len(list)) - for i := 0; i < len(list); i++ { + for i := range list { labels[i] = &issues_model.Label{ - Name: list[i].Name, - Exclusive: list[i].Exclusive, - Description: list[i].Description, - Color: list[i].Color, + Name: list[i].Name, + Exclusive: list[i].Exclusive, + ExclusiveOrder: list[i].ExclusiveOrder, + Description: list[i].Description, + Color: list[i].Color, } if isOrg { labels[i].OrgID = id diff --git a/modules/repository/init_test.go b/modules/repository/init_test.go index 227efdc1db..1fa928105c 100644 --- a/modules/repository/init_test.go +++ b/modules/repository/init_test.go @@ -14,17 +14,17 @@ func TestMergeCustomLabels(t *testing.T) { all: []string{"a", "a.yaml", "a.yml"}, custom: nil, }) - assert.EqualValues(t, []string{"a.yaml"}, files, "yaml file should win") + assert.Equal(t, []string{"a.yaml"}, files, "yaml file should win") files = mergeCustomLabelFiles(optionFileList{ all: []string{"a", "a.yaml"}, custom: []string{"a"}, }) - assert.EqualValues(t, []string{"a"}, files, "custom file should win") + assert.Equal(t, []string{"a"}, files, "custom file should win") files = mergeCustomLabelFiles(optionFileList{ all: []string{"a", "a.yml", "a.yaml"}, custom: []string{"a", "a.yml"}, }) - assert.EqualValues(t, []string{"a.yml"}, files, "custom yml file should win if no yaml") + assert.Equal(t, []string{"a.yml"}, files, "custom yml file should win if no yaml") } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 97b0343381..ad4a53b858 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -9,13 +9,10 @@ import ( "fmt" "io" "strings" - "time" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" @@ -59,118 +56,6 @@ func SyncRepoTags(ctx context.Context, repoID int64) error { return SyncReleasesWithTags(ctx, repo, gitRepo) } -// SyncReleasesWithTags synchronizes release table with repository tags -func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) - - // optimized procedure for pull-mirrors which saves a lot of time (in - // particular for repos with many tags). - if repo.IsMirror { - return pullMirrorReleaseSync(ctx, repo, gitRepo) - } - - existingRelTags := make(container.Set[string]) - opts := repo_model.FindReleasesOptions{ - IncludeDrafts: true, - IncludeTags: true, - ListOptions: db.ListOptions{PageSize: 50}, - RepoID: repo.ID, - } - for page := 1; ; page++ { - opts.Page = page - rels, err := db.Find[repo_model.Release](gitRepo.Ctx, opts) - if err != nil { - return fmt.Errorf("unable to GetReleasesByRepoID in Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) - } - if len(rels) == 0 { - break - } - for _, rel := range rels { - if rel.IsDraft { - continue - } - commitID, err := gitRepo.GetTagCommitID(rel.TagName) - if err != nil && !git.IsErrNotExist(err) { - return fmt.Errorf("unable to GetTagCommitID for %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) - } - if git.IsErrNotExist(err) || commitID != rel.Sha1 { - if err := repo_model.PushUpdateDeleteTag(ctx, repo, rel.TagName); err != nil { - return fmt.Errorf("unable to PushUpdateDeleteTag: %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) - } - } else { - existingRelTags.Add(strings.ToLower(rel.TagName)) - } - } - } - - _, err := gitRepo.WalkReferences(git.ObjectTag, 0, 0, func(sha1, refname string) error { - tagName := strings.TrimPrefix(refname, git.TagPrefix) - if existingRelTags.Contains(strings.ToLower(tagName)) { - return nil - } - - if err := PushUpdateAddTag(ctx, repo, gitRepo, tagName, sha1, refname); err != nil { - // sometimes, some tags will be sync failed. i.e. https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tag/?h=v2.6.11 - // this is a tree object, not a tag object which created before git - log.Error("unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %v", tagName, repo.ID, repo.OwnerName, repo.Name, err) - } - - return nil - }) - return err -} - -// PushUpdateAddTag must be called for any push actions to add tag -func PushUpdateAddTag(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tagName, sha1, refname string) error { - tag, err := gitRepo.GetTagWithID(sha1, tagName) - if err != nil { - return fmt.Errorf("unable to GetTag: %w", err) - } - commit, err := tag.Commit(gitRepo) - if err != nil { - return fmt.Errorf("unable to get tag Commit: %w", err) - } - - sig := tag.Tagger - if sig == nil { - sig = commit.Author - } - if sig == nil { - sig = commit.Committer - } - - var author *user_model.User - createdAt := time.Unix(1, 0) - - if sig != nil { - author, err = user_model.GetUserByEmail(ctx, sig.Email) - if err != nil && !user_model.IsErrUserNotExist(err) { - return fmt.Errorf("unable to GetUserByEmail for %q: %w", sig.Email, err) - } - createdAt = sig.When - } - - commitsCount, err := commit.CommitsCount() - if err != nil { - return fmt.Errorf("unable to get CommitsCount: %w", err) - } - - rel := repo_model.Release{ - RepoID: repo.ID, - TagName: tagName, - LowerTagName: strings.ToLower(tagName), - Sha1: commit.ID.String(), - NumCommits: commitsCount, - CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), - IsTag: true, - } - if author != nil { - rel.PublisherID = author.ID - } - - return repo_model.SaveOrUpdateTag(ctx, repo, &rel) -} - // StoreMissingLfsObjectsInRepository downloads missing LFS objects func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error { contentStore := lfs.NewContentStore() @@ -286,18 +171,19 @@ func (shortRelease) TableName() string { return "release" } -// pullMirrorReleaseSync is a pull-mirror specific tag<->release table +// SyncReleasesWithTags is a tag<->release table // synchronization which overwrites all Releases from the repository tags. This // can be relied on since a pull-mirror is always identical to its -// upstream. Hence, after each sync we want the pull-mirror release set to be +// upstream. Hence, after each sync we want the release set to be // identical to the upstream tag set. This is much more efficient for // repositories like https://github.com/vim/vim (with over 13000 tags). -func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) - tags, numTags, err := gitRepo.GetTagInfos(0, 0) +func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) + tags, _, err := gitRepo.GetTagInfos(0, 0) if err != nil { return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) } + var added, deleted, updated int err = db.WithTx(ctx, func(ctx context.Context) error { dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{ RepoID: repo.ID, @@ -318,9 +204,7 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git TagName: tag.Name, LowerTagName: strings.ToLower(tag.Name), Sha1: tag.Object.String(), - // NOTE: ignored, since NumCommits are unused - // for pull-mirrors (only relevant when - // displaying releases, IsTag: false) + // NOTE: ignored, The NumCommits value is calculated and cached on demand when the UI requires it. NumCommits: -1, CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()), IsTag: true, @@ -349,13 +233,14 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err) } } + added, deleted, updated = len(deletes), len(updates), len(inserts) return nil }) if err != nil { return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) } - log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags) + log.Trace("SyncReleasesWithTags: %d tags added, %d tags deleted, %d tags updated", added, deleted, updated) return nil } diff --git a/modules/repository/repo_test.go b/modules/repository/repo_test.go index f3e7be6d7d..f79a79ccbd 100644 --- a/modules/repository/repo_test.go +++ b/modules/repository/repo_test.go @@ -63,7 +63,7 @@ func Test_calcSync(t *testing.T) { inserts, deletes, updates := calcSync(gitTags, dbReleases) if assert.Len(t, inserts, 1, "inserts") { - assert.EqualValues(t, *gitTags[2], *inserts[0], "inserts equal") + assert.Equal(t, *gitTags[2], *inserts[0], "inserts equal") } if assert.Len(t, deletes, 1, "deletes") { @@ -71,6 +71,6 @@ func Test_calcSync(t *testing.T) { } if assert.Len(t, updates, 1, "updates") { - assert.EqualValues(t, *gitTags[1], *updates[0], "updates equal") + assert.Equal(t, *gitTags[1], *updates[0], "updates equal") } } diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 04faa9db3d..d7253d9e02 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -4,42 +4,19 @@ package repository import ( + "context" "fmt" - "os" - "path" - "path/filepath" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) -// LocalCopyPath returns the local repository temporary copy path. -func LocalCopyPath() string { - if filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { - return setting.Repository.Local.LocalCopyPath - } - return path.Join(setting.AppDataPath, setting.Repository.Local.LocalCopyPath) -} - // CreateTemporaryPath creates a temporary path -func CreateTemporaryPath(prefix string) (string, error) { - if err := os.MkdirAll(LocalCopyPath(), os.ModePerm); err != nil { - log.Error("Unable to create localcopypath directory: %s (%v)", LocalCopyPath(), err) - return "", fmt.Errorf("Failed to create localcopypath directory %s: %w", LocalCopyPath(), err) - } - basePath, err := os.MkdirTemp(LocalCopyPath(), prefix+".git") +func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { + basePath, cleanup, err := setting.AppDataTempDir("local-repo").MkdirTempRandom(prefix + ".git") if err != nil { log.Error("Unable to create temporary directory: %s-*.git (%v)", prefix, err) - return "", fmt.Errorf("Failed to create dir %s-*.git: %w", prefix, err) - } - return basePath, nil -} - -// RemoveTemporaryPath removes the temporary path -func RemoveTemporaryPath(basePath string) error { - if _, err := os.Stat(basePath); !os.IsNotExist(err) { - return util.RemoveAll(basePath) + return "", nil, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) } - return nil + return basePath, cleanup, nil } diff --git a/modules/reqctx/datastore.go b/modules/reqctx/datastore.go index d025dad7f3..1d4bee613f 100644 --- a/modules/reqctx/datastore.go +++ b/modules/reqctx/datastore.go @@ -6,6 +6,7 @@ package reqctx import ( "context" "io" + "maps" "sync" "code.gitea.io/gitea/modules/process" @@ -22,9 +23,7 @@ func (ds ContextData) GetData() ContextData { } func (ds ContextData) MergeFrom(other ContextData) ContextData { - for k, v := range other { - ds[k] = v - } + maps.Copy(ds, other) return ds } diff --git a/modules/session/key.go b/modules/session/key.go new file mode 100644 index 0000000000..c3da997c67 --- /dev/null +++ b/modules/session/key.go @@ -0,0 +1,11 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package session + +const ( + KeyUID = "uid" + KeyUname = "uname" + + KeyUserHasTwoFactorAuth = "userHasTwoFactorAuth" +) diff --git a/modules/setting/actions.go b/modules/setting/actions.go index 913872eaf2..8bace1f750 100644 --- a/modules/setting/actions.go +++ b/modules/setting/actions.go @@ -62,11 +62,11 @@ func (c logCompression) IsValid() bool { } func (c logCompression) IsNone() bool { - return strings.ToLower(string(c)) == "none" + return string(c) == "none" } func (c logCompression) IsZstd() bool { - return c == "" || strings.ToLower(string(c)) == "zstd" + return c == "" || string(c) == "zstd" } func loadActionsFrom(rootCfg ConfigProvider) error { diff --git a/modules/setting/actions_test.go b/modules/setting/actions_test.go index 3645a3f5da..353cc657fa 100644 --- a/modules/setting/actions_test.go +++ b/modules/setting/actions_test.go @@ -21,9 +21,9 @@ func Test_getStorageInheritNameSectionTypeForActions(t *testing.T) { assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) iniStr = ` [storage.actions_log] @@ -34,9 +34,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + assert.Equal(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) iniStr = ` [storage.actions_log] @@ -50,9 +50,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + assert.Equal(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) iniStr = ` [storage.actions_artifacts] @@ -66,9 +66,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "local", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.Equal(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) iniStr = ` [storage.actions_artifacts] @@ -82,9 +82,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "local", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.Equal(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) iniStr = `` cfg, err = NewConfigProviderFromData(iniStr) @@ -92,9 +92,9 @@ STORAGE_TYPE = minio assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "local", Actions.LogStorage.Type) - assert.EqualValues(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) + assert.Equal(t, "actions_log", filepath.Base(Actions.LogStorage.Path)) assert.EqualValues(t, "local", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) + assert.Equal(t, "actions_artifacts", filepath.Base(Actions.ArtifactStorage.Path)) } func Test_getDefaultActionsURLForActions(t *testing.T) { @@ -175,7 +175,7 @@ DEFAULT_ACTIONS_URL = gitea if !tt.wantErr(t, loadActionsFrom(cfg)) { return } - assert.EqualValues(t, tt.wantURL, Actions.DefaultActionsURL.URL()) + assert.Equal(t, tt.wantURL, Actions.DefaultActionsURL.URL()) }) } } diff --git a/modules/setting/api.go b/modules/setting/api.go index c36f05cfd1..cdad474cb9 100644 --- a/modules/setting/api.go +++ b/modules/setting/api.go @@ -18,6 +18,7 @@ var API = struct { DefaultPagingNum int DefaultGitTreesPerPage int DefaultMaxBlobSize int64 + DefaultMaxResponseSize int64 }{ EnableSwagger: true, SwaggerURL: "", @@ -25,6 +26,7 @@ var API = struct { DefaultPagingNum: 30, DefaultGitTreesPerPage: 1000, DefaultMaxBlobSize: 10485760, + DefaultMaxResponseSize: 104857600, } func loadAPIFrom(rootCfg ConfigProvider) { diff --git a/modules/setting/attachment_test.go b/modules/setting/attachment_test.go index 3e8d2da4d9..c566dfa60c 100644 --- a/modules/setting/attachment_test.go +++ b/modules/setting/attachment_test.go @@ -25,9 +25,9 @@ MINIO_ENDPOINT = my_minio:9000 assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint) - assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_minio:9000", Attachment.Storage.MinioConfig.Endpoint) + assert.Equal(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } func Test_getStorageTypeSectionOverridesStorageSection(t *testing.T) { @@ -47,8 +47,8 @@ MINIO_BUCKET = gitea assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-minio", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } func Test_getStorageSpecificOverridesStorage(t *testing.T) { @@ -69,8 +69,8 @@ STORAGE_TYPE = local assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } func Test_getStorageGetDefaults(t *testing.T) { @@ -80,7 +80,7 @@ func Test_getStorageGetDefaults(t *testing.T) { assert.NoError(t, loadAttachmentFrom(cfg)) // default storage is local, so bucket is empty - assert.EqualValues(t, "", Attachment.Storage.MinioConfig.Bucket) + assert.Empty(t, Attachment.Storage.MinioConfig.Bucket) } func Test_getStorageInheritNameSectionType(t *testing.T) { @@ -115,7 +115,7 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Attachment.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) } func Test_AttachmentStorage1(t *testing.T) { @@ -128,6 +128,6 @@ STORAGE_TYPE = minio assert.NoError(t, loadAttachmentFrom(cfg)) assert.EqualValues(t, "minio", Attachment.Storage.Type) - assert.EqualValues(t, "gitea", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) } diff --git a/modules/setting/config_env.go b/modules/setting/config_env.go index 5d94a9641f..409588dc44 100644 --- a/modules/setting/config_env.go +++ b/modules/setting/config_env.go @@ -97,7 +97,7 @@ func decodeEnvSectionKey(encoded string) (ok bool, section, key string) { // decodeEnvironmentKey decode the environment key to section and key // The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE -func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) { //nolint:unparam +func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) { if !strings.HasPrefix(envKey, prefixGitea) { return false, "", "", false } diff --git a/modules/setting/config_env_test.go b/modules/setting/config_env_test.go index 7d07c479a1..7d270ac21a 100644 --- a/modules/setting/config_env_test.go +++ b/modules/setting/config_env_test.go @@ -28,8 +28,8 @@ func TestDecodeEnvSectionKey(t *testing.T) { ok, section, key = decodeEnvSectionKey("SEC") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) } func TestDecodeEnvironmentKey(t *testing.T) { @@ -38,19 +38,19 @@ func TestDecodeEnvironmentKey(t *testing.T) { ok, section, key, file := decodeEnvironmentKey(prefix, suffix, "SEC__KEY") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) assert.False(t, file) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) assert.False(t, file) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA____KEY") assert.True(t, ok) - assert.Equal(t, "", section) + assert.Empty(t, section) assert.Equal(t, "KEY", key) assert.False(t, file) @@ -64,8 +64,8 @@ func TestDecodeEnvironmentKey(t *testing.T) { // but it could be fixed in the future by adding a new suffix like "__VALUE" (no such key VALUE is used in Gitea either) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__FILE") assert.False(t, ok) - assert.Equal(t, "", section) - assert.Equal(t, "", key) + assert.Empty(t, section) + assert.Empty(t, key) assert.True(t, file) ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY__FILE") @@ -73,6 +73,9 @@ func TestDecodeEnvironmentKey(t *testing.T) { assert.Equal(t, "sec", section) assert.Equal(t, "KEY", key) assert.True(t, file) + + ok, _, _, _ = decodeEnvironmentKey("PREFIX__", "", "PREFIX__SEC__KEY") + assert.True(t, ok) } func TestEnvironmentToConfig(t *testing.T) { diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index 3138f8a63e..09eaaefdaf 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -15,7 +15,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" - "gopkg.in/ini.v1" //nolint:depguard + "gopkg.in/ini.v1" //nolint:depguard // wrapper for this package ) type ConfigKey interface { @@ -26,6 +26,7 @@ type ConfigKey interface { In(defaultVal string, candidates []string) string String() string Strings(delim string) []string + Bool() (bool, error) MustString(defaultVal string) string MustBool(defaultVal ...bool) bool @@ -257,7 +258,7 @@ func (p *iniConfigProvider) Save() error { } filename := p.file if filename == "" { - return fmt.Errorf("config file path must not be empty") + return errors.New("config file path must not be empty") } if p.loadedFromEmpty { if err := os.MkdirAll(filepath.Dir(filename), os.ModePerm); err != nil { diff --git a/modules/setting/config_provider_test.go b/modules/setting/config_provider_test.go index a666d124c7..63121f0074 100644 --- a/modules/setting/config_provider_test.go +++ b/modules/setting/config_provider_test.go @@ -62,17 +62,17 @@ key = 123 // test default behavior assert.Equal(t, "123", ConfigSectionKeyString(sec, "key")) - assert.Equal(t, "", ConfigSectionKeyString(secSub, "key")) + assert.Empty(t, ConfigSectionKeyString(secSub, "key")) assert.Equal(t, "def", ConfigSectionKeyString(secSub, "key", "def")) assert.Equal(t, "123", ConfigInheritedKeyString(secSub, "key")) // Workaround for ini package's BuggyKeyOverwritten behavior - assert.Equal(t, "", ConfigSectionKeyString(sec, "empty")) - assert.Equal(t, "", ConfigSectionKeyString(secSub, "empty")) + assert.Empty(t, ConfigSectionKeyString(sec, "empty")) + assert.Empty(t, ConfigSectionKeyString(secSub, "empty")) assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("def")) assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("xyz")) - assert.Equal(t, "", ConfigSectionKeyString(sec, "empty")) + assert.Empty(t, ConfigSectionKeyString(sec, "empty")) assert.Equal(t, "def", ConfigSectionKeyString(secSub, "empty")) } diff --git a/modules/setting/cron_test.go b/modules/setting/cron_test.go index 55244d7075..39a228068a 100644 --- a/modules/setting/cron_test.go +++ b/modules/setting/cron_test.go @@ -38,6 +38,6 @@ EXTEND = true _, err = getCronSettings(cfg, "test", extended) assert.NoError(t, err) assert.True(t, extended.Base) - assert.EqualValues(t, "white rabbit", extended.Second) + assert.Equal(t, "white rabbit", extended.Second) assert.True(t, extended.Extend) } diff --git a/modules/setting/git_test.go b/modules/setting/git_test.go index 441c514d8c..0d7f634abf 100644 --- a/modules/setting/git_test.go +++ b/modules/setting/git_test.go @@ -6,6 +6,8 @@ package setting import ( "testing" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" ) @@ -23,8 +25,8 @@ a.b = 1 `) assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "1", GitConfig.Options["a.b"]) - assert.EqualValues(t, "histogram", GitConfig.Options["diff.algorithm"]) + assert.Equal(t, "1", GitConfig.Options["a.b"]) + assert.Equal(t, "histogram", GitConfig.Options["diff.algorithm"]) cfg, err = NewConfigProviderFromData(` [git.config] @@ -32,24 +34,20 @@ diff.algorithm = other `) assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "other", GitConfig.Options["diff.algorithm"]) + assert.Equal(t, "other", GitConfig.Options["diff.algorithm"]) } func TestGitReflog(t *testing.T) { - oldGit := Git - oldGitConfig := GitConfig - defer func() { - Git = oldGit - GitConfig = oldGitConfig - }() + defer test.MockVariableValue(&Git) + defer test.MockVariableValue(&GitConfig) // default reflog config without legacy options cfg, err := NewConfigProviderFromData(``) assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "true", GitConfig.GetOption("core.logAllRefUpdates")) - assert.EqualValues(t, "90", GitConfig.GetOption("gc.reflogExpire")) + assert.Equal(t, "true", GitConfig.GetOption("core.logAllRefUpdates")) + assert.Equal(t, "90", GitConfig.GetOption("gc.reflogExpire")) // custom reflog config by legacy options cfg, err = NewConfigProviderFromData(` @@ -60,6 +58,6 @@ EXPIRATION = 123 assert.NoError(t, err) loadGitFrom(cfg) - assert.EqualValues(t, "false", GitConfig.GetOption("core.logAllRefUpdates")) - assert.EqualValues(t, "123", GitConfig.GetOption("gc.reflogExpire")) + assert.Equal(t, "false", GitConfig.GetOption("core.logAllRefUpdates")) + assert.Equal(t, "123", GitConfig.GetOption("gc.reflogExpire")) } diff --git a/modules/setting/global_lock_test.go b/modules/setting/global_lock_test.go index 5eeb275523..5e15eb3483 100644 --- a/modules/setting/global_lock_test.go +++ b/modules/setting/global_lock_test.go @@ -16,7 +16,7 @@ func TestLoadGlobalLockConfig(t *testing.T) { assert.NoError(t, err) loadGlobalLockFrom(cfg) - assert.EqualValues(t, "memory", GlobalLock.ServiceType) + assert.Equal(t, "memory", GlobalLock.ServiceType) }) t.Run("RedisGlobalLockConfig", func(t *testing.T) { @@ -29,7 +29,7 @@ SERVICE_CONN_STR = addrs=127.0.0.1:6379 db=0 assert.NoError(t, err) loadGlobalLockFrom(cfg) - assert.EqualValues(t, "redis", GlobalLock.ServiceType) - assert.EqualValues(t, "addrs=127.0.0.1:6379 db=0", GlobalLock.ServiceConnStr) + assert.Equal(t, "redis", GlobalLock.ServiceType) + assert.Equal(t, "addrs=127.0.0.1:6379 db=0", GlobalLock.ServiceConnStr) }) } diff --git a/modules/setting/incoming_email.go b/modules/setting/incoming_email.go index bf81f292a2..4e433dde60 100644 --- a/modules/setting/incoming_email.go +++ b/modules/setting/incoming_email.go @@ -4,6 +4,7 @@ package setting import ( + "errors" "fmt" "net/mail" "strings" @@ -50,7 +51,7 @@ func checkReplyToAddress() error { } if parsed.Name != "" { - return fmt.Errorf("name must not be set") + return errors.New("name must not be set") } c := strings.Count(IncomingEmail.ReplyToAddress, IncomingEmail.TokenPlaceholder) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index e34baae012..ace7eec70e 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -96,7 +96,7 @@ func loadIndexerFrom(rootCfg ConfigProvider) { // IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing func IndexerGlobFromString(globstr string) []*GlobMatcher { extarr := make([]*GlobMatcher, 0, 10) - for _, expr := range strings.Split(strings.ToLower(globstr), ",") { + for expr := range strings.SplitSeq(strings.ToLower(globstr), ",") { expr = strings.TrimSpace(expr) if expr != "" { if g, err := GlobMatcherCompile(expr, '.', '/'); err != nil { diff --git a/modules/setting/lfs_test.go b/modules/setting/lfs_test.go index d27dd7c5bf..1b829d8839 100644 --- a/modules/setting/lfs_test.go +++ b/modules/setting/lfs_test.go @@ -19,7 +19,7 @@ func Test_getStorageInheritNameSectionTypeForLFS(t *testing.T) { assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) iniStr = ` [server] @@ -54,7 +54,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) iniStr = ` [lfs] @@ -68,7 +68,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) iniStr = ` [lfs] @@ -83,7 +83,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_lfs/", LFS.Storage.MinioConfig.BasePath) } func Test_LFSStorage1(t *testing.T) { @@ -96,8 +96,8 @@ STORAGE_TYPE = minio assert.NoError(t, loadLFSFrom(cfg)) assert.EqualValues(t, "minio", LFS.Storage.Type) - assert.EqualValues(t, "gitea", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", LFS.Storage.MinioConfig.Bucket) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) } func Test_LFSClientServerConfigs(t *testing.T) { @@ -112,9 +112,9 @@ BATCH_SIZE = 0 assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, 100, LFS.MaxBatchSize) - assert.EqualValues(t, 20, LFSClient.BatchSize) - assert.EqualValues(t, 8, LFSClient.BatchOperationConcurrency) + assert.Equal(t, 100, LFS.MaxBatchSize) + assert.Equal(t, 20, LFSClient.BatchSize) + assert.Equal(t, 8, LFSClient.BatchOperationConcurrency) iniStr = ` [lfs_client] @@ -125,6 +125,6 @@ BATCH_OPERATION_CONCURRENCY = 10 assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, 50, LFSClient.BatchSize) - assert.EqualValues(t, 10, LFSClient.BatchOperationConcurrency) + assert.Equal(t, 50, LFSClient.BatchSize) + assert.Equal(t, 10, LFSClient.BatchOperationConcurrency) } diff --git a/modules/setting/log.go b/modules/setting/log.go index 50c5779994..59866c7605 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -7,7 +7,6 @@ import ( "fmt" golog "log" "os" - "path" "path/filepath" "strings" @@ -41,7 +40,7 @@ func loadLogGlobalFrom(rootCfg ConfigProvider) { Log.BufferLen = sec.Key("BUFFER_LEN").MustInt(10000) Log.Mode = sec.Key("MODE").MustString("console") - Log.RootPath = sec.Key("ROOT_PATH").MustString(path.Join(AppWorkPath, "log")) + Log.RootPath = sec.Key("ROOT_PATH").MustString(filepath.Join(AppWorkPath, "log")) if !filepath.IsAbs(Log.RootPath) { Log.RootPath = filepath.Join(AppWorkPath, Log.RootPath) } @@ -228,8 +227,8 @@ func initLoggerByName(manager *log.LoggerManager, rootCfg ConfigProvider, logger } var eventWriters []log.EventWriter - modes := strings.Split(modeVal, ",") - for _, modeName := range modes { + modes := strings.SplitSeq(modeVal, ",") + for modeName := range modes { modeName = strings.TrimSpace(modeName) if modeName == "" { continue diff --git a/modules/setting/mailer_test.go b/modules/setting/mailer_test.go index fbabf11378..ceef35b051 100644 --- a/modules/setting/mailer_test.go +++ b/modules/setting/mailer_test.go @@ -34,8 +34,8 @@ func Test_loadMailerFrom(t *testing.T) { // Check mailer setting loadMailerFrom(cfg) - assert.EqualValues(t, kase.SMTPAddr, MailService.SMTPAddr) - assert.EqualValues(t, kase.SMTPPort, MailService.SMTPPort) + assert.Equal(t, kase.SMTPAddr, MailService.SMTPAddr) + assert.Equal(t, kase.SMTPPort, MailService.SMTPPort) }) } } diff --git a/modules/setting/markup.go b/modules/setting/markup.go index dfce8afa77..057b0650c3 100644 --- a/modules/setting/markup.go +++ b/modules/setting/markup.go @@ -8,6 +8,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" ) // ExternalMarkupRenderers represents the external markup renderers @@ -23,18 +24,33 @@ const ( RenderContentModeIframe = "iframe" ) +type MarkdownRenderOptions struct { + NewLineHardBreak bool + ShortIssuePattern bool // Actually it is a "markup" option because it is used in "post processor" +} + +type MarkdownMathCodeBlockOptions struct { + ParseInlineDollar bool + ParseInlineParentheses bool + ParseBlockDollar bool + ParseBlockSquareBrackets bool +} + // Markdown settings var Markdown = struct { - EnableHardLineBreakInComments bool - EnableHardLineBreakInDocuments bool - CustomURLSchemes []string `ini:"CUSTOM_URL_SCHEMES"` - FileExtensions []string - EnableMath bool + RenderOptionsComment MarkdownRenderOptions `ini:"-"` + RenderOptionsWiki MarkdownRenderOptions `ini:"-"` + RenderOptionsRepoFile MarkdownRenderOptions `ini:"-"` + + CustomURLSchemes []string `ini:"CUSTOM_URL_SCHEMES"` // Actually it is a "markup" option because it is used in "post processor" + FileExtensions []string + + EnableMath bool + MathCodeBlockDetection []string + MathCodeBlockOptions MarkdownMathCodeBlockOptions `ini:"-"` }{ - EnableHardLineBreakInComments: true, - EnableHardLineBreakInDocuments: false, - FileExtensions: strings.Split(".md,.markdown,.mdown,.mkd,.livemd", ","), - EnableMath: true, + FileExtensions: strings.Split(".md,.markdown,.mdown,.mkd,.livemd", ","), + EnableMath: true, } // MarkupRenderer defines the external parser configured in ini @@ -60,8 +76,58 @@ type MarkupSanitizerRule struct { func loadMarkupFrom(rootCfg ConfigProvider) { mustMapSetting(rootCfg, "markdown", &Markdown) + const none = "none" + + const renderOptionShortIssuePattern = "short-issue-pattern" + const renderOptionNewLineHardBreak = "new-line-hard-break" + cfgMarkdown := rootCfg.Section("markdown") + parseMarkdownRenderOptions := func(key string, defaults []string) (ret MarkdownRenderOptions) { + options := cfgMarkdown.Key(key).Strings(",") + options = util.IfEmpty(options, defaults) + for _, opt := range options { + switch opt { + case renderOptionShortIssuePattern: + ret.ShortIssuePattern = true + case renderOptionNewLineHardBreak: + ret.NewLineHardBreak = true + case none: + ret = MarkdownRenderOptions{} + case "": + default: + log.Error("Unknown markdown render option in %s: %s", key, opt) + } + } + return ret + } + Markdown.RenderOptionsComment = parseMarkdownRenderOptions("RENDER_OPTIONS_COMMENT", []string{renderOptionShortIssuePattern, renderOptionNewLineHardBreak}) + Markdown.RenderOptionsWiki = parseMarkdownRenderOptions("RENDER_OPTIONS_WIKI", []string{renderOptionShortIssuePattern}) + Markdown.RenderOptionsRepoFile = parseMarkdownRenderOptions("RENDER_OPTIONS_REPO_FILE", nil) + + const mathCodeInlineDollar = "inline-dollar" + const mathCodeInlineParentheses = "inline-parentheses" + const mathCodeBlockDollar = "block-dollar" + const mathCodeBlockSquareBrackets = "block-square-brackets" + Markdown.MathCodeBlockDetection = util.IfEmpty(Markdown.MathCodeBlockDetection, []string{mathCodeInlineDollar, mathCodeBlockDollar}) + Markdown.MathCodeBlockOptions = MarkdownMathCodeBlockOptions{} + for _, s := range Markdown.MathCodeBlockDetection { + switch s { + case mathCodeInlineDollar: + Markdown.MathCodeBlockOptions.ParseInlineDollar = true + case mathCodeInlineParentheses: + Markdown.MathCodeBlockOptions.ParseInlineParentheses = true + case mathCodeBlockDollar: + Markdown.MathCodeBlockOptions.ParseBlockDollar = true + case mathCodeBlockSquareBrackets: + Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets = true + case none: + Markdown.MathCodeBlockOptions = MarkdownMathCodeBlockOptions{} + case "": + default: + log.Error("Unknown math code block detection option: %s", s) + } + } - MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(5000) + MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(50000) ExternalMarkupRenderers = make([]*MarkupRenderer, 0, 10) ExternalSanitizerRules = make([]MarkupSanitizerRule, 0, 10) @@ -83,8 +149,8 @@ func loadMarkupFrom(rootCfg ConfigProvider) { func newMarkupSanitizer(name string, sec ConfigSection) { rule, ok := createMarkupSanitizerRule(name, sec) if ok { - if strings.HasPrefix(name, "sanitizer.") { - names := strings.SplitN(strings.TrimPrefix(name, "sanitizer."), ".", 2) + if after, found := strings.CutPrefix(name, "sanitizer."); found { + names := strings.SplitN(after, ".", 2) name = names[0] } for _, renderer := range ExternalMarkupRenderers { diff --git a/modules/setting/markup_test.go b/modules/setting/markup_test.go new file mode 100644 index 0000000000..c47a38ce15 --- /dev/null +++ b/modules/setting/markup_test.go @@ -0,0 +1,51 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLoadMarkup(t *testing.T) { + cfg, _ := NewConfigProviderFromData(``) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseBlockDollar: true}, Markdown.MathCodeBlockOptions) + assert.Equal(t, MarkdownRenderOptions{NewLineHardBreak: true, ShortIssuePattern: true}, Markdown.RenderOptionsComment) + assert.Equal(t, MarkdownRenderOptions{ShortIssuePattern: true}, Markdown.RenderOptionsWiki) + assert.Equal(t, MarkdownRenderOptions{}, Markdown.RenderOptionsRepoFile) + + t.Run("Math", func(t *testing.T) { + cfg, _ = NewConfigProviderFromData(` +[markdown] +MATH_CODE_BLOCK_DETECTION = none +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownMathCodeBlockOptions{}, Markdown.MathCodeBlockOptions) + + cfg, _ = NewConfigProviderFromData(` +[markdown] +MATH_CODE_BLOCK_DETECTION = inline-dollar, inline-parentheses, block-dollar, block-square-brackets +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseInlineParentheses: true, ParseBlockDollar: true, ParseBlockSquareBrackets: true}, Markdown.MathCodeBlockOptions) + }) + + t.Run("Render", func(t *testing.T) { + cfg, _ = NewConfigProviderFromData(` +[markdown] +RENDER_OPTIONS_COMMENT = none +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownRenderOptions{}, Markdown.RenderOptionsComment) + + cfg, _ = NewConfigProviderFromData(` +[markdown] +RENDER_OPTIONS_REPO_FILE = short-issue-pattern, new-line-hard-break +`) + loadMarkupFrom(cfg) + assert.Equal(t, MarkdownRenderOptions{NewLineHardBreak: true, ShortIssuePattern: true}, Markdown.RenderOptionsRepoFile) + }) +} diff --git a/modules/setting/mirror.go b/modules/setting/mirror.go index 3aa530a1f4..300711789d 100644 --- a/modules/setting/mirror.go +++ b/modules/setting/mirror.go @@ -48,11 +48,7 @@ func loadMirrorFrom(rootCfg ConfigProvider) { Mirror.MinInterval = 1 * time.Minute } if Mirror.DefaultInterval < Mirror.MinInterval { - if time.Hour*8 < Mirror.MinInterval { - Mirror.DefaultInterval = Mirror.MinInterval - } else { - Mirror.DefaultInterval = time.Hour * 8 - } + Mirror.DefaultInterval = max(time.Hour*8, Mirror.MinInterval) log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval, set to %s", Mirror.DefaultInterval.String()) } } diff --git a/modules/setting/oauth2.go b/modules/setting/oauth2.go index 0d3e63e0b4..1a88f3cb08 100644 --- a/modules/setting/oauth2.go +++ b/modules/setting/oauth2.go @@ -12,7 +12,7 @@ import ( "code.gitea.io/gitea/modules/log" ) -// OAuth2UsernameType is enum describing the way gitea 'name' should be generated from oauth2 data +// OAuth2UsernameType is enum describing the way gitea generates its 'username' from oauth2 data type OAuth2UsernameType string const ( diff --git a/modules/setting/oauth2_test.go b/modules/setting/oauth2_test.go index d0e5ccf13d..c6e66cad02 100644 --- a/modules/setting/oauth2_test.go +++ b/modules/setting/oauth2_test.go @@ -31,7 +31,7 @@ JWT_SECRET = BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB actual := GetGeneralTokenSigningSecret() expected, _ := generate.DecodeJwtSecretBase64("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB") assert.Len(t, actual, 32) - assert.EqualValues(t, expected, actual) + assert.Equal(t, expected, actual) } func TestGetGeneralSigningSecretSave(t *testing.T) { diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 3f618cfd64..b598424064 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -6,8 +6,6 @@ package setting import ( "fmt" "math" - "os" - "path/filepath" "github.com/dustin/go-humanize" ) @@ -15,9 +13,8 @@ import ( // Package registry settings var ( Packages = struct { - Storage *Storage - Enabled bool - ChunkedUploadPath string + Storage *Storage + Enabled bool LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 @@ -67,17 +64,6 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { return err } - Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload")) - if !filepath.IsAbs(Packages.ChunkedUploadPath) { - Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath)) - } - - if HasInstallLock(rootCfg) { - if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { - return fmt.Errorf("unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) - } - } - Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeAlpine = mustBytes(sec, "LIMIT_SIZE_ALPINE") Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH") diff --git a/modules/setting/packages_test.go b/modules/setting/packages_test.go index 87de276041..47378f35ad 100644 --- a/modules/setting/packages_test.go +++ b/modules/setting/packages_test.go @@ -41,7 +41,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) // we can also configure packages storage directly iniStr = ` @@ -53,7 +53,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) // or we can indicate the storage type in the packages section iniStr = ` @@ -68,7 +68,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) // or we can indicate the storage type and minio base path in the packages section iniStr = ` @@ -84,7 +84,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "my_packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_packages/", Packages.Storage.MinioConfig.BasePath) } func Test_PackageStorage1(t *testing.T) { @@ -109,8 +109,8 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } @@ -136,8 +136,8 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } @@ -164,8 +164,8 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "my_packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } @@ -192,7 +192,7 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := Packages.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) - assert.EqualValues(t, "my_packages/", storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "my_packages/", storage.MinioConfig.BasePath) assert.True(t, storage.MinioConfig.ServeDirect) } diff --git a/modules/setting/path.go b/modules/setting/path.go index 0fdc305aa1..f51457a620 100644 --- a/modules/setting/path.go +++ b/modules/setting/path.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/tempdir" ) var ( @@ -196,3 +197,18 @@ func InitWorkPathAndCfgProvider(getEnvFn func(name string) string, args ArgWorkP CustomPath = tmpCustomPath.Value CustomConf = tmpCustomConf.Value } + +// AppDataTempDir returns a managed temporary directory for the application data. +// Using empty sub will get the managed base temp directory, and it's safe to delete it. +// Gitea only creates subdirectories under it, but not the APP_TEMP_PATH directory itself. +// * When APP_TEMP_PATH="/tmp": the managed temp directory is "/tmp/gitea-tmp" +// * When APP_TEMP_PATH is not set: the managed temp directory is "/{APP_DATA_PATH}/tmp" +func AppDataTempDir(sub string) *tempdir.TempDir { + if appTempPathInternal != "" { + return tempdir.New(appTempPathInternal, "gitea-tmp/"+sub) + } + if AppDataPath == "" { + panic("setting.AppDataPath is not set") + } + return tempdir.New(AppDataPath, "tmp/"+sub) +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index c5619d0f04..318cf41108 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -5,7 +5,6 @@ package setting import ( "os/exec" - "path" "path/filepath" "strings" @@ -63,17 +62,11 @@ var ( // Repository upload settings Upload struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int } `ini:"-"` - // Repository local settings - Local struct { - LocalCopyPath string - } `ini:"-"` - // Pull request settings PullRequest struct { WorkInProgressPrefixes []string @@ -89,6 +82,7 @@ var ( AddCoCommitterTrailers bool TestConflictingPatchesWithGitApply bool RetargetChildrenOnMerge bool + DelayCheckForInactiveDays int } `ini:"repository.pull-request"` // Issue Setting @@ -106,11 +100,13 @@ var ( SigningKey string SigningName string SigningEmail string + SigningFormat string InitialCommit []string CRUDActions []string `ini:"CRUD_ACTIONS"` Merges []string Wiki []string DefaultTrustModel string + TrustedSSHKeys []string `ini:"TRUSTED_SSH_KEYS"` } `ini:"repository.signing"` }{ DetectedCharsetsOrder: []string{ @@ -182,25 +178,16 @@ var ( // Repository upload settings Upload: struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int }{ Enabled: true, - TempPath: "data/tmp/uploads", AllowedTypes: "", FileMaxSize: 50, MaxFiles: 5, }, - // Repository local settings - Local: struct { - LocalCopyPath string - }{ - LocalCopyPath: "tmp/local-repo", - }, - // Pull request settings PullRequest: struct { WorkInProgressPrefixes []string @@ -216,6 +203,7 @@ var ( AddCoCommitterTrailers bool TestConflictingPatchesWithGitApply bool RetargetChildrenOnMerge bool + DelayCheckForInactiveDays int }{ WorkInProgressPrefixes: []string{"WIP:", "[WIP]"}, // Same as GitHub. See @@ -231,6 +219,7 @@ var ( PopulateSquashCommentWithCommitMessages: false, AddCoCommitterTrailers: true, RetargetChildrenOnMerge: true, + DelayCheckForInactiveDays: 7, }, // Issue settings @@ -255,20 +244,24 @@ var ( SigningKey string SigningName string SigningEmail string + SigningFormat string InitialCommit []string CRUDActions []string `ini:"CRUD_ACTIONS"` Merges []string Wiki []string DefaultTrustModel string + TrustedSSHKeys []string `ini:"TRUSTED_SSH_KEYS"` }{ SigningKey: "default", SigningName: "", SigningEmail: "", + SigningFormat: "openpgp", // git.SigningKeyFormatOpenPGP InitialCommit: []string{"always"}, CRUDActions: []string{"pubkey", "twofa", "parentsigned"}, Merges: []string{"pubkey", "twofa", "basesigned", "commitssigned"}, Wiki: []string{"never"}, DefaultTrustModel: "collaborator", + TrustedSSHKeys: []string{}, }, } RepoRootPath string @@ -284,7 +277,7 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { Repository.GoGetCloneURLProtocol = sec.Key("GO_GET_CLONE_URL_PROTOCOL").MustString("https") Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) Repository.DefaultBranch = sec.Key("DEFAULT_BRANCH").MustString(Repository.DefaultBranch) - RepoRootPath = sec.Key("ROOT").MustString(path.Join(AppDataPath, "gitea-repositories")) + RepoRootPath = sec.Key("ROOT").MustString(filepath.Join(AppDataPath, "gitea-repositories")) if !filepath.IsAbs(RepoRootPath) { RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath) } else { @@ -309,8 +302,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { log.Fatal("Failed to map Repository.Editor settings: %v", err) } else if err = rootCfg.Section("repository.upload").MapTo(&Repository.Upload); err != nil { log.Fatal("Failed to map Repository.Upload settings: %v", err) - } else if err = rootCfg.Section("repository.local").MapTo(&Repository.Local); err != nil { - log.Fatal("Failed to map Repository.Local settings: %v", err) } else if err = rootCfg.Section("repository.pull-request").MapTo(&Repository.PullRequest); err != nil { log.Fatal("Failed to map Repository.PullRequest settings: %v", err) } @@ -362,10 +353,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { } } - if !filepath.IsAbs(Repository.Upload.TempPath) { - Repository.Upload.TempPath = path.Join(AppWorkPath, Repository.Upload.TempPath) - } - if err := loadRepoArchiveFrom(rootCfg); err != nil { log.Fatal("loadRepoArchiveFrom: %v", err) } diff --git a/modules/setting/repository_archive_test.go b/modules/setting/repository_archive_test.go index a0f91f0da1..d5b95272d6 100644 --- a/modules/setting/repository_archive_test.go +++ b/modules/setting/repository_archive_test.go @@ -20,7 +20,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) // we can also configure packages storage directly iniStr = ` @@ -32,7 +32,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) // or we can indicate the storage type in the packages section iniStr = ` @@ -47,7 +47,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) // or we can indicate the storage type and minio base path in the packages section iniStr = ` @@ -63,7 +63,7 @@ STORAGE_TYPE = minio assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "my_archive/", RepoArchive.Storage.MinioConfig.BasePath) } func Test_RepoArchiveStorage(t *testing.T) { @@ -85,7 +85,7 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage := RepoArchive.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) iniStr = ` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -107,5 +107,5 @@ MINIO_SECRET_ACCESS_KEY = correct_key storage = RepoArchive.Storage assert.EqualValues(t, "minio", storage.Type) - assert.EqualValues(t, "gitea", storage.MinioConfig.Bucket) + assert.Equal(t, "gitea", storage.MinioConfig.Bucket) } diff --git a/modules/setting/security.go b/modules/setting/security.go index 2f798b75c7..153b6bc944 100644 --- a/modules/setting/security.go +++ b/modules/setting/security.go @@ -39,6 +39,7 @@ var ( CSRFCookieName = "_csrf" CSRFCookieHTTPOnly = true RecordUserSignupMetadata = false + TwoFactorAuthEnforced = false ) // loadSecret load the secret from ini by uriKey or verbatimKey, only one of them could be set @@ -110,7 +111,7 @@ func loadSecurityFrom(rootCfg ConfigProvider) { if SecretKey == "" { // FIXME: https://github.com/go-gitea/gitea/issues/16832 // Until it supports rotating an existing secret key, we shouldn't move users off of the widely used default value - SecretKey = "!#@FDEWREWR&*(" //nolint:gosec + SecretKey = "!#@FDEWREWR&*(" } CookieRememberName = sec.Key("COOKIE_REMEMBER_NAME").MustString("gitea_incredible") @@ -142,6 +143,15 @@ func loadSecurityFrom(rootCfg ConfigProvider) { PasswordCheckPwn = sec.Key("PASSWORD_CHECK_PWN").MustBool(false) SuccessfulTokensCacheSize = sec.Key("SUCCESSFUL_TOKENS_CACHE_SIZE").MustInt(20) + twoFactorAuth := sec.Key("TWO_FACTOR_AUTH").String() + switch twoFactorAuth { + case "": + case "enforced": + TwoFactorAuthEnforced = true + default: + log.Fatal("Invalid two-factor auth option: %s", twoFactorAuth) + } + InternalToken = loadSecret(sec, "INTERNAL_TOKEN_URI", "INTERNAL_TOKEN") if InstallLock && InternalToken == "" { // if Gitea has been installed but the InternalToken hasn't been generated (upgrade from an old release), we should generate diff --git a/modules/setting/server.go b/modules/setting/server.go index e15b790906..38e166e02a 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "net" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -40,28 +41,47 @@ const ( LandingPageLogin LandingPage = "/user/login" ) +const ( + PublicURLAuto = "auto" + PublicURLLegacy = "legacy" +) + // Server settings var ( // AppURL is the Application ROOT_URL. It always has a '/' suffix // It maps to ini:"ROOT_URL" AppURL string - // AppSubURL represents the sub-url mounting point for gitea. It is either "" or starts with '/' and ends without '/', such as '/{subpath}'. + + // PublicURLDetection controls how to use the HTTP request headers to detect public URL + PublicURLDetection string + + // AppSubURL represents the sub-url mounting point for gitea, parsed from "ROOT_URL" + // It is either "" or starts with '/' and ends without '/', such as '/{sub-path}'. // This value is empty if site does not have sub-url. AppSubURL string - // UseSubURLPath makes Gitea handle requests with sub-path like "/sub-path/owner/repo/...", to make it easier to debug sub-path related problems without a reverse proxy. + + // UseSubURLPath makes Gitea handle requests with sub-path like "/sub-path/owner/repo/...", + // to make it easier to debug sub-path related problems without a reverse proxy. UseSubURLPath bool + // AppDataPath is the default path for storing data. // It maps to ini:"APP_DATA_PATH" in [server] and defaults to AppWorkPath + "/data" AppDataPath string + // LocalURL is the url for locally running applications to contact Gitea. It always has a '/' suffix // It maps to ini:"LOCAL_ROOT_URL" in [server] LocalURL string - // AssetVersion holds a opaque value that is used for cache-busting assets + + // AssetVersion holds an opaque value that is used for cache-busting assets AssetVersion string + // appTempPathInternal is the temporary path for the app, it is only an internal variable + // DO NOT use it directly, always use AppDataTempDir + appTempPathInternal string + Protocol Scheme - UseProxyProtocol bool // `ini:"USE_PROXY_PROTOCOL"` - ProxyProtocolTLSBridging bool //`ini:"PROXY_PROTOCOL_TLS_BRIDGING"` + UseProxyProtocol bool + ProxyProtocolTLSBridging bool ProxyProtocolHeaderTimeout time.Duration ProxyProtocolAcceptUnknown bool Domain string @@ -178,13 +198,14 @@ func loadServerFrom(rootCfg ConfigProvider) { EnableAcme = sec.Key("ENABLE_LETSENCRYPT").MustBool(false) } - Protocol = HTTP protocolCfg := sec.Key("PROTOCOL").String() if protocolCfg != "https" && EnableAcme { log.Fatal("ACME could only be used with HTTPS protocol") } switch protocolCfg { + case "", "http": + Protocol = HTTP case "https": Protocol = HTTPS if EnableAcme { @@ -240,7 +261,7 @@ func loadServerFrom(rootCfg ConfigProvider) { case "unix": log.Warn("unix PROTOCOL value is deprecated, please use http+unix") fallthrough - case "http+unix": + default: // "http+unix" Protocol = HTTPUnix } UnixSocketPermissionRaw := sec.Key("UNIX_SOCKET_PERMISSION").MustString("666") @@ -253,6 +274,8 @@ func loadServerFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(HTTPAddr) { HTTPAddr = filepath.Join(AppWorkPath, HTTPAddr) } + default: + log.Fatal("Invalid PROTOCOL %q", protocolCfg) } UseProxyProtocol = sec.Key("USE_PROXY_PROTOCOL").MustBool(false) ProxyProtocolTLSBridging = sec.Key("PROXY_PROTOCOL_TLS_BRIDGING").MustBool(false) @@ -266,11 +289,15 @@ func loadServerFrom(rootCfg ConfigProvider) { defaultAppURL := string(Protocol) + "://" + Domain + ":" + HTTPPort AppURL = sec.Key("ROOT_URL").MustString(defaultAppURL) + PublicURLDetection = sec.Key("PUBLIC_URL_DETECTION").MustString(PublicURLLegacy) + if PublicURLDetection != PublicURLAuto && PublicURLDetection != PublicURLLegacy { + log.Fatal("Invalid PUBLIC_URL_DETECTION value: %s", PublicURLDetection) + } // Check validity of AppURL appURL, err := url.Parse(AppURL) if err != nil { - log.Fatal("Invalid ROOT_URL '%s': %s", AppURL, err) + log.Fatal("Invalid ROOT_URL %q: %s", AppURL, err) } // Remove default ports from AppURL. // (scheme-based URL normalization, RFC 3986 section 6.2.3) @@ -306,13 +333,15 @@ func loadServerFrom(rootCfg ConfigProvider) { defaultLocalURL = AppURL case FCGIUnix: defaultLocalURL = AppURL - default: + case HTTP, HTTPS: defaultLocalURL = string(Protocol) + "://" if HTTPAddr == "0.0.0.0" { defaultLocalURL += net.JoinHostPort("localhost", HTTPPort) + "/" } else { defaultLocalURL += net.JoinHostPort(HTTPAddr, HTTPPort) + "/" } + default: + log.Fatal("Invalid PROTOCOL %q", Protocol) } LocalURL = sec.Key("LOCAL_ROOT_URL").MustString(defaultLocalURL) LocalURL = strings.TrimRight(LocalURL, "/") + "/" @@ -330,6 +359,19 @@ func loadServerFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(AppDataPath) { AppDataPath = filepath.ToSlash(filepath.Join(AppWorkPath, AppDataPath)) } + if IsInTesting && HasInstallLock(rootCfg) { + // FIXME: in testing, the "app data" directory is not correctly initialized before loading settings + if _, err := os.Stat(AppDataPath); err != nil { + _ = os.MkdirAll(AppDataPath, os.ModePerm) + } + } + + appTempPathInternal = sec.Key("APP_TEMP_PATH").String() + if appTempPathInternal != "" { + if _, err := os.Stat(appTempPathInternal); err != nil { + log.Fatal("APP_TEMP_PATH %q is not accessible: %v", appTempPathInternal, err) + } + } EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) diff --git a/modules/setting/service.go b/modules/setting/service.go index 8c1843eeb7..b1b9fedd62 100644 --- a/modules/setting/service.go +++ b/modules/setting/service.go @@ -5,6 +5,7 @@ package setting import ( "regexp" + "runtime" "strings" "time" @@ -43,7 +44,8 @@ var Service = struct { ShowRegistrationButton bool EnablePasswordSignInForm bool ShowMilestonesDashboardPage bool - RequireSignInView bool + RequireSignInViewStrict bool + BlockAnonymousAccessExpensive bool EnableNotifyMail bool EnableBasicAuth bool EnablePasskeyAuth bool @@ -97,6 +99,13 @@ var Service = struct { DisableOrganizationsPage bool `ini:"DISABLE_ORGANIZATIONS_PAGE"` DisableCodePage bool `ini:"DISABLE_CODE_PAGE"` } `ini:"service.explore"` + + QoS struct { + Enabled bool + MaxInFlightRequests int + MaxWaitingRequests int + TargetWaitTime time.Duration + } }{ AllowedUserVisibilityModesSlice: []bool{true, true, true}, } @@ -159,7 +168,18 @@ func loadServiceFrom(rootCfg ConfigProvider) { Service.EmailDomainBlockList = CompileEmailGlobList(sec, "EMAIL_DOMAIN_BLOCKLIST") Service.ShowRegistrationButton = sec.Key("SHOW_REGISTRATION_BUTTON").MustBool(!(Service.DisableRegistration || Service.AllowOnlyExternalRegistration)) Service.ShowMilestonesDashboardPage = sec.Key("SHOW_MILESTONES_DASHBOARD_PAGE").MustBool(true) - Service.RequireSignInView = sec.Key("REQUIRE_SIGNIN_VIEW").MustBool() + + // boolean values are considered as "strict" + var err error + Service.RequireSignInViewStrict, err = sec.Key("REQUIRE_SIGNIN_VIEW").Bool() + if s := sec.Key("REQUIRE_SIGNIN_VIEW").String(); err != nil && s != "" { + // non-boolean value only supports "expensive" at the moment + Service.BlockAnonymousAccessExpensive = s == "expensive" + if !Service.BlockAnonymousAccessExpensive { + log.Fatal("Invalid config option: REQUIRE_SIGNIN_VIEW = %s", s) + } + } + Service.EnableBasicAuth = sec.Key("ENABLE_BASIC_AUTHENTICATION").MustBool(true) Service.EnablePasswordSignInForm = sec.Key("ENABLE_PASSWORD_SIGNIN_FORM").MustBool(true) Service.EnablePasskeyAuth = sec.Key("ENABLE_PASSKEY_AUTHENTICATION").MustBool(true) @@ -243,6 +263,7 @@ func loadServiceFrom(rootCfg ConfigProvider) { mustMapSetting(rootCfg, "service.explore", &Service.Explore) loadOpenIDSetting(rootCfg) + loadQosSetting(rootCfg) } func loadOpenIDSetting(rootCfg ConfigProvider) { @@ -264,3 +285,11 @@ func loadOpenIDSetting(rootCfg ConfigProvider) { } } } + +func loadQosSetting(rootCfg ConfigProvider) { + sec := rootCfg.Section("qos") + Service.QoS.Enabled = sec.Key("ENABLED").MustBool(false) + Service.QoS.MaxInFlightRequests = sec.Key("MAX_INFLIGHT").MustInt(4 * runtime.NumCPU()) + Service.QoS.MaxWaitingRequests = sec.Key("MAX_WAITING").MustInt(100) + Service.QoS.TargetWaitTime = sec.Key("TARGET_WAIT_TIME").MustDuration(250 * time.Millisecond) +} diff --git a/modules/setting/service_test.go b/modules/setting/service_test.go index 1647bcec16..73736b793a 100644 --- a/modules/setting/service_test.go +++ b/modules/setting/service_test.go @@ -7,16 +7,14 @@ import ( "testing" "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/test" "github.com/gobwas/glob" "github.com/stretchr/testify/assert" ) func TestLoadServices(t *testing.T) { - oldService := Service - defer func() { - Service = oldService - }() + defer test.MockVariableValue(&Service)() cfg, err := NewConfigProviderFromData(` [service] @@ -48,10 +46,7 @@ EMAIL_DOMAIN_BLOCKLIST = d3, *.b } func TestLoadServiceVisibilityModes(t *testing.T) { - oldService := Service - defer func() { - Service = oldService - }() + defer test.MockVariableValue(&Service)() kases := map[string]func(){ ` @@ -130,3 +125,33 @@ ALLOWED_USER_VISIBILITY_MODES = public, limit, privated }) } } + +func TestLoadServiceRequireSignInView(t *testing.T) { + defer test.MockVariableValue(&Service)() + + cfg, err := NewConfigProviderFromData(` +[service] +`) + assert.NoError(t, err) + loadServiceFrom(cfg) + assert.False(t, Service.RequireSignInViewStrict) + assert.False(t, Service.BlockAnonymousAccessExpensive) + + cfg, err = NewConfigProviderFromData(` +[service] +REQUIRE_SIGNIN_VIEW = true +`) + assert.NoError(t, err) + loadServiceFrom(cfg) + assert.True(t, Service.RequireSignInViewStrict) + assert.False(t, Service.BlockAnonymousAccessExpensive) + + cfg, err = NewConfigProviderFromData(` +[service] +REQUIRE_SIGNIN_VIEW = expensive +`) + assert.NoError(t, err) + loadServiceFrom(cfg) + assert.False(t, Service.RequireSignInViewStrict) + assert.True(t, Service.BlockAnonymousAccessExpensive) +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 20da796b58..e14997801f 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -12,8 +12,8 @@ import ( "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/user" - "code.gitea.io/gitea/modules/util" ) // settings @@ -159,7 +159,7 @@ func loadRunModeFrom(rootCfg ConfigProvider) { // The following is a purposefully undocumented option. Please do not run Gitea as root. It will only cause future headaches. // Please don't use root as a bandaid to "fix" something that is broken, instead the broken thing should instead be fixed properly. unsafeAllowRunAsRoot := ConfigSectionKeyBool(rootSec, "I_AM_BEING_UNSAFE_RUNNING_AS_ROOT") - unsafeAllowRunAsRoot = unsafeAllowRunAsRoot || util.OptionalBoolParse(os.Getenv("GITEA_I_AM_BEING_UNSAFE_RUNNING_AS_ROOT")).Value() + unsafeAllowRunAsRoot = unsafeAllowRunAsRoot || optional.ParseBool(os.Getenv("GITEA_I_AM_BEING_UNSAFE_RUNNING_AS_ROOT")).Value() RunMode = os.Getenv("GITEA_RUN_MODE") if RunMode == "" { RunMode = rootSec.Key("RUN_MODE").MustString("prod") diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index ea387e521f..900fc6ade2 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -4,8 +4,6 @@ package setting import ( - "os" - "path" "path/filepath" "strings" "text/template" @@ -32,8 +30,6 @@ var SSH = struct { ServerKeyExchanges []string `ini:"SSH_SERVER_KEY_EXCHANGES"` ServerMACs []string `ini:"SSH_SERVER_MACS"` ServerHostKeys []string `ini:"SSH_SERVER_HOST_KEYS"` - KeyTestPath string `ini:"SSH_KEY_TEST_PATH"` - KeygenPath string `ini:"SSH_KEYGEN_PATH"` AuthorizedKeysBackup bool `ini:"SSH_AUTHORIZED_KEYS_BACKUP"` AuthorizedPrincipalsBackup bool `ini:"SSH_AUTHORIZED_PRINCIPALS_BACKUP"` AuthorizedKeysCommandTemplate string `ini:"SSH_AUTHORIZED_KEYS_COMMAND_TEMPLATE"` @@ -55,10 +51,6 @@ var SSH = struct { StartBuiltinServer: false, Domain: "", Port: 22, - ServerCiphers: []string{"chacha20-poly1305@openssh.com", "aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "aes256-gcm@openssh.com"}, - ServerKeyExchanges: []string{"curve25519-sha256", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521", "diffie-hellman-group14-sha256", "diffie-hellman-group14-sha1"}, - ServerMACs: []string{"hmac-sha2-256-etm@openssh.com", "hmac-sha2-256", "hmac-sha1"}, - KeygenPath: "", MinimumKeySizeCheck: true, MinimumKeySizes: map[string]int{"ed25519": 256, "ed25519-sk": 256, "ecdsa": 256, "ecdsa-sk": 256, "rsa": 3071}, ServerHostKeys: []string{"ssh/gitea.rsa", "ssh/gogs.rsa"}, @@ -111,30 +103,27 @@ func loadSSHFrom(rootCfg ConfigProvider) { } homeDir = strings.ReplaceAll(homeDir, "\\", "/") - SSH.RootPath = path.Join(homeDir, ".ssh") - serverCiphers := sec.Key("SSH_SERVER_CIPHERS").Strings(",") - if len(serverCiphers) > 0 { - SSH.ServerCiphers = serverCiphers - } - serverKeyExchanges := sec.Key("SSH_SERVER_KEY_EXCHANGES").Strings(",") - if len(serverKeyExchanges) > 0 { - SSH.ServerKeyExchanges = serverKeyExchanges - } - serverMACs := sec.Key("SSH_SERVER_MACS").Strings(",") - if len(serverMACs) > 0 { - SSH.ServerMACs = serverMACs - } - SSH.KeyTestPath = os.TempDir() + SSH.RootPath = filepath.Join(homeDir, ".ssh") + if err = sec.MapTo(&SSH); err != nil { log.Fatal("Failed to map SSH settings: %v", err) } + + serverCiphers := sec.Key("SSH_SERVER_CIPHERS").Strings(",") + SSH.ServerCiphers = util.Iif(len(serverCiphers) > 0, serverCiphers, nil) + + serverKeyExchanges := sec.Key("SSH_SERVER_KEY_EXCHANGES").Strings(",") + SSH.ServerKeyExchanges = util.Iif(len(serverKeyExchanges) > 0, serverKeyExchanges, nil) + + serverMACs := sec.Key("SSH_SERVER_MACS").Strings(",") + SSH.ServerMACs = util.Iif(len(serverMACs) > 0, serverMACs, nil) + for i, key := range SSH.ServerHostKeys { if !filepath.IsAbs(key) { SSH.ServerHostKeys[i] = filepath.Join(AppDataPath, key) } } - SSH.KeygenPath = sec.Key("SSH_KEYGEN_PATH").String() SSH.Port = sec.Key("SSH_PORT").MustInt(22) SSH.ListenPort = sec.Key("SSH_LISTEN_PORT").MustInt(SSH.Port) SSH.UseProxyProtocol = sec.Key("SSH_SERVER_USE_PROXY_PROTOCOL").MustBool(false) diff --git a/modules/setting/storage.go b/modules/setting/storage.go index d3d1fb9f30..ee246158d9 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "path/filepath" + "slices" "strings" ) @@ -30,12 +31,7 @@ var storageTypes = []StorageType{ // IsValidStorageType returns true if the given storage type is valid func IsValidStorageType(storageType StorageType) bool { - for _, t := range storageTypes { - if t == storageType { - return true - } - } - return false + return slices.Contains(storageTypes, storageType) } // MinioStorageConfig represents the configuration for a minio storage @@ -162,7 +158,7 @@ const ( targetSecIsSec // target section is from the name seciont [name] ) -func getStorageSectionByType(rootCfg ConfigProvider, typ string) (ConfigSection, targetSecType, error) { //nolint:unparam +func getStorageSectionByType(rootCfg ConfigProvider, typ string) (ConfigSection, targetSecType, error) { //nolint:unparam // FIXME: targetSecType is always 0, wrong design? targetSec, err := rootCfg.GetSection(storageSectionName + "." + typ) if err != nil { if !IsValidStorageType(StorageType(typ)) { @@ -210,8 +206,8 @@ func getStorageTargetSection(rootCfg ConfigProvider, name, typ string, sec Confi targetSec, _ := rootCfg.GetSection(storageSectionName + "." + name) if targetSec != nil { targetType := targetSec.Key("STORAGE_TYPE").String() - switch { - case targetType == "": + switch targetType { + case "": if targetSec.Key("PATH").String() == "" { // both storage type and path are empty, use default return getDefaultStorageSection(rootCfg), targetSecIsDefault, nil } @@ -287,7 +283,7 @@ func getStorageForLocal(targetSec, overrideSec ConfigSection, tp targetSecType, return &storage, nil } -func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl +func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl // duplicates azure setup var storage Storage storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String()) if err := targetSec.MapTo(&storage.MinioConfig); err != nil { @@ -316,7 +312,7 @@ func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, return &storage, nil } -func getStorageForAzureBlob(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl +func getStorageForAzureBlob(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl // duplicates minio setup var storage Storage storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String()) if err := targetSec.MapTo(&storage.AzureBlobConfig); err != nil { diff --git a/modules/setting/storage_test.go b/modules/setting/storage_test.go index afff85537e..6f5a54c41c 100644 --- a/modules/setting/storage_test.go +++ b/modules/setting/storage_test.go @@ -26,16 +26,16 @@ MINIO_BUCKET = gitea-storage assert.NoError(t, err) assert.NoError(t, loadAttachmentFrom(cfg)) - assert.EqualValues(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-attachment", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-lfs", LFS.Storage.MinioConfig.Bucket) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) assert.NoError(t, loadAvatarsFrom(cfg)) - assert.EqualValues(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-storage", Avatar.Storage.MinioConfig.Bucket) + assert.Equal(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) } func Test_getStorageUseOtherNameAsType(t *testing.T) { @@ -51,12 +51,12 @@ MINIO_BUCKET = gitea-storage assert.NoError(t, err) assert.NoError(t, loadAttachmentFrom(cfg)) - assert.EqualValues(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-storage", Attachment.Storage.MinioConfig.Bucket) + assert.Equal(t, "attachments/", Attachment.Storage.MinioConfig.BasePath) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea-storage", LFS.Storage.MinioConfig.Bucket) + assert.Equal(t, "lfs/", LFS.Storage.MinioConfig.BasePath) } func Test_getStorageInheritStorageType(t *testing.T) { @@ -69,32 +69,32 @@ STORAGE_TYPE = minio assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "minio", Packages.Storage.Type) - assert.EqualValues(t, "gitea", Packages.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "packages/", Packages.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Packages.Storage.MinioConfig.Bucket) + assert.Equal(t, "packages/", Packages.Storage.MinioConfig.BasePath) assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "minio", RepoArchive.Storage.Type) - assert.EqualValues(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", RepoArchive.Storage.MinioConfig.Bucket) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "minio", Actions.LogStorage.Type) - assert.EqualValues(t, "gitea", Actions.LogStorage.MinioConfig.Bucket) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Actions.LogStorage.MinioConfig.Bucket) + assert.Equal(t, "actions_log/", Actions.LogStorage.MinioConfig.BasePath) assert.EqualValues(t, "minio", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Actions.ArtifactStorage.MinioConfig.Bucket) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.MinioConfig.BasePath) assert.NoError(t, loadAvatarsFrom(cfg)) assert.EqualValues(t, "minio", Avatar.Storage.Type) - assert.EqualValues(t, "gitea", Avatar.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", Avatar.Storage.MinioConfig.Bucket) + assert.Equal(t, "avatars/", Avatar.Storage.MinioConfig.BasePath) assert.NoError(t, loadRepoAvatarFrom(cfg)) assert.EqualValues(t, "minio", RepoAvatar.Storage.Type) - assert.EqualValues(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket) - assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath) + assert.Equal(t, "gitea", RepoAvatar.Storage.MinioConfig.Bucket) + assert.Equal(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath) } func Test_getStorageInheritStorageTypeAzureBlob(t *testing.T) { @@ -107,32 +107,32 @@ STORAGE_TYPE = azureblob assert.NoError(t, loadPackagesFrom(cfg)) assert.EqualValues(t, "azureblob", Packages.Storage.Type) - assert.EqualValues(t, "gitea", Packages.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "packages/", Packages.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Packages.Storage.AzureBlobConfig.Container) + assert.Equal(t, "packages/", Packages.Storage.AzureBlobConfig.BasePath) assert.NoError(t, loadRepoArchiveFrom(cfg)) assert.EqualValues(t, "azureblob", RepoArchive.Storage.Type) - assert.EqualValues(t, "gitea", RepoArchive.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", RepoArchive.Storage.AzureBlobConfig.Container) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) assert.NoError(t, loadActionsFrom(cfg)) assert.EqualValues(t, "azureblob", Actions.LogStorage.Type) - assert.EqualValues(t, "gitea", Actions.LogStorage.AzureBlobConfig.Container) - assert.EqualValues(t, "actions_log/", Actions.LogStorage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Actions.LogStorage.AzureBlobConfig.Container) + assert.Equal(t, "actions_log/", Actions.LogStorage.AzureBlobConfig.BasePath) assert.EqualValues(t, "azureblob", Actions.ArtifactStorage.Type) - assert.EqualValues(t, "gitea", Actions.ArtifactStorage.AzureBlobConfig.Container) - assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Actions.ArtifactStorage.AzureBlobConfig.Container) + assert.Equal(t, "actions_artifacts/", Actions.ArtifactStorage.AzureBlobConfig.BasePath) assert.NoError(t, loadAvatarsFrom(cfg)) assert.EqualValues(t, "azureblob", Avatar.Storage.Type) - assert.EqualValues(t, "gitea", Avatar.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "avatars/", Avatar.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", Avatar.Storage.AzureBlobConfig.Container) + assert.Equal(t, "avatars/", Avatar.Storage.AzureBlobConfig.BasePath) assert.NoError(t, loadRepoAvatarFrom(cfg)) assert.EqualValues(t, "azureblob", RepoAvatar.Storage.Type) - assert.EqualValues(t, "gitea", RepoAvatar.Storage.AzureBlobConfig.Container) - assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "gitea", RepoAvatar.Storage.AzureBlobConfig.Container) + assert.Equal(t, "repo-avatars/", RepoAvatar.Storage.AzureBlobConfig.BasePath) } type testLocalStoragePathCase struct { @@ -151,7 +151,7 @@ func testLocalStoragePath(t *testing.T, appDataPath, iniStr string, cases []test assert.EqualValues(t, "local", storage.Type) assert.True(t, filepath.IsAbs(storage.Path)) - assert.EqualValues(t, filepath.Clean(c.expectedPath), filepath.Clean(storage.Path)) + assert.Equal(t, filepath.Clean(c.expectedPath), filepath.Clean(storage.Path)) } } @@ -389,8 +389,8 @@ MINIO_SECRET_ACCESS_KEY = my_secret_key assert.NoError(t, loadRepoArchiveFrom(cfg)) cp := RepoArchive.Storage.ToShadowCopy() - assert.EqualValues(t, "******", cp.MinioConfig.AccessKeyID) - assert.EqualValues(t, "******", cp.MinioConfig.SecretAccessKey) + assert.Equal(t, "******", cp.MinioConfig.AccessKeyID) + assert.Equal(t, "******", cp.MinioConfig.SecretAccessKey) } func Test_getStorageConfiguration24(t *testing.T) { @@ -445,10 +445,10 @@ MINIO_USE_SSL = true `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) assert.True(t, RepoArchive.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) } func Test_getStorageConfiguration28(t *testing.T) { @@ -462,10 +462,10 @@ MINIO_BASE_PATH = /prefix `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", RepoArchive.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", RepoArchive.Storage.MinioConfig.SecretAccessKey) assert.True(t, RepoArchive.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -476,9 +476,9 @@ MINIO_BASE_PATH = /prefix `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "127.0.0.1", RepoArchive.Storage.MinioConfig.IamEndpoint) + assert.Equal(t, "127.0.0.1", RepoArchive.Storage.MinioConfig.IamEndpoint) assert.True(t, RepoArchive.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) + assert.Equal(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -493,10 +493,10 @@ MINIO_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) assert.True(t, LFS.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "/lfs", LFS.Storage.MinioConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -511,10 +511,10 @@ MINIO_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) - assert.EqualValues(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) + assert.Equal(t, "my_access_key", LFS.Storage.MinioConfig.AccessKeyID) + assert.Equal(t, "my_secret_key", LFS.Storage.MinioConfig.SecretAccessKey) assert.True(t, LFS.Storage.MinioConfig.UseSSL) - assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath) + assert.Equal(t, "/lfs", LFS.Storage.MinioConfig.BasePath) } func Test_getStorageConfiguration29(t *testing.T) { @@ -539,9 +539,9 @@ AZURE_BLOB_ACCOUNT_KEY = my_account_key `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) } func Test_getStorageConfiguration31(t *testing.T) { @@ -554,9 +554,9 @@ AZURE_BLOB_BASE_PATH = /prefix `) assert.NoError(t, err) assert.NoError(t, loadRepoArchiveFrom(cfg)) - assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "/prefix/repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -570,9 +570,9 @@ AZURE_BLOB_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) cfg, err = NewConfigProviderFromData(` [storage] @@ -586,7 +586,7 @@ AZURE_BLOB_BASE_PATH = /lfs `) assert.NoError(t, err) assert.NoError(t, loadLFSFrom(cfg)) - assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) - assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) - assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) + assert.Equal(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName) + assert.Equal(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey) + assert.Equal(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath) } diff --git a/modules/ssh/init.go b/modules/ssh/init.go index 21d4f89936..cfb0d5693a 100644 --- a/modules/ssh/init.go +++ b/modules/ssh/init.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) func Init() error { @@ -23,20 +24,17 @@ func Init() error { if setting.SSH.StartBuiltinServer { Listen(setting.SSH.ListenHost, setting.SSH.ListenPort, setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs) - log.Info("SSH server started on %s. Cipher list (%v), key exchange algorithms (%v), MACs (%v)", + log.Info("SSH server started on %q. Ciphers: %v, key exchange algorithms: %v, MACs: %v", net.JoinHostPort(setting.SSH.ListenHost, strconv.Itoa(setting.SSH.ListenPort)), - setting.SSH.ServerCiphers, setting.SSH.ServerKeyExchanges, setting.SSH.ServerMACs, + util.Iif[any](setting.SSH.ServerCiphers == nil, "default", setting.SSH.ServerCiphers), + util.Iif[any](setting.SSH.ServerKeyExchanges == nil, "default", setting.SSH.ServerKeyExchanges), + util.Iif[any](setting.SSH.ServerMACs == nil, "default", setting.SSH.ServerMACs), ) return nil } builtinUnused() - // FIXME: why 0o644 for a directory ..... - if err := os.MkdirAll(setting.SSH.KeyTestPath, 0o644); err != nil { - return fmt.Errorf("failed to create directory %q for ssh key test: %w", setting.SSH.KeyTestPath, err) - } - if len(setting.SSH.TrustedUserCAKeys) > 0 && setting.SSH.AuthorizedPrincipalsEnabled { caKeysFileName := setting.SSH.TrustedUserCAKeysFile caKeysFileDir := filepath.Dir(caKeysFileName) diff --git a/modules/ssh/ssh.go b/modules/ssh/ssh.go index 7479cfbd95..3fea4851c7 100644 --- a/modules/ssh/ssh.go +++ b/modules/ssh/ssh.go @@ -11,7 +11,6 @@ import ( "crypto/x509" "encoding/pem" "errors" - "fmt" "io" "net" "os" @@ -216,7 +215,7 @@ func publicKeyHandler(ctx ssh.Context, key ssh.PublicKey) bool { ctx.Permissions().Permissions = &gossh.Permissions{} setPermExt := func(keyID int64) { ctx.Permissions().Permissions.Extensions = map[string]string{ - giteaPermissionExtensionKeyID: fmt.Sprint(keyID), + giteaPermissionExtensionKeyID: strconv.FormatInt(keyID, 10), } } @@ -334,7 +333,7 @@ func sshConnectionFailed(conn net.Conn, err error) { log.Warn("Failed authentication attempt from %s", conn.RemoteAddr()) } -// Listen starts a SSH server listens on given port. +// Listen starts an SSH server listening on given port. func Listen(host string, port int, ciphers, keyExchanges, macs []string) { srv := ssh.Server{ Addr: net.JoinHostPort(host, strconv.Itoa(port)), diff --git a/modules/storage/azureblob_test.go b/modules/storage/azureblob_test.go index 6905db5008..b3791b4916 100644 --- a/modules/storage/azureblob_test.go +++ b/modules/storage/azureblob_test.go @@ -4,9 +4,9 @@ package storage import ( - "bytes" "io" "os" + "strings" "testing" "code.gitea.io/gitea/modules/setting" @@ -33,14 +33,14 @@ func TestAzureBlobStorageIterator(t *testing.T) { func TestAzureBlobStoragePath(t *testing.T) { m := &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: ""}} - assert.Equal(t, "", m.buildAzureBlobPath("/")) - assert.Equal(t, "", m.buildAzureBlobPath(".")) + assert.Empty(t, m.buildAzureBlobPath("/")) + assert.Empty(t, m.buildAzureBlobPath(".")) assert.Equal(t, "a", m.buildAzureBlobPath("/a")) assert.Equal(t, "a/b", m.buildAzureBlobPath("/a/b/")) m = &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: "/"}} - assert.Equal(t, "", m.buildAzureBlobPath("/")) - assert.Equal(t, "", m.buildAzureBlobPath(".")) + assert.Empty(t, m.buildAzureBlobPath("/")) + assert.Empty(t, m.buildAzureBlobPath(".")) assert.Equal(t, "a", m.buildAzureBlobPath("/a")) assert.Equal(t, "a/b", m.buildAzureBlobPath("/a/b/")) @@ -76,7 +76,7 @@ func Test_azureBlobObject(t *testing.T) { assert.NoError(t, err) data := "Q2xTckt6Y1hDOWh0" - _, err = s.Save("test.txt", bytes.NewBufferString(data), int64(len(data))) + _, err = s.Save("test.txt", strings.NewReader(data), int64(len(data))) assert.NoError(t, err) obj, err := s.Open("test.txt") assert.NoError(t, err) @@ -86,7 +86,7 @@ func Test_azureBlobObject(t *testing.T) { buf1 := make([]byte, 3) read, err := obj.Read(buf1) assert.NoError(t, err) - assert.EqualValues(t, 3, read) + assert.Equal(t, 3, read) assert.Equal(t, data[2:5], string(buf1)) offset, err = obj.Seek(-5, io.SeekEnd) assert.NoError(t, err) @@ -94,7 +94,7 @@ func Test_azureBlobObject(t *testing.T) { buf2 := make([]byte, 4) read, err = obj.Read(buf2) assert.NoError(t, err) - assert.EqualValues(t, 4, read) + assert.Equal(t, 4, read) assert.Equal(t, data[11:15], string(buf2)) assert.NoError(t, obj.Close()) assert.NoError(t, s.Delete("test.txt")) diff --git a/modules/storage/local_test.go b/modules/storage/local_test.go index 540ced1655..0592fd716b 100644 --- a/modules/storage/local_test.go +++ b/modules/storage/local_test.go @@ -48,7 +48,7 @@ func TestBuildLocalPath(t *testing.T) { t.Run(k.path, func(t *testing.T) { l := LocalStorage{dir: k.localDir} - assert.EqualValues(t, k.expected, l.buildLocalPath(k.path)) + assert.Equal(t, k.expected, l.buildLocalPath(k.path)) }) } } diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 6b92be61fb..1c5d25b2d4 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -86,13 +86,14 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath) var lookup minio.BucketLookupType - if config.BucketLookUpType == "auto" || config.BucketLookUpType == "" { + switch config.BucketLookUpType { + case "auto", "": lookup = minio.BucketLookupAuto - } else if config.BucketLookUpType == "dns" { + case "dns": lookup = minio.BucketLookupDNS - } else if config.BucketLookUpType == "path" { + case "path": lookup = minio.BucketLookupPath - } else { + default: return nil, fmt.Errorf("invalid minio bucket lookup type: %s", config.BucketLookUpType) } diff --git a/modules/storage/minio_test.go b/modules/storage/minio_test.go index 395da051e8..2726d765dd 100644 --- a/modules/storage/minio_test.go +++ b/modules/storage/minio_test.go @@ -34,19 +34,19 @@ func TestMinioStorageIterator(t *testing.T) { func TestMinioStoragePath(t *testing.T) { m := &MinioStorage{basePath: ""} - assert.Equal(t, "", m.buildMinioPath("/")) - assert.Equal(t, "", m.buildMinioPath(".")) + assert.Empty(t, m.buildMinioPath("/")) + assert.Empty(t, m.buildMinioPath(".")) assert.Equal(t, "a", m.buildMinioPath("/a")) assert.Equal(t, "a/b", m.buildMinioPath("/a/b/")) - assert.Equal(t, "", m.buildMinioDirPrefix("")) + assert.Empty(t, m.buildMinioDirPrefix("")) assert.Equal(t, "a/", m.buildMinioDirPrefix("/a/")) m = &MinioStorage{basePath: "/"} - assert.Equal(t, "", m.buildMinioPath("/")) - assert.Equal(t, "", m.buildMinioPath(".")) + assert.Empty(t, m.buildMinioPath("/")) + assert.Empty(t, m.buildMinioPath(".")) assert.Equal(t, "a", m.buildMinioPath("/a")) assert.Equal(t, "a/b", m.buildMinioPath("/a/b/")) - assert.Equal(t, "", m.buildMinioDirPrefix("")) + assert.Empty(t, m.buildMinioDirPrefix("")) assert.Equal(t, "a/", m.buildMinioDirPrefix("/a/")) m = &MinioStorage{basePath: "/base"} diff --git a/modules/storage/storage_test.go b/modules/storage/storage_test.go index 7edde558f3..08f274e74b 100644 --- a/modules/storage/storage_test.go +++ b/modules/storage/storage_test.go @@ -4,7 +4,7 @@ package storage import ( - "bytes" + "strings" "testing" "code.gitea.io/gitea/modules/setting" @@ -26,7 +26,7 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) { {"b/x 4.txt", "bx4"}, } for _, f := range testFiles { - _, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1) + _, err = l.Save(f[0], strings.NewReader(f[1]), -1) assert.NoError(t, err) } diff --git a/modules/structs/admin_user.go b/modules/structs/admin_user.go index f7c6d10ba0..c68b59a897 100644 --- a/modules/structs/admin_user.go +++ b/modules/structs/admin_user.go @@ -8,8 +8,11 @@ import "time" // CreateUserOption create user options type CreateUserOption struct { - SourceID int64 `json:"source_id"` + SourceID int64 `json:"source_id"` + // identifier of the user, provided by the external authenticator (if configured) + // default: empty LoginName string `json:"login_name"` + // username of the user // required: true Username string `json:"username" binding:"Required;Username;MaxSize(40)"` FullName string `json:"full_name" binding:"MaxSize(100)"` @@ -32,6 +35,8 @@ type CreateUserOption struct { type EditUserOption struct { // required: true SourceID int64 `json:"source_id"` + // identifier of the user, provided by the external authenticator (if configured) + // default: empty // required: true LoginName string `json:"login_name" binding:"Required"` // swagger:strfmt email diff --git a/modules/structs/commit_status_test.go b/modules/structs/commit_status_test.go deleted file mode 100644 index 88e09aadc1..0000000000 --- a/modules/structs/commit_status_test.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package structs - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestNoBetterThan(t *testing.T) { - type args struct { - css CommitStatusState - css2 CommitStatusState - } - var unExpectedState CommitStatusState - tests := []struct { - name string - args args - want bool - }{ - { - name: "success is no better than success", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "success is no better than pending", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusPending, - }, - want: false, - }, - { - name: "success is no better than failure", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusFailure, - }, - want: false, - }, - { - name: "success is no better than error", - args: args{ - css: CommitStatusSuccess, - css2: CommitStatusError, - }, - want: false, - }, - { - name: "pending is no better than success", - args: args{ - css: CommitStatusPending, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "pending is no better than pending", - args: args{ - css: CommitStatusPending, - css2: CommitStatusPending, - }, - want: true, - }, - { - name: "pending is no better than failure", - args: args{ - css: CommitStatusPending, - css2: CommitStatusFailure, - }, - want: false, - }, - { - name: "pending is no better than error", - args: args{ - css: CommitStatusPending, - css2: CommitStatusError, - }, - want: false, - }, - { - name: "failure is no better than success", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "failure is no better than pending", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusPending, - }, - want: true, - }, - { - name: "failure is no better than failure", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusFailure, - }, - want: true, - }, - { - name: "failure is no better than error", - args: args{ - css: CommitStatusFailure, - css2: CommitStatusError, - }, - want: false, - }, - { - name: "error is no better than success", - args: args{ - css: CommitStatusError, - css2: CommitStatusSuccess, - }, - want: true, - }, - { - name: "error is no better than pending", - args: args{ - css: CommitStatusError, - css2: CommitStatusPending, - }, - want: true, - }, - { - name: "error is no better than failure", - args: args{ - css: CommitStatusError, - css2: CommitStatusFailure, - }, - want: true, - }, - { - name: "error is no better than error", - args: args{ - css: CommitStatusError, - css2: CommitStatusError, - }, - want: true, - }, - { - name: "unExpectedState is no better than success", - args: args{ - css: unExpectedState, - css2: CommitStatusSuccess, - }, - want: false, - }, - { - name: "unExpectedState is no better than unExpectedState", - args: args{ - css: unExpectedState, - css2: unExpectedState, - }, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := tt.args.css.NoBetterThan(tt.args.css2) - assert.Equal(t, tt.want, result) - }) - } -} diff --git a/modules/structs/git_blob.go b/modules/structs/git_blob.go index 96c7a271a9..643b69ed37 100644 --- a/modules/structs/git_blob.go +++ b/modules/structs/git_blob.go @@ -5,9 +5,12 @@ package structs // GitBlobResponse represents a git blob type GitBlobResponse struct { - Content string `json:"content"` - Encoding string `json:"encoding"` - URL string `json:"url"` - SHA string `json:"sha"` - Size int64 `json:"size"` + Content *string `json:"content"` + Encoding *string `json:"encoding"` + URL string `json:"url"` + SHA string `json:"sha"` + Size int64 `json:"size"` + + LfsOid *string `json:"lfs_oid,omitempty"` + LfsSize *int64 `json:"lfs_size,omitempty"` } diff --git a/modules/structs/hook.go b/modules/structs/hook.go index aaa9fbc9d3..ac779a5740 100644 --- a/modules/structs/hook.go +++ b/modules/structs/hook.go @@ -71,7 +71,8 @@ type PayloadUser struct { // Full name of the commit author Name string `json:"name"` // swagger:strfmt email - Email string `json:"email"` + Email string `json:"email"` + // username of the user UserName string `json:"username"` } @@ -286,6 +287,8 @@ const ( HookIssueReOpened HookIssueAction = "reopened" // HookIssueEdited edited HookIssueEdited HookIssueAction = "edited" + // HookIssueDeleted is an issue action for deleting an issue + HookIssueDeleted HookIssueAction = "deleted" // HookIssueAssigned assigned HookIssueAssigned HookIssueAction = "assigned" // HookIssueUnassigned unassigned @@ -470,6 +473,22 @@ func (p *CommitStatusPayload) JSONPayload() ([]byte, error) { return json.MarshalIndent(p, "", " ") } +// WorkflowRunPayload represents a payload information of workflow run event. +type WorkflowRunPayload struct { + Action string `json:"action"` + Workflow *ActionWorkflow `json:"workflow"` + WorkflowRun *ActionWorkflowRun `json:"workflow_run"` + PullRequest *PullRequest `json:"pull_request,omitempty"` + Organization *Organization `json:"organization,omitempty"` + Repo *Repository `json:"repository"` + Sender *User `json:"sender"` +} + +// JSONPayload implements Payload +func (p *WorkflowRunPayload) JSONPayload() ([]byte, error) { + return json.MarshalIndent(p, "", " ") +} + // WorkflowJobPayload represents a payload information of workflow job event. type WorkflowJobPayload struct { Action string `json:"action"` diff --git a/modules/structs/issue.go b/modules/structs/issue.go index 3682191be5..df0be8f9ec 100644 --- a/modules/structs/issue.go +++ b/modules/structs/issue.go @@ -203,7 +203,7 @@ func (l *IssueTemplateStringSlice) UnmarshalYAML(value *yaml.Node) error { if err != nil { return err } - for _, v := range strings.Split(str, ",") { + for v := range strings.SplitSeq(str, ",") { if v = strings.TrimSpace(v); v == "" { continue } @@ -266,3 +266,8 @@ type IssueMeta struct { Owner string `json:"owner"` Name string `json:"repo"` } + +// LockIssueOption options to lock an issue +type LockIssueOption struct { + Reason string `json:"lock_reason"` +} diff --git a/modules/structs/issue_tracked_time.go b/modules/structs/issue_tracked_time.go index a3904af80e..befcfb323d 100644 --- a/modules/structs/issue_tracked_time.go +++ b/modules/structs/issue_tracked_time.go @@ -14,7 +14,7 @@ type AddTimeOption struct { Time int64 `json:"time" binding:"Required"` // swagger:strfmt date-time Created time.Time `json:"created"` - // User who spent the time (optional) + // username of the user who spent the time working on the issue (optional) User string `json:"user_name"` } @@ -26,7 +26,8 @@ type TrackedTime struct { // Time in seconds Time int64 `json:"time"` // deprecated (only for backwards compatibility) - UserID int64 `json:"user_id"` + UserID int64 `json:"user_id"` + // username of the user UserName string `json:"user_name"` // deprecated (only for backwards compatibility) IssueID int64 `json:"issue_id"` diff --git a/modules/structs/org.go b/modules/structs/org.go index f93b3b6493..33b45c6344 100644 --- a/modules/structs/org.go +++ b/modules/structs/org.go @@ -15,6 +15,7 @@ type Organization struct { Location string `json:"location"` Visibility string `json:"visibility"` RepoAdminChangeTeamAccess bool `json:"repo_admin_change_team_access"` + // username of the organization // deprecated UserName string `json:"username"` } @@ -30,6 +31,7 @@ type OrganizationPermissions struct { // CreateOrgOption options for creating an organization type CreateOrgOption struct { + // username of the organization // required: true UserName string `json:"username" binding:"Required;Username;MaxSize(40)"` FullName string `json:"full_name" binding:"MaxSize(100)"` diff --git a/modules/structs/package.go b/modules/structs/package.go index a9a9429de2..1973f925a5 100644 --- a/modules/structs/package.go +++ b/modules/structs/package.go @@ -23,8 +23,8 @@ type Package struct { // PackageFile represents a package file type PackageFile struct { - ID int64 `json:"id"` - Size int64 + ID int64 `json:"id"` + Size int64 `json:"size"` Name string `json:"name"` HashMD5 string `json:"md5"` HashSHA1 string `json:"sha1"` diff --git a/modules/structs/release.go b/modules/structs/release.go index c7378645c2..fac86ca7a2 100644 --- a/modules/structs/release.go +++ b/modules/structs/release.go @@ -33,6 +33,7 @@ type Release struct { type CreateReleaseOption struct { // required: true TagName string `json:"tag_name" binding:"Required"` + TagMessage string `json:"tag_message"` Target string `json:"target_commitish"` Title string `json:"name"` Note string `json:"body"` diff --git a/modules/structs/repo.go b/modules/structs/repo.go index fb784bd8b3..aca5d9c3f4 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -57,7 +57,7 @@ type Repository struct { Private bool `json:"private"` Fork bool `json:"fork"` Template bool `json:"template"` - Parent *Repository `json:"parent"` + Parent *Repository `json:"parent,omitempty"` Mirror bool `json:"mirror"` Size int `json:"size"` Language string `json:"language"` @@ -101,6 +101,8 @@ type Repository struct { AllowSquash bool `json:"allow_squash_merge"` AllowFastForwardOnly bool `json:"allow_fast_forward_only_merge"` AllowRebaseUpdate bool `json:"allow_rebase_update"` + AllowManualMerge bool `json:"allow_manual_merge"` + AutodetectManualMerge bool `json:"autodetect_manual_merge"` DefaultDeleteBranchAfterMerge bool `json:"default_delete_branch_after_merge"` DefaultMergeStyle string `json:"default_merge_style"` DefaultAllowMaintainerEdit bool `json:"default_allow_maintainer_edit"` @@ -111,8 +113,8 @@ type Repository struct { // enum: sha1,sha256 ObjectFormatName string `json:"object_format_name"` // swagger:strfmt date-time - MirrorUpdated time.Time `json:"mirror_updated,omitempty"` - RepoTransfer *RepoTransfer `json:"repo_transfer"` + MirrorUpdated time.Time `json:"mirror_updated"` + RepoTransfer *RepoTransfer `json:"repo_transfer,omitempty"` Topics []string `json:"topics"` Licenses []string `json:"licenses"` } @@ -357,7 +359,7 @@ type MigrateRepoOptions struct { // required: true RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"` - // enum: git,github,gitea,gitlab,gogs,onedev,gitbucket,codebase + // enum: git,github,gitea,gitlab,gogs,onedev,gitbucket,codebase,codecommit Service string `json:"service"` AuthUsername string `json:"auth_username"` AuthPassword string `json:"auth_password"` diff --git a/modules/structs/repo_actions.go b/modules/structs/repo_actions.go index 22409b4aff..ac1c288270 100644 --- a/modules/structs/repo_actions.go +++ b/modules/structs/repo_actions.go @@ -57,7 +57,7 @@ type ActionWorkflow struct { HTMLURL string `json:"html_url"` BadgeURL string `json:"badge_url"` // swagger:strfmt date-time - DeletedAt time.Time `json:"deleted_at,omitempty"` + DeletedAt time.Time `json:"deleted_at"` } // ActionWorkflowResponse returns a ActionWorkflow @@ -86,9 +86,39 @@ type ActionArtifact struct { // ActionWorkflowRun represents a WorkflowRun type ActionWorkflowRun struct { - ID int64 `json:"id"` - RepositoryID int64 `json:"repository_id"` - HeadSha string `json:"head_sha"` + ID int64 `json:"id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + DisplayTitle string `json:"display_title"` + Path string `json:"path"` + Event string `json:"event"` + RunAttempt int64 `json:"run_attempt"` + RunNumber int64 `json:"run_number"` + RepositoryID int64 `json:"repository_id,omitempty"` + HeadSha string `json:"head_sha"` + HeadBranch string `json:"head_branch,omitempty"` + Status string `json:"status"` + Actor *User `json:"actor,omitempty"` + TriggerActor *User `json:"trigger_actor,omitempty"` + Repository *Repository `json:"repository,omitempty"` + HeadRepository *Repository `json:"head_repository,omitempty"` + Conclusion string `json:"conclusion,omitempty"` + // swagger:strfmt date-time + StartedAt time.Time `json:"started_at"` + // swagger:strfmt date-time + CompletedAt time.Time `json:"completed_at"` +} + +// ActionWorkflowRunsResponse returns ActionWorkflowRuns +type ActionWorkflowRunsResponse struct { + Entries []*ActionWorkflowRun `json:"workflow_runs"` + TotalCount int64 `json:"total_count"` +} + +// ActionWorkflowJobsResponse returns ActionWorkflowJobs +type ActionWorkflowJobsResponse struct { + Entries []*ActionWorkflowJob `json:"jobs"` + TotalCount int64 `json:"total_count"` } // ActionArtifactsResponse returns ActionArtifacts @@ -104,9 +134,9 @@ type ActionWorkflowStep struct { Status string `json:"status"` Conclusion string `json:"conclusion,omitempty"` // swagger:strfmt date-time - StartedAt time.Time `json:"started_at,omitempty"` + StartedAt time.Time `json:"started_at"` // swagger:strfmt date-time - CompletedAt time.Time `json:"completed_at,omitempty"` + CompletedAt time.Time `json:"completed_at"` } // ActionWorkflowJob represents a WorkflowJob @@ -129,7 +159,30 @@ type ActionWorkflowJob struct { // swagger:strfmt date-time CreatedAt time.Time `json:"created_at"` // swagger:strfmt date-time - StartedAt time.Time `json:"started_at,omitempty"` + StartedAt time.Time `json:"started_at"` // swagger:strfmt date-time - CompletedAt time.Time `json:"completed_at,omitempty"` + CompletedAt time.Time `json:"completed_at"` +} + +// ActionRunnerLabel represents a Runner Label +type ActionRunnerLabel struct { + ID int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` +} + +// ActionRunner represents a Runner +type ActionRunner struct { + ID int64 `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Busy bool `json:"busy"` + Ephemeral bool `json:"ephemeral"` + Labels []*ActionRunnerLabel `json:"labels"` +} + +// ActionRunnersResponse returns Runners +type ActionRunnersResponse struct { + Entries []*ActionRunner `json:"runners"` + TotalCount int64 `json:"total_count"` } diff --git a/modules/structs/repo_branch.go b/modules/structs/repo_branch.go index 55c98d60b9..5416f43b0d 100644 --- a/modules/structs/repo_branch.go +++ b/modules/structs/repo_branch.go @@ -136,6 +136,7 @@ type UpdateBranchProtectionPriories struct { type MergeUpstreamRequest struct { Branch string `json:"branch"` + FfOnly bool `json:"ff_only"` } type MergeUpstreamResponse struct { diff --git a/modules/structs/repo_file.go b/modules/structs/repo_file.go index 82bde96ab6..5a86db868b 100644 --- a/modules/structs/repo_file.go +++ b/modules/structs/repo_file.go @@ -4,6 +4,8 @@ package structs +import "time" + // FileOptions options for all file APIs type FileOptions struct { // message (optional) for the commit of this file. if not supplied, a default message will be used @@ -20,6 +22,23 @@ type FileOptions struct { Signoff bool `json:"signoff"` } +type FileOptionsWithSHA struct { + FileOptions + // the blob ID (SHA) for the file that already exists, it is required for changing existing files + // required: true + SHA string `json:"sha" binding:"Required"` +} + +func (f *FileOptions) GetFileOptions() *FileOptions { + return f +} + +type FileOptionsInterface interface { + GetFileOptions() *FileOptions +} + +var _ FileOptionsInterface = (*FileOptions)(nil) + // CreateFileOptions options for creating files // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type CreateFileOptions struct { @@ -29,29 +48,16 @@ type CreateFileOptions struct { ContentBase64 string `json:"content"` } -// Branch returns branch name -func (o *CreateFileOptions) Branch() string { - return o.FileOptions.BranchName -} - // DeleteFileOptions options for deleting files (used for other File structs below) // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type DeleteFileOptions struct { - FileOptions - // sha is the SHA for the file that already exists - // required: true - SHA string `json:"sha" binding:"Required"` -} - -// Branch returns branch name -func (o *DeleteFileOptions) Branch() string { - return o.FileOptions.BranchName + FileOptionsWithSHA } // UpdateFileOptions options for updating files // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type UpdateFileOptions struct { - DeleteFileOptions + FileOptionsWithSHA // content must be base64 encoded // required: true ContentBase64 string `json:"content"` @@ -59,23 +65,21 @@ type UpdateFileOptions struct { FromPath string `json:"from_path" binding:"MaxSize(500)"` } -// Branch returns branch name -func (o *UpdateFileOptions) Branch() string { - return o.FileOptions.BranchName -} +// FIXME: there is no LastCommitID in FileOptions, actually it should be an alternative to the SHA in ChangeFileOperation // ChangeFileOperation for creating, updating or deleting a file type ChangeFileOperation struct { - // indicates what to do with the file + // indicates what to do with the file: "create" for creating a new file, "update" for updating an existing file, + // "upload" for creating or updating a file, "rename" for renaming a file, and "delete" for deleting an existing file. // required: true - // enum: create,update,delete + // enum: create,update,upload,rename,delete Operation string `json:"operation" binding:"Required"` // path to the existing or new file // required: true Path string `json:"path" binding:"Required;MaxSize(500)"` - // new or updated file content, must be base64 encoded + // new or updated file content, it must be base64 encoded ContentBase64 string `json:"content"` - // sha is the SHA for the file that already exists, required for update or delete + // the blob ID (SHA) for the file that already exists, required for changing existing files SHA string `json:"sha"` // old path of the file to move FromPath string `json:"from_path"` @@ -90,20 +94,10 @@ type ChangeFilesOptions struct { Files []*ChangeFileOperation `json:"files" binding:"Required"` } -// Branch returns branch name -func (o *ChangeFilesOptions) Branch() string { - return o.FileOptions.BranchName -} - -// FileOptionInterface provides a unified interface for the different file options -type FileOptionInterface interface { - Branch() string -} - // ApplyDiffPatchFileOptions options for applying a diff patch // Note: `author` and `committer` are optional (if only one is given, it will be used for the other, otherwise the authenticated user will be used) type ApplyDiffPatchFileOptions struct { - DeleteFileOptions + FileOptions // required: true Content string `json:"content"` } @@ -115,12 +109,24 @@ type FileLinksResponse struct { HTMLURL *string `json:"html"` } +type ContentsExtResponse struct { + FileContents *ContentsResponse `json:"file_contents,omitempty"` + DirContents []*ContentsResponse `json:"dir_contents,omitempty"` +} + // ContentsResponse contains information about a repo's entry's (dir, file, symlink, submodule) metadata and content type ContentsResponse struct { - Name string `json:"name"` - Path string `json:"path"` - SHA string `json:"sha"` - LastCommitSHA string `json:"last_commit_sha"` + Name string `json:"name"` + Path string `json:"path"` + SHA string `json:"sha"` + + LastCommitSHA *string `json:"last_commit_sha,omitempty"` + // swagger:strfmt date-time + LastCommitterDate *time.Time `json:"last_committer_date,omitempty"` + // swagger:strfmt date-time + LastAuthorDate *time.Time `json:"last_author_date,omitempty"` + LastCommitMessage *string `json:"last_commit_message,omitempty"` + // `type` will be `file`, `dir`, `symlink`, or `submodule` Type string `json:"type"` Size int64 `json:"size"` @@ -137,6 +143,9 @@ type ContentsResponse struct { // `submodule_git_url` is populated when `type` is `submodule`, otherwise null SubmoduleGitURL *string `json:"submodule_git_url"` Links *FileLinksResponse `json:"_links"` + + LfsOid *string `json:"lfs_oid,omitempty"` + LfsSize *int64 `json:"lfs_size,omitempty"` } // FileCommitResponse contains information generated from a Git commit for a repo's file. @@ -170,3 +179,8 @@ type FileDeleteResponse struct { Commit *FileCommitResponse `json:"commit"` Verification *PayloadCommitVerification `json:"verification"` } + +// GetFilesOptions options for retrieving metadate and content of multiple files +type GetFilesOptions struct { + Files []string `json:"files" binding:"Required"` +} diff --git a/modules/structs/repo_tag.go b/modules/structs/repo_tag.go index 5722513f4f..bb8bfd10cb 100644 --- a/modules/structs/repo_tag.go +++ b/modules/structs/repo_tag.go @@ -11,8 +11,8 @@ type Tag struct { Message string `json:"message"` ID string `json:"id"` Commit *CommitMeta `json:"commit"` - ZipballURL string `json:"zipball_url"` - TarballURL string `json:"tarball_url"` + ZipballURL string `json:"zipball_url,omitempty"` + TarballURL string `json:"tarball_url,omitempty"` } // AnnotatedTag represents an annotated tag diff --git a/modules/structs/settings.go b/modules/structs/settings.go index e48b1a493d..59176210e6 100644 --- a/modules/structs/settings.go +++ b/modules/structs/settings.go @@ -26,6 +26,7 @@ type GeneralAPISettings struct { DefaultPagingNum int `json:"default_paging_num"` DefaultGitTreesPerPage int `json:"default_git_trees_per_page"` DefaultMaxBlobSize int64 `json:"default_max_blob_size"` + DefaultMaxResponseSize int64 `json:"default_max_response_size"` } // GeneralAttachmentSettings contains global Attachment settings exposed by API diff --git a/modules/structs/status.go b/modules/structs/status.go index c1d8b902ec..a9779541ff 100644 --- a/modules/structs/status.go +++ b/modules/structs/status.go @@ -5,17 +5,19 @@ package structs import ( "time" + + "code.gitea.io/gitea/modules/commitstatus" ) // CommitStatus holds a single status of a single Commit type CommitStatus struct { - ID int64 `json:"id"` - State CommitStatusState `json:"status"` - TargetURL string `json:"target_url"` - Description string `json:"description"` - URL string `json:"url"` - Context string `json:"context"` - Creator *User `json:"creator"` + ID int64 `json:"id"` + State commitstatus.CommitStatusState `json:"status"` + TargetURL string `json:"target_url"` + Description string `json:"description"` + URL string `json:"url"` + Context string `json:"context"` + Creator *User `json:"creator"` // swagger:strfmt date-time Created time.Time `json:"created_at"` // swagger:strfmt date-time @@ -24,19 +26,19 @@ type CommitStatus struct { // CombinedStatus holds the combined state of several statuses for a single commit type CombinedStatus struct { - State CommitStatusState `json:"state"` - SHA string `json:"sha"` - TotalCount int `json:"total_count"` - Statuses []*CommitStatus `json:"statuses"` - Repository *Repository `json:"repository"` - CommitURL string `json:"commit_url"` - URL string `json:"url"` + State commitstatus.CommitStatusState `json:"state"` + SHA string `json:"sha"` + TotalCount int `json:"total_count"` + Statuses []*CommitStatus `json:"statuses"` + Repository *Repository `json:"repository"` + CommitURL string `json:"commit_url"` + URL string `json:"url"` } // CreateStatusOption holds the information needed to create a new CommitStatus for a Commit type CreateStatusOption struct { - State CommitStatusState `json:"state"` - TargetURL string `json:"target_url"` - Description string `json:"description"` - Context string `json:"context"` + State commitstatus.CommitStatusState `json:"state"` + TargetURL string `json:"target_url"` + Description string `json:"description"` + Context string `json:"context"` } diff --git a/modules/structs/user.go b/modules/structs/user.go index 5ed677f239..89349cda2c 100644 --- a/modules/structs/user.go +++ b/modules/structs/user.go @@ -15,9 +15,9 @@ import ( type User struct { // the user's id ID int64 `json:"id"` - // the user's username + // login of the user, same as `username` UserName string `json:"login"` - // the user's authentication sign-in name. + // identifier of the user, provided by the external authenticator (if configured) // default: empty LoginName string `json:"login_name"` // The ID of the user's Authentication Source @@ -35,9 +35,9 @@ type User struct { // Is the user an administrator IsAdmin bool `json:"is_admin"` // swagger:strfmt date-time - LastLogin time.Time `json:"last_login,omitempty"` + LastLogin time.Time `json:"last_login"` // swagger:strfmt date-time - Created time.Time `json:"created,omitempty"` + Created time.Time `json:"created"` // Is user restricted Restricted bool `json:"restricted"` // Is user active diff --git a/modules/structs/user_app.go b/modules/structs/user_app.go index a7d2e28b41..15811ceb66 100644 --- a/modules/structs/user_app.go +++ b/modules/structs/user_app.go @@ -11,11 +11,13 @@ import ( // AccessToken represents an API access token. // swagger:response AccessToken type AccessToken struct { - ID int64 `json:"id"` - Name string `json:"name"` - Token string `json:"sha1"` - TokenLastEight string `json:"token_last_eight"` - Scopes []string `json:"scopes"` + ID int64 `json:"id"` + Name string `json:"name"` + Token string `json:"sha1"` + TokenLastEight string `json:"token_last_eight"` + Scopes []string `json:"scopes"` + Created time.Time `json:"created_at"` + Updated time.Time `json:"last_used_at"` } // AccessTokenList represents a list of API access token. @@ -23,9 +25,11 @@ type AccessToken struct { type AccessTokenList []*AccessToken // CreateAccessTokenOption options when create access token +// swagger:model CreateAccessTokenOption type CreateAccessTokenOption struct { // required: true - Name string `json:"name" binding:"Required"` + Name string `json:"name" binding:"Required"` + // example: ["all", "read:activitypub","read:issue", "write:misc", "read:notification", "read:organization", "read:package", "read:repository", "read:user"] Scopes []string `json:"scopes"` } diff --git a/modules/structs/user_email.go b/modules/structs/user_email.go index 9319667e8f..01895a0058 100644 --- a/modules/structs/user_email.go +++ b/modules/structs/user_email.go @@ -11,6 +11,7 @@ type Email struct { Verified bool `json:"verified"` Primary bool `json:"primary"` UserID int64 `json:"user_id"` + // username of the user UserName string `json:"username"` } diff --git a/modules/structs/user_gpgkey.go b/modules/structs/user_gpgkey.go index ff9b0aea1d..deae70de33 100644 --- a/modules/structs/user_gpgkey.go +++ b/modules/structs/user_gpgkey.go @@ -21,9 +21,9 @@ type GPGKey struct { CanCertify bool `json:"can_certify"` Verified bool `json:"verified"` // swagger:strfmt date-time - Created time.Time `json:"created_at,omitempty"` + Created time.Time `json:"created_at"` // swagger:strfmt date-time - Expires time.Time `json:"expires_at,omitempty"` + Expires time.Time `json:"expires_at"` } // GPGKeyEmail an email attached to a GPGKey diff --git a/modules/structs/user_key.go b/modules/structs/user_key.go index 08eed59a89..16225a852a 100644 --- a/modules/structs/user_key.go +++ b/modules/structs/user_key.go @@ -15,7 +15,8 @@ type PublicKey struct { Title string `json:"title,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` // swagger:strfmt date-time - Created time.Time `json:"created_at,omitempty"` + Created time.Time `json:"created_at"` + Updated time.Time `json:"last_used_at"` Owner *User `json:"user,omitempty"` ReadOnly bool `json:"read_only,omitempty"` KeyType string `json:"key_type,omitempty"` diff --git a/modules/system/appstate_test.go b/modules/system/appstate_test.go index 911319d00a..b5c057cf88 100644 --- a/modules/system/appstate_test.go +++ b/modules/system/appstate_test.go @@ -38,8 +38,8 @@ func TestAppStateDB(t *testing.T) { item1 := new(testItem1) assert.NoError(t, as.Get(db.DefaultContext, item1)) - assert.Equal(t, "", item1.Val1) - assert.EqualValues(t, 0, item1.Val2) + assert.Empty(t, item1.Val1) + assert.Equal(t, 0, item1.Val2) item1 = new(testItem1) item1.Val1 = "a" @@ -53,7 +53,7 @@ func TestAppStateDB(t *testing.T) { item1 = new(testItem1) assert.NoError(t, as.Get(db.DefaultContext, item1)) assert.Equal(t, "a", item1.Val1) - assert.EqualValues(t, 2, item1.Val2) + assert.Equal(t, 2, item1.Val2) item2 = new(testItem2) assert.NoError(t, as.Get(db.DefaultContext, item2)) diff --git a/modules/tempdir/tempdir.go b/modules/tempdir/tempdir.go new file mode 100644 index 0000000000..22c2e4ea16 --- /dev/null +++ b/modules/tempdir/tempdir.go @@ -0,0 +1,112 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" +) + +type TempDir struct { + // base is the base directory for temporary files, it must exist before accessing and won't be created automatically. + // for example: base="/system-tmpdir", sub="gitea-tmp" + base, sub string +} + +func (td *TempDir) JoinPath(elems ...string) string { + return filepath.Join(append([]string{td.base, td.sub}, elems...)...) +} + +// MkdirAllSub works like os.MkdirAll, but the base directory must exist +func (td *TempDir) MkdirAllSub(dir string) (string, error) { + if _, err := os.Stat(td.base); err != nil { + return "", err + } + full := filepath.Join(td.base, td.sub, dir) + if err := os.MkdirAll(full, os.ModePerm); err != nil { + return "", err + } + return full, nil +} + +func (td *TempDir) prepareDirWithPattern(elems ...string) (dir, pattern string, err error) { + if _, err = os.Stat(td.base); err != nil { + return "", "", err + } + dir, pattern = filepath.Split(filepath.Join(append([]string{td.base, td.sub}, elems...)...)) + if err = os.MkdirAll(dir, os.ModePerm); err != nil { + return "", "", err + } + return dir, pattern, nil +} + +// MkdirTempRandom works like os.MkdirTemp, the last path field is the "pattern" +func (td *TempDir) MkdirTempRandom(elems ...string) (string, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return "", nil, err + } + dir, err = os.MkdirTemp(dir, pattern) + if err != nil { + return "", nil, err + } + return dir, func() { + if err := util.RemoveAll(dir); err != nil { + log.Error("Failed to remove temp directory %s: %v", dir, err) + } + }, nil +} + +// CreateTempFileRandom works like os.CreateTemp, the last path field is the "pattern" +func (td *TempDir) CreateTempFileRandom(elems ...string) (*os.File, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return nil, nil, err + } + f, err := os.CreateTemp(dir, pattern) + if err != nil { + return nil, nil, err + } + filename := f.Name() + return f, func() { + _ = f.Close() + if err := util.Remove(filename); err != nil { + log.Error("Unable to remove temporary file: %s: Error: %v", filename, err) + } + }, err +} + +func (td *TempDir) RemoveOutdated(d time.Duration) { + var remove func(path string) + remove = func(path string) { + entries, _ := os.ReadDir(path) + for _, entry := range entries { + full := filepath.Join(path, entry.Name()) + if entry.IsDir() { + remove(full) + _ = os.Remove(full) + continue + } + info, err := entry.Info() + if err == nil && time.Since(info.ModTime()) > d { + _ = os.Remove(full) + } + } + } + remove(td.JoinPath("")) +} + +// New create a new TempDir instance, "base" must be an existing directory, +// "sub" could be a multi-level directory and will be created if not exist +func New(base, sub string) *TempDir { + return &TempDir{base: base, sub: sub} +} + +func OsTempDir(sub string) *TempDir { + return New(os.TempDir(), sub) +} diff --git a/modules/tempdir/tempdir_test.go b/modules/tempdir/tempdir_test.go new file mode 100644 index 0000000000..d6afcb7bed --- /dev/null +++ b/modules/tempdir/tempdir_test.go @@ -0,0 +1,75 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestTempDir(t *testing.T) { + base := t.TempDir() + + t.Run("Create", func(t *testing.T) { + td := New(base, "sub1/sub2") // make sure the sub dir supports "/" in the path + assert.Equal(t, filepath.Join(base, "sub1", "sub2"), td.JoinPath()) + assert.Equal(t, filepath.Join(base, "sub1", "sub2/test"), td.JoinPath("test")) + + t.Run("MkdirTempRandom", func(t *testing.T) { + s, cleanup, err := td.MkdirTempRandom("foo") + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(s, filepath.Join(base, "sub1/sub2", "foo"))) + + _, err = os.Stat(s) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(s) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("CreateTempFileRandom", func(t *testing.T) { + f, cleanup, err := td.CreateTempFileRandom("foo", "bar") + filename := f.Name() + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(filename, filepath.Join(base, "sub1/sub2", "foo", "bar"))) + _, err = os.Stat(filename) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(filename) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("RemoveOutDated", func(t *testing.T) { + fa1, _, err := td.CreateTempFileRandom("dir-a", "f1") + assert.NoError(t, err) + fa2, _, err := td.CreateTempFileRandom("dir-a", "f2") + assert.NoError(t, err) + _ = os.Chtimes(fa2.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + fb1, _, err := td.CreateTempFileRandom("dir-b", "f1") + assert.NoError(t, err) + _ = os.Chtimes(fb1.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + _, _, _ = fa1.Close(), fa2.Close(), fb1.Close() + + td.RemoveOutdated(time.Minute) + + _, err = os.Stat(fa1.Name()) + assert.NoError(t, err) + _, err = os.Stat(fa2.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(fb1.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + }) + + t.Run("BaseNotExist", func(t *testing.T) { + td := New(filepath.Join(base, "not-exist"), "sub") + _, _, err := td.MkdirTempRandom("foo") + assert.ErrorIs(t, err, os.ErrNotExist) + }) +} diff --git a/modules/templates/eval/eval_test.go b/modules/templates/eval/eval_test.go index c9e514b5eb..f956f6cbdf 100644 --- a/modules/templates/eval/eval_test.go +++ b/modules/templates/eval/eval_test.go @@ -12,7 +12,7 @@ import ( ) func tokens(s string) (a []any) { - for _, v := range strings.Fields(s) { + for v := range strings.FieldsSeq(s) { a = append(a, v) } return a diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 3237f8b295..e454bce4bd 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -6,9 +6,9 @@ package templates import ( "fmt" - "html" "html/template" "net/url" + "strconv" "strings" "time" @@ -37,12 +37,9 @@ func NewFuncMap() template.FuncMap { "dict": dict, // it's lowercase because this name has been widely used. Our other functions should have uppercase names. "Iif": iif, "Eval": evalTokens, - "SafeHTML": safeHTML, "HTMLFormat": htmlFormat, - "HTMLEscape": htmlEscape, "QueryEscape": queryEscape, "QueryBuild": QueryBuild, - "JSEscape": jsEscapeSafe, "SanitizeHTML": SanitizeHTML, "URLJoin": util.URLJoin, "DotEscape": dotEscape, @@ -73,7 +70,7 @@ func NewFuncMap() template.FuncMap { "TimeEstimateString": timeEstimateString, "LoadTimes": func(startTime time.Time) string { - return fmt.Sprint(time.Since(startTime).Nanoseconds()/1e6) + "ms" + return strconv.FormatInt(time.Since(startTime).Nanoseconds()/1e6, 10) + "ms" }, // ----------------------------------------------------------------- @@ -161,49 +158,12 @@ func NewFuncMap() template.FuncMap { "FilenameIsImage": filenameIsImage, "TabSizeClass": tabSizeClass, - - // for backward compatibility only, do not use them anymore - "TimeSince": timeSinceLegacy, - "TimeSinceUnix": timeSinceLegacy, - "DateTime": dateTimeLegacy, - - "RenderEmoji": renderEmojiLegacy, - "RenderLabel": renderLabelLegacy, - "RenderLabels": renderLabelsLegacy, - "RenderIssueTitle": renderIssueTitleLegacy, - - "RenderMarkdownToHtml": renderMarkdownToHtmlLegacy, - - "RenderCommitMessage": renderCommitMessageLegacy, - "RenderCommitMessageLinkSubject": renderCommitMessageLinkSubjectLegacy, - "RenderCommitBody": renderCommitBodyLegacy, } } -// safeHTML render raw as HTML -func safeHTML(s any) template.HTML { - switch v := s.(type) { - case string: - return template.HTML(v) - case template.HTML: - return v - } - panic(fmt.Sprintf("unexpected type %T", s)) -} - -// SanitizeHTML sanitizes the input by pre-defined markdown rules +// SanitizeHTML sanitizes the input by default sanitization rules. func SanitizeHTML(s string) template.HTML { - return template.HTML(markup.Sanitize(s)) -} - -func htmlEscape(s any) template.HTML { - switch v := s.(type) { - case string: - return template.HTML(html.EscapeString(v)) - case template.HTML: - return v - } - panic(fmt.Sprintf("unexpected type %T", s)) + return markup.Sanitize(s) } func htmlFormat(s any, args ...any) template.HTML { @@ -220,10 +180,6 @@ func htmlFormat(s any, args ...any) template.HTML { panic(fmt.Sprintf("unexpected type %T", s)) } -func jsEscapeSafe(s string) template.HTML { - return template.HTML(template.JSEscapeString(s)) -} - func queryEscape(s string) template.URL { return template.URL(url.QueryEscape(s)) } @@ -366,7 +322,3 @@ func QueryBuild(a ...any) template.URL { } return template.URL(s) } - -func panicIfDevOrTesting() { - setting.PanicInDevOrTesting("legacy template functions are for backward compatibility only, do not use them in new code") -} diff --git a/modules/templates/helper_test.go b/modules/templates/helper_test.go index 5d7bc93622..7e3a952e7b 100644 --- a/modules/templates/helper_test.go +++ b/modules/templates/helper_test.go @@ -15,7 +15,7 @@ import ( func TestSubjectBodySeparator(t *testing.T) { test := func(input, subject, body string) { - loc := mailSubjectSplit.FindIndex([]byte(input)) + loc := mailSubjectSplit.FindStringIndex(input) if loc == nil { assert.Empty(t, subject, "no subject found, but one expected") assert.Equal(t, body, input) @@ -57,10 +57,6 @@ func TestSubjectBodySeparator(t *testing.T) { "Insufficient\n--\nSeparators") } -func TestJSEscapeSafe(t *testing.T) { - assert.EqualValues(t, `\u0026\u003C\u003E\'\"`, jsEscapeSafe(`&<>'"`)) -} - func TestSanitizeHTML(t *testing.T) { assert.Equal(t, template.HTML(`<a href="/" rel="nofollow">link</a> xss <div>inline</div>`), SanitizeHTML(`<a href="/">link</a> <a href="javascript:">xss</a> <div style="dangerous">inline</div>`)) } @@ -120,8 +116,8 @@ func TestTemplateEscape(t *testing.T) { func TestQueryBuild(t *testing.T) { t.Run("construct", func(t *testing.T) { - assert.Equal(t, "", string(QueryBuild())) - assert.Equal(t, "", string(QueryBuild("a", nil, "b", false, "c", 0, "d", ""))) + assert.Empty(t, string(QueryBuild())) + assert.Empty(t, string(QueryBuild("a", nil, "b", false, "c", 0, "d", ""))) assert.Equal(t, "a=1&b=true", string(QueryBuild("a", 1, "b", "true"))) // path with query parameters @@ -136,9 +132,9 @@ func TestQueryBuild(t *testing.T) { // only query parameters assert.Equal(t, "&k=1", string(QueryBuild("&", "k", 1))) - assert.Equal(t, "", string(QueryBuild("&", "k", 0))) - assert.Equal(t, "", string(QueryBuild("&k=a", "k", 0))) - assert.Equal(t, "", string(QueryBuild("k=a&", "k", 0))) + assert.Empty(t, string(QueryBuild("&", "k", 0))) + assert.Empty(t, string(QueryBuild("&k=a", "k", 0))) + assert.Empty(t, string(QueryBuild("k=a&", "k", 0))) assert.Equal(t, "a=1&b=2", string(QueryBuild("a=1", "b", 2))) assert.Equal(t, "&a=1&b=2", string(QueryBuild("&a=1", "b", 2))) assert.Equal(t, "a=1&b=2&", string(QueryBuild("a=1&", "b", 2))) diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go index 529284f7e8..8073a6e5f5 100644 --- a/modules/templates/htmlrenderer.go +++ b/modules/templates/htmlrenderer.go @@ -42,7 +42,7 @@ var ( var ErrTemplateNotInitialized = errors.New("template system is not initialized, check your log for errors") -func (h *HTMLRender) HTML(w io.Writer, status int, tplName TplName, data any, ctx context.Context) error { //nolint:revive +func (h *HTMLRender) HTML(w io.Writer, status int, tplName TplName, data any, ctx context.Context) error { //nolint:revive // we don't use ctx, only pass it to the template executor name := string(tplName) if respWriter, ok := w.(http.ResponseWriter); ok { if respWriter.Header().Get("Content-Type") == "" { @@ -57,7 +57,7 @@ func (h *HTMLRender) HTML(w io.Writer, status int, tplName TplName, data any, ct return t.Execute(w, data) } -func (h *HTMLRender) TemplateLookup(name string, ctx context.Context) (TemplateExecutor, error) { //nolint:revive +func (h *HTMLRender) TemplateLookup(name string, ctx context.Context) (TemplateExecutor, error) { //nolint:revive // we don't use ctx, only pass it to the template executor tmpls := h.templates.Load() if tmpls == nil { return nil, ErrTemplateNotInitialized @@ -251,7 +251,7 @@ func extractErrorLine(code []byte, lineNum, posNum int, target string) string { b := bufio.NewReader(bytes.NewReader(code)) var line []byte var err error - for i := 0; i < lineNum; i++ { + for i := range lineNum { if line, err = b.ReadBytes('\n'); err != nil { if i == lineNum-1 && errors.Is(err, io.EOF) { err = nil diff --git a/modules/templates/htmlrenderer_test.go b/modules/templates/htmlrenderer_test.go index 2a74b74c23..e8b01c30fe 100644 --- a/modules/templates/htmlrenderer_test.go +++ b/modules/templates/htmlrenderer_test.go @@ -65,7 +65,7 @@ func TestHandleError(t *testing.T) { _, err = tmpl.Parse(s) assert.Error(t, err) msg := h(err) - assert.EqualValues(t, strings.TrimSpace(expect), strings.TrimSpace(msg)) + assert.Equal(t, strings.TrimSpace(expect), strings.TrimSpace(msg)) } test("{{", p.handleGenericTemplateError, ` @@ -102,5 +102,5 @@ god knows XXX ---------------------------------------------------------------------- ` actualMsg := p.handleExpectedEndError(errors.New("template: test:1: expected end; found XXX")) - assert.EqualValues(t, strings.TrimSpace(expectedMsg), strings.TrimSpace(actualMsg)) + assert.Equal(t, strings.TrimSpace(expectedMsg), strings.TrimSpace(actualMsg)) } diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go index 310d645328..c43b760777 100644 --- a/modules/templates/mailer.go +++ b/modules/templates/mailer.go @@ -9,6 +9,7 @@ import ( "html/template" "regexp" "strings" + "sync/atomic" texttmpl "text/template" "code.gitea.io/gitea/modules/log" @@ -16,6 +17,12 @@ import ( "code.gitea.io/gitea/modules/util" ) +type MailTemplates struct { + TemplateNames []string + BodyTemplates *template.Template + SubjectTemplates *texttmpl.Template +} + var mailSubjectSplit = regexp.MustCompile(`(?m)^-{3,}\s*$`) // mailSubjectTextFuncMap returns functions for injecting to text templates, it's only used for mail subject @@ -52,16 +59,17 @@ func buildSubjectBodyTemplate(stpl *texttmpl.Template, btpl *template.Template, return nil } -// Mailer provides the templates required for sending notification mails. -func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { - subjectTemplates := texttmpl.New("") - bodyTemplates := template.New("") - - subjectTemplates.Funcs(mailSubjectTextFuncMap()) - bodyTemplates.Funcs(NewFuncMap()) - +// LoadMailTemplates provides the templates required for sending notification mails. +func LoadMailTemplates(ctx context.Context, loadedTemplates *atomic.Pointer[MailTemplates]) { assetFS := AssetFS() refreshTemplates := func(firstRun bool) { + var templateNames []string + subjectTemplates := texttmpl.New("") + bodyTemplates := template.New("") + + subjectTemplates.Funcs(mailSubjectTextFuncMap()) + bodyTemplates.Funcs(NewFuncMap()) + if !firstRun { log.Trace("Reloading mail templates") } @@ -81,6 +89,7 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { if firstRun { log.Trace("Adding mail template %s: %s by %s", tmplName, assetPath, layerName) } + templateNames = append(templateNames, tmplName) if err = buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content); err != nil { if firstRun { log.Fatal("Failed to parse mail template, err: %v", err) @@ -88,6 +97,12 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { log.Error("Failed to parse mail template, err: %v", err) } } + loaded := &MailTemplates{ + TemplateNames: templateNames, + BodyTemplates: bodyTemplates, + SubjectTemplates: subjectTemplates, + } + loadedTemplates.Store(loaded) } refreshTemplates(true) @@ -99,6 +114,4 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { refreshTemplates(false) }) } - - return subjectTemplates, bodyTemplates } diff --git a/modules/templates/scopedtmpl/scopedtmpl.go b/modules/templates/scopedtmpl/scopedtmpl.go index 2722ba97a2..34e8b9ad70 100644 --- a/modules/templates/scopedtmpl/scopedtmpl.go +++ b/modules/templates/scopedtmpl/scopedtmpl.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "io" + "maps" "reflect" "sync" texttemplate "text/template" @@ -40,9 +41,7 @@ func (t *ScopedTemplate) Funcs(funcMap template.FuncMap) { panic("cannot add new functions to frozen template set") } t.all.Funcs(funcMap) - for k, v := range funcMap { - t.parseFuncs[k] = v - } + maps.Copy(t.parseFuncs, funcMap) } func (t *ScopedTemplate) New(name string) *template.Template { @@ -103,31 +102,28 @@ func escapeTemplate(t *template.Template) error { return nil } -//nolint:unused type htmlTemplate struct { - escapeErr error - text *texttemplate.Template + _/*escapeErr*/ error + text *texttemplate.Template } -//nolint:unused type textTemplateCommon struct { - tmpl map[string]*template.Template // Map from name to defined templates. - muTmpl sync.RWMutex // protects tmpl - option struct { + _/*tmpl*/ map[string]*template.Template + _/*muTmpl*/ sync.RWMutex + _/*option*/ struct { missingKey int } - muFuncs sync.RWMutex // protects parseFuncs and execFuncs - parseFuncs texttemplate.FuncMap - execFuncs map[string]reflect.Value + muFuncs sync.RWMutex + _/*parseFuncs*/ texttemplate.FuncMap + execFuncs map[string]reflect.Value } -//nolint:unused type textTemplate struct { - name string + _/*name*/ string *parse.Tree *textTemplateCommon - leftDelim string - rightDelim string + _/*leftDelim*/ string + _/*rightDelim*/ string } func ptr[T, P any](ptr *P) *T { @@ -159,9 +155,7 @@ func newScopedTemplateSet(all *template.Template, name string) (*scopedTemplateS textTmplPtr.muFuncs.Lock() ts.execFuncs = map[string]reflect.Value{} - for k, v := range textTmplPtr.execFuncs { - ts.execFuncs[k] = v - } + maps.Copy(ts.execFuncs, textTmplPtr.execFuncs) textTmplPtr.muFuncs.Unlock() var collectTemplates func(nodes []parse.Node) @@ -220,9 +214,7 @@ func (ts *scopedTemplateSet) newExecutor(funcMap map[string]any) TemplateExecuto tmpl := texttemplate.New("") tmplPtr := ptr[textTemplate](tmpl) tmplPtr.execFuncs = map[string]reflect.Value{} - for k, v := range ts.execFuncs { - tmplPtr.execFuncs[k] = v - } + maps.Copy(tmplPtr.execFuncs, ts.execFuncs) if funcMap != nil { tmpl.Funcs(funcMap) } diff --git a/modules/templates/static.go b/modules/templates/static.go deleted file mode 100644 index b5a7e561ec..0000000000 --- a/modules/templates/static.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2016 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build bindata - -package templates - -import ( - "time" - - "code.gitea.io/gitea/modules/assetfs" - "code.gitea.io/gitea/modules/timeutil" -) - -// GlobalModTime provide a global mod time for embedded asset files -func GlobalModTime(filename string) time.Time { - return timeutil.GetExecutableModTime() -} - -func BuiltinAssets() *assetfs.Layer { - return assetfs.Bindata("builtin(bindata)", Assets) -} diff --git a/modules/templates/templates_bindata.go b/modules/templates/templates_bindata.go index 6f1d3cf539..a919591ecf 100644 --- a/modules/templates/templates_bindata.go +++ b/modules/templates/templates_bindata.go @@ -3,6 +3,21 @@ //go:build bindata +//go:generate go run ../../build/generate-bindata.go ../../templates bindata.dat + package templates -//go:generate go run ../../build/generate-bindata.go ../../templates templates bindata.go true +import ( + "sync" + + _ "embed" + + "code.gitea.io/gitea/modules/assetfs" +) + +//go:embed bindata.dat +var bindata []byte + +var BuiltinAssets = sync.OnceValue(func() *assetfs.Layer { + return assetfs.Bindata("builtin(bindata)", assetfs.NewEmbeddedFS(bindata)) +}) diff --git a/modules/templates/dynamic.go b/modules/templates/templates_dynamic.go index e1babd83c9..e1babd83c9 100644 --- a/modules/templates/dynamic.go +++ b/modules/templates/templates_dynamic.go diff --git a/modules/templates/util_avatar.go b/modules/templates/util_avatar.go index f7dd408ee2..ee9994ab0b 100644 --- a/modules/templates/util_avatar.go +++ b/modules/templates/util_avatar.go @@ -5,9 +5,9 @@ package templates import ( "context" - "fmt" "html" "html/template" + "strconv" activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/avatars" @@ -28,13 +28,14 @@ func NewAvatarUtils(ctx context.Context) *AvatarUtils { // AvatarHTML creates the HTML for an avatar func AvatarHTML(src string, size int, class, name string) template.HTML { - sizeStr := fmt.Sprintf(`%d`, size) + sizeStr := strconv.Itoa(size) if name == "" { name = "avatar" } - return template.HTML(`<img loading="lazy" class="` + class + `" src="` + src + `" title="` + html.EscapeString(name) + `" width="` + sizeStr + `" height="` + sizeStr + `"/>`) + // use empty alt, otherwise if the image fails to load, the width will follow the "alt" text's width + return template.HTML(`<img loading="lazy" alt class="` + class + `" src="` + src + `" title="` + html.EscapeString(name) + `" width="` + sizeStr + `" height="` + sizeStr + `"/>`) } // Avatar renders user avatars. args: user, size (int), class (string) diff --git a/modules/templates/util_date.go b/modules/templates/util_date.go index 658691ee40..fc3f3f2339 100644 --- a/modules/templates/util_date.go +++ b/modules/templates/util_date.go @@ -99,7 +99,7 @@ func dateTimeFormat(format string, datetime any) template.HTML { attrs = append(attrs, `format="datetime"`, `month="short"`, `day="numeric"`, `hour="numeric"`, `minute="numeric"`, `second="numeric"`, `data-tooltip-content`, `data-tooltip-interactive="true"`) return template.HTML(fmt.Sprintf(`<relative-time %s datetime="%s">%s</relative-time>`, strings.Join(attrs, " "), datetimeEscaped, textEscaped)) default: - panic(fmt.Sprintf("Unsupported format %s", format)) + panic("Unsupported format " + format) } } diff --git a/modules/templates/util_date_legacy.go b/modules/templates/util_date_legacy.go deleted file mode 100644 index ceefb00447..0000000000 --- a/modules/templates/util_date_legacy.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package templates - -import ( - "html/template" - - "code.gitea.io/gitea/modules/translation" -) - -func dateTimeLegacy(format string, datetime any, _ ...string) template.HTML { - panicIfDevOrTesting() - if s, ok := datetime.(string); ok { - datetime = parseLegacy(s) - } - return dateTimeFormat(format, datetime) -} - -func timeSinceLegacy(time any, _ translation.Locale) template.HTML { - panicIfDevOrTesting() - return TimeSince(time) -} diff --git a/modules/templates/util_date_test.go b/modules/templates/util_date_test.go index f3a2409a9f..2c1f2d242e 100644 --- a/modules/templates/util_date_test.go +++ b/modules/templates/util_date_test.go @@ -17,12 +17,12 @@ import ( func TestDateTime(t *testing.T) { testTz, _ := time.LoadLocation("America/New_York") defer test.MockVariableValue(&setting.DefaultUILocation, testTz)() + defer test.MockVariableValue(&setting.IsProd, true)() defer test.MockVariableValue(&setting.IsInTesting, false)() du := NewDateUtils() refTimeStr := "2018-01-01T00:00:00Z" - refDateStr := "2018-01-01" refTime, _ := time.Parse(time.RFC3339, refTimeStr) refTimeStamp := timeutil.TimeStamp(refTime.Unix()) @@ -31,18 +31,9 @@ func TestDateTime(t *testing.T) { assert.EqualValues(t, "-", du.AbsoluteShort(time.Time{})) assert.EqualValues(t, "-", du.AbsoluteShort(timeutil.TimeStamp(0))) - actual := dateTimeLegacy("short", "invalid") - assert.EqualValues(t, `-`, actual) - - actual = dateTimeLegacy("short", refTimeStr) - assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2018-01-01T00:00:00Z">2018-01-01</absolute-date>`, actual) - - actual = du.AbsoluteShort(refTime) + actual := du.AbsoluteShort(refTime) assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2018-01-01T00:00:00Z">2018-01-01</absolute-date>`, actual) - actual = dateTimeLegacy("short", refDateStr) - assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2018-01-01T00:00:00-05:00">2018-01-01</absolute-date>`, actual) - actual = du.AbsoluteShort(refTimeStamp) assert.EqualValues(t, `<absolute-date weekday="" year="numeric" month="short" day="numeric" date="2017-12-31T19:00:00-05:00">2017-12-31</absolute-date>`, actual) @@ -53,6 +44,7 @@ func TestDateTime(t *testing.T) { func TestTimeSince(t *testing.T) { testTz, _ := time.LoadLocation("America/New_York") defer test.MockVariableValue(&setting.DefaultUILocation, testTz)() + defer test.MockVariableValue(&setting.IsProd, true)() defer test.MockVariableValue(&setting.IsInTesting, false)() du := NewDateUtils() @@ -67,6 +59,6 @@ func TestTimeSince(t *testing.T) { actual = timeSinceTo(&refTime, time.Time{}) assert.EqualValues(t, `<relative-time prefix="" tense="future" datetime="2018-01-01T00:00:00Z" data-tooltip-content data-tooltip-interactive="true">2018-01-01 00:00:00 +00:00</relative-time>`, actual) - actual = timeSinceLegacy(timeutil.TimeStampNano(refTime.UnixNano()), nil) + actual = du.TimeSince(timeutil.TimeStampNano(refTime.UnixNano())) assert.EqualValues(t, `<relative-time prefix="" tense="past" datetime="2017-12-31T19:00:00-05:00" data-tooltip-content data-tooltip-interactive="true">2017-12-31 19:00:00 -05:00</relative-time>`, actual) } diff --git a/modules/templates/util_dict.go b/modules/templates/util_dict.go index 8d6376b522..cc3018a71c 100644 --- a/modules/templates/util_dict.go +++ b/modules/templates/util_dict.go @@ -4,6 +4,7 @@ package templates import ( + "errors" "fmt" "html" "html/template" @@ -33,7 +34,7 @@ func dictMerge(base map[string]any, arg any) bool { // The dot syntax is highly discouraged because it might cause unclear key conflicts. It's always good to use explicit keys. func dict(args ...any) (map[string]any, error) { if len(args)%2 != 0 { - return nil, fmt.Errorf("invalid dict constructor syntax: must have key-value pairs") + return nil, errors.New("invalid dict constructor syntax: must have key-value pairs") } m := make(map[string]any, len(args)/2) for i := 0; i < len(args); i += 2 { diff --git a/modules/templates/util_format.go b/modules/templates/util_format.go index bee6fb7b75..3485e3251e 100644 --- a/modules/templates/util_format.go +++ b/modules/templates/util_format.go @@ -5,6 +5,7 @@ package templates import ( "fmt" + "strconv" "code.gitea.io/gitea/modules/util" ) @@ -24,7 +25,7 @@ func countFmt(data any) string { return "" } if num < 1000 { - return fmt.Sprintf("%d", num) + return strconv.FormatInt(num, 10) } else if num < 1_000_000 { num2 := float32(num) / 1000.0 return fmt.Sprintf("%.1fk", num2) diff --git a/modules/templates/util_format_test.go b/modules/templates/util_format_test.go index 8d466faff0..13a57c24e2 100644 --- a/modules/templates/util_format_test.go +++ b/modules/templates/util_format_test.go @@ -14,5 +14,5 @@ func TestCountFmt(t *testing.T) { assert.Equal(t, "1.3k", countFmt(int64(1317))) assert.Equal(t, "21.3M", countFmt(21317675)) assert.Equal(t, "45.7G", countFmt(45721317675)) - assert.Equal(t, "", countFmt("test")) + assert.Empty(t, countFmt("test")) } diff --git a/modules/templates/util_json.go b/modules/templates/util_json.go index 71a4e23d36..29a04290fa 100644 --- a/modules/templates/util_json.go +++ b/modules/templates/util_json.go @@ -9,11 +9,11 @@ import ( "code.gitea.io/gitea/modules/json" ) -type JsonUtils struct{} //nolint:revive +type JsonUtils struct{} //nolint:revive // variable naming triggers on Json, wants JSON var jsonUtils = JsonUtils{} -func NewJsonUtils() *JsonUtils { //nolint:revive +func NewJsonUtils() *JsonUtils { //nolint:revive // variable naming triggers on Json, wants JSON return &jsonUtils } diff --git a/modules/templates/util_misc.go b/modules/templates/util_misc.go index 2d42bc76b5..cc5bf67b42 100644 --- a/modules/templates/util_misc.go +++ b/modules/templates/util_misc.go @@ -38,10 +38,11 @@ func sortArrow(normSort, revSort, urlSort string, isDefault bool) template.HTML } else { // if sort arg is in url test if it correlates with column header sort arguments // the direction of the arrow should indicate the "current sort order", up means ASC(normal), down means DESC(rev) - if urlSort == normSort { + switch urlSort { + case normSort: // the table is sorted with this header normal return svg.RenderHTML("octicon-triangle-up", 16) - } else if urlSort == revSort { + case revSort: // the table is sorted with this header reverse return svg.RenderHTML("octicon-triangle-down", 16) } diff --git a/modules/templates/util_render.go b/modules/templates/util_render.go index 27316bbfec..1056c42643 100644 --- a/modules/templates/util_render.go +++ b/modules/templates/util_render.go @@ -14,9 +14,9 @@ import ( "unicode" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/renderhelper" + "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/emoji" - "code.gitea.io/gitea/modules/fileicon" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/htmlutil" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" @@ -36,25 +36,25 @@ func NewRenderUtils(ctx reqctx.RequestContext) *RenderUtils { } // RenderCommitMessage renders commit message with XSS-safe and special links. -func (ut *RenderUtils) RenderCommitMessage(msg string, metas map[string]string) template.HTML { +func (ut *RenderUtils) RenderCommitMessage(msg string, repo *repo.Repository) template.HTML { cleanMsg := template.HTMLEscapeString(msg) - // we can safely assume that it will not return any error, since there - // shouldn't be any special HTML. - fullMessage, err := markup.PostProcessCommitMessage(markup.NewRenderContext(ut.ctx).WithMetas(metas), cleanMsg) + // we can safely assume that it will not return any error, since there shouldn't be any special HTML. + // "repo" can be nil when rendering commit messages for deleted repositories in a user's dashboard feed. + fullMessage, err := markup.PostProcessCommitMessage(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), cleanMsg) if err != nil { log.Error("PostProcessCommitMessage: %v", err) return "" } msgLines := strings.Split(strings.TrimSpace(fullMessage), "\n") if len(msgLines) == 0 { - return template.HTML("") + return "" } return renderCodeBlock(template.HTML(msgLines[0])) } // RenderCommitMessageLinkSubject renders commit message as a XSS-safe link to // the provided default url, handling for special links without email to links. -func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, metas map[string]string) template.HTML { +func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, repo *repo.Repository) template.HTML { msgLine := strings.TrimLeftFunc(msg, unicode.IsSpace) lineEnd := strings.IndexByte(msgLine, '\n') if lineEnd > 0 { @@ -65,9 +65,8 @@ func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, me return "" } - // we can safely assume that it will not return any error, since there - // shouldn't be any special HTML. - renderedMessage, err := markup.PostProcessCommitMessageSubject(markup.NewRenderContext(ut.ctx).WithMetas(metas), urlDefault, template.HTMLEscapeString(msgLine)) + // we can safely assume that it will not return any error, since there shouldn't be any special HTML. + renderedMessage, err := markup.PostProcessCommitMessageSubject(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), urlDefault, template.HTMLEscapeString(msgLine)) if err != nil { log.Error("PostProcessCommitMessageSubject: %v", err) return "" @@ -76,7 +75,7 @@ func (ut *RenderUtils) RenderCommitMessageLinkSubject(msg, urlDefault string, me } // RenderCommitBody extracts the body of a commit message without its title. -func (ut *RenderUtils) RenderCommitBody(msg string, metas map[string]string) template.HTML { +func (ut *RenderUtils) RenderCommitBody(msg string, repo *repo.Repository) template.HTML { msgLine := strings.TrimSpace(msg) lineEnd := strings.IndexByte(msgLine, '\n') if lineEnd > 0 { @@ -89,7 +88,7 @@ func (ut *RenderUtils) RenderCommitBody(msg string, metas map[string]string) tem return "" } - renderedMessage, err := markup.PostProcessCommitMessage(markup.NewRenderContext(ut.ctx).WithMetas(metas), template.HTMLEscapeString(msgLine)) + renderedMessage, err := markup.PostProcessCommitMessage(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), template.HTMLEscapeString(msgLine)) if err != nil { log.Error("PostProcessCommitMessage: %v", err) return "" @@ -107,8 +106,8 @@ func renderCodeBlock(htmlEscapedTextToRender template.HTML) template.HTML { } // RenderIssueTitle renders issue/pull title with defined post processors -func (ut *RenderUtils) RenderIssueTitle(text string, metas map[string]string) template.HTML { - renderedText, err := markup.PostProcessIssueTitle(markup.NewRenderContext(ut.ctx).WithMetas(metas), template.HTMLEscapeString(text)) +func (ut *RenderUtils) RenderIssueTitle(text string, repo *repo.Repository) template.HTML { + renderedText, err := markup.PostProcessIssueTitle(renderhelper.NewRenderContextRepoComment(ut.ctx, repo), template.HTMLEscapeString(text)) if err != nil { log.Error("PostProcessIssueTitle: %v", err) return "" @@ -123,8 +122,23 @@ func (ut *RenderUtils) RenderIssueSimpleTitle(text string) template.HTML { return ret } -// RenderLabel renders a label +func (ut *RenderUtils) RenderLabelWithLink(label *issues_model.Label, link any) template.HTML { + var attrHref template.HTML + switch link.(type) { + case template.URL, string: + attrHref = htmlutil.HTMLFormat(`href="%s"`, link) + default: + panic(fmt.Sprintf("unexpected type %T for link", link)) + } + return ut.renderLabelWithTag(label, "a", attrHref) +} + func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { + return ut.renderLabelWithTag(label, "span", "") +} + +// RenderLabel renders a label +func (ut *RenderUtils) renderLabelWithTag(label *issues_model.Label, tagName, tagAttrs template.HTML) template.HTML { locale := ut.ctx.Value(translation.ContextKey).(translation.Locale) var extraCSSClasses string textColor := util.ContrastColor(label.Color) @@ -138,8 +152,8 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { if labelScope == "" { // Regular label - return htmlutil.HTMLFormat(`<div class="ui label %s" style="color: %s !important; background-color: %s !important;" data-tooltip-content title="%s">%s</div>`, - extraCSSClasses, textColor, label.Color, descriptionText, ut.RenderEmoji(label.Name)) + return htmlutil.HTMLFormat(`<%s %s class="ui label %s" style="color: %s !important; background-color: %s !important;" data-tooltip-content title="%s"><span class="gt-ellipsis">%s</span></%s>`, + tagName, tagAttrs, extraCSSClasses, textColor, label.Color, descriptionText, ut.RenderEmoji(label.Name), tagName) } // Scoped label @@ -153,7 +167,7 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { // Ensure we add the same amount of contrast also near 0 and 1. darken := contrast + math.Max(luminance+contrast-1.0, 0.0) lighten := contrast + math.Max(contrast-luminance, 0.0) - // Compute factor to keep RGB values proportional. + // Compute the factor to keep RGB values proportional. darkenFactor := math.Max(luminance-darken, 0.0) / math.Max(luminance, 1.0/255.0) lightenFactor := math.Min(luminance+lighten, 1.0) / math.Max(luminance, 1.0/255.0) @@ -172,20 +186,31 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { itemColor := "#" + hex.EncodeToString(itemBytes) scopeColor := "#" + hex.EncodeToString(scopeBytes) - return htmlutil.HTMLFormat(`<span class="ui label %s scope-parent" data-tooltip-content title="%s">`+ + if label.ExclusiveOrder > 0 { + // <scope> | <label> | <order> + return htmlutil.HTMLFormat(`<%s %s class="ui label %s scope-parent" data-tooltip-content title="%s">`+ + `<div class="ui label scope-left" style="color: %s !important; background-color: %s !important">%s</div>`+ + `<div class="ui label scope-middle" style="color: %s !important; background-color: %s !important">%s</div>`+ + `<div class="ui label scope-right">%d</div>`+ + `</%s>`, + tagName, tagAttrs, + extraCSSClasses, descriptionText, + textColor, scopeColor, scopeHTML, + textColor, itemColor, itemHTML, + label.ExclusiveOrder, + tagName) + } + + // <scope> | <label> + return htmlutil.HTMLFormat(`<%s %s class="ui label %s scope-parent" data-tooltip-content title="%s">`+ `<div class="ui label scope-left" style="color: %s !important; background-color: %s !important">%s</div>`+ `<div class="ui label scope-right" style="color: %s !important; background-color: %s !important">%s</div>`+ - `</span>`, + `</%s>`, + tagName, tagAttrs, extraCSSClasses, descriptionText, textColor, scopeColor, scopeHTML, - textColor, itemColor, itemHTML) -} - -func (ut *RenderUtils) RenderFileIcon(entry *git.TreeEntry) template.HTML { - if setting.UI.FileIconTheme == "material" { - return fileicon.DefaultMaterialIconProvider().FileIcon(ut.ctx, entry) - } - return fileicon.BasicThemeIcon(entry) + textColor, itemColor, itemHTML, + tagName) } // RenderEmoji renders html text with emoji post processors @@ -211,7 +236,7 @@ func reactionToEmoji(reaction string) template.HTML { return template.HTML(fmt.Sprintf(`<img alt=":%s:" src="%s/assets/img/emoji/%s.png"></img>`, reaction, setting.StaticURLPrefix, url.PathEscape(reaction))) } -func (ut *RenderUtils) MarkdownToHtml(input string) template.HTML { //nolint:revive +func (ut *RenderUtils) MarkdownToHtml(input string) template.HTML { //nolint:revive // variable naming triggers on Html, wants HTML output, err := markdown.RenderString(markup.NewRenderContext(ut.ctx).WithMetas(markup.ComposeSimpleDocumentMetas()), input) if err != nil { log.Error("RenderString: %v", err) @@ -228,7 +253,8 @@ func (ut *RenderUtils) RenderLabels(labels []*issues_model.Label, repoLink strin if label == nil { continue } - htmlCode += fmt.Sprintf(`<a href="%s?labels=%d">%s</a>`, baseLink, label.ID, ut.RenderLabel(label)) + link := fmt.Sprintf("%s?labels=%d", baseLink, label.ID) + htmlCode += string(ut.RenderLabelWithLink(label, template.URL(link))) } htmlCode += "</span>" return template.HTML(htmlCode) diff --git a/modules/templates/util_render_legacy.go b/modules/templates/util_render_legacy.go deleted file mode 100644 index 8f7b84c83d..0000000000 --- a/modules/templates/util_render_legacy.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package templates - -import ( - "context" - "html/template" - - issues_model "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/modules/reqctx" - "code.gitea.io/gitea/modules/translation" -) - -func renderEmojiLegacy(ctx context.Context, text string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(reqctx.FromContext(ctx)).RenderEmoji(text) -} - -func renderLabelLegacy(ctx context.Context, locale translation.Locale, label *issues_model.Label) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(reqctx.FromContext(ctx)).RenderLabel(label) -} - -func renderLabelsLegacy(ctx context.Context, locale translation.Locale, labels []*issues_model.Label, repoLink string, issue *issues_model.Issue) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(reqctx.FromContext(ctx)).RenderLabels(labels, repoLink, issue) -} - -func renderMarkdownToHtmlLegacy(ctx context.Context, input string) template.HTML { //nolint:revive - panicIfDevOrTesting() - return NewRenderUtils(reqctx.FromContext(ctx)).MarkdownToHtml(input) -} - -func renderCommitMessageLegacy(ctx context.Context, msg string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(reqctx.FromContext(ctx)).RenderCommitMessage(msg, metas) -} - -func renderCommitMessageLinkSubjectLegacy(ctx context.Context, msg, urlDefault string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(reqctx.FromContext(ctx)).RenderCommitMessageLinkSubject(msg, urlDefault, metas) -} - -func renderIssueTitleLegacy(ctx context.Context, text string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(reqctx.FromContext(ctx)).RenderIssueTitle(text, metas) -} - -func renderCommitBodyLegacy(ctx context.Context, msg string, metas map[string]string) template.HTML { - panicIfDevOrTesting() - return NewRenderUtils(reqctx.FromContext(ctx)).RenderCommitBody(msg, metas) -} diff --git a/modules/templates/util_render_test.go b/modules/templates/util_render_test.go index 617021e510..5c37f084df 100644 --- a/modules/templates/util_render_test.go +++ b/modules/templates/util_render_test.go @@ -11,11 +11,11 @@ import ( "testing" "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/reqctx" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/translation" @@ -47,19 +47,8 @@ mail@domain.com return strings.ReplaceAll(s, "<SPACE>", " ") } -var testMetas = map[string]string{ - "user": "user13", - "repo": "repo11", - "repoPath": "../../tests/gitea-repositories-meta/user13/repo11.git/", - "markdownLineBreakStyle": "comment", - "markupAllowShortIssuePattern": "true", -} - func TestMain(m *testing.M) { - unittest.InitSettings() - if err := git.InitSimple(context.Background()); err != nil { - log.Fatal("git init failed, err: %v", err) - } + setting.Markdown.RenderOptionsComment.ShortIssuePattern = true markup.Init(&markup.RenderHelperFuncs{ IsUsernameMentionable: func(ctx context.Context, username string) bool { return username == "mention-user" @@ -74,46 +63,52 @@ func newTestRenderUtils(t *testing.T) *RenderUtils { return NewRenderUtils(ctx) } -func TestRenderCommitBody(t *testing.T) { - defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() - type args struct { - msg string +func TestRenderRepoComment(t *testing.T) { + mockRepo := &repo.Repository{ + ID: 1, OwnerName: "user13", Name: "repo11", + Owner: &user_model.User{ID: 13, Name: "user13"}, + Units: []*repo.RepoUnit{}, } - tests := []struct { - name string - args args - want template.HTML - }{ - { - name: "multiple lines", - args: args{ - msg: "first line\nsecond line", + t.Run("RenderCommitBody", func(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + type args struct { + msg string + } + tests := []struct { + name string + args args + want template.HTML + }{ + { + name: "multiple lines", + args: args{ + msg: "first line\nsecond line", + }, + want: "second line", }, - want: "second line", - }, - { - name: "multiple lines with leading newlines", - args: args{ - msg: "\n\n\n\nfirst line\nsecond line", + { + name: "multiple lines with leading newlines", + args: args{ + msg: "\n\n\n\nfirst line\nsecond line", + }, + want: "second line", }, - want: "second line", - }, - { - name: "multiple lines with trailing newlines", - args: args{ - msg: "first line\nsecond line\n\n\n", + { + name: "multiple lines with trailing newlines", + args: args{ + msg: "first line\nsecond line\n\n\n", + }, + want: "second line", }, - want: "second line", - }, - } - ut := newTestRenderUtils(t) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, ut.RenderCommitBody(tt.args.msg, nil), "RenderCommitBody(%v, %v)", tt.args.msg, nil) - }) - } - - expected := `/just/a/path.bin + } + ut := newTestRenderUtils(t) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, ut.RenderCommitBody(tt.args.msg, mockRepo), "RenderCommitBody(%v, %v)", tt.args.msg, nil) + }) + } + + expected := `/just/a/path.bin <a href="https://example.com/file.bin">https://example.com/file.bin</a> [local link](file.bin) [remote link](<a href="https://example.com">https://example.com</a>) @@ -123,31 +118,31 @@ func TestRenderCommitBody(t *testing.T) { ![remote image](<a href="https://example.com/image.jpg">https://example.com/image.jpg</a>) [[local image|image.jpg]] [[remote link|<a href="https://example.com/image.jpg">https://example.com/image.jpg</a>]] -<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" class="compare"><code class="nohighlight">88fc37a3c0...12fc37a3c0 (hash)</code></a> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" class="compare"><code>88fc37a3c0...12fc37a3c0 (hash)</code></a> com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare -<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" class="commit"><code class="nohighlight">88fc37a3c0</code></a> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" class="commit"><code>88fc37a3c0</code></a> com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit <span class="emoji" aria-label="thumbs up">👍</span> <a href="mailto:mail@domain.com">mail@domain.com</a> <a href="/mention-user">@mention-user</a> test <a href="/user13/repo11/issues/123" class="ref-issue">#123</a> space` - assert.EqualValues(t, expected, string(newTestRenderUtils(t).RenderCommitBody(testInput(), testMetas))) -} + assert.Equal(t, expected, string(newTestRenderUtils(t).RenderCommitBody(testInput(), mockRepo))) + }) -func TestRenderCommitMessage(t *testing.T) { - expected := `space <a href="/mention-user" data-markdown-generated-content="">@mention-user</a> ` - assert.EqualValues(t, expected, newTestRenderUtils(t).RenderCommitMessage(testInput(), testMetas)) -} + t.Run("RenderCommitMessage", func(t *testing.T) { + expected := `space <a href="/mention-user" data-markdown-generated-content="">@mention-user</a> ` + assert.EqualValues(t, expected, newTestRenderUtils(t).RenderCommitMessage(testInput(), mockRepo)) + }) -func TestRenderCommitMessageLinkSubject(t *testing.T) { - expected := `<a href="https://example.com/link" class="muted">space </a><a href="/mention-user" data-markdown-generated-content="">@mention-user</a>` - assert.EqualValues(t, expected, newTestRenderUtils(t).RenderCommitMessageLinkSubject(testInput(), "https://example.com/link", testMetas)) -} + t.Run("RenderCommitMessageLinkSubject", func(t *testing.T) { + expected := `<a href="https://example.com/link" class="muted">space </a><a href="/mention-user" data-markdown-generated-content="">@mention-user</a>` + assert.EqualValues(t, expected, newTestRenderUtils(t).RenderCommitMessageLinkSubject(testInput(), "https://example.com/link", mockRepo)) + }) -func TestRenderIssueTitle(t *testing.T) { - defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() - expected := ` space @mention-user<SPACE><SPACE> + t.Run("RenderIssueTitle", func(t *testing.T) { + defer test.MockVariableValue(&markup.RenderBehaviorForTesting.DisableAdditionalAttributes, true)() + expected := ` space @mention-user<SPACE><SPACE> /just/a/path.bin https://example.com/file.bin [local link](file.bin) @@ -168,8 +163,9 @@ mail@domain.com <a href="/user13/repo11/issues/123" class="ref-issue">#123</a> space<SPACE><SPACE> ` - expected = strings.ReplaceAll(expected, "<SPACE>", " ") - assert.EqualValues(t, expected, string(newTestRenderUtils(t).RenderIssueTitle(testInput(), testMetas))) + expected = strings.ReplaceAll(expected, "<SPACE>", " ") + assert.Equal(t, expected, string(newTestRenderUtils(t).RenderIssueTitle(testInput(), mockRepo))) + }) } func TestRenderMarkdownToHtml(t *testing.T) { @@ -209,10 +205,21 @@ func TestRenderLabels(t *testing.T) { issue = &issues.Issue{IsPull: true} expected = `/owner/repo/pulls?labels=123` assert.Contains(t, ut.RenderLabels([]*issues.Label{label}, "/owner/repo", issue), expected) + + expectedLabel := `<a href="<>" class="ui label " style="color: #fff !important; background-color: label-color !important;" data-tooltip-content title=""><span class="gt-ellipsis">label-name</span></a>` + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, "<>"))) + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, template.URL("<>")))) + + label = &issues.Label{ID: 123, Name: "</>", Exclusive: true} + expectedLabel = `<a href="" class="ui label scope-parent" data-tooltip-content title=""><div class="ui label scope-left" style="color: #fff !important; background-color: #000000 !important"><</div><div class="ui label scope-right" style="color: #fff !important; background-color: #000000 !important">></div></a>` + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, ""))) + label = &issues.Label{ID: 123, Name: "</>", Exclusive: true, ExclusiveOrder: 1} + expectedLabel = `<a href="" class="ui label scope-parent" data-tooltip-content title=""><div class="ui label scope-left" style="color: #fff !important; background-color: #000000 !important"><</div><div class="ui label scope-middle" style="color: #fff !important; background-color: #000000 !important">></div><div class="ui label scope-right">1</div></a>` + assert.Equal(t, expectedLabel, string(ut.RenderLabelWithLink(label, ""))) } func TestUserMention(t *testing.T) { markup.RenderBehaviorForTesting.DisableAdditionalAttributes = true rendered := newTestRenderUtils(t).MarkdownToHtml("@no-such-user @mention-user @mention-user") - assert.EqualValues(t, `<p>@no-such-user <a href="/mention-user" rel="nofollow">@mention-user</a> <a href="/mention-user" rel="nofollow">@mention-user</a></p>`, strings.TrimSpace(string(rendered))) + assert.Equal(t, `<p>@no-such-user <a href="/mention-user" rel="nofollow">@mention-user</a> <a href="/mention-user" rel="nofollow">@mention-user</a></p>`, strings.TrimSpace(string(rendered))) } diff --git a/modules/templates/util_test.go b/modules/templates/util_test.go index febaf7fa88..a6448a6ff2 100644 --- a/modules/templates/util_test.go +++ b/modules/templates/util_test.go @@ -28,7 +28,7 @@ func TestDict(t *testing.T) { for _, c := range cases { got, err := dict(c.args...) if assert.NoError(t, err) { - assert.EqualValues(t, c.want, got) + assert.Equal(t, c.want, got) } } diff --git a/modules/templates/vars/vars.go b/modules/templates/vars/vars.go index cc9d0e976f..500078d4b8 100644 --- a/modules/templates/vars/vars.go +++ b/modules/templates/vars/vars.go @@ -16,7 +16,7 @@ type ErrWrongSyntax struct { } func (err ErrWrongSyntax) Error() string { - return fmt.Sprintf("wrong syntax found in %s", err.Template) + return "wrong syntax found in " + err.Template } // ErrVarMissing represents an error that no matched variable diff --git a/modules/templates/vars/vars_test.go b/modules/templates/vars/vars_test.go index 8f421d9e4b..9b48167237 100644 --- a/modules/templates/vars/vars_test.go +++ b/modules/templates/vars/vars_test.go @@ -60,7 +60,7 @@ func TestExpandVars(t *testing.T) { for _, kase := range kases { t.Run(kase.tmpl, func(t *testing.T) { res, err := Expand(kase.tmpl, kase.data) - assert.EqualValues(t, kase.out, res) + assert.Equal(t, kase.out, res) if kase.error { assert.Error(t, err) } else { diff --git a/modules/test/logchecker.go b/modules/test/logchecker.go index 7bf234f560..829f735c7c 100644 --- a/modules/test/logchecker.go +++ b/modules/test/logchecker.go @@ -5,7 +5,7 @@ package test import ( "context" - "fmt" + "strconv" "strings" "sync" "sync/atomic" @@ -58,7 +58,7 @@ var checkerIndex int64 func NewLogChecker(namePrefix string) (logChecker *LogChecker, cancel func()) { logger := log.GetManager().GetLogger(namePrefix) newCheckerIndex := atomic.AddInt64(&checkerIndex, 1) - writerName := namePrefix + "-" + fmt.Sprint(newCheckerIndex) + writerName := namePrefix + "-" + strconv.FormatInt(newCheckerIndex, 10) lc := &LogChecker{} lc.EventWriterBaseImpl = log.NewEventWriterBase(writerName, "test-log-checker", log.WriterMode{}) diff --git a/modules/test/utils.go b/modules/test/utils.go index ec4c976388..53c6a3ed52 100644 --- a/modules/test/utils.go +++ b/modules/test/utils.go @@ -4,7 +4,6 @@ package test import ( - "fmt" "net/http" "net/http/httptest" "os" @@ -18,6 +17,7 @@ import ( // RedirectURL returns the redirect URL of a http response. // It also works for JSONRedirect: `{"redirect": "..."}` +// FIXME: it should separate the logic of checking from header and JSON body func RedirectURL(resp http.ResponseWriter) string { loc := resp.Header().Get("Location") if loc != "" { @@ -35,6 +35,15 @@ func RedirectURL(resp http.ResponseWriter) string { return "" } +func ParseJSONError(buf []byte) (ret struct { + ErrorMessage string `json:"errorMessage"` + RenderFormat string `json:"renderFormat"` +}, +) { + _ = json.Unmarshal(buf, &ret) + return ret +} + func IsNormalPageCompleted(s string) bool { return strings.Contains(s, `<footer class="page-footer"`) && strings.Contains(s, `</html>`) } @@ -57,7 +66,7 @@ func SetupGiteaRoot() string { giteaRoot = filepath.Dir(filepath.Dir(filepath.Dir(filename))) fixturesDir := filepath.Join(giteaRoot, "models", "fixtures") if exist, _ := util.IsDir(fixturesDir); !exist { - panic(fmt.Sprintf("fixtures directory not found: %s", fixturesDir)) + panic("fixtures directory not found: " + fixturesDir) } _ = os.Setenv("GITEA_ROOT", giteaRoot) return giteaRoot diff --git a/modules/testlogger/testlogger.go b/modules/testlogger/testlogger.go index 8e970aa2be..60e281d403 100644 --- a/modules/testlogger/testlogger.go +++ b/modules/testlogger/testlogger.go @@ -92,7 +92,7 @@ func (w *testLoggerWriterCloser) Reset() { // Printf takes a format and args and prints the string to os.Stdout func Printf(format string, args ...any) { if !log.CanColorStdout { - for i := 0; i < len(args); i++ { + for i := range args { if c, ok := args[i].(*log.ColoredValue); ok { args[i] = c.Value() } diff --git a/modules/timeutil/executable.go b/modules/timeutil/executable.go deleted file mode 100644 index 57ae8b2a9d..0000000000 --- a/modules/timeutil/executable.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package timeutil - -import ( - "os" - "path/filepath" - "sync" - "time" - - "code.gitea.io/gitea/modules/log" -) - -var ( - executablModTime = time.Now() - executablModTimeOnce sync.Once -) - -// GetExecutableModTime get executable file modified time of current process. -func GetExecutableModTime() time.Time { - executablModTimeOnce.Do(func() { - exePath, err := os.Executable() - if err != nil { - log.Error("os.Executable: %v", err) - return - } - - exePath, err = filepath.Abs(exePath) - if err != nil { - log.Error("filepath.Abs: %v", err) - return - } - - exePath, err = filepath.EvalSymlinks(exePath) - if err != nil { - log.Error("filepath.EvalSymlinks: %v", err) - return - } - - st, err := os.Stat(exePath) - if err != nil { - log.Error("os.Stat: %v", err) - return - } - - executablModTime = st.ModTime() - }) - return executablModTime -} diff --git a/modules/translation/translation_test.go b/modules/translation/translation_test.go index 464aa32661..87df9eb825 100644 --- a/modules/translation/translation_test.go +++ b/modules/translation/translation_test.go @@ -20,13 +20,13 @@ func TestPrettyNumber(t *testing.T) { allLangMap["id-ID"] = &LangType{Lang: "id-ID", Name: "Bahasa Indonesia"} l := NewLocale("id-ID") - assert.EqualValues(t, "1.000.000", l.PrettyNumber(1000000)) - assert.EqualValues(t, "1.000.000,1", l.PrettyNumber(1000000.1)) - assert.EqualValues(t, "1.000.000", l.PrettyNumber("1000000")) - assert.EqualValues(t, "1.000.000", l.PrettyNumber("1000000.0")) - assert.EqualValues(t, "1.000.000,1", l.PrettyNumber("1000000.1")) + assert.Equal(t, "1.000.000", l.PrettyNumber(1000000)) + assert.Equal(t, "1.000.000,1", l.PrettyNumber(1000000.1)) + assert.Equal(t, "1.000.000", l.PrettyNumber("1000000")) + assert.Equal(t, "1.000.000", l.PrettyNumber("1000000.0")) + assert.Equal(t, "1.000.000,1", l.PrettyNumber("1000000.1")) l = NewLocale("nosuch") - assert.EqualValues(t, "1,000,000", l.PrettyNumber(1000000)) - assert.EqualValues(t, "1,000,000.1", l.PrettyNumber(1000000.1)) + assert.Equal(t, "1,000,000", l.PrettyNumber(1000000)) + assert.Equal(t, "1,000,000.1", l.PrettyNumber(1000000.1)) } diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go index 8cb3d278ce..2e8d9c4a1e 100644 --- a/modules/typesniffer/typesniffer.go +++ b/modules/typesniffer/typesniffer.go @@ -6,18 +6,14 @@ package typesniffer import ( "bytes" "encoding/binary" - "fmt" - "io" "net/http" "regexp" "slices" "strings" - - "code.gitea.io/gitea/modules/util" + "sync" ) -// Use at most this many bytes to determine Content Type. -const sniffLen = 1024 +const SniffContentSize = 1024 const ( MimeTypeImageSvg = "image/svg+xml" @@ -26,22 +22,30 @@ const ( MimeTypeApplicationOctetStream = "application/octet-stream" ) -var ( - svgComment = regexp.MustCompile(`(?s)<!--.*?-->`) - svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) - svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) -) - -// SniffedType contains information about a blobs type. +var globalVars = sync.OnceValue(func() (ret struct { + svgComment, svgTagRegex, svgTagInXMLRegex *regexp.Regexp +}, +) { + ret.svgComment = regexp.MustCompile(`(?s)<!--.*?-->`) + ret.svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) + ret.svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) + return ret +}) + +// SniffedType contains information about a blob's type. type SniffedType struct { contentType string } -// IsText etects if content format is plain text. +// IsText detects if the content format is text family, including text/plain, text/html, text/css, etc. func (ct SniffedType) IsText() bool { return strings.Contains(ct.contentType, "text/") } +func (ct SniffedType) IsTextPlain() bool { + return strings.Contains(ct.contentType, "text/plain") +} + // IsImage detects if data is an image format func (ct SniffedType) IsImage() bool { return strings.Contains(ct.contentType, "image/") @@ -57,12 +61,12 @@ func (ct SniffedType) IsPDF() bool { return strings.Contains(ct.contentType, "application/pdf") } -// IsVideo detects if data is an video format +// IsVideo detects if data is a video format func (ct SniffedType) IsVideo() bool { return strings.Contains(ct.contentType, "video/") } -// IsAudio detects if data is an video format +// IsAudio detects if data is a video format func (ct SniffedType) IsAudio() bool { return strings.Contains(ct.contentType, "audio/") } @@ -103,33 +107,34 @@ func detectFileTypeBox(data []byte) (brands []string, found bool) { return brands, true } -// DetectContentType extends http.DetectContentType with more content types. Defaults to text/unknown if input is empty. +// DetectContentType extends http.DetectContentType with more content types. Defaults to text/plain if input is empty. func DetectContentType(data []byte) SniffedType { if len(data) == 0 { - return SniffedType{"text/unknown"} + return SniffedType{"text/plain"} } ct := http.DetectContentType(data) - if len(data) > sniffLen { - data = data[:sniffLen] + if len(data) > SniffContentSize { + data = data[:SniffContentSize] } + vars := globalVars() // SVG is unsupported by http.DetectContentType, https://github.com/golang/go/issues/15888 detectByHTML := strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html") detectByXML := strings.Contains(ct, "text/xml") if detectByHTML || detectByXML { - dataProcessed := svgComment.ReplaceAll(data, nil) + dataProcessed := vars.svgComment.ReplaceAll(data, nil) dataProcessed = bytes.TrimSpace(dataProcessed) - if detectByHTML && svgTagRegex.Match(dataProcessed) || - detectByXML && svgTagInXMLRegex.Match(dataProcessed) { + if detectByHTML && vars.svgTagRegex.Match(dataProcessed) || + detectByXML && vars.svgTagInXMLRegex.Match(dataProcessed) { ct = MimeTypeImageSvg } } if strings.HasPrefix(ct, "audio/") && bytes.HasPrefix(data, []byte("ID3")) { // The MP3 detection is quite inaccurate, any content with "ID3" prefix will result in "audio/mpeg". - // So remove the "ID3" prefix and detect again, if result is text, then it must be text content. + // So remove the "ID3" prefix and detect again, then if the result is "text", it must be text content. // This works especially because audio files contain many unprintable/invalid characters like `0x00` ct2 := http.DetectContentType(data[3:]) if strings.HasPrefix(ct2, "text/") { @@ -155,15 +160,3 @@ func DetectContentType(data []byte) SniffedType { } return SniffedType{ct} } - -// DetectContentTypeFromReader guesses the content type contained in the reader. -func DetectContentTypeFromReader(r io.Reader) (SniffedType, error) { - buf := make([]byte, sniffLen) - n, err := util.ReadAtMost(r, buf) - if err != nil { - return SniffedType{}, fmt.Errorf("DetectContentTypeFromReader io error: %w", err) - } - buf = buf[:n] - - return DetectContentType(buf), nil -} diff --git a/modules/typesniffer/typesniffer_test.go b/modules/typesniffer/typesniffer_test.go index 3e5db3308b..a0c824b912 100644 --- a/modules/typesniffer/typesniffer_test.go +++ b/modules/typesniffer/typesniffer_test.go @@ -4,7 +4,6 @@ package typesniffer import ( - "bytes" "encoding/base64" "encoding/hex" "strings" @@ -17,7 +16,7 @@ func TestDetectContentTypeLongerThanSniffLen(t *testing.T) { // Pre-condition: Shorter than sniffLen detects SVG. assert.Equal(t, "image/svg+xml", DetectContentType([]byte(`<!-- Comment --><svg></svg>`)).contentType) // Longer than sniffLen detects something else. - assert.NotEqual(t, "image/svg+xml", DetectContentType([]byte(`<!-- `+strings.Repeat("x", sniffLen)+` --><svg></svg>`)).contentType) + assert.NotEqual(t, "image/svg+xml", DetectContentType([]byte(`<!-- `+strings.Repeat("x", SniffContentSize)+` --><svg></svg>`)).contentType) } func TestIsTextFile(t *testing.T) { @@ -116,22 +115,13 @@ func TestIsAudio(t *testing.T) { assert.True(t, DetectContentType([]byte("ID3Toy\n====\t* hi 🌞, ..."+"🌛"[0:2])).IsText()) // test ID3 tag with incomplete UTF8 char } -func TestDetectContentTypeFromReader(t *testing.T) { - mp3, _ := base64.StdEncoding.DecodeString("SUQzBAAAAAABAFRYWFgAAAASAAADbWFqb3JfYnJhbmQAbXA0MgBUWFhYAAAAEQAAA21pbm9yX3Zl") - st, err := DetectContentTypeFromReader(bytes.NewReader(mp3)) - assert.NoError(t, err) - assert.True(t, st.IsAudio()) -} - func TestDetectContentTypeOgg(t *testing.T) { oggAudio, _ := hex.DecodeString("4f67675300020000000000000000352f0000000000007dc39163011e01766f72626973000000000244ac0000000000000071020000000000b8014f6767530000") - st, err := DetectContentTypeFromReader(bytes.NewReader(oggAudio)) - assert.NoError(t, err) + st := DetectContentType(oggAudio) assert.True(t, st.IsAudio()) oggVideo, _ := hex.DecodeString("4f676753000200000000000000007d9747ef000000009b59daf3012a807468656f7261030201001e00110001e000010e00020000001e00000001000001000001") - st, err = DetectContentTypeFromReader(bytes.NewReader(oggVideo)) - assert.NoError(t, err) + st = DetectContentType(oggVideo) assert.True(t, st.IsVideo()) } diff --git a/modules/updatechecker/update_checker.go b/modules/updatechecker/update_checker.go index 3c1e05d060..f0686c0f78 100644 --- a/modules/updatechecker/update_checker.go +++ b/modules/updatechecker/update_checker.go @@ -34,7 +34,7 @@ func GiteaUpdateChecker(httpEndpoint string) error { }, } - req, err := http.NewRequest("GET", httpEndpoint, nil) + req, err := http.NewRequest(http.MethodGet, httpEndpoint, nil) if err != nil { return err } diff --git a/modules/util/error.go b/modules/util/error.go index 8e67d5a82f..6b2721618e 100644 --- a/modules/util/error.go +++ b/modules/util/error.go @@ -17,8 +17,8 @@ var ( ErrNotExist = errors.New("resource does not exist") // also implies HTTP 404 ErrAlreadyExist = errors.New("resource already exists") // also implies HTTP 409 - // ErrUnprocessableContent implies HTTP 422, syntax of the request content was correct, - // but server was unable to process the contained instructions + // ErrUnprocessableContent implies HTTP 422, the syntax of the request content is correct, + // but the server is unable to process the contained instructions ErrUnprocessableContent = errors.New("unprocessable content") ) diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index 739543e297..0731ba30c8 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -7,16 +7,10 @@ import ( "bytes" "errors" "io" - "math" "os" ) -var ( - // ErrInvalidMemorySize occurs if the memory size is not in a valid range - ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") - // ErrWriteAfterRead occurs if Write is called after a read operation - ErrWriteAfterRead = errors.New("Write is unsupported after a read operation") -) +var ErrWriteAfterRead = errors.New("write is unsupported after a read operation") // occurs if Write is called after a read operation type readAtSeeker interface { io.ReadSeeker @@ -30,34 +24,17 @@ type FileBackedBuffer struct { maxMemorySize int64 size int64 buffer bytes.Buffer + tempDir string file *os.File reader readAtSeeker } // New creates a file backed buffer with a specific maximum memory size -func New(maxMemorySize int) (*FileBackedBuffer, error) { - if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 { - return nil, ErrInvalidMemorySize - } - +func New(maxMemorySize int, tempDir string) *FileBackedBuffer { return &FileBackedBuffer{ maxMemorySize: int64(maxMemorySize), - }, nil -} - -// CreateFromReader creates a file backed buffer and copies the provided reader data into it. -func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) { - b, err := New(maxMemorySize) - if err != nil { - return nil, err + tempDir: tempDir, } - - _, err = io.Copy(b, r) - if err != nil { - return nil, err - } - - return b, nil } // Write implements io.Writer @@ -73,7 +50,7 @@ func (b *FileBackedBuffer) Write(p []byte) (int, error) { n, err = b.file.Write(p) } else { if b.size+int64(len(p)) > b.maxMemorySize { - b.file, err = os.CreateTemp("", "gitea-buffer-") + b.file, err = os.CreateTemp(b.tempDir, "gitea-buffer-") if err != nil { return 0, err } @@ -148,7 +125,7 @@ func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { func (b *FileBackedBuffer) Close() error { if b.file != nil { err := b.file.Close() - os.Remove(b.file.Name()) + _ = os.Remove(b.file.Name()) b.file = nil return err } diff --git a/modules/util/filebuffer/file_backed_buffer_test.go b/modules/util/filebuffer/file_backed_buffer_test.go index 16d5a1965f..3f13c6ac7b 100644 --- a/modules/util/filebuffer/file_backed_buffer_test.go +++ b/modules/util/filebuffer/file_backed_buffer_test.go @@ -21,7 +21,8 @@ func TestFileBackedBuffer(t *testing.T) { } for _, c := range cases { - buf, err := CreateFromReader(strings.NewReader(c.Data), c.MaxMemorySize) + buf := New(c.MaxMemorySize, t.TempDir()) + _, err := io.Copy(buf, strings.NewReader(c.Data)) assert.NoError(t, err) assert.EqualValues(t, len(c.Data), buf.Size()) diff --git a/modules/util/map.go b/modules/util/map.go new file mode 100644 index 0000000000..f307faad1f --- /dev/null +++ b/modules/util/map.go @@ -0,0 +1,13 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +func GetMapValueOrDefault[T any](m map[string]any, key string, defaultValue T) T { + if value, ok := m[key]; ok { + if v, ok := value.(T); ok { + return v + } + } + return defaultValue +} diff --git a/modules/util/map_test.go b/modules/util/map_test.go new file mode 100644 index 0000000000..1a141cec88 --- /dev/null +++ b/modules/util/map_test.go @@ -0,0 +1,26 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetMapValueOrDefault(t *testing.T) { + testMap := map[string]any{ + "key1": "value1", + "key2": 42, + "key3": nil, + } + + assert.Equal(t, "value1", GetMapValueOrDefault(testMap, "key1", "default")) + assert.Equal(t, 42, GetMapValueOrDefault(testMap, "key2", 0)) + + assert.Equal(t, "default", GetMapValueOrDefault(testMap, "key4", "default")) + assert.Equal(t, 100, GetMapValueOrDefault(testMap, "key5", 100)) + + assert.Equal(t, "default", GetMapValueOrDefault(testMap, "key3", "default")) +} diff --git a/modules/util/paginate_test.go b/modules/util/paginate_test.go index 6e69dd19cc..3dc5095071 100644 --- a/modules/util/paginate_test.go +++ b/modules/util/paginate_test.go @@ -13,23 +13,23 @@ func TestPaginateSlice(t *testing.T) { stringSlice := []string{"a", "b", "c", "d", "e"} result, ok := PaginateSlice(stringSlice, 1, 2).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"a", "b"}, result) + assert.Equal(t, []string{"a", "b"}, result) result, ok = PaginateSlice(stringSlice, 100, 2).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{}, result) + assert.Equal(t, []string{}, result) result, ok = PaginateSlice(stringSlice, 3, 2).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"e"}, result) + assert.Equal(t, []string{"e"}, result) result, ok = PaginateSlice(stringSlice, 1, 0).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"a", "b", "c", "d", "e"}, result) + assert.Equal(t, []string{"a", "b", "c", "d", "e"}, result) result, ok = PaginateSlice(stringSlice, 1, -1).([]string) assert.True(t, ok) - assert.EqualValues(t, []string{"a", "b", "c", "d", "e"}, result) + assert.Equal(t, []string{"a", "b", "c", "d", "e"}, result) type Test struct { Val int @@ -38,9 +38,9 @@ func TestPaginateSlice(t *testing.T) { testVar := []*Test{{Val: 2}, {Val: 3}, {Val: 4}} testVar, ok = PaginateSlice(testVar, 1, 50).([]*Test) assert.True(t, ok) - assert.EqualValues(t, []*Test{{Val: 2}, {Val: 3}, {Val: 4}}, testVar) + assert.Equal(t, []*Test{{Val: 2}, {Val: 3}, {Val: 4}}, testVar) testVar, ok = PaginateSlice(testVar, 2, 2).([]*Test) assert.True(t, ok) - assert.EqualValues(t, []*Test{{Val: 4}}, testVar) + assert.Equal(t, []*Test{{Val: 4}}, testVar) } diff --git a/modules/util/path.go b/modules/util/path.go index d9f17bd124..0e56348978 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -36,9 +36,10 @@ func PathJoinRel(elem ...string) string { elems[i] = path.Clean("/" + e) } p := path.Join(elems...) - if p == "" { + switch p { + case "": return "" - } else if p == "/" { + case "/": return "." } return p[1:] diff --git a/modules/util/remove.go b/modules/util/remove.go index d1e38faf5f..3db0b5a796 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -15,7 +15,7 @@ const windowsSharingViolationError syscall.Errno = 32 // Remove removes the named file or (empty) directory with at most 5 attempts. func Remove(name string) error { var err error - for i := 0; i < 5; i++ { + for range 5 { err = os.Remove(name) if err == nil { break @@ -44,7 +44,7 @@ func Remove(name string) error { // RemoveAll removes the named file or (empty) directory with at most 5 attempts. func RemoveAll(name string) error { var err error - for i := 0; i < 5; i++ { + for range 5 { err = os.RemoveAll(name) if err == nil { break @@ -73,7 +73,7 @@ func RemoveAll(name string) error { // Rename renames (moves) oldpath to newpath with at most 5 attempts. func Rename(oldpath, newpath string) error { var err error - for i := 0; i < 5; i++ { + for i := range 5 { err = os.Rename(oldpath, newpath) if err == nil { break diff --git a/modules/util/rotatingfilewriter/writer_test.go b/modules/util/rotatingfilewriter/writer_test.go index 88392797b3..f6ea1d50ae 100644 --- a/modules/util/rotatingfilewriter/writer_test.go +++ b/modules/util/rotatingfilewriter/writer_test.go @@ -23,7 +23,7 @@ func TestCompressOldFile(t *testing.T) { ng, err := os.OpenFile(nonGzip, os.O_CREATE|os.O_WRONLY, 0o660) assert.NoError(t, err) - for i := 0; i < 999; i++ { + for range 999 { f.WriteString("This is a test file\n") ng.WriteString("This is a test file\n") } diff --git a/modules/util/sec_to_time_test.go b/modules/util/sec_to_time_test.go index b67926bbcf..84e767c6e0 100644 --- a/modules/util/sec_to_time_test.go +++ b/modules/util/sec_to_time_test.go @@ -24,5 +24,5 @@ func TestSecToHours(t *testing.T) { assert.Equal(t, "672 hours", SecToHours(4*7*day)) assert.Equal(t, "1 second", SecToHours(1)) assert.Equal(t, "2 seconds", SecToHours(2)) - assert.Equal(t, "", SecToHours(nil)) // old behavior, empty means no output + assert.Empty(t, SecToHours(nil)) // old behavior, empty means no output } diff --git a/modules/util/slice.go b/modules/util/slice.go index da6886491e..aaa729c1c9 100644 --- a/modules/util/slice.go +++ b/modules/util/slice.go @@ -12,8 +12,7 @@ import ( // SliceContainsString sequential searches if string exists in slice. func SliceContainsString(slice []string, target string, insensitive ...bool) bool { if len(insensitive) != 0 && insensitive[0] { - target = strings.ToLower(target) - return slices.ContainsFunc(slice, func(t string) bool { return strings.ToLower(t) == target }) + return slices.ContainsFunc(slice, func(t string) bool { return strings.EqualFold(t, target) }) } return slices.Contains(slice, target) diff --git a/modules/util/string.go b/modules/util/string.go index 19cf75b8b3..b9b59df3ef 100644 --- a/modules/util/string.go +++ b/modules/util/string.go @@ -103,10 +103,31 @@ func UnsafeStringToBytes(s string) []byte { func SplitTrimSpace(input, sep string) []string { input = strings.TrimSpace(input) var stringList []string - for _, s := range strings.Split(input, sep) { + for s := range strings.SplitSeq(input, sep) { if s = strings.TrimSpace(s); s != "" { stringList = append(stringList, s) } } return stringList } + +func asciiLower(b byte) byte { + if 'A' <= b && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +// AsciiEqualFold is from Golang https://cs.opensource.google/go/go/+/refs/tags/go1.24.4:src/net/http/internal/ascii/print.go +// ASCII only. In most cases for protocols, we should only use this but not [strings.EqualFold] +func AsciiEqualFold(s, t string) bool { //nolint:revive // PascalCase + if len(s) != len(t) { + return false + } + for i := 0; i < len(s); i++ { + if asciiLower(s[i]) != asciiLower(t[i]) { + return false + } + } + return true +} diff --git a/modules/util/time_str.go b/modules/util/time_str.go index 0fccfe82cc..81b132c3db 100644 --- a/modules/util/time_str.go +++ b/modules/util/time_str.go @@ -59,7 +59,7 @@ func TimeEstimateParse(timeStr string) (int64, error) { unit := timeStr[match[4]:match[5]] found := false for _, u := range timeStrGlobalVars().units { - if strings.ToLower(unit) == u.name { + if strings.EqualFold(unit, u.name) { total += amount * u.num found = true break diff --git a/modules/util/truncate_test.go b/modules/util/truncate_test.go index 8789c824f5..9f4ad7dc20 100644 --- a/modules/util/truncate_test.go +++ b/modules/util/truncate_test.go @@ -5,6 +5,7 @@ package util import ( "fmt" + "strconv" "strings" "testing" @@ -100,7 +101,7 @@ func TestEllipsisString(t *testing.T) { {limit: 7, left: "\xef\x03\xfe\xef\x03\xfe", right: ""}, } for _, c := range invalidCases { - t.Run(fmt.Sprintf("%d", c.limit), func(t *testing.T) { + t.Run(strconv.Itoa(c.limit), func(t *testing.T) { left, right := EllipsisDisplayStringX("\xef\x03\xfe\xef\x03\xfe", c.limit) assert.Equal(t, c.left, left, "left") assert.Equal(t, c.right, right, "right") @@ -115,15 +116,15 @@ func TestEllipsisString(t *testing.T) { } func TestTruncateRunes(t *testing.T) { - assert.Equal(t, "", TruncateRunes("", 0)) - assert.Equal(t, "", TruncateRunes("", 1)) + assert.Empty(t, TruncateRunes("", 0)) + assert.Empty(t, TruncateRunes("", 1)) - assert.Equal(t, "", TruncateRunes("ab", 0)) + assert.Empty(t, TruncateRunes("ab", 0)) assert.Equal(t, "a", TruncateRunes("ab", 1)) assert.Equal(t, "ab", TruncateRunes("ab", 2)) assert.Equal(t, "ab", TruncateRunes("ab", 3)) - assert.Equal(t, "", TruncateRunes("测试", 0)) + assert.Empty(t, TruncateRunes("测试", 0)) assert.Equal(t, "测", TruncateRunes("测试", 1)) assert.Equal(t, "测试", TruncateRunes("测试", 2)) assert.Equal(t, "测试", TruncateRunes("测试", 3)) diff --git a/modules/util/util.go b/modules/util/util.go index 1fb4cb21cb..dd8e073888 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -11,21 +11,10 @@ import ( "strconv" "strings" - "code.gitea.io/gitea/modules/optional" - "golang.org/x/text/cases" "golang.org/x/text/language" ) -// OptionalBoolParse get the corresponding optional.Option[bool] of a string using strconv.ParseBool -func OptionalBoolParse(s string) optional.Option[bool] { - v, e := strconv.ParseBool(s) - if e != nil { - return optional.None[bool]() - } - return optional.Some(v) -} - // IsEmptyString checks if the provided string is empty func IsEmptyString(s string) bool { return len(strings.TrimSpace(s)) == 0 @@ -230,6 +219,13 @@ func IfZero[T comparable](v, def T) T { return v } +func IfEmpty[T any](v, def []T) []T { + if len(v) == 0 { + return def + } + return v +} + // OptionalArg helps the "optional argument" in Golang: // // func foo(optArg ...int) { return OptionalArg(optArg) } diff --git a/modules/util/util_test.go b/modules/util/util_test.go index effbc6da1e..fe4125cdb5 100644 --- a/modules/util/util_test.go +++ b/modules/util/util_test.go @@ -8,8 +8,6 @@ import ( "strings" "testing" - "code.gitea.io/gitea/modules/optional" - "github.com/stretchr/testify/assert" ) @@ -175,19 +173,6 @@ func Test_RandomBytes(t *testing.T) { assert.NotEqual(t, bytes3, bytes4) } -func TestOptionalBoolParse(t *testing.T) { - assert.Equal(t, optional.None[bool](), OptionalBoolParse("")) - assert.Equal(t, optional.None[bool](), OptionalBoolParse("x")) - - assert.Equal(t, optional.Some(false), OptionalBoolParse("0")) - assert.Equal(t, optional.Some(false), OptionalBoolParse("f")) - assert.Equal(t, optional.Some(false), OptionalBoolParse("False")) - - assert.Equal(t, optional.Some(true), OptionalBoolParse("1")) - assert.Equal(t, optional.Some(true), OptionalBoolParse("t")) - assert.Equal(t, optional.Some(true), OptionalBoolParse("True")) -} - // Test case for any function which accepts and returns a single string. type StringTest struct { in, out string diff --git a/modules/validation/binding_test.go b/modules/validation/binding_test.go index 28d0f57b5c..0cd328f312 100644 --- a/modules/validation/binding_test.go +++ b/modules/validation/binding_test.go @@ -47,7 +47,7 @@ func performValidationTest(t *testing.T, testCase validationTestCase) { assert.Equal(t, testCase.expectedErrors, actual) }) - req, err := http.NewRequest("POST", testRoute, nil) + req, err := http.NewRequest(http.MethodPost, testRoute, nil) if err != nil { panic(err) } diff --git a/modules/validation/helpers.go b/modules/validation/helpers.go index 9f6cf5201a..ba383ba195 100644 --- a/modules/validation/helpers.go +++ b/modules/validation/helpers.go @@ -7,6 +7,7 @@ import ( "net" "net/url" "regexp" + "slices" "strings" "sync" @@ -55,12 +56,7 @@ func IsValidSiteURL(uri string) bool { return false } - for _, scheme := range setting.Service.ValidSiteURLSchemes { - if scheme == u.Scheme { - return true - } - } - return false + return slices.Contains(setting.Service.ValidSiteURLSchemes, u.Scheme) } // IsEmailDomainListed checks whether the domain of an email address diff --git a/modules/validation/helpers_test.go b/modules/validation/helpers_test.go index 52f383f698..6a982965f6 100644 --- a/modules/validation/helpers_test.go +++ b/modules/validation/helpers_test.go @@ -7,6 +7,7 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" ) @@ -47,7 +48,7 @@ func Test_IsValidURL(t *testing.T) { } func Test_IsValidExternalURL(t *testing.T) { - setting.AppURL = "https://try.gitea.io/" + defer test.MockVariableValue(&setting.AppURL, "https://try.gitea.io/")() cases := []struct { description string @@ -89,7 +90,7 @@ func Test_IsValidExternalURL(t *testing.T) { } func Test_IsValidExternalTrackerURLFormat(t *testing.T) { - setting.AppURL = "https://try.gitea.io/" + defer test.MockVariableValue(&setting.AppURL, "https://try.gitea.io/")() cases := []struct { description string diff --git a/modules/web/middleware/binding.go b/modules/web/middleware/binding.go index 03e188f509..ee4eca976e 100644 --- a/modules/web/middleware/binding.go +++ b/modules/web/middleware/binding.go @@ -50,7 +50,7 @@ func AssignForm(form any, data map[string]any) { } func getRuleBody(field reflect.StructField, prefix string) string { - for _, rule := range strings.Split(field.Tag.Get("binding"), ";") { + for rule := range strings.SplitSeq(field.Tag.Get("binding"), ";") { if strings.HasPrefix(rule, prefix) { return rule[len(prefix) : len(rule)-1] } diff --git a/modules/web/routemock_test.go b/modules/web/routemock_test.go index 89cfaacdd1..a0949bf622 100644 --- a/modules/web/routemock_test.go +++ b/modules/web/routemock_test.go @@ -30,13 +30,13 @@ func TestRouteMock(t *testing.T) { // normal request recorder := httptest.NewRecorder() - req, err := http.NewRequest("GET", "http://localhost:8000/foo", nil) + req, err := http.NewRequest(http.MethodGet, "http://localhost:8000/foo", nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Len(t, recorder.Header(), 3) - assert.EqualValues(t, "m1", recorder.Header().Get("X-Test-Middleware1")) - assert.EqualValues(t, "m2", recorder.Header().Get("X-Test-Middleware2")) - assert.EqualValues(t, "h", recorder.Header().Get("X-Test-Handler")) + assert.Equal(t, "m1", recorder.Header().Get("X-Test-Middleware1")) + assert.Equal(t, "m2", recorder.Header().Get("X-Test-Middleware2")) + assert.Equal(t, "h", recorder.Header().Get("X-Test-Handler")) RouteMockReset() // mock at "mock-point" @@ -45,12 +45,12 @@ func TestRouteMock(t *testing.T) { resp.WriteHeader(http.StatusOK) }) recorder = httptest.NewRecorder() - req, err = http.NewRequest("GET", "http://localhost:8000/foo", nil) + req, err = http.NewRequest(http.MethodGet, "http://localhost:8000/foo", nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Len(t, recorder.Header(), 2) - assert.EqualValues(t, "m1", recorder.Header().Get("X-Test-Middleware1")) - assert.EqualValues(t, "a", recorder.Header().Get("X-Test-MockPoint")) + assert.Equal(t, "m1", recorder.Header().Get("X-Test-Middleware1")) + assert.Equal(t, "a", recorder.Header().Get("X-Test-MockPoint")) RouteMockReset() // mock at MockAfterMiddlewares @@ -59,12 +59,12 @@ func TestRouteMock(t *testing.T) { resp.WriteHeader(http.StatusOK) }) recorder = httptest.NewRecorder() - req, err = http.NewRequest("GET", "http://localhost:8000/foo", nil) + req, err = http.NewRequest(http.MethodGet, "http://localhost:8000/foo", nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Len(t, recorder.Header(), 3) - assert.EqualValues(t, "m1", recorder.Header().Get("X-Test-Middleware1")) - assert.EqualValues(t, "m2", recorder.Header().Get("X-Test-Middleware2")) - assert.EqualValues(t, "b", recorder.Header().Get("X-Test-MockPoint")) + assert.Equal(t, "m1", recorder.Header().Get("X-Test-Middleware1")) + assert.Equal(t, "m2", recorder.Header().Get("X-Test-Middleware2")) + assert.Equal(t, "b", recorder.Header().Get("X-Test-MockPoint")) RouteMockReset() } diff --git a/modules/web/router.go b/modules/web/router.go index da06b955b1..5812ff69d4 100644 --- a/modules/web/router.go +++ b/modules/web/router.go @@ -125,8 +125,8 @@ func (r *Router) Methods(methods, pattern string, h ...any) { middlewares, handlerFunc := wrapMiddlewareAndHandler(r.curMiddlewares, h) fullPattern := r.getPattern(pattern) if strings.Contains(methods, ",") { - methods := strings.Split(methods, ",") - for _, method := range methods { + methods := strings.SplitSeq(methods, ",") + for method := range methods { r.chiRouter.With(middlewares...).Method(strings.TrimSpace(method), fullPattern, handlerFunc) } } else { diff --git a/modules/web/router_path.go b/modules/web/router_path.go index b59948581a..64154c34a5 100644 --- a/modules/web/router_path.go +++ b/modules/web/router_path.go @@ -4,9 +4,9 @@ package web import ( - "fmt" "net/http" "regexp" + "slices" "strings" "code.gitea.io/gitea/modules/container" @@ -26,6 +26,7 @@ func (g *RouterPathGroup) ServeHTTP(resp http.ResponseWriter, req *http.Request) path := chiCtx.URLParam(g.pathParam) for _, m := range g.matchers { if m.matchPath(chiCtx, path) { + chiCtx.RoutePatterns = append(chiCtx.RoutePatterns, m.pattern) handler := m.handlerFunc for i := len(m.middlewares) - 1; i >= 0; i-- { handler = m.middlewares[i](handler).ServeHTTP @@ -37,11 +38,22 @@ func (g *RouterPathGroup) ServeHTTP(resp http.ResponseWriter, req *http.Request) g.r.chiRouter.NotFoundHandler().ServeHTTP(resp, req) } +type RouterPathGroupPattern struct { + pattern string + re *regexp.Regexp + params []routerPathParam + middlewares []any +} + // MatchPath matches the request method, and uses regexp to match the path. -// The pattern uses "<...>" to define path parameters, for example: "/<name>" (different from chi router) -// It is only designed to resolve some special cases which chi router can't handle. +// The pattern uses "<...>" to define path parameters, for example, "/<name>" (different from chi router) +// It is only designed to resolve some special cases that chi router can't handle. // For most cases, it shouldn't be used because it needs to iterate all rules to find the matched one (inefficient). func (g *RouterPathGroup) MatchPath(methods, pattern string, h ...any) { + g.MatchPattern(methods, g.PatternRegexp(pattern), h...) +} + +func (g *RouterPathGroup) MatchPattern(methods string, pattern *RouterPathGroupPattern, h ...any) { g.matchers = append(g.matchers, newRouterPathMatcher(methods, pattern, h...)) } @@ -52,6 +64,7 @@ type routerPathParam struct { type routerPathMatcher struct { methods container.Set[string] + pattern string re *regexp.Regexp params []routerPathParam middlewares []func(http.Handler) http.Handler @@ -97,29 +110,35 @@ func isValidMethod(name string) bool { return false } -func newRouterPathMatcher(methods, pattern string, h ...any) *routerPathMatcher { - middlewares, handlerFunc := wrapMiddlewareAndHandler(nil, h) +func newRouterPathMatcher(methods string, patternRegexp *RouterPathGroupPattern, h ...any) *routerPathMatcher { + middlewares, handlerFunc := wrapMiddlewareAndHandler(patternRegexp.middlewares, h) p := &routerPathMatcher{methods: make(container.Set[string]), middlewares: middlewares, handlerFunc: handlerFunc} - for _, method := range strings.Split(methods, ",") { + for method := range strings.SplitSeq(methods, ",") { method = strings.TrimSpace(method) if !isValidMethod(method) { - panic(fmt.Sprintf("invalid HTTP method: %s", method)) + panic("invalid HTTP method: " + method) } p.methods.Add(method) } + p.pattern, p.re, p.params = patternRegexp.pattern, patternRegexp.re, patternRegexp.params + return p +} + +func patternRegexp(pattern string, h ...any) *RouterPathGroupPattern { + p := &RouterPathGroupPattern{middlewares: slices.Clone(h)} re := []byte{'^'} lastEnd := 0 for lastEnd < len(pattern) { start := strings.IndexByte(pattern[lastEnd:], '<') if start == -1 { - re = append(re, pattern[lastEnd:]...) + re = append(re, regexp.QuoteMeta(pattern[lastEnd:])...) break } end := strings.IndexByte(pattern[lastEnd+start:], '>') if end == -1 { - panic(fmt.Sprintf("invalid pattern: %s", pattern)) + panic("invalid pattern: " + pattern) } - re = append(re, pattern[lastEnd:lastEnd+start]...) + re = append(re, regexp.QuoteMeta(pattern[lastEnd:lastEnd+start])...) partName, partExp, _ := strings.Cut(pattern[lastEnd+start+1:lastEnd+start+end], ":") lastEnd += start + end + 1 @@ -141,7 +160,10 @@ func newRouterPathMatcher(methods, pattern string, h ...any) *routerPathMatcher p.params = append(p.params, param) } re = append(re, '$') - reStr := string(re) - p.re = regexp.MustCompile(reStr) + p.pattern, p.re = pattern, regexp.MustCompile(string(re)) return p } + +func (g *RouterPathGroup) PatternRegexp(pattern string, h ...any) *RouterPathGroupPattern { + return patternRegexp(pattern, h...) +} diff --git a/modules/web/router_test.go b/modules/web/router_test.go index 582980a27a..f216aa6180 100644 --- a/modules/web/router_test.go +++ b/modules/web/router_test.go @@ -34,7 +34,7 @@ func TestPathProcessor(t *testing.T) { testProcess := func(pattern, uri string, expectedPathParams map[string]string) { chiCtx := chi.NewRouteContext() chiCtx.RouteMethod = "GET" - p := newRouterPathMatcher("GET", pattern, http.NotFound) + p := newRouterPathMatcher("GET", patternRegexp(pattern), http.NotFound) assert.True(t, p.matchPath(chiCtx, uri), "use pattern %s to process uri %s", pattern, uri) assert.Equal(t, expectedPathParams, chiURLParamsToMap(chiCtx), "use pattern %s to process uri %s", pattern, uri) } @@ -51,23 +51,28 @@ func TestPathProcessor(t *testing.T) { } func TestRouter(t *testing.T) { - buff := bytes.NewBufferString("") + buff := &bytes.Buffer{} recorder := httptest.NewRecorder() recorder.Body = buff type resultStruct struct { - method string - pathParams map[string]string - handlerMark string + method string + pathParams map[string]string + handlerMarks []string + chiRoutePattern *string } - var res resultStruct + var res resultStruct h := func(optMark ...string) func(resp http.ResponseWriter, req *http.Request) { mark := util.OptionalArg(optMark, "") return func(resp http.ResponseWriter, req *http.Request) { + chiCtx := chi.RouteContext(req.Context()) res.method = req.Method - res.pathParams = chiURLParamsToMap(chi.RouteContext(req.Context())) - res.handlerMark = mark + res.pathParams = chiURLParamsToMap(chiCtx) + res.chiRoutePattern = util.ToPointer(chiCtx.RoutePattern()) + if mark != "" { + res.handlerMarks = append(res.handlerMarks, mark) + } } } @@ -77,6 +82,8 @@ func TestRouter(t *testing.T) { if stop := req.FormValue("stop"); stop != "" && (mark == "" || mark == stop) { h(stop)(resp, req) resp.WriteHeader(http.StatusOK) + } else if mark != "" { + res.handlerMarks = append(res.handlerMarks, mark) } } } @@ -108,7 +115,7 @@ func TestRouter(t *testing.T) { m.Delete("", h()) }) m.PathGroup("/*", func(g *RouterPathGroup) { - g.MatchPath("GET", `/<dir:*>/<file:[a-z]{1,2}>`, stopMark("s2"), h("match-path")) + g.MatchPattern("GET", g.PatternRegexp(`/<dir:*>/<file:[a-z]{1,2}>`, stopMark("s2")), stopMark("s3"), h("match-path")) }, stopMark("s1")) }) }) @@ -121,36 +128,47 @@ func TestRouter(t *testing.T) { req, err := http.NewRequest(methodPathFields[0], methodPathFields[1], nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) - assert.EqualValues(t, expected, res) + if expected.chiRoutePattern == nil { + res.chiRoutePattern = nil + } + assert.Equal(t, expected, res) }) } t.Run("RootRouter", func(t *testing.T) { - testRoute(t, "GET /the-user/the-repo/other", resultStruct{method: "GET", handlerMark: "not-found:/"}) + testRoute(t, "GET /the-user/the-repo/other", resultStruct{ + method: "GET", + handlerMarks: []string{"not-found:/"}, + chiRoutePattern: util.ToPointer(""), + }) testRoute(t, "GET /the-user/the-repo/pulls", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "pulls"}, - handlerMark: "list-issues-b", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "pulls"}, + handlerMarks: []string{"list-issues-b"}, }) testRoute(t, "GET /the-user/the-repo/issues/123", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, - handlerMark: "view-issue", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, + handlerMarks: []string{"view-issue"}, + chiRoutePattern: util.ToPointer("/{username}/{reponame}/{type:issues|pulls}/{index}"), }) testRoute(t, "GET /the-user/the-repo/issues/123?stop=hijack", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, - handlerMark: "hijack", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, + handlerMarks: []string{"hijack"}, }) testRoute(t, "POST /the-user/the-repo/issues/123/update", resultStruct{ - method: "POST", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "index": "123"}, - handlerMark: "update-issue", + method: "POST", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "index": "123"}, + handlerMarks: []string{"update-issue"}, }) }) t.Run("Sub Router", func(t *testing.T) { - testRoute(t, "GET /api/v1/other", resultStruct{method: "GET", handlerMark: "not-found:/api/v1"}) + testRoute(t, "GET /api/v1/other", resultStruct{ + method: "GET", + handlerMarks: []string{"not-found:/api/v1"}, + }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches", resultStruct{ method: "GET", pathParams: map[string]string{"username": "the-user", "reponame": "the-repo"}, @@ -179,31 +197,38 @@ func TestRouter(t *testing.T) { t.Run("MatchPath", func(t *testing.T) { testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, - handlerMark: "match-path", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3", "match-path"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1%2fd2/fn", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1%2fd2/fn", "dir": "d1%2fd2", "file": "fn"}, - handlerMark: "match-path", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1%2fd2/fn", "dir": "d1%2fd2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3", "match-path"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/000", resultStruct{ - method: "GET", - pathParams: map[string]string{"reponame": "the-repo", "username": "the-user", "*": "d1/d2/000"}, - handlerMark: "not-found:/api/v1", + method: "GET", + pathParams: map[string]string{"reponame": "the-repo", "username": "the-user", "*": "d1/d2/000"}, + handlerMarks: []string{"s1", "not-found:/api/v1"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s1", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn"}, - handlerMark: "s1", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn"}, + handlerMarks: []string{"s1"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s2", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, - handlerMark: "s2", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2"}, + }) + + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s3", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3"}, + chiRoutePattern: util.ToPointer("/api/v1/repos/{username}/{reponame}/branches/<dir:*>/<file:[a-z]{1,2}>"), }) }) } @@ -224,7 +249,7 @@ func TestRouteNormalizePath(t *testing.T) { actualPaths.Path = req.URL.Path }) - req, err := http.NewRequest("GET", reqPath, nil) + req, err := http.NewRequest(http.MethodGet, reqPath, nil) assert.NoError(t, err) r.ServeHTTP(recorder, req) assert.Equal(t, expectedPaths, actualPaths, "req path = %q", reqPath) diff --git a/modules/web/routing/logger.go b/modules/web/routing/logger.go index e3843b1402..3bca9b3420 100644 --- a/modules/web/routing/logger.go +++ b/modules/web/routing/logger.go @@ -103,7 +103,10 @@ func logPrinter(logger log.Logger) func(trigger Event, record *requestRecord) { status = v.WrittenStatus() } logf := logInfo - if strings.HasPrefix(req.RequestURI, "/assets/") { + // lower the log level for some specific requests, in most cases these logs are not useful + if strings.HasPrefix(req.RequestURI, "/assets/") /* static assets */ || + req.RequestURI == "/user/events" /* Server-Sent Events (SSE) handler */ || + req.RequestURI == "/api/actions/runner.v1.RunnerService/FetchTask" /* Actions Runner polling */ { logf = logTrace } message := completedMessage diff --git a/modules/webhook/type.go b/modules/webhook/type.go index 72ffde26a1..89c6a4bfe5 100644 --- a/modules/webhook/type.go +++ b/modules/webhook/type.go @@ -38,6 +38,7 @@ const ( HookEventPullRequestReview HookEventType = "pull_request_review" // Actions event only HookEventSchedule HookEventType = "schedule" + HookEventWorkflowRun HookEventType = "workflow_run" HookEventWorkflowJob HookEventType = "workflow_job" ) @@ -67,6 +68,7 @@ func AllEvents() []HookEventType { HookEventRelease, HookEventPackage, HookEventStatus, + HookEventWorkflowRun, HookEventWorkflowJob, } } diff --git a/modules/zstd/zstd_test.go b/modules/zstd/zstd_test.go index c3ca8e78f7..7fd30484ca 100644 --- a/modules/zstd/zstd_test.go +++ b/modules/zstd/zstd_test.go @@ -16,7 +16,7 @@ import ( ) func TestWriterReader(t *testing.T) { - testData := prepareTestData(t, 20_000_000) + testData := prepareTestData(t, 1_000_000) result := bytes.NewBuffer(nil) @@ -64,7 +64,7 @@ func TestWriterReader(t *testing.T) { } func TestSeekableWriterReader(t *testing.T) { - testData := prepareTestData(t, 20_000_000) + testData := prepareTestData(t, 2_000_000) result := bytes.NewBuffer(nil) @@ -109,7 +109,7 @@ func TestSeekableWriterReader(t *testing.T) { reader, err := NewSeekableReader(assertReader) require.NoError(t, err) - _, err = reader.Seek(10_000_000, io.SeekStart) + _, err = reader.Seek(1_000_000, io.SeekStart) require.NoError(t, err) data := make([]byte, 1000) @@ -117,7 +117,7 @@ func TestSeekableWriterReader(t *testing.T) { require.NoError(t, err) require.NoError(t, reader.Close()) - assert.Equal(t, testData[10_000_000:10_000_000+1000], data) + assert.Equal(t, testData[1_000_000:1_000_000+1000], data) // Should seek 3 times, // the first two times are for getting the index, |