From 654cfd1dfbd3f3f1d94addee50b6fe2b018a49c3 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 3 Apr 2024 10:16:46 +0800 Subject: Refactor "dump" sub-command (#30240) Major changes: * Move some functions like "addReader" / "isSubDir" / "addRecursiveExclude" to a separate package, and add tests * Clarify the filename&dump type logic and add tests * Clarify the logger behavior and remove FIXME comments Co-authored-by: Giteabot --- modules/dump/dumper.go | 174 ++++++++++++++++++++++++++++++++++++++++++ modules/dump/dumper_test.go | 113 +++++++++++++++++++++++++++ modules/setting/log.go | 9 +++ modules/timeutil/timestamp.go | 3 +- modules/util/util.go | 8 ++ 5 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 modules/dump/dumper.go create mode 100644 modules/dump/dumper_test.go (limited to 'modules') diff --git a/modules/dump/dumper.go b/modules/dump/dumper.go new file mode 100644 index 0000000000..47730851fb --- /dev/null +++ b/modules/dump/dumper.go @@ -0,0 +1,174 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package dump + +import ( + "fmt" + "io" + "os" + "path" + "path/filepath" + "slices" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/mholt/archiver/v3" +) + +var SupportedOutputTypes = []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"} + +// PrepareFileNameAndType prepares the output file name and type, if the type is not supported, it returns an empty "outType" +func PrepareFileNameAndType(argFile, argType string) (outFileName, outType string) { + if argFile == "" && argType == "" { + outType = SupportedOutputTypes[0] + outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType) + } else if argFile == "" { + outType = argType + outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType) + } else if argType == "" { + if filepath.Ext(outFileName) == "" { + outType = SupportedOutputTypes[0] + outFileName = argFile + } else { + for _, t := range SupportedOutputTypes { + if strings.HasSuffix(argFile, "."+t) { + outFileName = argFile + outType = t + } + } + } + } else { + outFileName, outType = argFile, argType + } + if !slices.Contains(SupportedOutputTypes, outType) { + return "", "" + } + return outFileName, outType +} + +func IsSubdir(upper, lower string) (bool, error) { + if relPath, err := filepath.Rel(upper, lower); err != nil { + return false, err + } else if relPath == "." || !strings.HasPrefix(relPath, ".") { + return true, nil + } + return false, nil +} + +type Dumper struct { + Writer archiver.Writer + Verbose bool + + globalExcludeAbsPaths []string +} + +func (dumper *Dumper) AddReader(r io.ReadCloser, info os.FileInfo, customName string) error { + if dumper.Verbose { + log.Info("Adding file %s", customName) + } + + return dumper.Writer.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: info, + CustomName: customName, + }, + ReadCloser: r, + }) +} + +func (dumper *Dumper) AddFile(filePath, absPath string) error { + file, err := os.Open(absPath) + if err != nil { + return err + } + defer file.Close() + fileInfo, err := file.Stat() + if err != nil { + return err + } + return dumper.AddReader(file, fileInfo, filePath) +} + +func (dumper *Dumper) normalizeFilePath(absPath string) string { + absPath = filepath.Clean(absPath) + if setting.IsWindows { + absPath = strings.ToLower(absPath) + } + return absPath +} + +func (dumper *Dumper) GlobalExcludeAbsPath(absPaths ...string) { + for _, absPath := range absPaths { + dumper.globalExcludeAbsPaths = append(dumper.globalExcludeAbsPaths, dumper.normalizeFilePath(absPath)) + } +} + +func (dumper *Dumper) shouldExclude(absPath string, excludes []string) bool { + norm := dumper.normalizeFilePath(absPath) + return slices.Contains(dumper.globalExcludeAbsPaths, norm) || slices.Contains(excludes, norm) +} + +func (dumper *Dumper) AddRecursiveExclude(insidePath, absPath string, excludes []string) error { + excludes = slices.Clone(excludes) + for i := range excludes { + excludes[i] = dumper.normalizeFilePath(excludes[i]) + } + return dumper.addFileOrDir(insidePath, absPath, excludes) +} + +func (dumper *Dumper) addFileOrDir(insidePath, absPath string, excludes []string) error { + absPath, err := filepath.Abs(absPath) + if err != nil { + return err + } + dir, err := os.Open(absPath) + if err != nil { + return err + } + defer dir.Close() + + files, err := dir.Readdir(0) + if err != nil { + return err + } + for _, file := range files { + currentAbsPath := filepath.Join(absPath, file.Name()) + if dumper.shouldExclude(currentAbsPath, excludes) { + continue + } + + currentInsidePath := path.Join(insidePath, file.Name()) + if file.IsDir() { + if err := dumper.AddFile(currentInsidePath, currentAbsPath); err != nil { + return err + } + if err = dumper.addFileOrDir(currentInsidePath, currentAbsPath, excludes); err != nil { + return err + } + } else { + // only copy regular files and symlink regular files, skip non-regular files like socket/pipe/... + shouldAdd := file.Mode().IsRegular() + if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink { + target, err := filepath.EvalSymlinks(currentAbsPath) + if err != nil { + return err + } + targetStat, err := os.Stat(target) + if err != nil { + return err + } + shouldAdd = targetStat.Mode().IsRegular() + } + if shouldAdd { + if err = dumper.AddFile(currentInsidePath, currentAbsPath); err != nil { + return err + } + } + } + } + return nil +} diff --git a/modules/dump/dumper_test.go b/modules/dump/dumper_test.go new file mode 100644 index 0000000000..b444fa2de5 --- /dev/null +++ b/modules/dump/dumper_test.go @@ -0,0 +1,113 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package dump + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "code.gitea.io/gitea/modules/timeutil" + + "github.com/mholt/archiver/v3" + "github.com/stretchr/testify/assert" +) + +func TestPrepareFileNameAndType(t *testing.T) { + defer timeutil.MockSet(time.Unix(1234, 0))() + test := func(argFile, argType, expFile, expType string) { + outFile, outType := PrepareFileNameAndType(argFile, argType) + assert.Equal(t, + fmt.Sprintf("outFile=%s, outType=%s", expFile, expType), + fmt.Sprintf("outFile=%s, outType=%s", outFile, outType), + fmt.Sprintf("argFile=%s, argType=%s", argFile, argType), + ) + } + + test("", "", "gitea-dump-1234.zip", "zip") + test("", "tar.gz", "gitea-dump-1234.tar.gz", "tar.gz") + test("", "no-such", "", "") + + test("-", "", "-", "zip") + test("-", "tar.gz", "-", "tar.gz") + test("-", "no-such", "", "") + + test("a", "", "a", "zip") + test("a", "tar.gz", "a", "tar.gz") + test("a", "no-such", "", "") + + test("a.zip", "", "a.zip", "zip") + test("a.zip", "tar.gz", "a.zip", "tar.gz") + test("a.zip", "no-such", "", "") + + test("a.tar.gz", "", "a.tar.gz", "zip") + test("a.tar.gz", "tar.gz", "a.tar.gz", "tar.gz") + test("a.tar.gz", "no-such", "", "") +} + +func TestIsSubDir(t *testing.T) { + tmpDir := t.TempDir() + _ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755) + + isSub, err := IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include")) + assert.NoError(t, err) + assert.True(t, isSub) + + isSub, err = IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include/sub")) + assert.NoError(t, err) + assert.True(t, isSub) + + isSub, err = IsSubdir(filepath.Join(tmpDir, "include/sub"), filepath.Join(tmpDir, "include")) + assert.NoError(t, err) + assert.False(t, isSub) +} + +type testWriter struct { + added []string +} + +func (t *testWriter) Create(out io.Writer) error { + return nil +} + +func (t *testWriter) Write(f archiver.File) error { + t.added = append(t.added, f.Name()) + return nil +} + +func (t *testWriter) Close() error { + return nil +} + +func TestDumper(t *testing.T) { + sortStrings := func(s []string) []string { + sort.Strings(s) + return s + } + tmpDir := t.TempDir() + _ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude1"), 0o755) + _ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude2"), 0o755) + _ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755) + _ = os.WriteFile(filepath.Join(tmpDir, "include/a"), nil, 0o644) + _ = os.WriteFile(filepath.Join(tmpDir, "include/sub/b"), nil, 0o644) + _ = os.WriteFile(filepath.Join(tmpDir, "include/exclude1/a-1"), nil, 0o644) + _ = os.WriteFile(filepath.Join(tmpDir, "include/exclude2/a-2"), nil, 0o644) + + tw := &testWriter{} + d := &Dumper{Writer: tw} + d.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1")) + err := d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")}) + assert.NoError(t, err) + assert.EqualValues(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added)) + + tw = &testWriter{} + d = &Dumper{Writer: tw} + err = d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil) + assert.NoError(t, err) + assert.EqualValues(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added)) +} diff --git a/modules/setting/log.go b/modules/setting/log.go index e404074b72..50c5779994 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -185,8 +185,13 @@ func InitLoggersForTest() { initAllLoggers() } +var initLoggerDisabled bool + // initAllLoggers creates all the log services func initAllLoggers() { + if initLoggerDisabled { + return + } initManagedLoggers(log.GetManager(), CfgProvider) golog.SetFlags(0) @@ -194,6 +199,10 @@ func initAllLoggers() { golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info)) } +func DisableLoggerInit() { + initLoggerDisabled = true +} + func initManagedLoggers(manager *log.LoggerManager, cfg ConfigProvider) { loadLogGlobalFrom(cfg) prepareLoggerConfig(cfg) diff --git a/modules/timeutil/timestamp.go b/modules/timeutil/timestamp.go index 27a80b6682..e77652b24f 100644 --- a/modules/timeutil/timestamp.go +++ b/modules/timeutil/timestamp.go @@ -21,8 +21,9 @@ var ( ) // MockSet sets the time to a mocked time.Time -func MockSet(now time.Time) { +func MockSet(now time.Time) func() { mockNow = now + return MockUnset } // MockUnset will unset the mocked time.Time diff --git a/modules/util/util.go b/modules/util/util.go index b6e730eb54..3921002e2a 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -213,6 +213,14 @@ func ToPointer[T any](val T) *T { return &val } +// Iif is an "inline-if", it returns "trueVal" if "condition" is true, otherwise "falseVal" +func Iif[T comparable](condition bool, trueVal, falseVal T) T { + if condition { + return trueVal + } + return falseVal +} + // IfZero returns "def" if "v" is a zero value, otherwise "v" func IfZero[T comparable](v, def T) T { var zero T -- cgit v1.2.3