Refactor "dump" sub-command (#30240)

Major changes:

* Move some functions like "addReader" / "isSubDir" /
"addRecursiveExclude" to a separate package, and add tests
* Clarify the filename&dump type logic and add tests
* Clarify the logger behavior and remove FIXME comments

Co-authored-by: Giteabot <teabot@gitea.io>

1 files changed, 63 insertions, 223 deletions

diff --git a/cmd/dump.go b/cmd/dump.go
index 69ecdcec12..da0a51d9ce 100644
--- a/cmd/dump.go
+++ b/cmd/dump.go
@@ -6,14 +6,13 @@ package cmd
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/dump"
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -25,89 +24,17 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-func addReader(w archiver.Writer, r io.ReadCloser, info os.FileInfo, customName string, verbose bool) error {
-	if verbose {
-		log.Info("Adding file %s", customName)
-	}
-
-	return w.Write(archiver.File{
-		FileInfo: archiver.FileInfo{
-			FileInfo:   info,
-			CustomName: customName,
-		},
-		ReadCloser: r,
-	})
-}
-
-func addFile(w archiver.Writer, filePath, absPath string, verbose bool) error {
-	file, err := os.Open(absPath)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-	fileInfo, err := file.Stat()
-	if err != nil {
-		return err
-	}
-
-	return addReader(w, file, fileInfo, filePath, verbose)
-}
-
-func isSubdir(upper, lower string) (bool, error) {
-	if relPath, err := filepath.Rel(upper, lower); err != nil {
-		return false, err
-	} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
-		return true, nil
-	}
-	return false, nil
-}
-
-type outputType struct {
-	Enum     []string
-	Default  string
-	selected string
-}
-
-func (o outputType) Join() string {
-	return strings.Join(o.Enum, ", ")
-}
-
-func (o *outputType) Set(value string) error {
-	for _, enum := range o.Enum {
-		if enum == value {
-			o.selected = value
-			return nil
-		}
-	}
-
-	return fmt.Errorf("allowed values are %s", o.Join())
-}
-
-func (o outputType) String() string {
-	if o.selected == "" {
-		return o.Default
-	}
-	return o.selected
-}
-
-var outputTypeEnum = &outputType{
-	Enum:    []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"},
-	Default: "zip",
-}
-
 // CmdDump represents the available dump sub-command.
 var CmdDump = &cli.Command{
-	Name:  "dump",
-	Usage: "Dump Gitea files and database",
-	Description: `Dump compresses all related files and database into zip file.
-It can be used for backup and capture Gitea server image to send to maintainer`,
-	Action: runDump,
+	Name:        "dump",
+	Usage:       "Dump Gitea files and database",
+	Description: `Dump compresses all related files and database into zip file. It can be used for backup and capture Gitea server image to send to maintainer`,
+	Action:      runDump,
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:    "file",
 			Aliases: []string{"f"},
-			Value:   fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()),
-			Usage:   "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.",
+			Usage:   `Name of the dump file which will be created, default to "gitea-dump-{time}.zip". Supply '-' for stdout. See type for available types.`,
 		},
 		&cli.BoolFlag{
 			Name:    "verbose",
@@ -160,63 +87,51 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
 			Name:  "skip-index",
 			Usage: "Skip bleve index data",
 		},
-		&cli.GenericFlag{
+		&cli.StringFlag{
 			Name:  "type",
-			Value: outputTypeEnum,
-			Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()),
+			Usage: fmt.Sprintf(`Dump output format, default to "zip", supported types: %s`, strings.Join(dump.SupportedOutputTypes, ", ")),
 		},
 	},
 }
 
 func fatal(format string, args ...any) {
-	fmt.Fprintf(os.Stderr, format+"\n", args...)
 	log.Fatal(format, args...)
 }
 
 func runDump(ctx *cli.Context) error {
-	var file *os.File
-	fileName := ctx.String("file")
-	outType := ctx.String("type")
-	if fileName == "-" {
-		file = os.Stdout
-		setupConsoleLogger(log.FATAL, log.CanColorStderr, os.Stderr)
-	} else {
-		for _, suffix := range outputTypeEnum.Enum {
-			if strings.HasSuffix(fileName, "."+suffix) {
-				fileName = strings.TrimSuffix(fileName, "."+suffix)
-				break
-			}
-		}
-		fileName += "." + outType
-	}
 	setting.MustInstalled()
 
-	// make sure we are logging to the console no matter what the configuration tells us do to
-	// FIXME: don't use CfgProvider directly
-	if _, err := setting.CfgProvider.Section("log").NewKey("MODE", "console"); err != nil {
-		fatal("Setting logging mode to console failed: %v", err)
+	quite := ctx.Bool("quiet")
+	verbose := ctx.Bool("verbose")
+	if verbose && quite {
+		fatal("Option --quiet and --verbose cannot both be set")
 	}
-	if _, err := setting.CfgProvider.Section("log.console").NewKey("STDERR", "true"); err != nil {
-		fatal("Setting console logger to stderr failed: %v", err)
+
+	// outFileName is either "-" or a file name (will be made absolute)
+	outFileName, outType := dump.PrepareFileNameAndType(ctx.String("file"), ctx.String("type"))
+	if outType == "" {
+		fatal("Invalid output type")
 	}
 
-	// Set loglevel to Warn if quiet-mode is requested
-	if ctx.Bool("quiet") {
-		if _, err := setting.CfgProvider.Section("log.console").NewKey("LEVEL", "Warn"); err != nil {
-			fatal("Setting console log-level failed: %v", err)
+	outFile := os.Stdout
+	if outFileName != "-" {
+		var err error
+		if outFileName, err = filepath.Abs(outFileName); err != nil {
+			fatal("Unable to get absolute path of dump file: %v", err)
+		}
+		if exist, _ := util.IsExist(outFileName); exist {
+			fatal("Dump file %q exists", outFileName)
 		}
+		if outFile, err = os.Create(outFileName); err != nil {
+			fatal("Unable to create dump file %q: %v", outFileName, err)
+		}
+		defer outFile.Close()
 	}
 
-	if !setting.InstallLock {
-		log.Error("Is '%s' really the right config path?\n", setting.CustomConf)
-		return fmt.Errorf("gitea is not initialized")
-	}
-	setting.LoadSettings() // cannot access session settings otherwise
+	setupConsoleLogger(util.Iif(quite, log.WARN, log.INFO), log.CanColorStderr, os.Stderr)
 
-	verbose := ctx.Bool("verbose")
-	if verbose && ctx.Bool("quiet") {
-		return fmt.Errorf("--quiet and --verbose cannot both be set")
-	}
+	setting.DisableLoggerInit()
+	setting.LoadSettings() // cannot access session settings otherwise
 
 	stdCtx, cancel := installSignals()
 	defer cancel()
@@ -226,44 +141,32 @@ func runDump(ctx *cli.Context) error {
 		return err
 	}
 
-	if err := storage.Init(); err != nil {
+	if err = storage.Init(); err != nil {
 		return err
 	}
 
-	if file == nil {
-		file, err = os.Create(fileName)
-		if err != nil {
-			fatal("Unable to open %s: %v", fileName, err)
-		}
-	}
-	defer file.Close()
-
-	absFileName, err := filepath.Abs(fileName)
-	if err != nil {
-		return err
-	}
-
-	var iface any
-	if fileName == "-" {
-		iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType))
-	} else {
-		iface, err = archiver.ByExtension(fileName)
-	}
+	archiverGeneric, err := archiver.ByExtension("." + outType)
 	if err != nil {
 		fatal("Unable to get archiver for extension: %v", err)
 	}
 
-	w, _ := iface.(archiver.Writer)
-	if err := w.Create(file); err != nil {
+	archiverWriter := archiverGeneric.(archiver.Writer)
+	if err := archiverWriter.Create(outFile); err != nil {
 		fatal("Creating archiver.Writer failed: %v", err)
 	}
-	defer w.Close()
+	defer archiverWriter.Close()
+
+	dumper := &dump.Dumper{
+		Writer:  archiverWriter,
+		Verbose: verbose,
+	}
+	dumper.GlobalExcludeAbsPath(outFileName)
 
 	if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
 		log.Info("Skip dumping local repositories")
 	} else {
 		log.Info("Dumping local repositories... %s", setting.RepoRootPath)
-		if err := addRecursiveExclude(w, "repos", setting.RepoRootPath, []string{absFileName}, verbose); err != nil {
+		if err := dumper.AddRecursiveExclude("repos", setting.RepoRootPath, nil); err != nil {
 			fatal("Failed to include repositories: %v", err)
 		}
 
@@ -276,8 +179,7 @@ func runDump(ctx *cli.Context) error {
 			if err != nil {
 				return err
 			}
-
-			return addReader(w, object, info, path.Join("data", "lfs", objPath), verbose)
+			return dumper.AddReader(object, info, path.Join("data", "lfs", objPath))
 		}); err != nil {
 			fatal("Failed to dump LFS objects: %v", err)
 		}
@@ -310,15 +212,13 @@ func runDump(ctx *cli.Context) error {
 		fatal("Failed to dump database: %v", err)
 	}
 
-	if err := addFile(w, "gitea-db.sql", dbDump.Name(), verbose); err != nil {
+	if err = dumper.AddFile("gitea-db.sql", dbDump.Name()); err != nil {
 		fatal("Failed to include gitea-db.sql: %v", err)
 	}
 
-	if len(setting.CustomConf) > 0 {
-		log.Info("Adding custom configuration file from %s", setting.CustomConf)
-		if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil {
-			fatal("Failed to include specified app.ini: %v", err)
-		}
+	log.Info("Adding custom configuration file from %s", setting.CustomConf)
+	if err = dumper.AddFile("app.ini", setting.CustomConf); err != nil {
+		fatal("Failed to include specified app.ini: %v", err)
 	}
 
 	if ctx.IsSet("skip-custom-dir") && ctx.Bool("skip-custom-dir") {
@@ -326,8 +226,8 @@ func runDump(ctx *cli.Context) error {
 	} else {
 		customDir, err := os.Stat(setting.CustomPath)
 		if err == nil && customDir.IsDir() {
-			if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is {
-				if err := addRecursiveExclude(w, "custom", setting.CustomPath, []string{absFileName}, verbose); err != nil {
+			if is, _ := dump.IsSubdir(setting.AppDataPath, setting.CustomPath); !is {
+				if err := dumper.AddRecursiveExclude("custom", setting.CustomPath, nil); err != nil {
 					fatal("Failed to include custom: %v", err)
 				}
 			} else {
@@ -364,8 +264,7 @@ func runDump(ctx *cli.Context) error {
 		excludes = append(excludes, setting.Attachment.Storage.Path)
 		excludes = append(excludes, setting.Packages.Storage.Path)
 		excludes = append(excludes, setting.Log.RootPath)
-		excludes = append(excludes, absFileName)
-		if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
+		if err := dumper.AddRecursiveExclude("data", setting.AppDataPath, excludes); err != nil {
 			fatal("Failed to include data directory: %v", err)
 		}
 	}
@@ -377,8 +276,7 @@ func runDump(ctx *cli.Context) error {
 		if err != nil {
 			return err
 		}
-
-		return addReader(w, object, info, path.Join("data", "attachments", objPath), verbose)
+		return dumper.AddReader(object, info, path.Join("data", "attachments", objPath))
 	}); err != nil {
 		fatal("Failed to dump attachments: %v", err)
 	}
@@ -392,8 +290,7 @@ func runDump(ctx *cli.Context) error {
 		if err != nil {
 			return err
 		}
-
-		return addReader(w, object, info, path.Join("data", "packages", objPath), verbose)
+		return dumper.AddReader(object, info, path.Join("data", "packages", objPath))
 	}); err != nil {
 		fatal("Failed to dump packages: %v", err)
 	}
@@ -409,80 +306,23 @@ func runDump(ctx *cli.Context) error {
 			log.Error("Unable to check if %s exists. Error: %v", setting.Log.RootPath, err)
 		}
 		if isExist {
-			if err := addRecursiveExclude(w, "log", setting.Log.RootPath, []string{absFileName}, verbose); err != nil {
+			if err := dumper.AddRecursiveExclude("log", setting.Log.RootPath, nil); err != nil {
 				fatal("Failed to include log: %v", err)
 			}
 		}
 	}
 
-	if fileName != "-" {
-		if err = w.Close(); err != nil {
-			_ = util.Remove(fileName)
-			fatal("Failed to save %s: %v", fileName, err)
+	if outFileName == "-" {
+		log.Info("Finish dumping to stdout")
+	} else {
+		if err = archiverWriter.Close(); err != nil {
+			_ = os.Remove(outFileName)
+			fatal("Failed to save %q: %v", outFileName, err)
 		}
-
-		if err := os.Chmod(fileName, 0o600); err != nil {
+		if err = os.Chmod(outFileName, 0o600); err != nil {
 			log.Info("Can't change file access permissions mask to 0600: %v", err)
 		}
-	}
-
-	if fileName != "-" {
-		log.Info("Finish dumping in file %s", fileName)
-	} else {
-		log.Info("Finish dumping to stdout")
-	}
-
-	return nil
-}
-
-// addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath
-func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error {
-	absPath, err := filepath.Abs(absPath)
-	if err != nil {
-		return err
-	}
-	dir, err := os.Open(absPath)
-	if err != nil {
-		return err
-	}
-	defer dir.Close()
-
-	files, err := dir.Readdir(0)
-	if err != nil {
-		return err
-	}
-	for _, file := range files {
-		currentAbsPath := filepath.Join(absPath, file.Name())
-		currentInsidePath := path.Join(insidePath, file.Name())
-		if file.IsDir() {
-			if !util.SliceContainsString(excludeAbsPath, currentAbsPath) {
-				if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil {
-					return err
-				}
-				if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil {
-					return err
-				}
-			}
-		} else {
-			// only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
-			shouldAdd := file.Mode().IsRegular()
-			if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink {
-				target, err := filepath.EvalSymlinks(currentAbsPath)
-				if err != nil {
-					return err
-				}
-				targetStat, err := os.Stat(target)
-				if err != nil {
-					return err
-				}
-				shouldAdd = targetStat.Mode().IsRegular()
-			}
-			if shouldAdd {
-				if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil {
-					return err
-				}
-			}
-		}
+		log.Info("Finish dumping in file %s", outFileName)
 	}
 	return nil
 }