123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462 |
- // Copyright 2014 The Gogs Authors. All rights reserved.
- // Copyright 2016 The Gitea Authors. All rights reserved.
- // SPDX-License-Identifier: MIT
-
- package cmd
-
- import (
- "fmt"
- "io"
- "os"
- "path"
- "path/filepath"
- "strings"
- "time"
-
- "code.gitea.io/gitea/models/db"
- "code.gitea.io/gitea/modules/json"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
- "code.gitea.io/gitea/modules/util"
-
- "gitea.com/go-chi/session"
- "github.com/mholt/archiver/v3"
- "github.com/urfave/cli"
- )
-
- func addReader(w archiver.Writer, r io.ReadCloser, info os.FileInfo, customName string, verbose bool) error {
- if verbose {
- log.Info("Adding file %s", customName)
- }
-
- return w.Write(archiver.File{
- FileInfo: archiver.FileInfo{
- FileInfo: info,
- CustomName: customName,
- },
- ReadCloser: r,
- })
- }
-
- func addFile(w archiver.Writer, filePath, absPath string, verbose bool) error {
- file, err := os.Open(absPath)
- if err != nil {
- return err
- }
- defer file.Close()
- fileInfo, err := file.Stat()
- if err != nil {
- return err
- }
-
- return addReader(w, file, fileInfo, filePath, verbose)
- }
-
- func isSubdir(upper, lower string) (bool, error) {
- if relPath, err := filepath.Rel(upper, lower); err != nil {
- return false, err
- } else if relPath == "." || !strings.HasPrefix(relPath, ".") {
- return true, nil
- }
- return false, nil
- }
-
- type outputType struct {
- Enum []string
- Default string
- selected string
- }
-
- func (o outputType) Join() string {
- return strings.Join(o.Enum, ", ")
- }
-
- func (o *outputType) Set(value string) error {
- for _, enum := range o.Enum {
- if enum == value {
- o.selected = value
- return nil
- }
- }
-
- return fmt.Errorf("allowed values are %s", o.Join())
- }
-
- func (o outputType) String() string {
- if o.selected == "" {
- return o.Default
- }
- return o.selected
- }
-
- var outputTypeEnum = &outputType{
- Enum: []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"},
- Default: "zip",
- }
-
- // CmdDump represents the available dump sub-command.
- var CmdDump = cli.Command{
- Name: "dump",
- Usage: "Dump Gitea files and database",
- Description: `Dump compresses all related files and database into zip file.
- It can be used for backup and capture Gitea server image to send to maintainer`,
- Action: runDump,
- Flags: []cli.Flag{
- cli.StringFlag{
- Name: "file, f",
- Value: fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()),
- Usage: "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.",
- },
- cli.BoolFlag{
- Name: "verbose, V",
- Usage: "Show process details",
- },
- cli.StringFlag{
- Name: "tempdir, t",
- Value: os.TempDir(),
- Usage: "Temporary dir path",
- },
- cli.StringFlag{
- Name: "database, d",
- Usage: "Specify the database SQL syntax",
- },
- cli.BoolFlag{
- Name: "skip-repository, R",
- Usage: "Skip the repository dumping",
- },
- cli.BoolFlag{
- Name: "skip-log, L",
- Usage: "Skip the log dumping",
- },
- cli.BoolFlag{
- Name: "skip-custom-dir",
- Usage: "Skip custom directory",
- },
- cli.BoolFlag{
- Name: "skip-lfs-data",
- Usage: "Skip LFS data",
- },
- cli.BoolFlag{
- Name: "skip-attachment-data",
- Usage: "Skip attachment data",
- },
- cli.BoolFlag{
- Name: "skip-package-data",
- Usage: "Skip package data",
- },
- cli.BoolFlag{
- Name: "skip-index",
- Usage: "Skip bleve index data",
- },
- cli.GenericFlag{
- Name: "type",
- Value: outputTypeEnum,
- Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()),
- },
- },
- }
-
- func fatal(format string, args ...interface{}) {
- fmt.Fprintf(os.Stderr, format+"\n", args...)
- log.Fatal(format, args...)
- }
-
- func runDump(ctx *cli.Context) error {
- var file *os.File
- fileName := ctx.String("file")
- outType := ctx.String("type")
- if fileName == "-" {
- file = os.Stdout
- err := log.DelLogger("console")
- if err != nil {
- fatal("Deleting default logger failed. Can not write to stdout: %v", err)
- }
- } else {
- for _, suffix := range outputTypeEnum.Enum {
- if strings.HasSuffix(fileName, "."+suffix) {
- fileName = strings.TrimSuffix(fileName, "."+suffix)
- break
- }
- }
- fileName += "." + outType
- }
- setting.LoadFromExisting()
-
- // make sure we are logging to the console no matter what the configuration tells us do to
- if _, err := setting.Cfg.Section("log").NewKey("MODE", "console"); err != nil {
- fatal("Setting logging mode to console failed: %v", err)
- }
- if _, err := setting.Cfg.Section("log.console").NewKey("STDERR", "true"); err != nil {
- fatal("Setting console logger to stderr failed: %v", err)
- }
- if !setting.InstallLock {
- log.Error("Is '%s' really the right config path?\n", setting.CustomConf)
- return fmt.Errorf("gitea is not initialized")
- }
- setting.NewServices() // cannot access session settings otherwise
-
- stdCtx, cancel := installSignals()
- defer cancel()
-
- err := db.InitEngine(stdCtx)
- if err != nil {
- return err
- }
-
- if err := storage.Init(); err != nil {
- return err
- }
-
- if file == nil {
- file, err = os.Create(fileName)
- if err != nil {
- fatal("Unable to open %s: %v", fileName, err)
- }
- }
- defer file.Close()
-
- absFileName, err := filepath.Abs(fileName)
- if err != nil {
- return err
- }
-
- verbose := ctx.Bool("verbose")
- var iface interface{}
- if fileName == "-" {
- iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType))
- } else {
- iface, err = archiver.ByExtension(fileName)
- }
- if err != nil {
- fatal("Unable to get archiver for extension: %v", err)
- }
-
- w, _ := iface.(archiver.Writer)
- if err := w.Create(file); err != nil {
- fatal("Creating archiver.Writer failed: %v", err)
- }
- defer w.Close()
-
- if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
- log.Info("Skip dumping local repositories")
- } else {
- log.Info("Dumping local repositories... %s", setting.RepoRootPath)
- if err := addRecursiveExclude(w, "repos", setting.RepoRootPath, []string{absFileName}, verbose); err != nil {
- fatal("Failed to include repositories: %v", err)
- }
-
- if ctx.IsSet("skip-lfs-data") && ctx.Bool("skip-lfs-data") {
- log.Info("Skip dumping LFS data")
- } else if err := storage.LFS.IterateObjects(func(objPath string, object storage.Object) error {
- info, err := object.Stat()
- if err != nil {
- return err
- }
-
- return addReader(w, object, info, path.Join("data", "lfs", objPath), verbose)
- }); err != nil {
- fatal("Failed to dump LFS objects: %v", err)
- }
- }
-
- tmpDir := ctx.String("tempdir")
- if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
- fatal("Path does not exist: %s", tmpDir)
- }
-
- dbDump, err := os.CreateTemp(tmpDir, "gitea-db.sql")
- if err != nil {
- fatal("Failed to create tmp file: %v", err)
- }
- defer func() {
- if err := util.Remove(dbDump.Name()); err != nil {
- log.Warn("Unable to remove temporary file: %s: Error: %v", dbDump.Name(), err)
- }
- }()
-
- targetDBType := ctx.String("database")
- if len(targetDBType) > 0 && targetDBType != setting.Database.Type {
- log.Info("Dumping database %s => %s...", setting.Database.Type, targetDBType)
- } else {
- log.Info("Dumping database...")
- }
-
- if err := db.DumpDatabase(dbDump.Name(), targetDBType); err != nil {
- fatal("Failed to dump database: %v", err)
- }
-
- if err := addFile(w, "gitea-db.sql", dbDump.Name(), verbose); err != nil {
- fatal("Failed to include gitea-db.sql: %v", err)
- }
-
- if len(setting.CustomConf) > 0 {
- log.Info("Adding custom configuration file from %s", setting.CustomConf)
- if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil {
- fatal("Failed to include specified app.ini: %v", err)
- }
- }
-
- if ctx.IsSet("skip-custom-dir") && ctx.Bool("skip-custom-dir") {
- log.Info("Skipping custom directory")
- } else {
- customDir, err := os.Stat(setting.CustomPath)
- if err == nil && customDir.IsDir() {
- if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is {
- if err := addRecursiveExclude(w, "custom", setting.CustomPath, []string{absFileName}, verbose); err != nil {
- fatal("Failed to include custom: %v", err)
- }
- } else {
- log.Info("Custom dir %s is inside data dir %s, skipped", setting.CustomPath, setting.AppDataPath)
- }
- } else {
- log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath)
- }
- }
-
- isExist, err := util.IsExist(setting.AppDataPath)
- if err != nil {
- log.Error("Unable to check if %s exists. Error: %v", setting.AppDataPath, err)
- }
- if isExist {
- log.Info("Packing data directory...%s", setting.AppDataPath)
-
- var excludes []string
- if setting.Cfg.Section("session").Key("PROVIDER").Value() == "file" {
- var opts session.Options
- if err = json.Unmarshal([]byte(setting.SessionConfig.ProviderConfig), &opts); err != nil {
- return err
- }
- excludes = append(excludes, opts.ProviderConfig)
- }
-
- if ctx.IsSet("skip-index") && ctx.Bool("skip-index") {
- excludes = append(excludes, setting.Indexer.RepoPath)
- excludes = append(excludes, setting.Indexer.IssuePath)
- }
-
- excludes = append(excludes, setting.RepoRootPath)
- excludes = append(excludes, setting.LFS.Path)
- excludes = append(excludes, setting.Attachment.Path)
- excludes = append(excludes, setting.Packages.Path)
- excludes = append(excludes, setting.LogRootPath)
- excludes = append(excludes, absFileName)
- if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
- fatal("Failed to include data directory: %v", err)
- }
- }
-
- if ctx.IsSet("skip-attachment-data") && ctx.Bool("skip-attachment-data") {
- log.Info("Skip dumping attachment data")
- } else if err := storage.Attachments.IterateObjects(func(objPath string, object storage.Object) error {
- info, err := object.Stat()
- if err != nil {
- return err
- }
-
- return addReader(w, object, info, path.Join("data", "attachments", objPath), verbose)
- }); err != nil {
- fatal("Failed to dump attachments: %v", err)
- }
-
- if ctx.IsSet("skip-package-data") && ctx.Bool("skip-package-data") {
- log.Info("Skip dumping package data")
- } else if err := storage.Packages.IterateObjects(func(objPath string, object storage.Object) error {
- info, err := object.Stat()
- if err != nil {
- return err
- }
-
- return addReader(w, object, info, path.Join("data", "packages", objPath), verbose)
- }); err != nil {
- fatal("Failed to dump packages: %v", err)
- }
-
- // Doesn't check if LogRootPath exists before processing --skip-log intentionally,
- // ensuring that it's clear the dump is skipped whether the directory's initialized
- // yet or not.
- if ctx.IsSet("skip-log") && ctx.Bool("skip-log") {
- log.Info("Skip dumping log files")
- } else {
- isExist, err := util.IsExist(setting.LogRootPath)
- if err != nil {
- log.Error("Unable to check if %s exists. Error: %v", setting.LogRootPath, err)
- }
- if isExist {
- if err := addRecursiveExclude(w, "log", setting.LogRootPath, []string{absFileName}, verbose); err != nil {
- fatal("Failed to include log: %v", err)
- }
- }
- }
-
- if fileName != "-" {
- if err = w.Close(); err != nil {
- _ = util.Remove(fileName)
- fatal("Failed to save %s: %v", fileName, err)
- }
-
- if err := os.Chmod(fileName, 0o600); err != nil {
- log.Info("Can't change file access permissions mask to 0600: %v", err)
- }
- }
-
- if fileName != "-" {
- log.Info("Finish dumping in file %s", fileName)
- } else {
- log.Info("Finish dumping to stdout")
- }
-
- return nil
- }
-
- // addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath
- func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error {
- absPath, err := filepath.Abs(absPath)
- if err != nil {
- return err
- }
- dir, err := os.Open(absPath)
- if err != nil {
- return err
- }
- defer dir.Close()
-
- files, err := dir.Readdir(0)
- if err != nil {
- return err
- }
- for _, file := range files {
- currentAbsPath := path.Join(absPath, file.Name())
- currentInsidePath := path.Join(insidePath, file.Name())
- if file.IsDir() {
- if !util.SliceContainsString(excludeAbsPath, currentAbsPath) {
- if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil {
- return err
- }
- if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil {
- return err
- }
- }
- } else {
- // only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
- shouldAdd := file.Mode().IsRegular()
- if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink {
- target, err := filepath.EvalSymlinks(currentAbsPath)
- if err != nil {
- return err
- }
- targetStat, err := os.Stat(target)
- if err != nil {
- return err
- }
- shouldAdd = targetStat.Mode().IsRegular()
- }
- if shouldAdd {
- if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil {
- return err
- }
- }
- }
- }
- return nil
- }
|