As suggest by Go developers, use `filepath.WalkDir` instead of `filepath.Walk` because [*Walk is less efficient than WalkDir, introduced in Go 1.16, which avoids calling `os.Lstat` on every file or directory visited](https://pkg.go.dev/path/filepath#Walk). This proposition address that, in a similar way as https://github.com/go-gitea/gitea/pull/22392 did. Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>tags/v1.19.0-rc0
hasher := sha1.New() | hasher := sha1.New() | ||||
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { | |||||
err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { | |||||
if err != nil { | if err != nil { | ||||
return err | return err | ||||
} | } | ||||
_, _ = hasher.Write([]byte(info.Name())) | |||||
info, err := d.Info() | |||||
if err != nil { | |||||
return err | |||||
} | |||||
_, _ = hasher.Write([]byte(d.Name())) | |||||
_, _ = hasher.Write([]byte(info.ModTime().String())) | _, _ = hasher.Write([]byte(info.ModTime().String())) | ||||
_, _ = hasher.Write([]byte(strconv.FormatInt(info.Size(), 16))) | _, _ = hasher.Write([]byte(strconv.FormatInt(info.Size(), 16))) | ||||
return nil | return nil |
func (log *FileLogger) deleteOldLog() { | func (log *FileLogger) deleteOldLog() { | ||||
dir := filepath.Dir(log.Filename) | dir := filepath.Dir(log.Filename) | ||||
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) { | |||||
_ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) (returnErr error) { | |||||
defer func() { | defer func() { | ||||
if r := recover(); r != nil { | if r := recover(); r != nil { | ||||
returnErr = fmt.Errorf("Unable to delete old log '%s', error: %+v", path, r) | returnErr = fmt.Errorf("Unable to delete old log '%s', error: %+v", path, r) | ||||
} | } | ||||
}() | }() | ||||
if !info.IsDir() && info.ModTime().Unix() < (time.Now().Unix()-60*60*24*log.Maxdays) { | |||||
if err != nil { | |||||
return err | |||||
} | |||||
if d.IsDir() { | |||||
return nil | |||||
} | |||||
info, err := d.Info() | |||||
if err != nil { | |||||
return err | |||||
} | |||||
if info.ModTime().Unix() < (time.Now().Unix() - 60*60*24*log.Maxdays) { | |||||
if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) { | if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) { | ||||
if err := util.Remove(path); err != nil { | if err := util.Remove(path); err != nil { | ||||
returnErr = fmt.Errorf("Failed to remove %s: %w", path, err) | returnErr = fmt.Errorf("Failed to remove %s: %w", path, err) |
// Avoid walking tree if there are no globs | // Avoid walking tree if there are no globs | ||||
if len(gt.Globs()) > 0 { | if len(gt.Globs()) > 0 { | ||||
tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" | tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" | ||||
if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error { | |||||
if err := filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error { | |||||
if walkErr != nil { | if walkErr != nil { | ||||
return walkErr | return walkErr | ||||
} | } | ||||
if info.IsDir() { | |||||
if d.IsDir() { | |||||
return nil | return nil | ||||
} | } | ||||
// IterateObjects iterates across the objects in the local storage | // IterateObjects iterates across the objects in the local storage | ||||
func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error { | func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error { | ||||
return filepath.Walk(l.dir, func(path string, info os.FileInfo, err error) error { | |||||
return filepath.WalkDir(l.dir, func(path string, d os.DirEntry, err error) error { | |||||
if err != nil { | if err != nil { | ||||
return err | return err | ||||
} | } | ||||
if path == l.dir { | if path == l.dir { | ||||
return nil | return nil | ||||
} | } | ||||
if info.IsDir() { | |||||
if d.IsDir() { | |||||
return nil | return nil | ||||
} | } | ||||
relPath, err := filepath.Rel(l.dir, path) | relPath, err := filepath.Rel(l.dir, path) |
repos := map[string]*repo_model.Repository{} | repos := map[string]*repo_model.Repository{} | ||||
// We're going to iterate by pagesize. | // We're going to iterate by pagesize. | ||||
root := user_model.UserPath(ctxUser.Name) | root := user_model.UserPath(ctxUser.Name) | ||||
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { | |||||
if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { | |||||
if err != nil { | if err != nil { | ||||
if os.IsNotExist(err) { | if os.IsNotExist(err) { | ||||
return nil | return nil | ||||
} | } | ||||
return err | return err | ||||
} | } | ||||
if !info.IsDir() || path == root { | |||||
if !d.IsDir() || path == root { | |||||
return nil | return nil | ||||
} | } | ||||
name := info.Name() | |||||
name := d.Name() | |||||
if !strings.HasSuffix(name, ".git") { | if !strings.HasSuffix(name, ".git") { | ||||
return filepath.SkipDir | return filepath.SkipDir | ||||
} | } | ||||
count++ | count++ | ||||
return filepath.SkipDir | return filepath.SkipDir | ||||
}); err != nil { | }); err != nil { | ||||
ctx.ServerError("filepath.Walk", err) | |||||
ctx.ServerError("filepath.WalkDir", err) | |||||
return | return | ||||
} | } | ||||
// We're going to iterate by pagesize. | // We're going to iterate by pagesize. | ||||
root := filepath.Clean(setting.RepoRootPath) | root := filepath.Clean(setting.RepoRootPath) | ||||
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { | |||||
if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { | |||||
if err != nil { | if err != nil { | ||||
return err | return err | ||||
} | } | ||||
if !info.IsDir() || path == root { | |||||
if !d.IsDir() || path == root { | |||||
return nil | return nil | ||||
} | } | ||||
name := d.Name() | |||||
if !strings.ContainsRune(path[len(root)+1:], filepath.Separator) { | if !strings.ContainsRune(path[len(root)+1:], filepath.Separator) { | ||||
// Got a new user | // Got a new user | ||||
if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil { | if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil { | ||||
} | } | ||||
repoNamesToCheck = repoNamesToCheck[:0] | repoNamesToCheck = repoNamesToCheck[:0] | ||||
if !globUser.Match(info.Name()) { | |||||
if !globUser.Match(name) { | |||||
return filepath.SkipDir | return filepath.SkipDir | ||||
} | } | ||||
userName = info.Name() | |||||
userName = name | |||||
return nil | return nil | ||||
} | } | ||||
name := info.Name() | |||||
if !strings.HasSuffix(name, ".git") { | if !strings.HasSuffix(name, ".git") { | ||||
return filepath.SkipDir | return filepath.SkipDir | ||||
} | } |