The idea is to use a Layered Asset File-system (modules/assetfs/layered.go) For example: when there are 2 layers: "custom", "builtin", when access to asset "my/page.tmpl", the Layered Asset File-system will first try to use "custom" assets, if not found, then use "builtin" assets. This approach will hugely simplify a lot of code, make them testable. Other changes: * Simplify the AssetsHandlerFunc code * Simplify the `gitea embedded` sub-command code --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>tags/v1.20.0-rc0
@@ -1,8 +1,6 @@ | |||
// Copyright 2020 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
//go:build bindata | |||
package cmd | |||
import ( | |||
@@ -10,9 +8,9 @@ import ( | |||
"fmt" | |||
"os" | |||
"path/filepath" | |||
"sort" | |||
"strings" | |||
"code.gitea.io/gitea/modules/assetfs" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/options" | |||
"code.gitea.io/gitea/modules/public" | |||
@@ -89,24 +87,20 @@ var ( | |||
}, | |||
} | |||
sections map[string]*section | |||
assets []asset | |||
matchedAssetFiles []assetFile | |||
) | |||
type section struct { | |||
Path string | |||
Names func() []string | |||
IsDir func(string) (bool, error) | |||
Asset func(string) ([]byte, error) | |||
} | |||
type asset struct { | |||
Section *section | |||
Name string | |||
Path string | |||
type assetFile struct { | |||
fs *assetfs.LayeredFS | |||
name string | |||
path string | |||
} | |||
func initEmbeddedExtractor(c *cli.Context) error { | |||
// FIXME: there is a bug, if the user runs `gitea embedded` with a different user or root, | |||
// The setting.Init (loadRunModeFrom) will fail and do log.Fatal | |||
// But the console logger has been deleted, so nothing is printed, the user sees nothing and Gitea just exits. | |||
// Silence the console logger | |||
log.DelNamedLogger("console") | |||
log.DelNamedLogger(log.DEFAULT) | |||
@@ -115,24 +109,14 @@ func initEmbeddedExtractor(c *cli.Context) error { | |||
setting.InitProviderAllowEmpty() | |||
setting.LoadCommonSettings() | |||
pats, err := getPatterns(c.Args()) | |||
patterns, err := compileCollectPatterns(c.Args()) | |||
if err != nil { | |||
return err | |||
} | |||
sections := make(map[string]*section, 3) | |||
sections["public"] = §ion{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset} | |||
sections["options"] = §ion{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset} | |||
sections["templates"] = §ion{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset} | |||
for _, sec := range sections { | |||
assets = append(assets, buildAssetList(sec, pats, c)...) | |||
} | |||
// Sort assets | |||
sort.SliceStable(assets, func(i, j int) bool { | |||
return assets[i].Path < assets[j].Path | |||
}) | |||
collectAssetFilesByPattern(c, patterns, "options", options.BuiltinAssets()) | |||
collectAssetFilesByPattern(c, patterns, "public", public.BuiltinAssets()) | |||
collectAssetFilesByPattern(c, patterns, "templates", templates.BuiltinAssets()) | |||
return nil | |||
} | |||
@@ -166,8 +150,8 @@ func runListDo(c *cli.Context) error { | |||
return err | |||
} | |||
for _, a := range assets { | |||
fmt.Println(a.Path) | |||
for _, a := range matchedAssetFiles { | |||
fmt.Println(a.path) | |||
} | |||
return nil | |||
@@ -178,19 +162,19 @@ func runViewDo(c *cli.Context) error { | |||
return err | |||
} | |||
if len(assets) == 0 { | |||
return fmt.Errorf("No files matched the given pattern") | |||
} else if len(assets) > 1 { | |||
return fmt.Errorf("Too many files matched the given pattern; try to be more specific") | |||
if len(matchedAssetFiles) == 0 { | |||
return fmt.Errorf("no files matched the given pattern") | |||
} else if len(matchedAssetFiles) > 1 { | |||
return fmt.Errorf("too many files matched the given pattern, try to be more specific") | |||
} | |||
data, err := assets[0].Section.Asset(assets[0].Name) | |||
data, err := matchedAssetFiles[0].fs.ReadFile(matchedAssetFiles[0].name) | |||
if err != nil { | |||
return fmt.Errorf("%s: %w", assets[0].Path, err) | |||
return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err) | |||
} | |||
if _, err = os.Stdout.Write(data); err != nil { | |||
return fmt.Errorf("%s: %w", assets[0].Path, err) | |||
return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err) | |||
} | |||
return nil | |||
@@ -202,7 +186,7 @@ func runExtractDo(c *cli.Context) error { | |||
} | |||
if len(c.Args()) == 0 { | |||
return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)") | |||
return fmt.Errorf("a list of pattern of files to extract is mandatory (e.g. '**' for all)") | |||
} | |||
destdir := "." | |||
@@ -227,7 +211,7 @@ func runExtractDo(c *cli.Context) error { | |||
if err != nil { | |||
return fmt.Errorf("%s: %s", destdir, err) | |||
} else if !fi.IsDir() { | |||
return fmt.Errorf("%s is not a directory.", destdir) | |||
return fmt.Errorf("destination %q is not a directory", destdir) | |||
} | |||
fmt.Printf("Extracting to %s:\n", destdir) | |||
@@ -235,23 +219,23 @@ func runExtractDo(c *cli.Context) error { | |||
overwrite := c.Bool("overwrite") | |||
rename := c.Bool("rename") | |||
for _, a := range assets { | |||
for _, a := range matchedAssetFiles { | |||
if err := extractAsset(destdir, a, overwrite, rename); err != nil { | |||
// Non-fatal error | |||
fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err) | |||
fmt.Fprintf(os.Stderr, "%s: %v", a.path, err) | |||
} | |||
} | |||
return nil | |||
} | |||
func extractAsset(d string, a asset, overwrite, rename bool) error { | |||
dest := filepath.Join(d, filepath.FromSlash(a.Path)) | |||
func extractAsset(d string, a assetFile, overwrite, rename bool) error { | |||
dest := filepath.Join(d, filepath.FromSlash(a.path)) | |||
dir := filepath.Dir(dest) | |||
data, err := a.Section.Asset(a.Name) | |||
data, err := a.fs.ReadFile(a.name) | |||
if err != nil { | |||
return fmt.Errorf("%s: %w", a.Path, err) | |||
return fmt.Errorf("%s: %w", a.path, err) | |||
} | |||
if err := os.MkdirAll(dir, os.ModePerm); err != nil { | |||
@@ -272,7 +256,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { | |||
return fmt.Errorf("%s already exists, but it's not a regular file", dest) | |||
} else if rename { | |||
if err := util.Rename(dest, dest+".bak"); err != nil { | |||
return fmt.Errorf("Error creating backup for %s: %w", dest, err) | |||
return fmt.Errorf("error creating backup for %s: %w", dest, err) | |||
} | |||
// Attempt to respect file permissions mask (even if user:group will be set anew) | |||
perms = fi.Mode() | |||
@@ -293,32 +277,30 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { | |||
return nil | |||
} | |||
func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { | |||
results := make([]asset, 0, 64) | |||
for _, name := range sec.Names() { | |||
if isdir, err := sec.IsDir(name); !isdir && err == nil { | |||
if sec.Path == "public" && | |||
strings.HasPrefix(name, "vendor/") && | |||
!c.Bool("include-vendored") { | |||
continue | |||
} | |||
matchName := sec.Path + "/" + name | |||
for _, g := range globs { | |||
if g.Match(matchName) { | |||
results = append(results, asset{ | |||
Section: sec, | |||
Name: name, | |||
Path: sec.Path + "/" + name, | |||
}) | |||
break | |||
} | |||
func collectAssetFilesByPattern(c *cli.Context, globs []glob.Glob, path string, layer *assetfs.Layer) { | |||
fs := assetfs.Layered(layer) | |||
files, err := fs.ListAllFiles(".", true) | |||
if err != nil { | |||
log.Error("Error listing files in %q: %v", path, err) | |||
return | |||
} | |||
for _, name := range files { | |||
if path == "public" && | |||
strings.HasPrefix(name, "vendor/") && | |||
!c.Bool("include-vendored") { | |||
continue | |||
} | |||
matchName := path + "/" + name | |||
for _, g := range globs { | |||
if g.Match(matchName) { | |||
matchedAssetFiles = append(matchedAssetFiles, assetFile{fs: fs, name: name, path: path + "/" + name}) | |||
break | |||
} | |||
} | |||
} | |||
return results | |||
} | |||
func getPatterns(args []string) ([]glob.Glob, error) { | |||
func compileCollectPatterns(args []string) ([]glob.Glob, error) { | |||
if len(args) == 0 { | |||
args = []string{"**"} | |||
} | |||
@@ -326,7 +308,7 @@ func getPatterns(args []string) ([]glob.Glob, error) { | |||
for i := range args { | |||
if g, err := glob.Compile(args[i], '/'); err != nil { | |||
return nil, fmt.Errorf("'%s': Invalid glob pattern: %w", args[i], err) | |||
} else { | |||
} else { //nolint:revive | |||
pat[i] = g | |||
} | |||
} |
@@ -1,29 +0,0 @@ | |||
// Copyright 2020 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
//go:build !bindata | |||
package cmd | |||
import ( | |||
"fmt" | |||
"os" | |||
"github.com/urfave/cli" | |||
) | |||
// Cmdembedded represents the available extract sub-command. | |||
var ( | |||
Cmdembedded = cli.Command{ | |||
Name: "embedded", | |||
Usage: "Extract embedded resources", | |||
Description: "A command for extracting embedded resources, like templates and images", | |||
Action: extractorNotImplemented, | |||
} | |||
) | |||
func extractorNotImplemented(c *cli.Context) error { | |||
err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata") | |||
fmt.Fprintf(os.Stderr, "%s\n", err) | |||
return err | |||
} |
@@ -0,0 +1,260 @@ | |||
// Copyright 2023 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
package assetfs | |||
import ( | |||
"context" | |||
"fmt" | |||
"io" | |||
"io/fs" | |||
"net/http" | |||
"os" | |||
"path/filepath" | |||
"sort" | |||
"time" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/process" | |||
"code.gitea.io/gitea/modules/util" | |||
"github.com/fsnotify/fsnotify" | |||
) | |||
// Layer represents a layer in a layered asset file-system. It has a name and works like http.FileSystem | |||
type Layer struct { | |||
name string | |||
fs http.FileSystem | |||
localPath string | |||
} | |||
func (l *Layer) Name() string { | |||
return l.name | |||
} | |||
// Open opens the named file. The caller is responsible for closing the file. | |||
func (l *Layer) Open(name string) (http.File, error) { | |||
return l.fs.Open(name) | |||
} | |||
// Local returns a new Layer with the given name, it serves files from the given local path. | |||
func Local(name, base string, sub ...string) *Layer { | |||
// TODO: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before | |||
// Ideally, the caller should guarantee the base is absolute, guessing a relative path based on the current working directory is unreliable. | |||
base, err := filepath.Abs(base) | |||
if err != nil { | |||
// This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths. | |||
panic(fmt.Sprintf("Unable to get absolute path for %q: %v", base, err)) | |||
} | |||
root := util.FilePathJoinAbs(base, sub...) | |||
return &Layer{name: name, fs: http.Dir(root), localPath: root} | |||
} | |||
// Bindata returns a new Layer with the given name, it serves files from the given bindata asset. | |||
func Bindata(name string, fs http.FileSystem) *Layer { | |||
return &Layer{name: name, fs: fs} | |||
} | |||
// LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers. | |||
// The first layer is the top layer, and it will be used first. | |||
// If the file is not found in the top layer, it will be searched in the next layer. | |||
type LayeredFS struct { | |||
layers []*Layer | |||
} | |||
// Layered returns a new LayeredFS with the given layers. The first layer is the top layer. | |||
func Layered(layers ...*Layer) *LayeredFS { | |||
return &LayeredFS{layers: layers} | |||
} | |||
// Open opens the named file. The caller is responsible for closing the file. | |||
func (l *LayeredFS) Open(name string) (http.File, error) { | |||
for _, layer := range l.layers { | |||
f, err := layer.Open(name) | |||
if err == nil || !os.IsNotExist(err) { | |||
return f, err | |||
} | |||
} | |||
return nil, fs.ErrNotExist | |||
} | |||
// ReadFile reads the named file. | |||
func (l *LayeredFS) ReadFile(elems ...string) ([]byte, error) { | |||
bs, _, err := l.ReadLayeredFile(elems...) | |||
return bs, err | |||
} | |||
// ReadLayeredFile reads the named file, and returns the layer name. | |||
func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) { | |||
name := util.PathJoinRel(elems...) | |||
for _, layer := range l.layers { | |||
f, err := layer.Open(name) | |||
if os.IsNotExist(err) { | |||
continue | |||
} else if err != nil { | |||
return nil, layer.name, err | |||
} | |||
bs, err := io.ReadAll(f) | |||
_ = f.Close() | |||
return bs, layer.name, err | |||
} | |||
return nil, "", fs.ErrNotExist | |||
} | |||
func shouldInclude(info fs.FileInfo, fileMode ...bool) bool { | |||
if util.CommonSkip(info.Name()) { | |||
return false | |||
} | |||
if len(fileMode) == 0 { | |||
return true | |||
} else if len(fileMode) == 1 { | |||
return fileMode[0] == !info.Mode().IsDir() | |||
} | |||
panic("too many arguments for fileMode in shouldInclude") | |||
} | |||
func readDir(layer *Layer, name string) ([]fs.FileInfo, error) { | |||
f, err := layer.Open(name) | |||
if os.IsNotExist(err) { | |||
return nil, nil | |||
} else if err != nil { | |||
return nil, err | |||
} | |||
defer f.Close() | |||
return f.Readdir(-1) | |||
} | |||
// ListFiles lists files/directories in the given directory. The fileMode controls the returned files. | |||
// * omitted: all files and directories will be returned. | |||
// * true: only files will be returned. | |||
// * false: only directories will be returned. | |||
// The returned files are sorted by name. | |||
func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) { | |||
fileMap := map[string]bool{} | |||
for _, layer := range l.layers { | |||
infos, err := readDir(layer, name) | |||
if err != nil { | |||
return nil, err | |||
} | |||
for _, info := range infos { | |||
if shouldInclude(info, fileMode...) { | |||
fileMap[info.Name()] = true | |||
} | |||
} | |||
} | |||
files := make([]string, 0, len(fileMap)) | |||
for file := range fileMap { | |||
files = append(files, file) | |||
} | |||
sort.Strings(files) | |||
return files, nil | |||
} | |||
// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively. | |||
// The fileMode controls the returned files: | |||
// * omitted: all files and directories will be returned. | |||
// * true: only files will be returned. | |||
// * false: only directories will be returned. | |||
// The returned files are sorted by name. | |||
func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) { | |||
return listAllFiles(l.layers, name, fileMode...) | |||
} | |||
func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) { | |||
fileMap := map[string]bool{} | |||
var list func(dir string) error | |||
list = func(dir string) error { | |||
for _, layer := range layers { | |||
infos, err := readDir(layer, dir) | |||
if err != nil { | |||
return err | |||
} | |||
for _, info := range infos { | |||
path := util.PathJoinRelX(dir, info.Name()) | |||
if shouldInclude(info, fileMode...) { | |||
fileMap[path] = true | |||
} | |||
if info.IsDir() { | |||
if err = list(path); err != nil { | |||
return err | |||
} | |||
} | |||
} | |||
} | |||
return nil | |||
} | |||
if err := list(name); err != nil { | |||
return nil, err | |||
} | |||
var files []string | |||
for file := range fileMap { | |||
files = append(files, file) | |||
} | |||
sort.Strings(files) | |||
return files, nil | |||
} | |||
// WatchLocalChanges watches local changes in the file-system. It's used to help to reload assets when the local file-system changes. | |||
func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) { | |||
ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Asset Local FileSystem Watcher", process.SystemProcessType, true) | |||
defer finished() | |||
watcher, err := fsnotify.NewWatcher() | |||
if err != nil { | |||
log.Error("Unable to create watcher for asset local file-system: %v", err) | |||
return | |||
} | |||
defer watcher.Close() | |||
for _, layer := range l.layers { | |||
if layer.localPath == "" { | |||
continue | |||
} | |||
layerDirs, err := listAllFiles([]*Layer{layer}, ".", false) | |||
if err != nil { | |||
log.Error("Unable to list directories for asset local file-system %q: %v", layer.localPath, err) | |||
continue | |||
} | |||
for _, dir := range layerDirs { | |||
if err = watcher.Add(util.FilePathJoinAbs(layer.localPath, dir)); err != nil { | |||
log.Error("Unable to watch directory %s: %v", dir, err) | |||
} | |||
} | |||
} | |||
debounce := util.Debounce(100 * time.Millisecond) | |||
for { | |||
select { | |||
case <-ctx.Done(): | |||
return | |||
case event, ok := <-watcher.Events: | |||
if !ok { | |||
return | |||
} | |||
log.Trace("Watched asset local file-system had event: %v", event) | |||
debounce(callback) | |||
case err, ok := <-watcher.Errors: | |||
if !ok { | |||
return | |||
} | |||
log.Error("Watched asset local file-system had error: %v", err) | |||
} | |||
} | |||
} | |||
// GetFileLayerName returns the name of the first-seen layer that contains the given file. | |||
func (l *LayeredFS) GetFileLayerName(elems ...string) string { | |||
name := util.PathJoinRel(elems...) | |||
for _, layer := range l.layers { | |||
f, err := layer.Open(name) | |||
if os.IsNotExist(err) { | |||
continue | |||
} else if err != nil { | |||
return "" | |||
} | |||
_ = f.Close() | |||
return layer.name | |||
} | |||
return "" | |||
} |
@@ -0,0 +1,109 @@ | |||
// Copyright 2023 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
package assetfs | |||
import ( | |||
"io" | |||
"io/fs" | |||
"os" | |||
"path/filepath" | |||
"testing" | |||
"github.com/stretchr/testify/assert" | |||
) | |||
func TestLayered(t *testing.T) { | |||
dir := filepath.Join(t.TempDir(), "assetfs-layers") | |||
dir1 := filepath.Join(dir, "l1") | |||
dir2 := filepath.Join(dir, "l2") | |||
mkdir := func(elems ...string) { | |||
assert.NoError(t, os.MkdirAll(filepath.Join(elems...), 0o755)) | |||
} | |||
write := func(content string, elems ...string) { | |||
assert.NoError(t, os.WriteFile(filepath.Join(elems...), []byte(content), 0o644)) | |||
} | |||
// d1 & f1: only in "l1"; d2 & f2: only in "l2" | |||
// da & fa: in both "l1" and "l2" | |||
mkdir(dir1, "d1") | |||
mkdir(dir1, "da") | |||
mkdir(dir1, "da/sub1") | |||
mkdir(dir2, "d2") | |||
mkdir(dir2, "da") | |||
mkdir(dir2, "da/sub2") | |||
write("dummy", dir1, ".DS_Store") | |||
write("f1", dir1, "f1") | |||
write("fa-1", dir1, "fa") | |||
write("d1-f", dir1, "d1/f") | |||
write("da-f-1", dir1, "da/f") | |||
write("f2", dir2, "f2") | |||
write("fa-2", dir2, "fa") | |||
write("d2-f", dir2, "d2/f") | |||
write("da-f-2", dir2, "da/f") | |||
assets := Layered(Local("l1", dir1), Local("l2", dir2)) | |||
f, err := assets.Open("f1") | |||
assert.NoError(t, err) | |||
bs, err := io.ReadAll(f) | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, "f1", string(bs)) | |||
_ = f.Close() | |||
assertRead := func(expected string, expectedErr error, elems ...string) { | |||
bs, err := assets.ReadFile(elems...) | |||
if err != nil { | |||
assert.ErrorAs(t, err, &expectedErr) | |||
} else { | |||
assert.NoError(t, err) | |||
assert.Equal(t, expected, string(bs)) | |||
} | |||
} | |||
assertRead("f1", nil, "f1") | |||
assertRead("f2", nil, "f2") | |||
assertRead("fa-1", nil, "fa") | |||
assertRead("d1-f", nil, "d1/f") | |||
assertRead("d2-f", nil, "d2/f") | |||
assertRead("da-f-1", nil, "da/f") | |||
assertRead("", fs.ErrNotExist, "no-such") | |||
files, err := assets.ListFiles(".", true) | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, []string{"f1", "f2", "fa"}, files) | |||
files, err = assets.ListFiles(".", false) | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, []string{"d1", "d2", "da"}, files) | |||
files, err = assets.ListFiles(".") | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files) | |||
files, err = assets.ListAllFiles(".", true) | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files) | |||
files, err = assets.ListAllFiles(".", false) | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files) | |||
files, err = assets.ListAllFiles(".") | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, []string{ | |||
"d1", "d1/f", | |||
"d2", "d2/f", | |||
"da", "da/f", "da/sub1", "da/sub2", | |||
"f1", "f2", "fa", | |||
}, files) | |||
assert.Empty(t, assets.GetFileLayerName("no-such")) | |||
assert.EqualValues(t, "l1", assets.GetFileLayerName("f1")) | |||
assert.EqualValues(t, "l2", assets.GetFileLayerName("f2")) | |||
} |
@@ -14,5 +14,9 @@ var Supported = false | |||
// Auth not supported lack of pam tag | |||
func Auth(serviceName, userName, passwd string) (string, error) { | |||
return "", errors.New("PAM not supported") | |||
// bypass the lint on callers: SA4023: this comparison is always true (staticcheck) | |||
if !Supported { | |||
return "", errors.New("PAM not supported") | |||
} | |||
return "", nil | |||
} |
@@ -240,19 +240,15 @@ func (ctx *Context) HTML(status int, name base.TplName) { | |||
} | |||
line, _ := strconv.Atoi(lineStr) // Cannot error out as groups[2] is [1-9][0-9]* | |||
pos, _ := strconv.Atoi(posStr) // Cannot error out as groups[3] is [1-9][0-9]* | |||
filename, filenameErr := templates.GetAssetFilename("templates/" + errorTemplateName + ".tmpl") | |||
if filenameErr != nil { | |||
filename = "(template) " + errorTemplateName | |||
} | |||
assetLayerName := templates.AssetFS().GetFileLayerName(errorTemplateName + ".tmpl") | |||
filename := fmt.Sprintf("(%s) %s", assetLayerName, errorTemplateName) | |||
if errorTemplateName != string(name) { | |||
filename += " (subtemplate of " + string(name) + ")" | |||
} | |||
err = fmt.Errorf("failed to render %s, error: %w:\n%s", filename, err, templates.GetLineFromTemplate(errorTemplateName, line, target, pos)) | |||
} else { | |||
filename, filenameErr := templates.GetAssetFilename("templates/" + execErr.Name + ".tmpl") | |||
if filenameErr != nil { | |||
filename = "(template) " + execErr.Name | |||
} | |||
assetLayerName := templates.AssetFS().GetFileLayerName(execErr.Name + ".tmpl") | |||
filename := fmt.Sprintf("(%s) %s", assetLayerName, execErr.Name) | |||
if execErr.Name != string(name) { | |||
filename += " (subtemplate of " + string(name) + ")" | |||
} |
@@ -9,7 +9,6 @@ import ( | |||
"os" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/options" | |||
"code.gitea.io/gitea/modules/setting" | |||
) | |||
@@ -79,7 +78,7 @@ func checkConfigurationFiles(ctx context.Context, logger log.Logger, autofix boo | |||
{"Log Root Path", setting.Log.RootPath, true, true, true}, | |||
} | |||
if options.IsDynamic() { | |||
if !setting.HasBuiltinBindata { | |||
configurationFiles = append(configurationFiles, configurationFile{"Static File Root Path", setting.StaticRootPath, true, true, false}) | |||
} | |||
@@ -4,131 +4,39 @@ | |||
package options | |||
import ( | |||
"fmt" | |||
"io/fs" | |||
"os" | |||
"path/filepath" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/assetfs" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/util" | |||
) | |||
var directories = make(directorySet) | |||
func CustomAssets() *assetfs.Layer { | |||
return assetfs.Local("custom", setting.CustomPath, "options") | |||
} | |||
func AssetFS() *assetfs.LayeredFS { | |||
return assetfs.Layered(CustomAssets(), BuiltinAssets()) | |||
} | |||
// Locale reads the content of a specific locale from static/bindata or custom path. | |||
func Locale(name string) ([]byte, error) { | |||
return fileFromOptionsDir("locale", name) | |||
return AssetFS().ReadFile("locale", name) | |||
} | |||
// Readme reads the content of a specific readme from static/bindata or custom path. | |||
func Readme(name string) ([]byte, error) { | |||
return fileFromOptionsDir("readme", name) | |||
return AssetFS().ReadFile("readme", name) | |||
} | |||
// Gitignore reads the content of a gitignore locale from static/bindata or custom path. | |||
func Gitignore(name string) ([]byte, error) { | |||
return fileFromOptionsDir("gitignore", name) | |||
return AssetFS().ReadFile("gitignore", name) | |||
} | |||
// License reads the content of a specific license from static/bindata or custom path. | |||
func License(name string) ([]byte, error) { | |||
return fileFromOptionsDir("license", name) | |||
return AssetFS().ReadFile("license", name) | |||
} | |||
// Labels reads the content of a specific labels from static/bindata or custom path. | |||
func Labels(name string) ([]byte, error) { | |||
return fileFromOptionsDir("label", name) | |||
} | |||
// WalkLocales reads the content of a specific locale | |||
func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error { | |||
if IsDynamic() { | |||
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { | |||
return fmt.Errorf("failed to walk locales. Error: %w", err) | |||
} | |||
} | |||
if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { | |||
return fmt.Errorf("failed to walk locales. Error: %w", err) | |||
} | |||
return nil | |||
} | |||
func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error { | |||
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { | |||
// name is the path relative to the root | |||
name := path[len(root):] | |||
if len(name) > 0 && name[0] == '/' { | |||
name = name[1:] | |||
} | |||
if err != nil { | |||
if os.IsNotExist(err) { | |||
return callback(path, name, d, err) | |||
} | |||
return err | |||
} | |||
if util.CommonSkip(d.Name()) { | |||
if d.IsDir() { | |||
return fs.SkipDir | |||
} | |||
return nil | |||
} | |||
return callback(path, name, d, err) | |||
}); err != nil && !os.IsNotExist(err) { | |||
return fmt.Errorf("unable to get files for assets in %s: %w", root, err) | |||
} | |||
return nil | |||
} | |||
// mustLocalPathAbs coverts a path to absolute path | |||
// FIXME: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before | |||
func mustLocalPathAbs(s string) string { | |||
abs, err := filepath.Abs(s) | |||
if err != nil { | |||
// This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths. | |||
log.Fatal("Unable to get absolute path for %q: %v", s, err) | |||
} | |||
return abs | |||
} | |||
func joinLocalPaths(baseDirs []string, subDir string, elems ...string) (paths []string) { | |||
abs := make([]string, len(elems)+2) | |||
abs[1] = subDir | |||
copy(abs[2:], elems) | |||
for _, baseDir := range baseDirs { | |||
abs[0] = mustLocalPathAbs(baseDir) | |||
paths = append(paths, util.FilePathJoinAbs(abs...)) | |||
} | |||
return paths | |||
} | |||
func listLocalDirIfExist(baseDirs []string, subDir string, elems ...string) (files []string, err error) { | |||
for _, localPath := range joinLocalPaths(baseDirs, subDir, elems...) { | |||
isDir, err := util.IsDir(localPath) | |||
if err != nil { | |||
return nil, fmt.Errorf("unable to check if path %q is a directory. %w", localPath, err) | |||
} else if !isDir { | |||
continue | |||
} | |||
dirFiles, err := util.StatDir(localPath, true) | |||
if err != nil { | |||
return nil, fmt.Errorf("unable to read directory %q. %w", localPath, err) | |||
} | |||
files = append(files, dirFiles...) | |||
} | |||
return files, nil | |||
} | |||
func readLocalFile(baseDirs []string, subDir string, elems ...string) ([]byte, error) { | |||
for _, localPath := range joinLocalPaths(baseDirs, subDir, elems...) { | |||
data, err := os.ReadFile(localPath) | |||
if err == nil { | |||
return data, nil | |||
} else if !os.IsNotExist(err) { | |||
log.Error("Unable to read file %q. Error: %v", localPath, err) | |||
} | |||
} | |||
return nil, os.ErrNotExist | |||
return AssetFS().ReadFile("label", name) | |||
} |
@@ -6,29 +6,10 @@ | |||
package options | |||
import ( | |||
"code.gitea.io/gitea/modules/assetfs" | |||
"code.gitea.io/gitea/modules/setting" | |||
) | |||
// Dir returns all files from static or custom directory. | |||
func Dir(name string) ([]string, error) { | |||
if directories.Filled(name) { | |||
return directories.Get(name), nil | |||
} | |||
result, err := listLocalDirIfExist([]string{setting.CustomPath, setting.StaticRootPath}, "options", name) | |||
if err != nil { | |||
return nil, err | |||
} | |||
return directories.AddAndGet(name, result), nil | |||
} | |||
// fileFromOptionsDir is a helper to read files from custom or static path. | |||
func fileFromOptionsDir(elems ...string) ([]byte, error) { | |||
return readLocalFile([]string{setting.CustomPath, setting.StaticRootPath}, "options", elems...) | |||
} | |||
// IsDynamic will return false when using embedded data (-tags bindata) | |||
func IsDynamic() bool { | |||
return true | |||
func BuiltinAssets() *assetfs.Layer { | |||
return assetfs.Local("builtin(static)", setting.StaticRootPath, "options") | |||
} |
@@ -1,45 +0,0 @@ | |||
// Copyright 2016 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
package options | |||
type directorySet map[string][]string | |||
func (s directorySet) Add(key string, value []string) { | |||
_, ok := s[key] | |||
if !ok { | |||
s[key] = make([]string, 0, len(value)) | |||
} | |||
s[key] = append(s[key], value...) | |||
} | |||
func (s directorySet) Get(key string) []string { | |||
_, ok := s[key] | |||
if ok { | |||
result := []string{} | |||
seen := map[string]string{} | |||
for _, val := range s[key] { | |||
if _, ok := seen[val]; !ok { | |||
result = append(result, val) | |||
seen[val] = val | |||
} | |||
} | |||
return result | |||
} | |||
return []string{} | |||
} | |||
func (s directorySet) AddAndGet(key string, value []string) []string { | |||
s.Add(key, value) | |||
return s.Get(key) | |||
} | |||
func (s directorySet) Filled(key string) bool { | |||
return len(s[key]) > 0 | |||
} |
@@ -6,98 +6,9 @@ | |||
package options | |||
import ( | |||
"fmt" | |||
"io" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/util" | |||
"code.gitea.io/gitea/modules/assetfs" | |||
) | |||
// Dir returns all files from custom directory or bindata. | |||
func Dir(name string) ([]string, error) { | |||
if directories.Filled(name) { | |||
return directories.Get(name), nil | |||
} | |||
result, err := listLocalDirIfExist([]string{setting.CustomPath}, "options", name) | |||
if err != nil { | |||
return nil, err | |||
} | |||
files, err := AssetDir(name) | |||
if err != nil { | |||
return []string{}, fmt.Errorf("unable to read embedded directory %q. %w", name, err) | |||
} | |||
result = append(result, files...) | |||
return directories.AddAndGet(name, result), nil | |||
} | |||
func AssetDir(dirName string) ([]string, error) { | |||
d, err := Assets.Open(dirName) | |||
if err != nil { | |||
return nil, err | |||
} | |||
defer d.Close() | |||
files, err := d.Readdir(-1) | |||
if err != nil { | |||
return nil, err | |||
} | |||
results := make([]string, 0, len(files)) | |||
for _, file := range files { | |||
results = append(results, file.Name()) | |||
} | |||
return results, nil | |||
} | |||
// fileFromOptionsDir is a helper to read files from custom path or bindata. | |||
func fileFromOptionsDir(elems ...string) ([]byte, error) { | |||
// only try custom dir, no static dir | |||
if data, err := readLocalFile([]string{setting.CustomPath}, "options", elems...); err == nil { | |||
return data, nil | |||
} | |||
f, err := Assets.Open(util.PathJoinRelX(elems...)) | |||
if err != nil { | |||
return nil, err | |||
} | |||
defer f.Close() | |||
return io.ReadAll(f) | |||
} | |||
func Asset(name string) ([]byte, error) { | |||
f, err := Assets.Open("/" + name) | |||
if err != nil { | |||
return nil, err | |||
} | |||
defer f.Close() | |||
return io.ReadAll(f) | |||
} | |||
func AssetNames() []string { | |||
realFS := Assets.(vfsgen۰FS) | |||
results := make([]string, 0, len(realFS)) | |||
for k := range realFS { | |||
results = append(results, k[1:]) | |||
} | |||
return results | |||
} | |||
func AssetIsDir(name string) (bool, error) { | |||
if f, err := Assets.Open("/" + name); err != nil { | |||
return false, err | |||
} else { | |||
defer f.Close() | |||
if fi, err := f.Stat(); err != nil { | |||
return false, err | |||
} else { | |||
return fi.IsDir(), nil | |||
} | |||
} | |||
} | |||
// IsDynamic will return false when using embedded data (-tags bindata) | |||
func IsDynamic() bool { | |||
return false | |||
func BuiltinAssets() *assetfs.Layer { | |||
return assetfs.Bindata("builtin(bindata)", Assets) | |||
} |
@@ -4,11 +4,15 @@ | |||
package public | |||
import ( | |||
"bytes" | |||
"io" | |||
"net/http" | |||
"os" | |||
"path/filepath" | |||
"path" | |||
"strings" | |||
"time" | |||
"code.gitea.io/gitea/modules/assetfs" | |||
"code.gitea.io/gitea/modules/container" | |||
"code.gitea.io/gitea/modules/httpcache" | |||
"code.gitea.io/gitea/modules/log" | |||
@@ -16,55 +20,31 @@ import ( | |||
"code.gitea.io/gitea/modules/util" | |||
) | |||
// Options represents the available options to configure the handler. | |||
type Options struct { | |||
Directory string | |||
Prefix string | |||
CorsHandler func(http.Handler) http.Handler | |||
func CustomAssets() *assetfs.Layer { | |||
return assetfs.Local("custom", setting.CustomPath, "public") | |||
} | |||
// AssetsURLPathPrefix is the path prefix for static asset files | |||
const AssetsURLPathPrefix = "/assets/" | |||
func AssetFS() *assetfs.LayeredFS { | |||
return assetfs.Layered(CustomAssets(), BuiltinAssets()) | |||
} | |||
// AssetsHandlerFunc implements the static handler for serving custom or original assets. | |||
func AssetsHandlerFunc(opts *Options) http.HandlerFunc { | |||
custPath := filepath.Join(setting.CustomPath, "public") | |||
if !filepath.IsAbs(custPath) { | |||
custPath = filepath.Join(setting.AppWorkPath, custPath) | |||
} | |||
if !filepath.IsAbs(opts.Directory) { | |||
opts.Directory = filepath.Join(setting.AppWorkPath, opts.Directory) | |||
} | |||
if !strings.HasSuffix(opts.Prefix, "/") { | |||
opts.Prefix += "/" | |||
} | |||
func AssetsHandlerFunc(prefix string) http.HandlerFunc { | |||
assetFS := AssetFS() | |||
prefix = strings.TrimSuffix(prefix, "/") + "/" | |||
return func(resp http.ResponseWriter, req *http.Request) { | |||
if req.Method != "GET" && req.Method != "HEAD" { | |||
resp.WriteHeader(http.StatusNotFound) | |||
subPath := req.URL.Path | |||
if !strings.HasPrefix(subPath, prefix) { | |||
return | |||
} | |||
subPath = strings.TrimPrefix(subPath, prefix) | |||
if opts.CorsHandler != nil { | |||
var corsSent bool | |||
opts.CorsHandler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { | |||
corsSent = true | |||
})).ServeHTTP(resp, req) | |||
// If CORS is not sent, the response must have been written by other handlers | |||
if !corsSent { | |||
return | |||
} | |||
} | |||
file := req.URL.Path[len(opts.Prefix):] | |||
// custom files | |||
if opts.handle(resp, req, http.Dir(custPath), file) { | |||
if req.Method != "GET" && req.Method != "HEAD" { | |||
resp.WriteHeader(http.StatusNotFound) | |||
return | |||
} | |||
// internal files | |||
if opts.handle(resp, req, fileSystem(opts.Directory), file) { | |||
if handleRequest(resp, req, assetFS, subPath) { | |||
return | |||
} | |||
@@ -85,13 +65,13 @@ func parseAcceptEncoding(val string) container.Set[string] { | |||
// setWellKnownContentType will set the Content-Type if the file is a well-known type. | |||
// See the comments of detectWellKnownMimeType | |||
func setWellKnownContentType(w http.ResponseWriter, file string) { | |||
mimeType := detectWellKnownMimeType(filepath.Ext(file)) | |||
mimeType := detectWellKnownMimeType(path.Ext(file)) | |||
if mimeType != "" { | |||
w.Header().Set("Content-Type", mimeType) | |||
} | |||
} | |||
func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool { | |||
func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool { | |||
// actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here | |||
f, err := fs.Open(util.PathJoinRelX(file)) | |||
if err != nil { | |||
@@ -121,8 +101,34 @@ func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.Fi | |||
return true | |||
} | |||
setWellKnownContentType(w, file) | |||
serveContent(w, req, fi, fi.ModTime(), f) | |||
return true | |||
} | |||
type GzipBytesProvider interface { | |||
GzipBytes() []byte | |||
} | |||
// serveContent serve http content | |||
func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { | |||
setWellKnownContentType(w, fi.Name()) | |||
encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) | |||
if encodings.Contains("gzip") { | |||
// try to provide gzip content directly from bindata (provided by vfsgen۰CompressedFileInfo) | |||
if compressed, ok := fi.(GzipBytesProvider); ok { | |||
rdGzip := bytes.NewReader(compressed.GzipBytes()) | |||
// all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name | |||
// then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data | |||
if w.Header().Get("Content-Type") == "" { | |||
w.Header().Set("Content-Type", "application/octet-stream") | |||
} | |||
w.Header().Set("Content-Encoding", "gzip") | |||
http.ServeContent(w, req, fi.Name(), modtime, rdGzip) | |||
return | |||
} | |||
} | |||
http.ServeContent(w, req, fi.Name(), modtime, content) | |||
return | |||
} |
@@ -6,17 +6,10 @@ | |||
package public | |||
import ( | |||
"io" | |||
"net/http" | |||
"os" | |||
"time" | |||
"code.gitea.io/gitea/modules/assetfs" | |||
"code.gitea.io/gitea/modules/setting" | |||
) | |||
func fileSystem(dir string) http.FileSystem { | |||
return http.Dir(dir) | |||
} | |||
// serveContent serve http content | |||
func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { | |||
http.ServeContent(w, req, fi.Name(), modtime, content) | |||
func BuiltinAssets() *assetfs.Layer { | |||
return assetfs.Local("builtin(static)", setting.StaticRootPath, "public") | |||
} |
@@ -6,75 +6,19 @@ | |||
package public | |||
import ( | |||
"bytes" | |||
"io" | |||
"net/http" | |||
"os" | |||
"path/filepath" | |||
"time" | |||
"code.gitea.io/gitea/modules/assetfs" | |||
"code.gitea.io/gitea/modules/timeutil" | |||
) | |||
var _ GzipBytesProvider = (*vfsgen۰CompressedFileInfo)(nil) | |||
// GlobalModTime provide a global mod time for embedded asset files | |||
func GlobalModTime(filename string) time.Time { | |||
return timeutil.GetExecutableModTime() | |||
} | |||
func fileSystem(dir string) http.FileSystem { | |||
return Assets | |||
} | |||
func Asset(name string) ([]byte, error) { | |||
f, err := Assets.Open("/" + name) | |||
if err != nil { | |||
return nil, err | |||
} | |||
defer f.Close() | |||
return io.ReadAll(f) | |||
} | |||
func AssetNames() []string { | |||
realFS := Assets.(vfsgen۰FS) | |||
results := make([]string, 0, len(realFS)) | |||
for k := range realFS { | |||
results = append(results, k[1:]) | |||
} | |||
return results | |||
} | |||
func AssetIsDir(name string) (bool, error) { | |||
if f, err := Assets.Open("/" + name); err != nil { | |||
return false, err | |||
} else { | |||
defer f.Close() | |||
if fi, err := f.Stat(); err != nil { | |||
return false, err | |||
} else { | |||
return fi.IsDir(), nil | |||
} | |||
} | |||
} | |||
// serveContent serve http content | |||
func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { | |||
encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) | |||
if encodings.Contains("gzip") { | |||
if cf, ok := fi.(*vfsgen۰CompressedFileInfo); ok { | |||
rdGzip := bytes.NewReader(cf.GzipBytes()) | |||
// all static files are managed by Gitea, so we can make sure every file has the correct ext name | |||
// then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data | |||
mimeType := detectWellKnownMimeType(filepath.Ext(fi.Name())) | |||
if mimeType == "" { | |||
mimeType = "application/octet-stream" | |||
} | |||
w.Header().Set("Content-Type", mimeType) | |||
w.Header().Set("Content-Encoding", "gzip") | |||
http.ServeContent(w, req, fi.Name(), modtime, rdGzip) | |||
return | |||
} | |||
} | |||
http.ServeContent(w, req, fi.Name(), modtime, content) | |||
return | |||
func BuiltinAssets() *assetfs.Layer { | |||
return assetfs.Bindata("builtin(bindata)", Assets) | |||
} |
@@ -79,7 +79,7 @@ func LoadRepoConfig() error { | |||
typeFiles := make([]optionFileList, len(types)) | |||
for i, t := range types { | |||
var err error | |||
if typeFiles[i].all, err = options.Dir(t); err != nil { | |||
if typeFiles[i].all, err = options.AssetFS().ListFiles(t, true); err != nil { | |||
return fmt.Errorf("failed to list %s files: %w", t, err) | |||
} | |||
sort.Strings(typeFiles[i].all) |
@@ -0,0 +1,8 @@ | |||
// Copyright 2023 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
//go:build !bindata | |||
package setting | |||
const HasBuiltinBindata = false |
@@ -0,0 +1,8 @@ | |||
// Copyright 2023 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
//go:build bindata | |||
package setting | |||
const HasBuiltinBindata = true |
@@ -1,30 +0,0 @@ | |||
// Copyright 2020 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
//go:build bindata | |||
package svg | |||
import ( | |||
"path/filepath" | |||
"code.gitea.io/gitea/modules/public" | |||
) | |||
// Discover returns a map of discovered SVG icons in bindata | |||
func Discover() map[string]string { | |||
svgs := make(map[string]string) | |||
for _, file := range public.AssetNames() { | |||
matched, _ := filepath.Match("img/svg/*.svg", file) | |||
if matched { | |||
content, err := public.Asset(file) | |||
if err == nil { | |||
filename := filepath.Base(file) | |||
svgs[filename[:len(filename)-4]] = string(content) | |||
} | |||
} | |||
} | |||
return svgs | |||
} |
@@ -1,29 +0,0 @@ | |||
// Copyright 2020 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
//go:build !bindata | |||
package svg | |||
import ( | |||
"os" | |||
"path/filepath" | |||
"code.gitea.io/gitea/modules/setting" | |||
) | |||
// Discover returns a map of discovered SVG icons in the file system | |||
func Discover() map[string]string { | |||
svgs := make(map[string]string) | |||
files, _ := filepath.Glob(filepath.Join(setting.StaticRootPath, "public", "img", "svg", "*.svg")) | |||
for _, file := range files { | |||
content, err := os.ReadFile(file) | |||
if err == nil { | |||
filename := filepath.Base(file) | |||
svgs[filename[:len(filename)-4]] = string(content) | |||
} | |||
} | |||
return svgs | |||
} |
@@ -6,15 +6,18 @@ package svg | |||
import ( | |||
"fmt" | |||
"html/template" | |||
"path" | |||
"regexp" | |||
"strings" | |||
"code.gitea.io/gitea/modules/html" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/public" | |||
) | |||
var ( | |||
// SVGs contains discovered SVGs | |||
SVGs map[string]string | |||
SVGs = map[string]string{} | |||
widthRe = regexp.MustCompile(`width="[0-9]+?"`) | |||
heightRe = regexp.MustCompile(`height="[0-9]+?"`) | |||
@@ -23,17 +26,29 @@ var ( | |||
const defaultSize = 16 | |||
// Init discovers SVGs and populates the `SVGs` variable | |||
func Init() { | |||
SVGs = Discover() | |||
func Init() error { | |||
files, err := public.AssetFS().ListFiles("img/svg") | |||
if err != nil { | |||
return err | |||
} | |||
// Remove `xmlns` because inline SVG does not need it | |||
r := regexp.MustCompile(`(<svg\b[^>]*?)\s+xmlns="[^"]*"`) | |||
for name, svg := range SVGs { | |||
SVGs[name] = r.ReplaceAllString(svg, "$1") | |||
reXmlns := regexp.MustCompile(`(<svg\b[^>]*?)\s+xmlns="[^"]*"`) | |||
for _, file := range files { | |||
if path.Ext(file) != ".svg" { | |||
continue | |||
} | |||
bs, err := public.AssetFS().ReadFile("img/svg", file) | |||
if err != nil { | |||
log.Error("Failed to read SVG file %s: %v", file, err) | |||
} else { | |||
SVGs[file[:len(file)-4]] = reXmlns.ReplaceAllString(string(bs), "$1") | |||
} | |||
} | |||
return nil | |||
} | |||
// Render render icons - arguments icon name (string), size (int), class (string) | |||
// RenderHTML renders icons - arguments icon name (string), size (int), class (string) | |||
func RenderHTML(icon string, others ...interface{}) template.HTML { | |||
size, class := html.ParseSizeAndClass(defaultSize, "", others...) | |||
@@ -4,14 +4,10 @@ | |||
package templates | |||
import ( | |||
"fmt" | |||
"io/fs" | |||
"os" | |||
"path/filepath" | |||
"strings" | |||
"time" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/assetfs" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/util" | |||
) | |||
@@ -47,81 +43,30 @@ func BaseVars() Vars { | |||
} | |||
} | |||
func getDirTemplateAssetNames(dir string) []string { | |||
return getDirAssetNames(dir, false) | |||
func AssetFS() *assetfs.LayeredFS { | |||
return assetfs.Layered(CustomAssets(), BuiltinAssets()) | |||
} | |||
func getDirAssetNames(dir string, mailer bool) []string { | |||
var tmpls []string | |||
if mailer { | |||
dir += filepath.Join(dir, "mail") | |||
} | |||
f, err := os.Stat(dir) | |||
if err != nil { | |||
if os.IsNotExist(err) { | |||
return tmpls | |||
} | |||
log.Warn("Unable to check if templates dir %s is a directory. Error: %v", dir, err) | |||
return tmpls | |||
} | |||
if !f.IsDir() { | |||
log.Warn("Templates dir %s is a not directory.", dir) | |||
return tmpls | |||
} | |||
func CustomAssets() *assetfs.Layer { | |||
return assetfs.Local("custom", setting.CustomPath, "templates") | |||
} | |||
files, err := util.StatDir(dir) | |||
func ListWebTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) { | |||
files, err := assets.ListAllFiles(".", true) | |||
if err != nil { | |||
log.Warn("Failed to read %s templates dir. %v", dir, err) | |||
return tmpls | |||
return nil, err | |||
} | |||
prefix := "templates/" | |||
if mailer { | |||
prefix += "mail/" | |||
} | |||
for _, filePath := range files { | |||
if !mailer && strings.HasPrefix(filePath, "mail/") { | |||
continue | |||
} | |||
if !strings.HasSuffix(filePath, ".tmpl") { | |||
continue | |||
} | |||
tmpls = append(tmpls, prefix+filePath) | |||
} | |||
return tmpls | |||
return util.SliceRemoveAllFunc(files, func(file string) bool { | |||
return strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl") | |||
}), nil | |||
} | |||
func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error { | |||
mailRoot := filepath.Join(root, "mail") | |||
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { | |||
name := path[len(root):] | |||
if len(name) > 0 && name[0] == '/' { | |||
name = name[1:] | |||
} | |||
if err != nil { | |||
if os.IsNotExist(err) { | |||
return callback(path, name, d, err) | |||
} | |||
return err | |||
} | |||
if skipMail && path == mailRoot && d.IsDir() { | |||
return fs.SkipDir | |||
} | |||
if util.CommonSkip(d.Name()) { | |||
if d.IsDir() { | |||
return fs.SkipDir | |||
} | |||
return nil | |||
} | |||
if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() { | |||
return callback(path, name, d, err) | |||
} | |||
return nil | |||
}); err != nil && !os.IsNotExist(err) { | |||
return fmt.Errorf("unable to get files for template assets in %s: %w", root, err) | |||
func ListMailTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) { | |||
files, err := assets.ListAllFiles(".", true) | |||
if err != nil { | |||
return nil, err | |||
} | |||
return nil | |||
return util.SliceRemoveAllFunc(files, func(file string) bool { | |||
return !strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl") | |||
}), nil | |||
} |
@@ -6,76 +6,10 @@ | |||
package templates | |||
import ( | |||
"io/fs" | |||
"os" | |||
"path/filepath" | |||
"code.gitea.io/gitea/modules/assetfs" | |||
"code.gitea.io/gitea/modules/setting" | |||
) | |||
// GetAsset returns asset content via name | |||
func GetAsset(name string) ([]byte, error) { | |||
bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name)) | |||
if err != nil && !os.IsNotExist(err) { | |||
return nil, err | |||
} else if err == nil { | |||
return bs, nil | |||
} | |||
return os.ReadFile(filepath.Join(setting.StaticRootPath, name)) | |||
} | |||
// GetAssetFilename returns the filename of the provided asset | |||
func GetAssetFilename(name string) (string, error) { | |||
filename := filepath.Join(setting.CustomPath, name) | |||
_, err := os.Stat(filename) | |||
if err != nil && !os.IsNotExist(err) { | |||
return filename, err | |||
} else if err == nil { | |||
return filename, nil | |||
} | |||
filename = filepath.Join(setting.StaticRootPath, name) | |||
_, err = os.Stat(filename) | |||
return filename, err | |||
} | |||
// walkTemplateFiles calls a callback for each template asset | |||
func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error { | |||
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { | |||
return err | |||
} | |||
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { | |||
return err | |||
} | |||
return nil | |||
} | |||
// GetTemplateAssetNames returns list of template names | |||
func GetTemplateAssetNames() []string { | |||
tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates")) | |||
tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates")) | |||
return append(tmpls, tmpls2...) | |||
} | |||
func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error { | |||
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { | |||
return err | |||
} | |||
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { | |||
return err | |||
} | |||
return nil | |||
} | |||
// BuiltinAsset will read the provided asset from the embedded assets | |||
// (This always returns os.ErrNotExist) | |||
func BuiltinAsset(name string) ([]byte, error) { | |||
return nil, os.ErrNotExist | |||
} | |||
// BuiltinAssetNames returns the names of the embedded assets | |||
// (This always returns nil) | |||
func BuiltinAssetNames() []string { | |||
return nil | |||
func BuiltinAssets() *assetfs.Layer { | |||
return assetfs.Local("builtin(static)", setting.StaticRootPath, "templates") | |||
} |
@@ -21,7 +21,6 @@ import ( | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/util" | |||
"code.gitea.io/gitea/modules/watcher" | |||
) | |||
var ( | |||
@@ -66,20 +65,23 @@ func (h *HTMLRender) TemplateLookup(name string) (*template.Template, error) { | |||
} | |||
func (h *HTMLRender) CompileTemplates() error { | |||
dirPrefix := "templates/" | |||
extSuffix := ".tmpl" | |||
tmpls := template.New("") | |||
for _, path := range GetTemplateAssetNames() { | |||
if !strings.HasSuffix(path, extSuffix) { | |||
assets := AssetFS() | |||
files, err := ListWebTemplateAssetNames(assets) | |||
if err != nil { | |||
return nil | |||
} | |||
for _, file := range files { | |||
if !strings.HasSuffix(file, extSuffix) { | |||
continue | |||
} | |||
name := strings.TrimPrefix(path, dirPrefix) | |||
name = strings.TrimSuffix(name, extSuffix) | |||
name := strings.TrimSuffix(file, extSuffix) | |||
tmpl := tmpls.New(filepath.ToSlash(name)) | |||
for _, fm := range NewFuncMap() { | |||
tmpl.Funcs(fm) | |||
} | |||
buf, err := GetAsset(path) | |||
buf, err := assets.ReadFile(file) | |||
if err != nil { | |||
return err | |||
} | |||
@@ -112,13 +114,10 @@ func HTMLRenderer(ctx context.Context) (context.Context, *HTMLRender) { | |||
log.Fatal("HTMLRenderer error: %v", err) | |||
} | |||
if !setting.IsProd { | |||
watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{ | |||
PathsCallback: walkTemplateFiles, | |||
BetweenCallback: func() { | |||
if err := renderer.CompileTemplates(); err != nil { | |||
log.Error("Template error: %v\n%s", err, log.Stack(2)) | |||
} | |||
}, | |||
go AssetFS().WatchLocalChanges(ctx, func() { | |||
if err := renderer.CompileTemplates(); err != nil { | |||
log.Error("Template error: %v\n%s", err, log.Stack(2)) | |||
} | |||
}) | |||
} | |||
return context.WithValue(ctx, rendererKey, renderer), renderer | |||
@@ -138,14 +137,8 @@ func handleGenericTemplateError(err error) (string, []interface{}) { | |||
} | |||
templateName, lineNumberStr, message := groups[1], groups[2], groups[3] | |||
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") | |||
if assetErr != nil { | |||
return "", nil | |||
} | |||
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) | |||
lineNumber, _ := strconv.Atoi(lineNumberStr) | |||
line := GetLineFromTemplate(templateName, lineNumber, "", -1) | |||
return "PANIC: Unable to compile templates!\n%s in template file %s at line %d:\n\n%s\nStacktrace:\n\n%s", []interface{}{message, filename, lineNumber, log.NewColoredValue(line, log.Reset), log.Stack(2)} | |||
@@ -158,16 +151,9 @@ func handleNotDefinedPanicError(err error) (string, []interface{}) { | |||
} | |||
templateName, lineNumberStr, functionName := groups[1], groups[2], groups[3] | |||
functionName, _ = strconv.Unquote(`"` + functionName + `"`) | |||
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") | |||
if assetErr != nil { | |||
return "", nil | |||
} | |||
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) | |||
lineNumber, _ := strconv.Atoi(lineNumberStr) | |||
line := GetLineFromTemplate(templateName, lineNumber, functionName, -1) | |||
return "PANIC: Unable to compile templates!\nUndefined function %q in template file %s at line %d:\n\n%s", []interface{}{functionName, filename, lineNumber, log.NewColoredValue(line, log.Reset)} | |||
@@ -181,14 +167,8 @@ func handleUnexpected(err error) (string, []interface{}) { | |||
templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3] | |||
unexpected, _ = strconv.Unquote(`"` + unexpected + `"`) | |||
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") | |||
if assetErr != nil { | |||
return "", nil | |||
} | |||
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) | |||
lineNumber, _ := strconv.Atoi(lineNumberStr) | |||
line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1) | |||
return "PANIC: Unable to compile templates!\nUnexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)} | |||
@@ -201,14 +181,8 @@ func handleExpectedEnd(err error) (string, []interface{}) { | |||
} | |||
templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3] | |||
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl") | |||
if assetErr != nil { | |||
return "", nil | |||
} | |||
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl")) | |||
lineNumber, _ := strconv.Atoi(lineNumberStr) | |||
line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1) | |||
return "PANIC: Unable to compile templates!\nMissing end with unexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)} | |||
@@ -218,7 +192,7 @@ const dashSeparator = "--------------------------------------------------------- | |||
// GetLineFromTemplate returns a line from a template with some context | |||
func GetLineFromTemplate(templateName string, targetLineNum int, target string, position int) string { | |||
bs, err := GetAsset("templates/" + templateName + ".tmpl") | |||
bs, err := AssetFS().ReadFile(templateName + ".tmpl") | |||
if err != nil { | |||
return fmt.Sprintf("(unable to read template file: %v)", err) | |||
} |
@@ -6,15 +6,12 @@ package templates | |||
import ( | |||
"context" | |||
"html/template" | |||
"io/fs" | |||
"os" | |||
"strings" | |||
texttmpl "text/template" | |||
"code.gitea.io/gitea/modules/base" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/watcher" | |||
) | |||
// mailSubjectTextFuncMap returns functions for injecting to text templates, it's only used for mail subject | |||
@@ -62,54 +59,23 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { | |||
bodyTemplates.Funcs(funcs) | |||
} | |||
assetFS := AssetFS() | |||
refreshTemplates := func() { | |||
for _, assetPath := range BuiltinAssetNames() { | |||
if !strings.HasPrefix(assetPath, "mail/") { | |||
continue | |||
} | |||
if !strings.HasSuffix(assetPath, ".tmpl") { | |||
continue | |||
} | |||
content, err := BuiltinAsset(assetPath) | |||
if err != nil { | |||
log.Warn("Failed to read embedded %s template. %v", assetPath, err) | |||
continue | |||
} | |||
assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") | |||
log.Trace("Adding built-in mailer template for %s", assetName) | |||
buildSubjectBodyTemplate(subjectTemplates, | |||
bodyTemplates, | |||
assetName, | |||
content) | |||
assetPaths, err := ListMailTemplateAssetNames(assetFS) | |||
if err != nil { | |||
log.Error("Failed to list mail templates: %v", err) | |||
return | |||
} | |||
if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error { | |||
if err != nil { | |||
return err | |||
} | |||
if d.IsDir() { | |||
return nil | |||
} | |||
content, err := os.ReadFile(path) | |||
for _, assetPath := range assetPaths { | |||
content, layerName, err := assetFS.ReadLayeredFile(assetPath) | |||
if err != nil { | |||
log.Warn("Failed to read custom %s template. %v", path, err) | |||
return nil | |||
log.Warn("Failed to read mail template %s by %s: %v", assetPath, layerName, err) | |||
continue | |||
} | |||
assetName := strings.TrimSuffix(name, ".tmpl") | |||
log.Trace("Adding mailer template for %s from %q", assetName, path) | |||
buildSubjectBodyTemplate(subjectTemplates, | |||
bodyTemplates, | |||
assetName, | |||
content) | |||
return nil | |||
}); err != nil && !os.IsNotExist(err) { | |||
log.Warn("Error whilst walking mailer templates directories. %v", err) | |||
tmplName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") | |||
log.Trace("Adding mail template %s: %s by %s", tmplName, assetPath, layerName) | |||
buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content) | |||
} | |||
} | |||
@@ -118,10 +84,7 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { | |||
if !setting.IsProd { | |||
// Now subjectTemplates and bodyTemplates are both synchronized | |||
// thus it is safe to call refresh from a different goroutine | |||
watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{ | |||
PathsCallback: walkMailerTemplates, | |||
BetweenCallback: refreshTemplates, | |||
}) | |||
go assetFS.WatchLocalChanges(ctx, refreshTemplates) | |||
} | |||
return subjectTemplates, bodyTemplates |
@@ -6,114 +6,17 @@ | |||
package templates | |||
import ( | |||
"html/template" | |||
"io" | |||
"io/fs" | |||
"os" | |||
"path" | |||
"path/filepath" | |||
"strings" | |||
texttmpl "text/template" | |||
"time" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/assetfs" | |||
"code.gitea.io/gitea/modules/timeutil" | |||
) | |||
var ( | |||
subjectTemplates = texttmpl.New("") | |||
bodyTemplates = template.New("") | |||
) | |||
// GlobalModTime provide a global mod time for embedded asset files | |||
func GlobalModTime(filename string) time.Time { | |||
return timeutil.GetExecutableModTime() | |||
} | |||
// GetAssetFilename returns the filename of the provided asset | |||
func GetAssetFilename(name string) (string, error) { | |||
filename := filepath.Join(setting.CustomPath, name) | |||
_, err := os.Stat(filename) | |||
if err != nil && !os.IsNotExist(err) { | |||
return name, err | |||
} else if err == nil { | |||
return filename, nil | |||
} | |||
return "(builtin) " + name, nil | |||
} | |||
// GetAsset get a special asset, only for chi | |||
func GetAsset(name string) ([]byte, error) { | |||
bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name)) | |||
if err != nil && !os.IsNotExist(err) { | |||
return nil, err | |||
} else if err == nil { | |||
return bs, nil | |||
} | |||
return BuiltinAsset(strings.TrimPrefix(name, "templates/")) | |||
} | |||
// GetFiles calls a callback for each template asset | |||
func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error { | |||
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) { | |||
return err | |||
} | |||
return nil | |||
} | |||
// GetTemplateAssetNames only for chi | |||
func GetTemplateAssetNames() []string { | |||
realFS := Assets.(vfsgen۰FS) | |||
tmpls := make([]string, 0, len(realFS)) | |||
for k := range realFS { | |||
if strings.HasPrefix(k, "/mail/") { | |||
continue | |||
} | |||
tmpls = append(tmpls, "templates/"+k[1:]) | |||
} | |||
customDir := path.Join(setting.CustomPath, "templates") | |||
customTmpls := getDirTemplateAssetNames(customDir) | |||
return append(tmpls, customTmpls...) | |||
} | |||
func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error { | |||
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) { | |||
return err | |||
} | |||
return nil | |||
} | |||
// BuiltinAsset reads the provided asset from the builtin embedded assets | |||
func BuiltinAsset(name string) ([]byte, error) { | |||
f, err := Assets.Open("/" + name) | |||
if err != nil { | |||
return nil, err | |||
} | |||
defer f.Close() | |||
return io.ReadAll(f) | |||
} | |||
// BuiltinAssetNames returns the names of the built-in embedded assets | |||
func BuiltinAssetNames() []string { | |||
realFS := Assets.(vfsgen۰FS) | |||
results := make([]string, 0, len(realFS)) | |||
for k := range realFS { | |||
results = append(results, k[1:]) | |||
} | |||
return results | |||
} | |||
// BuiltinAssetIsDir returns if a provided asset is a directory | |||
func BuiltinAssetIsDir(name string) (bool, error) { | |||
if f, err := Assets.Open("/" + name); err != nil { | |||
return false, err | |||
} else { | |||
defer f.Close() | |||
if fi, err := f.Stat(); err != nil { | |||
return false, err | |||
} else { | |||
return fi.IsDir(), nil | |||
} | |||
} | |||
func BuiltinAssets() *assetfs.Layer { | |||
return assetfs.Bindata("builtin(bindata)", Assets) | |||
} |
@@ -13,7 +13,6 @@ import ( | |||
"code.gitea.io/gitea/modules/options" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/translation/i18n" | |||
"code.gitea.io/gitea/modules/watcher" | |||
"golang.org/x/text/language" | |||
) | |||
@@ -58,7 +57,7 @@ func InitLocales(ctx context.Context) { | |||
refreshLocales := func() { | |||
i18n.ResetDefaultLocales() | |||
localeNames, err := options.Dir("locale") | |||
localeNames, err := options.AssetFS().ListFiles("locale", true) | |||
if err != nil { | |||
log.Fatal("Failed to list locale files: %v", err) | |||
} | |||
@@ -118,13 +117,10 @@ func InitLocales(ctx context.Context) { | |||
}) | |||
if !setting.IsProd { | |||
watcher.CreateWatcher(ctx, "Locales", &watcher.CreateWatcherOpts{ | |||
PathsCallback: options.WalkLocales, | |||
BetweenCallback: func() { | |||
lock.Lock() | |||
defer lock.Unlock() | |||
refreshLocales() | |||
}, | |||
go options.AssetFS().WatchLocalChanges(ctx, func() { | |||
lock.Lock() | |||
defer lock.Unlock() | |||
refreshLocales() | |||
}) | |||
} | |||
} |
@@ -74,29 +74,28 @@ const pathSeparator = string(os.PathSeparator) | |||
// | |||
// {`/foo`, ``, `bar`} => `/foo/bar` | |||
// {`/foo`, `..`, `bar`} => `/foo/bar` | |||
func FilePathJoinAbs(elem ...string) string { | |||
elems := make([]string, len(elem)) | |||
func FilePathJoinAbs(base string, sub ...string) string { | |||
elems := make([]string, 1, len(sub)+1) | |||
// POISX filesystem can have `\` in file names. Windows: `\` and `/` are both used for path separators | |||
// POSIX filesystem can have `\` in file names. Windows: `\` and `/` are both used for path separators | |||
// to keep the behavior consistent, we do not allow `\` in file names, replace all `\` with `/` | |||
if isOSWindows() { | |||
elems[0] = filepath.Clean(elem[0]) | |||
elems[0] = filepath.Clean(base) | |||
} else { | |||
elems[0] = filepath.Clean(strings.ReplaceAll(elem[0], "\\", pathSeparator)) | |||
elems[0] = filepath.Clean(strings.ReplaceAll(base, "\\", pathSeparator)) | |||
} | |||
if !filepath.IsAbs(elems[0]) { | |||
// This shouldn't happen. If there is really necessary to pass in relative path, return the full path with filepath.Abs() instead | |||
panic(fmt.Sprintf("FilePathJoinAbs: %q (for path %v) is not absolute, do not guess a relative path based on current working directory", elems[0], elems)) | |||
} | |||
for i := 1; i < len(elem); i++ { | |||
if elem[i] == "" { | |||
for _, s := range sub { | |||
if s == "" { | |||
continue | |||
} | |||
if isOSWindows() { | |||
elems[i] = filepath.Clean(pathSeparator + elem[i]) | |||
elems = append(elems, filepath.Clean(pathSeparator+s)) | |||
} else { | |||
elems[i] = filepath.Clean(pathSeparator + strings.ReplaceAll(elem[i], "\\", pathSeparator)) | |||
elems = append(elems, filepath.Clean(pathSeparator+strings.ReplaceAll(s, "\\", pathSeparator))) | |||
} | |||
} | |||
// the elems[0] must be an absolute path, just join them together |
@@ -207,6 +207,6 @@ func TestCleanPath(t *testing.T) { | |||
} | |||
} | |||
for _, c := range cases { | |||
assert.Equal(t, c.expected, FilePathJoinAbs(c.elems...), "case: %v", c.elems) | |||
assert.Equal(t, c.expected, FilePathJoinAbs(c.elems[0], c.elems[1:]...), "case: %v", c.elems) | |||
} | |||
} |
@@ -4,6 +4,7 @@ | |||
package util | |||
import ( | |||
"sync" | |||
"time" | |||
) | |||
@@ -18,3 +19,30 @@ func StopTimer(t *time.Timer) bool { | |||
} | |||
return stopped | |||
} | |||
func Debounce(d time.Duration) func(f func()) { | |||
type debouncer struct { | |||
mu sync.Mutex | |||
t *time.Timer | |||
} | |||
db := &debouncer{} | |||
return func(f func()) { | |||
db.mu.Lock() | |||
defer db.mu.Unlock() | |||
if db.t != nil { | |||
db.t.Stop() | |||
} | |||
var trigger *time.Timer | |||
trigger = time.AfterFunc(d, func() { | |||
db.mu.Lock() | |||
defer db.mu.Unlock() | |||
if trigger == db.t { | |||
f() | |||
db.t = nil | |||
} | |||
}) | |||
db.t = trigger | |||
} | |||
} |
@@ -0,0 +1,30 @@ | |||
// Copyright 2023 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
package util | |||
import ( | |||
"sync/atomic" | |||
"testing" | |||
"time" | |||
"github.com/stretchr/testify/assert" | |||
) | |||
func TestDebounce(t *testing.T) { | |||
var c int64 | |||
d := Debounce(50 * time.Millisecond) | |||
d(func() { atomic.AddInt64(&c, 1) }) | |||
assert.EqualValues(t, 0, atomic.LoadInt64(&c)) | |||
d(func() { atomic.AddInt64(&c, 1) }) | |||
d(func() { atomic.AddInt64(&c, 1) }) | |||
time.Sleep(100 * time.Millisecond) | |||
assert.EqualValues(t, 1, atomic.LoadInt64(&c)) | |||
d(func() { atomic.AddInt64(&c, 1) }) | |||
assert.EqualValues(t, 1, atomic.LoadInt64(&c)) | |||
d(func() { atomic.AddInt64(&c, 1) }) | |||
d(func() { atomic.AddInt64(&c, 1) }) | |||
d(func() { atomic.AddInt64(&c, 1) }) | |||
time.Sleep(100 * time.Millisecond) | |||
assert.EqualValues(t, 2, atomic.LoadInt64(&c)) | |||
} |
@@ -1,114 +0,0 @@ | |||
// Copyright 2022 The Gitea Authors. All rights reserved. | |||
// SPDX-License-Identifier: MIT | |||
package watcher | |||
import ( | |||
"context" | |||
"io/fs" | |||
"os" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/process" | |||
"github.com/fsnotify/fsnotify" | |||
) | |||
// CreateWatcherOpts are options to configure the watcher | |||
type CreateWatcherOpts struct { | |||
// PathsCallback is used to set the required paths to watch | |||
PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error | |||
// BeforeCallback is called before any files are watched | |||
BeforeCallback func() | |||
// Between Callback is called between after a watched event has occurred | |||
BetweenCallback func() | |||
// AfterCallback is called as this watcher ends | |||
AfterCallback func() | |||
} | |||
// CreateWatcher creates a watcher labelled with the provided description and running with the provided options. | |||
// The created watcher will create a subcontext from the provided ctx and register it with the process manager. | |||
func CreateWatcher(ctx context.Context, desc string, opts *CreateWatcherOpts) { | |||
go run(ctx, desc, opts) | |||
} | |||
func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { | |||
if opts.BeforeCallback != nil { | |||
opts.BeforeCallback() | |||
} | |||
if opts.AfterCallback != nil { | |||
defer opts.AfterCallback() | |||
} | |||
ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Watcher: "+desc, process.SystemProcessType, true) | |||
defer finished() | |||
log.Trace("Watcher loop starting for %s", desc) | |||
defer log.Trace("Watcher loop ended for %s", desc) | |||
watcher, err := fsnotify.NewWatcher() | |||
if err != nil { | |||
log.Error("Unable to create watcher for %s: %v", desc, err) | |||
return | |||
} | |||
if err := opts.PathsCallback(func(path, _ string, d fs.DirEntry, err error) error { | |||
if err != nil && !os.IsNotExist(err) { | |||
return err | |||
} | |||
log.Trace("Watcher: %s watching %q", desc, path) | |||
_ = watcher.Add(path) | |||
return nil | |||
}); err != nil { | |||
log.Error("Unable to create watcher for %s: %v", desc, err) | |||
_ = watcher.Close() | |||
return | |||
} | |||
// Note we don't call the BetweenCallback here | |||
for { | |||
select { | |||
case event, ok := <-watcher.Events: | |||
if !ok { | |||
_ = watcher.Close() | |||
return | |||
} | |||
log.Debug("Watched file for %s had event: %v", desc, event) | |||
case err, ok := <-watcher.Errors: | |||
if !ok { | |||
_ = watcher.Close() | |||
return | |||
} | |||
log.Error("Error whilst watching files for %s: %v", desc, err) | |||
case <-ctx.Done(): | |||
_ = watcher.Close() | |||
return | |||
} | |||
// Recreate the watcher - only call the BetweenCallback after the new watcher is set-up | |||
_ = watcher.Close() | |||
watcher, err = fsnotify.NewWatcher() | |||
if err != nil { | |||
log.Error("Unable to create watcher for %s: %v", desc, err) | |||
return | |||
} | |||
if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { | |||
if err != nil { | |||
return err | |||
} | |||
_ = watcher.Add(path) | |||
return nil | |||
}); err != nil { | |||
log.Error("Unable to create watcher for %s: %v", desc, err) | |||
_ = watcher.Close() | |||
return | |||
} | |||
// Inform our BetweenCallback that there has been an event | |||
if opts.BetweenCallback != nil { | |||
opts.BetweenCallback() | |||
} | |||
} | |||
} |
@@ -71,13 +71,6 @@ func mustInitCtx(ctx context.Context, fn func(ctx context.Context) error) { | |||
} | |||
} | |||
// InitGitServices init new services for git, this is also called in `contrib/pr/checkout.go` | |||
func InitGitServices() { | |||
setting.LoadSettings() | |||
mustInit(storage.Init) | |||
mustInit(repo_service.Init) | |||
} | |||
func syncAppConfForGit(ctx context.Context) error { | |||
runtimeState := new(system.RuntimeState) | |||
if err := system.AppState.Get(runtimeState); err != nil { | |||
@@ -172,7 +165,7 @@ func GlobalInitInstalled(ctx context.Context) { | |||
mustInit(ssh.Init) | |||
auth.Init() | |||
svg.Init() | |||
mustInit(svg.Init) | |||
actions_service.Init() | |||
@@ -8,7 +8,6 @@ import ( | |||
"fmt" | |||
"html" | |||
"net/http" | |||
"path" | |||
"code.gitea.io/gitea/modules/httpcache" | |||
"code.gitea.io/gitea/modules/log" | |||
@@ -89,10 +88,7 @@ func Routes(ctx goctx.Context) *web.Route { | |||
r.Use(middle) | |||
} | |||
r.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{ | |||
Directory: path.Join(setting.StaticRootPath, "public"), | |||
Prefix: public.AssetsURLPathPrefix, | |||
}), "InstallAssetsHandler")) | |||
r.Use(web.WrapWithPrefix("/assets/", public.AssetsHandlerFunc("/assets/"), "AssetsHandler")) | |||
r.Use(session.Sessioner(session.Options{ | |||
Provider: setting.SessionConfig.Provider, |
@@ -30,7 +30,7 @@ func PreloadSettings(ctx context.Context) bool { | |||
} | |||
setting.LoadSettingsForInstall() | |||
svg.Init() | |||
_ = svg.Init() | |||
} | |||
return !setting.InstallLock | |||
@@ -47,6 +47,5 @@ func reloadSettings(ctx context.Context) { | |||
} else { | |||
log.Fatal("ORM engine initialization failed: %v", err) | |||
} | |||
svg.Init() | |||
} | |||
} |
@@ -15,15 +15,16 @@ import ( | |||
// List all devtest templates, they will be used for e2e tests for the UI components | |||
func List(ctx *context.Context) { | |||
templateNames := templates.GetTemplateAssetNames() | |||
templateNames, err := templates.AssetFS().ListFiles("devtest", true) | |||
if err != nil { | |||
ctx.ServerError("AssetFS().ListFiles", err) | |||
return | |||
} | |||
var subNames []string | |||
const prefix = "templates/devtest/" | |||
for _, tmplName := range templateNames { | |||
if strings.HasPrefix(tmplName, prefix) { | |||
subName := strings.TrimSuffix(strings.TrimPrefix(tmplName, prefix), ".tmpl") | |||
if subName != "list" { | |||
subNames = append(subNames, subName) | |||
} | |||
subName := strings.TrimSuffix(tmplName, ".tmpl") | |||
if subName != "list" { | |||
subNames = append(subNames, subName) | |||
} | |||
} | |||
ctx.Data["SubNames"] = subNames |
@@ -103,11 +103,7 @@ func buildAuthGroup() *auth_service.Group { | |||
func Routes(ctx gocontext.Context) *web.Route { | |||
routes := web.NewRoute() | |||
routes.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{ | |||
Directory: path.Join(setting.StaticRootPath, "public"), | |||
Prefix: public.AssetsURLPathPrefix, | |||
CorsHandler: CorsHandler(), | |||
}), "AssetsHandler")) | |||
routes.Use(web.WrapWithPrefix("/assets/", web.Wrap(CorsHandler(), public.AssetsHandlerFunc("/assets/")), "AssetsHandler")) | |||
sessioner := session.Sessioner(session.Options{ | |||
Provider: setting.SessionConfig.Provider, |