Use a general approach to access custom/static/builtin assets (#24022)

The idea is to use a Layered Asset File-system (modules/assetfs/layered.go)

For example: when there are 2 layers: "custom", "builtin", when access
to asset "my/page.tmpl", the Layered Asset File-system will first try to
use "custom" assets, if not found, then use "builtin" assets.

This approach will hugely simplify a lot of code, make them testable.

Other changes:

* Simplify the AssetsHandlerFunc code
* Simplify the `gitea embedded` sub-command code

---------

Co-authored-by: Jason Song <i@wolfogre.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

5 files changed, 56 insertions, 337 deletions

diff --git a/modules/templates/base.go b/modules/templates/base.go
index e0f8350afb..e95ce31cfc 100644
--- a/modules/templates/base.go
+++ b/modules/templates/base.go
@@ -4,14 +4,10 @@
 package templates
 
 import (
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
 	"strings"
 	"time"
 
-	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/assetfs"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 )
@@ -47,81 +43,30 @@ func BaseVars() Vars {
 	}
 }
 
-func getDirTemplateAssetNames(dir string) []string {
-	return getDirAssetNames(dir, false)
+func AssetFS() *assetfs.LayeredFS {
+	return assetfs.Layered(CustomAssets(), BuiltinAssets())
 }
 
-func getDirAssetNames(dir string, mailer bool) []string {
-	var tmpls []string
-
-	if mailer {
-		dir += filepath.Join(dir, "mail")
-	}
-	f, err := os.Stat(dir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return tmpls
-		}
-		log.Warn("Unable to check if templates dir %s is a directory. Error: %v", dir, err)
-		return tmpls
-	}
-	if !f.IsDir() {
-		log.Warn("Templates dir %s is a not directory.", dir)
-		return tmpls
-	}
+func CustomAssets() *assetfs.Layer {
+	return assetfs.Local("custom", setting.CustomPath, "templates")
+}
 
-	files, err := util.StatDir(dir)
+func ListWebTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) {
+	files, err := assets.ListAllFiles(".", true)
 	if err != nil {
-		log.Warn("Failed to read %s templates dir. %v", dir, err)
-		return tmpls
+		return nil, err
 	}
-
-	prefix := "templates/"
-	if mailer {
-		prefix += "mail/"
-	}
-	for _, filePath := range files {
-		if !mailer && strings.HasPrefix(filePath, "mail/") {
-			continue
-		}
-
-		if !strings.HasSuffix(filePath, ".tmpl") {
-			continue
-		}
-
-		tmpls = append(tmpls, prefix+filePath)
-	}
-	return tmpls
+	return util.SliceRemoveAllFunc(files, func(file string) bool {
+		return strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl")
+	}), nil
 }
 
-func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error {
-	mailRoot := filepath.Join(root, "mail")
-	if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
-		name := path[len(root):]
-		if len(name) > 0 && name[0] == '/' {
-			name = name[1:]
-		}
-		if err != nil {
-			if os.IsNotExist(err) {
-				return callback(path, name, d, err)
-			}
-			return err
-		}
-		if skipMail && path == mailRoot && d.IsDir() {
-			return fs.SkipDir
-		}
-		if util.CommonSkip(d.Name()) {
-			if d.IsDir() {
-				return fs.SkipDir
-			}
-			return nil
-		}
-		if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() {
-			return callback(path, name, d, err)
-		}
-		return nil
-	}); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("unable to get files for template assets in %s: %w", root, err)
+func ListMailTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) {
+	files, err := assets.ListAllFiles(".", true)
+	if err != nil {
+		return nil, err
 	}
-	return nil
+	return util.SliceRemoveAllFunc(files, func(file string) bool {
+		return !strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl")
+	}), nil
 }
diff --git a/modules/templates/dynamic.go b/modules/templates/dynamic.go
index 2f4f542e72..e1babd83c9 100644
--- a/modules/templates/dynamic.go
+++ b/modules/templates/dynamic.go
@@ -6,76 +6,10 @@
 package templates
 
 import (
-	"io/fs"
-	"os"
-	"path/filepath"
-
+	"code.gitea.io/gitea/modules/assetfs"
 	"code.gitea.io/gitea/modules/setting"
 )
 
-// GetAsset returns asset content via name
-func GetAsset(name string) ([]byte, error) {
-	bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name))
-	if err != nil && !os.IsNotExist(err) {
-		return nil, err
-	} else if err == nil {
-		return bs, nil
-	}
-
-	return os.ReadFile(filepath.Join(setting.StaticRootPath, name))
-}
-
-// GetAssetFilename returns the filename of the provided asset
-func GetAssetFilename(name string) (string, error) {
-	filename := filepath.Join(setting.CustomPath, name)
-	_, err := os.Stat(filename)
-	if err != nil && !os.IsNotExist(err) {
-		return filename, err
-	} else if err == nil {
-		return filename, nil
-	}
-
-	filename = filepath.Join(setting.StaticRootPath, name)
-	_, err = os.Stat(filename)
-	return filename, err
-}
-
-// walkTemplateFiles calls a callback for each template asset
-func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
-	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	return nil
-}
-
-// GetTemplateAssetNames returns list of template names
-func GetTemplateAssetNames() []string {
-	tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates"))
-	tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates"))
-	return append(tmpls, tmpls2...)
-}
-
-func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
-	if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	return nil
-}
-
-// BuiltinAsset will read the provided asset from the embedded assets
-// (This always returns os.ErrNotExist)
-func BuiltinAsset(name string) ([]byte, error) {
-	return nil, os.ErrNotExist
-}
-
-// BuiltinAssetNames returns the names of the embedded assets
-// (This always returns nil)
-func BuiltinAssetNames() []string {
-	return nil
+func BuiltinAssets() *assetfs.Layer {
+	return assetfs.Local("builtin(static)", setting.StaticRootPath, "templates")
 }
diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go
index 4e7b09a9ec..26dd365e4c 100644
--- a/modules/templates/htmlrenderer.go
+++ b/modules/templates/htmlrenderer.go
@@ -21,7 +21,6 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
-	"code.gitea.io/gitea/modules/watcher"
 )
 
 var (
@@ -66,20 +65,23 @@ func (h *HTMLRender) TemplateLookup(name string) (*template.Template, error) {
 }
 
 func (h *HTMLRender) CompileTemplates() error {
-	dirPrefix := "templates/"
 	extSuffix := ".tmpl"
 	tmpls := template.New("")
-	for _, path := range GetTemplateAssetNames() {
-		if !strings.HasSuffix(path, extSuffix) {
+	assets := AssetFS()
+	files, err := ListWebTemplateAssetNames(assets)
+	if err != nil {
+		return nil
+	}
+	for _, file := range files {
+		if !strings.HasSuffix(file, extSuffix) {
 			continue
 		}
-		name := strings.TrimPrefix(path, dirPrefix)
-		name = strings.TrimSuffix(name, extSuffix)
+		name := strings.TrimSuffix(file, extSuffix)
 		tmpl := tmpls.New(filepath.ToSlash(name))
 		for _, fm := range NewFuncMap() {
 			tmpl.Funcs(fm)
 		}
-		buf, err := GetAsset(path)
+		buf, err := assets.ReadFile(file)
 		if err != nil {
 			return err
 		}
@@ -112,13 +114,10 @@ func HTMLRenderer(ctx context.Context) (context.Context, *HTMLRender) {
 		log.Fatal("HTMLRenderer error: %v", err)
 	}
 	if !setting.IsProd {
-		watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{
-			PathsCallback: walkTemplateFiles,
-			BetweenCallback: func() {
-				if err := renderer.CompileTemplates(); err != nil {
-					log.Error("Template error: %v\n%s", err, log.Stack(2))
-				}
-			},
+		go AssetFS().WatchLocalChanges(ctx, func() {
+			if err := renderer.CompileTemplates(); err != nil {
+				log.Error("Template error: %v\n%s", err, log.Stack(2))
+			}
 		})
 	}
 	return context.WithValue(ctx, rendererKey, renderer), renderer
@@ -138,14 +137,8 @@ func handleGenericTemplateError(err error) (string, []interface{}) {
 	}
 
 	templateName, lineNumberStr, message := groups[1], groups[2], groups[3]
-
-	filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
-	if assetErr != nil {
-		return "", nil
-	}
-
+	filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
 	lineNumber, _ := strconv.Atoi(lineNumberStr)
-
 	line := GetLineFromTemplate(templateName, lineNumber, "", -1)
 
 	return "PANIC: Unable to compile templates!\n%s in template file %s at line %d:\n\n%s\nStacktrace:\n\n%s", []interface{}{message, filename, lineNumber, log.NewColoredValue(line, log.Reset), log.Stack(2)}
@@ -158,16 +151,9 @@ func handleNotDefinedPanicError(err error) (string, []interface{}) {
 	}
 
 	templateName, lineNumberStr, functionName := groups[1], groups[2], groups[3]
-
 	functionName, _ = strconv.Unquote(`"` + functionName + `"`)
-
-	filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
-	if assetErr != nil {
-		return "", nil
-	}
-
+	filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
 	lineNumber, _ := strconv.Atoi(lineNumberStr)
-
 	line := GetLineFromTemplate(templateName, lineNumber, functionName, -1)
 
 	return "PANIC: Unable to compile templates!\nUndefined function %q in template file %s at line %d:\n\n%s", []interface{}{functionName, filename, lineNumber, log.NewColoredValue(line, log.Reset)}
@@ -181,14 +167,8 @@ func handleUnexpected(err error) (string, []interface{}) {
 
 	templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3]
 	unexpected, _ = strconv.Unquote(`"` + unexpected + `"`)
-
-	filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
-	if assetErr != nil {
-		return "", nil
-	}
-
+	filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
 	lineNumber, _ := strconv.Atoi(lineNumberStr)
-
 	line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1)
 
 	return "PANIC: Unable to compile templates!\nUnexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)}
@@ -201,14 +181,8 @@ func handleExpectedEnd(err error) (string, []interface{}) {
 	}
 
 	templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3]
-
-	filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
-	if assetErr != nil {
-		return "", nil
-	}
-
+	filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
 	lineNumber, _ := strconv.Atoi(lineNumberStr)
-
 	line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1)
 
 	return "PANIC: Unable to compile templates!\nMissing end with unexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)}
@@ -218,7 +192,7 @@ const dashSeparator = "---------------------------------------------------------
 
 // GetLineFromTemplate returns a line from a template with some context
 func GetLineFromTemplate(templateName string, targetLineNum int, target string, position int) string {
-	bs, err := GetAsset("templates/" + templateName + ".tmpl")
+	bs, err := AssetFS().ReadFile(templateName + ".tmpl")
 	if err != nil {
 		return fmt.Sprintf("(unable to read template file: %v)", err)
 	}
diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go
index d0c49e1025..280ac0e587 100644
--- a/modules/templates/mailer.go
+++ b/modules/templates/mailer.go
@@ -6,15 +6,12 @@ package templates
 import (
 	"context"
 	"html/template"
-	"io/fs"
-	"os"
 	"strings"
 	texttmpl "text/template"
 
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/watcher"
 )
 
 // mailSubjectTextFuncMap returns functions for injecting to text templates, it's only used for mail subject
@@ -62,54 +59,23 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) {
 		bodyTemplates.Funcs(funcs)
 	}
 
+	assetFS := AssetFS()
 	refreshTemplates := func() {
-		for _, assetPath := range BuiltinAssetNames() {
-			if !strings.HasPrefix(assetPath, "mail/") {
-				continue
-			}
-
-			if !strings.HasSuffix(assetPath, ".tmpl") {
-				continue
-			}
-
-			content, err := BuiltinAsset(assetPath)
-			if err != nil {
-				log.Warn("Failed to read embedded %s template. %v", assetPath, err)
-				continue
-			}
-
-			assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/")
-
-			log.Trace("Adding built-in mailer template for %s", assetName)
-			buildSubjectBodyTemplate(subjectTemplates,
-				bodyTemplates,
-				assetName,
-				content)
+		assetPaths, err := ListMailTemplateAssetNames(assetFS)
+		if err != nil {
+			log.Error("Failed to list mail templates: %v", err)
+			return
 		}
 
-		if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error {
-			if err != nil {
-				return err
-			}
-			if d.IsDir() {
-				return nil
-			}
-
-			content, err := os.ReadFile(path)
+		for _, assetPath := range assetPaths {
+			content, layerName, err := assetFS.ReadLayeredFile(assetPath)
 			if err != nil {
-				log.Warn("Failed to read custom %s template. %v", path, err)
-				return nil
+				log.Warn("Failed to read mail template %s by %s: %v", assetPath, layerName, err)
+				continue
 			}
-
-			assetName := strings.TrimSuffix(name, ".tmpl")
-			log.Trace("Adding mailer template for %s from %q", assetName, path)
-			buildSubjectBodyTemplate(subjectTemplates,
-				bodyTemplates,
-				assetName,
-				content)
-			return nil
-		}); err != nil && !os.IsNotExist(err) {
-			log.Warn("Error whilst walking mailer templates directories. %v", err)
+			tmplName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/")
+			log.Trace("Adding mail template %s: %s by %s", tmplName, assetPath, layerName)
+			buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content)
 		}
 	}
 
@@ -118,10 +84,7 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) {
 	if !setting.IsProd {
 		// Now subjectTemplates and bodyTemplates are both synchronized
 		// thus it is safe to call refresh from a different goroutine
-		watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{
-			PathsCallback:   walkMailerTemplates,
-			BetweenCallback: refreshTemplates,
-		})
+		go assetFS.WatchLocalChanges(ctx, refreshTemplates)
 	}
 
 	return subjectTemplates, bodyTemplates
diff --git a/modules/templates/static.go b/modules/templates/static.go
index 7ebb327ae6..b5a7e561ec 100644
--- a/modules/templates/static.go
+++ b/modules/templates/static.go
@@ -6,114 +6,17 @@
 package templates
 
 import (
-	"html/template"
-	"io"
-	"io/fs"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-	texttmpl "text/template"
 	"time"
 
-	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/assetfs"
 	"code.gitea.io/gitea/modules/timeutil"
 )
 
-var (
-	subjectTemplates = texttmpl.New("")
-	bodyTemplates    = template.New("")
-)
-
 // GlobalModTime provide a global mod time for embedded asset files
 func GlobalModTime(filename string) time.Time {
 	return timeutil.GetExecutableModTime()
 }
 
-// GetAssetFilename returns the filename of the provided asset
-func GetAssetFilename(name string) (string, error) {
-	filename := filepath.Join(setting.CustomPath, name)
-	_, err := os.Stat(filename)
-	if err != nil && !os.IsNotExist(err) {
-		return name, err
-	} else if err == nil {
-		return filename, nil
-	}
-	return "(builtin) " + name, nil
-}
-
-// GetAsset get a special asset, only for chi
-func GetAsset(name string) ([]byte, error) {
-	bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name))
-	if err != nil && !os.IsNotExist(err) {
-		return nil, err
-	} else if err == nil {
-		return bs, nil
-	}
-	return BuiltinAsset(strings.TrimPrefix(name, "templates/"))
-}
-
-// GetFiles calls a callback for each template asset
-func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
-	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	return nil
-}
-
-// GetTemplateAssetNames only for chi
-func GetTemplateAssetNames() []string {
-	realFS := Assets.(vfsgen۰FS)
-	tmpls := make([]string, 0, len(realFS))
-	for k := range realFS {
-		if strings.HasPrefix(k, "/mail/") {
-			continue
-		}
-		tmpls = append(tmpls, "templates/"+k[1:])
-	}
-
-	customDir := path.Join(setting.CustomPath, "templates")
-	customTmpls := getDirTemplateAssetNames(customDir)
-	return append(tmpls, customTmpls...)
-}
-
-func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
-	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	return nil
-}
-
-// BuiltinAsset reads the provided asset from the builtin embedded assets
-func BuiltinAsset(name string) ([]byte, error) {
-	f, err := Assets.Open("/" + name)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	return io.ReadAll(f)
-}
-
-// BuiltinAssetNames returns the names of the built-in embedded assets
-func BuiltinAssetNames() []string {
-	realFS := Assets.(vfsgen۰FS)
-	results := make([]string, 0, len(realFS))
-	for k := range realFS {
-		results = append(results, k[1:])
-	}
-	return results
-}
-
-// BuiltinAssetIsDir returns if a provided asset is a directory
-func BuiltinAssetIsDir(name string) (bool, error) {
-	if f, err := Assets.Open("/" + name); err != nil {
-		return false, err
-	} else {
-		defer f.Close()
-		if fi, err := f.Stat(); err != nil {
-			return false, err
-		} else {
-			return fi.IsDir(), nil
-		}
-	}
+func BuiltinAssets() *assetfs.Layer {
+	return assetfs.Bindata("builtin(bindata)", Assets)
 }